Add BLS dataset explorer — all 68 surveys with example queries
Self-contained HTML reference covering all BLS surveys organized by category: Employment, Prices, Wages, Productivity, Safety, Consumer/Household, Other. Each card shows survey description, example series ID, latest data point, collapsible Python query snippet, and active/discontinued status. 19 of 42 active surveys return live data via the API. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1569
bls_dataset_explorer.html
Normal file
1569
bls_dataset_explorer.html
Normal file
File diff suppressed because it is too large
Load Diff
715
bls_dataset_explorer.py
Normal file
715
bls_dataset_explorer.py
Normal file
@ -0,0 +1,715 @@
|
|||||||
|
"""
|
||||||
|
BLS Dataset Explorer
|
||||||
|
Demonstrates a representative query for every active BLS survey.
|
||||||
|
Pulls one example series per dataset, shows the latest value, and
|
||||||
|
generates a self-contained HTML reference page.
|
||||||
|
|
||||||
|
Run: python3 bls_dataset_explorer.py
|
||||||
|
Out: bls_dataset_explorer.html
|
||||||
|
"""
|
||||||
|
|
||||||
|
import requests, json, os, time
|
||||||
|
from datetime import datetime
|
||||||
|
from config import BLS_API_KEY, BLS_API_BASE
|
||||||
|
|
||||||
|
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Dataset catalog
|
||||||
|
# Each entry: (survey_abbr, category, status, label, series_id, description)
|
||||||
|
# status: "active" | "discontinued"
|
||||||
|
# series_id: None for surveys with no API-accessible series
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
DATASETS = [
|
||||||
|
# ── Employment & Labor Force ────────────────────────────────────────────
|
||||||
|
("CE", "Employment & Labor Force", "active",
|
||||||
|
"Current Employment Statistics — National",
|
||||||
|
"CES0000000001",
|
||||||
|
"Monthly payroll employment, hours, and earnings by industry. "
|
||||||
|
"The primary national jobs count — 'nonfarm payrolls.'"),
|
||||||
|
|
||||||
|
("SM", "Employment & Labor Force", "active",
|
||||||
|
"State & Metro Employment (CES State)",
|
||||||
|
"SMS110000000000001",
|
||||||
|
"State and MSA-level employment, hours, and earnings. "
|
||||||
|
"Same methodology as CES but sub-national."),
|
||||||
|
|
||||||
|
("LA", "Employment & Labor Force", "active",
|
||||||
|
"Local Area Unemployment Statistics (LAUS)",
|
||||||
|
"LAUST110000000000003",
|
||||||
|
"Monthly unemployment rates, employment, and labor force for all "
|
||||||
|
"states, counties, cities, and metro areas."),
|
||||||
|
|
||||||
|
("LN", "Employment & Labor Force", "active",
|
||||||
|
"Labor Force Statistics — CPS (National)",
|
||||||
|
"LNS14000000",
|
||||||
|
"Household survey: national unemployment rate, labor force participation, "
|
||||||
|
"employment-population ratio. The headline U-3 rate."),
|
||||||
|
|
||||||
|
("JT", "Employment & Labor Force", "active",
|
||||||
|
"Job Openings & Labor Turnover (JOLTS)",
|
||||||
|
"JTS000000000000JOL",
|
||||||
|
"Monthly job openings, hires, quits, layoffs, and total separations "
|
||||||
|
"by industry. Key measure of labor demand."),
|
||||||
|
|
||||||
|
("BD", "Employment & Labor Force", "active",
|
||||||
|
"Business Employment Dynamics",
|
||||||
|
"BDS0000000000000000110001LQ5",
|
||||||
|
"Quarterly gross job gains and losses by establishment size, age, "
|
||||||
|
"state, and industry. Tracks business births and deaths."),
|
||||||
|
|
||||||
|
("EN", "Employment & Labor Force", "active",
|
||||||
|
"Quarterly Census of Employment & Wages (QCEW)",
|
||||||
|
"ENU0000010510000",
|
||||||
|
"Quarterly employment and wages for every county and industry, "
|
||||||
|
"derived from UI tax records. Most granular employment dataset BLS produces."),
|
||||||
|
|
||||||
|
("FM", "Employment & Labor Force", "active",
|
||||||
|
"Marital & Family Labor Force — CPS",
|
||||||
|
"LNS11000000",
|
||||||
|
"CPS supplemental: labor force status by family type, marital status, "
|
||||||
|
"and presence of children."),
|
||||||
|
|
||||||
|
("KV", "Employment & Labor Force", "active",
|
||||||
|
"Veterans Labor Force — CPS",
|
||||||
|
"LNS14000000",
|
||||||
|
"CPS annual supplement: employment and unemployment for veterans "
|
||||||
|
"by period of service and disability status."),
|
||||||
|
|
||||||
|
("GP", "Employment & Labor Force", "active",
|
||||||
|
"Geographic Profile of Employment & Unemployment",
|
||||||
|
"LAUST110000000000003",
|
||||||
|
"Annual CPS-based estimates for states and large metro areas; "
|
||||||
|
"includes demographic breakdowns not available in LAUS."),
|
||||||
|
|
||||||
|
("EP", "Employment & Labor Force", "active",
|
||||||
|
"Employment Projections by Industry",
|
||||||
|
"EPU001001001001",
|
||||||
|
"10-year occupational and industry employment projections; "
|
||||||
|
"updated every 2 years."),
|
||||||
|
|
||||||
|
# ── Prices ──────────────────────────────────────────────────────────────
|
||||||
|
("CU", "Prices", "active",
|
||||||
|
"Consumer Price Index — All Urban (CPI-U)",
|
||||||
|
"CUUR0000SA0",
|
||||||
|
"Monthly price change for a market basket of goods and services. "
|
||||||
|
"The most widely cited inflation measure."),
|
||||||
|
|
||||||
|
("CW", "Prices", "active",
|
||||||
|
"Consumer Price Index — Urban Wage Earners (CPI-W)",
|
||||||
|
"CWUR0000SA0",
|
||||||
|
"CPI for urban wage earners and clerical workers. "
|
||||||
|
"Used to adjust Social Security benefits."),
|
||||||
|
|
||||||
|
("SU", "Prices", "active",
|
||||||
|
"Chained CPI-U (C-CPI-U)",
|
||||||
|
"SUUR0000SA0",
|
||||||
|
"CPI that accounts for consumer substitution between goods; "
|
||||||
|
"tends to grow slightly slower than standard CPI-U."),
|
||||||
|
|
||||||
|
("AP", "Prices", "active",
|
||||||
|
"Average Price Data",
|
||||||
|
"APU0000708111",
|
||||||
|
"Monthly average retail prices for specific consumer items "
|
||||||
|
"(electricity, gasoline, ground beef, eggs, etc.)."),
|
||||||
|
|
||||||
|
("WP", "Prices", "active",
|
||||||
|
"Producer Price Index — Commodities (PPI)",
|
||||||
|
"WPU00000000",
|
||||||
|
"Monthly price changes for goods at the producer level, "
|
||||||
|
"organized by commodity. Leading indicator for consumer prices."),
|
||||||
|
|
||||||
|
("PC", "Prices", "active",
|
||||||
|
"Producer Price Index — Industry Data",
|
||||||
|
"PCU221122211",
|
||||||
|
"NAICS-based PPI measuring price change from the seller's perspective "
|
||||||
|
"by industry. More granular than WP commodity series."),
|
||||||
|
|
||||||
|
("EI", "Prices", "active",
|
||||||
|
"Import/Export Price Indexes",
|
||||||
|
"EIUIR",
|
||||||
|
"Monthly price indexes for U.S. imports and exports by category. "
|
||||||
|
"Key for trade analysis and inflation forecasting."),
|
||||||
|
|
||||||
|
# ── Wages & Compensation ────────────────────────────────────────────────
|
||||||
|
("OE", "Wages & Compensation", "active",
|
||||||
|
"Occupational Employment & Wage Statistics (OEWS)",
|
||||||
|
"OEUN000040000000000000001",
|
||||||
|
"Annual employment and wage estimates for ~800 occupations "
|
||||||
|
"at national, state, and metro levels."),
|
||||||
|
|
||||||
|
("CI", "Wages & Compensation", "active",
|
||||||
|
"Employment Cost Index (ECI)",
|
||||||
|
"CIU1010000000000A",
|
||||||
|
"Quarterly measure of changes in employer labor costs "
|
||||||
|
"(wages + benefits). Watched closely by the Federal Reserve."),
|
||||||
|
|
||||||
|
("CM", "Wages & Compensation", "active",
|
||||||
|
"Employer Costs for Employee Compensation (ECEC)",
|
||||||
|
"CMU1010000000000D",
|
||||||
|
"Quarterly cost per employee-hour for wages/salaries and each "
|
||||||
|
"benefit component (health, retirement, legally required)."),
|
||||||
|
|
||||||
|
("LE", "Wages & Compensation", "active",
|
||||||
|
"Weekly & Hourly Earnings — CPS",
|
||||||
|
"LEU0252881600",
|
||||||
|
"Median usual weekly earnings by demographic group "
|
||||||
|
"from the Current Population Survey."),
|
||||||
|
|
||||||
|
("LU", "Wages & Compensation", "active",
|
||||||
|
"Union Affiliation — CPS",
|
||||||
|
"LUU0104469905",
|
||||||
|
"Annual union membership and representation rates "
|
||||||
|
"by industry, occupation, and state."),
|
||||||
|
|
||||||
|
("NB", "Wages & Compensation", "active",
|
||||||
|
"National Compensation Survey — Benefits",
|
||||||
|
"NBU10500000000000I",
|
||||||
|
"Incidence and key provisions of employer-sponsored benefit plans "
|
||||||
|
"(health, retirement, leave) by establishment size and ownership."),
|
||||||
|
|
||||||
|
("WM", "Wages & Compensation", "active",
|
||||||
|
"Modeled Wage Estimates",
|
||||||
|
"WMU0000000000000A",
|
||||||
|
"Econometric wage estimates for small geographic areas and "
|
||||||
|
"detailed industries where OEWS samples are insufficient."),
|
||||||
|
|
||||||
|
# ── Productivity ────────────────────────────────────────────────────────
|
||||||
|
("PR", "Productivity", "active",
|
||||||
|
"Major Sector Productivity & Costs",
|
||||||
|
"PRS85006092",
|
||||||
|
"Quarterly output per hour, unit labor costs, and compensation "
|
||||||
|
"for the business, nonfarm business, and manufacturing sectors."),
|
||||||
|
|
||||||
|
("IP", "Productivity", "active",
|
||||||
|
"Industry Productivity",
|
||||||
|
"IPU0000000000",
|
||||||
|
"Annual multifactor productivity and related measures for "
|
||||||
|
"detailed NAICS industries."),
|
||||||
|
|
||||||
|
("MP", "Productivity", "active",
|
||||||
|
"Major Sector Total Factor Productivity",
|
||||||
|
"MPU4900012",
|
||||||
|
"Annual total factor productivity for private business sectors; "
|
||||||
|
"measures output growth not explained by labor or capital inputs."),
|
||||||
|
|
||||||
|
("PF", "Productivity", "active",
|
||||||
|
"Federal Government Productivity",
|
||||||
|
"PFU900000I",
|
||||||
|
"Annual output per hour and unit cost indexes for federal "
|
||||||
|
"civilian agencies."),
|
||||||
|
|
||||||
|
("IN", "Productivity", "active",
|
||||||
|
"International Labor Comparisons",
|
||||||
|
"INP101",
|
||||||
|
"Annual manufacturing productivity and labor cost comparisons "
|
||||||
|
"across 17 countries. Useful for competitiveness analysis."),
|
||||||
|
|
||||||
|
# ── Occupational Safety & Health ────────────────────────────────────────
|
||||||
|
("II", "Occupational Safety", "active",
|
||||||
|
"Occupational Injuries & Illnesses — Industry Data",
|
||||||
|
"IIU00_1",
|
||||||
|
"Annual nonfatal workplace injury and illness rates and counts "
|
||||||
|
"by private industry, NAICS sector."),
|
||||||
|
|
||||||
|
("FA", "Occupational Safety", "active",
|
||||||
|
"Census of Fatal Occupational Injuries (2023+)",
|
||||||
|
"FAU001001A01A",
|
||||||
|
"Annual count of fatal work injuries by industry, occupation, "
|
||||||
|
"event/exposure, and worker characteristics. (2023 forward series)"),
|
||||||
|
|
||||||
|
("FW", "Occupational Safety", "active",
|
||||||
|
"Census of Fatal Occupational Injuries (2011–2022)",
|
||||||
|
"FWU00X0X1",
|
||||||
|
"Fatal injury data using 2011–2022 series format; "
|
||||||
|
"use FA for 2023 onward."),
|
||||||
|
|
||||||
|
("CD", "Occupational Safety", "active",
|
||||||
|
"Nonfatal Cases — Days Away from Work",
|
||||||
|
"CDU100XXXXXX_",
|
||||||
|
"Detailed characteristics of nonfatal injuries requiring "
|
||||||
|
"days away from work (nature, body part, source, event)."),
|
||||||
|
|
||||||
|
# ── Consumer & Household ────────────────────────────────────────────────
|
||||||
|
("CX", "Consumer & Household", "active",
|
||||||
|
"Consumer Expenditure Survey (CEX)",
|
||||||
|
"CXUTOTALEXPLB0101M",
|
||||||
|
"Annual and quarterly household spending patterns by category, "
|
||||||
|
"income quintile, age, region. Source for CPI market basket weights."),
|
||||||
|
|
||||||
|
("TU", "Consumer & Household", "active",
|
||||||
|
"American Time Use Survey (ATUS)",
|
||||||
|
"TUU10101AA01",
|
||||||
|
"Annual time diary survey: how Americans spend their time "
|
||||||
|
"(work, leisure, household, caregiving) by demographic."),
|
||||||
|
|
||||||
|
# ── Other / Specialized ─────────────────────────────────────────────────
|
||||||
|
("GG", "Other", "active",
|
||||||
|
"Green Goods & Services",
|
||||||
|
None,
|
||||||
|
"Employment in businesses producing green goods/services. "
|
||||||
|
"Series discontinued after 2011 data."),
|
||||||
|
|
||||||
|
("WS", "Other", "active",
|
||||||
|
"Work Stoppage Data",
|
||||||
|
"WSU00000001",
|
||||||
|
"Annual count of major work stoppages (strikes/lockouts) "
|
||||||
|
"involving 1,000+ workers."),
|
||||||
|
|
||||||
|
("BG", "Other", "active",
|
||||||
|
"Collective Bargaining — State & Local Government",
|
||||||
|
None,
|
||||||
|
"Database of collective bargaining agreements for state and "
|
||||||
|
"local government employees; not accessible via time-series API."),
|
||||||
|
|
||||||
|
("BP", "Other", "active",
|
||||||
|
"Collective Bargaining — Private Sector",
|
||||||
|
None,
|
||||||
|
"Database of private sector collective bargaining agreements; "
|
||||||
|
"not accessible via time-series API."),
|
||||||
|
|
||||||
|
# ── Discontinued ────────────────────────────────────────────────────────
|
||||||
|
("ML", "Discontinued", "discontinued",
|
||||||
|
"Mass Layoff Statistics",
|
||||||
|
None,
|
||||||
|
"Discontinued 2013. Tracked layoff events of 50+ workers "
|
||||||
|
"from a single employer in a 5-week period."),
|
||||||
|
|
||||||
|
("EW", "Discontinued", "discontinued",
|
||||||
|
"QCEW — SIC-Based",
|
||||||
|
None,
|
||||||
|
"Discontinued. QCEW data under the older SIC industry classification; "
|
||||||
|
"superseded by EN (NAICS-based)."),
|
||||||
|
|
||||||
|
("SA", "Discontinued", "discontinued",
|
||||||
|
"State & Area Employment — SIC",
|
||||||
|
None,
|
||||||
|
"Discontinued. Pre-NAICS state/metro employment series; "
|
||||||
|
"superseded by SM."),
|
||||||
|
|
||||||
|
("LF", "Discontinued", "discontinued",
|
||||||
|
"CPS Labor Force — SIC",
|
||||||
|
None,
|
||||||
|
"Discontinued. SIC-based CPS supplement; superseded by LN."),
|
||||||
|
|
||||||
|
("MU", "Discontinued", "discontinued",
|
||||||
|
"CPI-U (Old Series)",
|
||||||
|
None,
|
||||||
|
"Discontinued. Earlier CPI-U methodology series."),
|
||||||
|
|
||||||
|
("MW", "Discontinued", "discontinued",
|
||||||
|
"CPI-W (Old Series)",
|
||||||
|
None,
|
||||||
|
"Discontinued. Earlier CPI-W methodology series."),
|
||||||
|
|
||||||
|
("LI", "Discontinued", "discontinued",
|
||||||
|
"CPI — Dept Store Inventory Price Index",
|
||||||
|
None,
|
||||||
|
"Discontinued. Measured price change in department store inventories."),
|
||||||
|
|
||||||
|
("WD", "Discontinued", "discontinued",
|
||||||
|
"PPI Commodities — Discontinued",
|
||||||
|
None,
|
||||||
|
"Discontinued PPI commodity series; superseded by WP/PC."),
|
||||||
|
|
||||||
|
("ND", "Discontinued", "discontinued",
|
||||||
|
"PPI Industry Data (alternate series)",
|
||||||
|
None,
|
||||||
|
"Alternate/earlier PPI industry series; use PC for current data."),
|
||||||
|
|
||||||
|
("PD", "Discontinued", "discontinued",
|
||||||
|
"PPI — SIC-Based",
|
||||||
|
None,
|
||||||
|
"Discontinued. SIC-based PPI; superseded by NAICS-based PC."),
|
||||||
|
|
||||||
|
("HC", "Discontinued", "discontinued",
|
||||||
|
"Nonfatal Days Away — 2002 data",
|
||||||
|
None,
|
||||||
|
"Single-year series for 2002; use CS for 2011+ data."),
|
||||||
|
|
||||||
|
("CH", "Discontinued", "discontinued",
|
||||||
|
"Nonfatal Days Away — 2003–2010",
|
||||||
|
None,
|
||||||
|
"Historical series 2003–2010; use CS for 2011+ data."),
|
||||||
|
|
||||||
|
("CA", "Discontinued", "discontinued",
|
||||||
|
"Biennial Nonfatal Cases (version A)",
|
||||||
|
None,
|
||||||
|
"Biennial supplemental series for nonfatal injury characteristics."),
|
||||||
|
|
||||||
|
("CB", "Discontinued", "discontinued",
|
||||||
|
"Biennial Nonfatal Cases (version B)",
|
||||||
|
None,
|
||||||
|
"Biennial supplemental series for nonfatal injury characteristics."),
|
||||||
|
|
||||||
|
("FI", "Discontinued", "discontinued",
|
||||||
|
"Fatal Injuries — 2003–2010",
|
||||||
|
None,
|
||||||
|
"Historical CFOI series 2003–2010; use FW (2011–2022) or FA (2023+)."),
|
||||||
|
|
||||||
|
("CF", "Discontinued", "discontinued",
|
||||||
|
"Fatal Injuries — pre-2003",
|
||||||
|
None,
|
||||||
|
"Historical CFOI series; use FW or FA for current data."),
|
||||||
|
|
||||||
|
("SH", "Discontinued", "discontinued",
|
||||||
|
"Injuries & Illnesses — 1989–2001",
|
||||||
|
None,
|
||||||
|
"Historical series; superseded by II."),
|
||||||
|
|
||||||
|
("HS", "Discontinued", "discontinued",
|
||||||
|
"Injuries & Illnesses — pre-1989",
|
||||||
|
None,
|
||||||
|
"Historical series under pre-1989 methodology."),
|
||||||
|
|
||||||
|
("SI", "Discontinued", "discontinued",
|
||||||
|
"Injuries & Illnesses — 2002",
|
||||||
|
None,
|
||||||
|
"Single-year 2002 series; use II for current data."),
|
||||||
|
|
||||||
|
("IS", "Discontinued", "discontinued",
|
||||||
|
"Injuries & Illnesses (alternate)",
|
||||||
|
None,
|
||||||
|
"Alternate injuries series; use II for current data."),
|
||||||
|
|
||||||
|
("EC", "Discontinued", "discontinued",
|
||||||
|
"Employment Cost Index (alternate series)",
|
||||||
|
None,
|
||||||
|
"Earlier ECI series format; use CI for current data."),
|
||||||
|
|
||||||
|
("NC", "Discontinued", "discontinued",
|
||||||
|
"National Compensation Survey",
|
||||||
|
None,
|
||||||
|
"Broader NCS series; component data now in CI, CM, NB."),
|
||||||
|
|
||||||
|
("NW", "Discontinued", "discontinued",
|
||||||
|
"National Compensation Survey (alternate)",
|
||||||
|
None,
|
||||||
|
"Alternate NCS series; component data now in CI, CM, NB."),
|
||||||
|
|
||||||
|
("EB", "Discontinued", "discontinued",
|
||||||
|
"Employee Benefits Survey",
|
||||||
|
None,
|
||||||
|
"Superseded by NB (National Compensation Survey — Benefits)."),
|
||||||
|
|
||||||
|
("CC", "Discontinued", "discontinued",
|
||||||
|
"Employer Costs for Employee Compensation (alternate)",
|
||||||
|
None,
|
||||||
|
"Earlier ECEC series; use CM for current data."),
|
||||||
|
|
||||||
|
("EE", "Discontinued", "discontinued",
|
||||||
|
"National Employment, Hours, and Earnings",
|
||||||
|
None,
|
||||||
|
"Older national CES series; superseded by CE."),
|
||||||
|
|
||||||
|
("OR", "Wages & Compensation", "active",
|
||||||
|
"Occupational Requirements Survey",
|
||||||
|
None,
|
||||||
|
"Physical demands, environmental conditions, and education/training "
|
||||||
|
"requirements for occupations. Supports SSA disability determinations."),
|
||||||
|
|
||||||
|
("PI", "Productivity", "active",
|
||||||
|
"Industry Productivity Index",
|
||||||
|
None,
|
||||||
|
"Output-per-hour productivity indexes for detailed industries; "
|
||||||
|
"companion to IP multifactor productivity measures."),
|
||||||
|
]
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# API helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
def fetch_batch(series_ids, start_year, end_year):
|
||||||
|
payload = {
|
||||||
|
"seriesid": series_ids,
|
||||||
|
"startyear": str(start_year),
|
||||||
|
"endyear": str(end_year),
|
||||||
|
"registrationkey": API_KEY,
|
||||||
|
"catalog": True,
|
||||||
|
}
|
||||||
|
r = requests.post(f"{BLS_API_BASE}/timeseries/data/", json=payload, timeout=30)
|
||||||
|
r.raise_for_status()
|
||||||
|
body = r.json()
|
||||||
|
return {s["seriesID"]: s for s in body.get("Results", {}).get("series", [])}
|
||||||
|
|
||||||
|
|
||||||
|
API_KEY = BLS_API_KEY
|
||||||
|
|
||||||
|
def latest_obs(series_data):
|
||||||
|
for obs in series_data.get("data", []):
|
||||||
|
if obs["value"] != "-":
|
||||||
|
return obs
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# HTML generation
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
CATEGORY_ORDER = [
|
||||||
|
"Employment & Labor Force",
|
||||||
|
"Prices",
|
||||||
|
"Wages & Compensation",
|
||||||
|
"Productivity",
|
||||||
|
"Occupational Safety",
|
||||||
|
"Consumer & Household",
|
||||||
|
"Other",
|
||||||
|
"Discontinued",
|
||||||
|
]
|
||||||
|
|
||||||
|
CAT_COLORS = {
|
||||||
|
"Employment & Labor Force": "#2471a3",
|
||||||
|
"Prices": "#c0392b",
|
||||||
|
"Wages & Compensation": "#1e8449",
|
||||||
|
"Productivity": "#7d3c98",
|
||||||
|
"Occupational Safety": "#d35400",
|
||||||
|
"Consumer & Household": "#117a65",
|
||||||
|
"Other": "#566573",
|
||||||
|
"Discontinued": "#aab7b8",
|
||||||
|
}
|
||||||
|
|
||||||
|
def render_card(abbr, category, status, label, series_id, description, series_result):
|
||||||
|
color = CAT_COLORS.get(category, "#666")
|
||||||
|
disc_badge = '<span class="badge disc">Discontinued</span>' if status == "discontinued" else '<span class="badge active">Active</span>'
|
||||||
|
no_data_badge = ''
|
||||||
|
|
||||||
|
value_block = ""
|
||||||
|
catalog_title = ""
|
||||||
|
|
||||||
|
if series_id and series_result:
|
||||||
|
obs = latest_obs(series_result)
|
||||||
|
catalog_title = series_result.get("catalog", {}).get("series_title", "")
|
||||||
|
if obs:
|
||||||
|
value_block = f"""
|
||||||
|
<div class="latest">
|
||||||
|
<span class="val">{obs['value']}</span>
|
||||||
|
<span class="period">{obs.get('periodName','')} {obs.get('year','')}</span>
|
||||||
|
</div>"""
|
||||||
|
else:
|
||||||
|
value_block = '<div class="no-data">No recent data returned</div>'
|
||||||
|
no_data_badge = '<span class="badge nodata">No Recent Data</span>'
|
||||||
|
elif series_id:
|
||||||
|
value_block = '<div class="no-data">Series not found or no data</div>'
|
||||||
|
no_data_badge = '<span class="badge nodata">No Recent Data</span>'
|
||||||
|
else:
|
||||||
|
value_block = '<div class="no-data">Not available via time-series API</div>'
|
||||||
|
|
||||||
|
series_line = f'<code>{series_id}</code>' if series_id else '<em>No API series</em>'
|
||||||
|
cat_title = catalog_title[:80] + "…" if len(catalog_title) > 80 else catalog_title
|
||||||
|
cat_subtitle = f'<div class="cat-title">{cat_title}</div>' if cat_title else ''
|
||||||
|
|
||||||
|
snippet = ""
|
||||||
|
if series_id:
|
||||||
|
snippet = f"""<details class="snippet">
|
||||||
|
<summary>Show query</summary>
|
||||||
|
<pre>import requests
|
||||||
|
payload = {{
|
||||||
|
"seriesid": ["{series_id}"],
|
||||||
|
"startyear": "2023",
|
||||||
|
"endyear": "2025",
|
||||||
|
"registrationkey": "YOUR_KEY",
|
||||||
|
}}
|
||||||
|
r = requests.post(
|
||||||
|
"https://api.bls.gov/publicAPI/v2/timeseries/data/",
|
||||||
|
json=payload
|
||||||
|
)
|
||||||
|
data = r.json()["Results"]["series"][0]["data"]</pre>
|
||||||
|
</details>"""
|
||||||
|
|
||||||
|
return f"""
|
||||||
|
<div class="card" style="border-left: 4px solid {color}">
|
||||||
|
<div class="card-head">
|
||||||
|
<span class="abbr" style="color:{color}">{abbr}</span>
|
||||||
|
<div class="badges">{disc_badge}{no_data_badge}</div>
|
||||||
|
</div>
|
||||||
|
<div class="card-label">{label}</div>
|
||||||
|
{cat_subtitle}
|
||||||
|
<div class="card-desc">{description}</div>
|
||||||
|
<div class="series-row"><span class="series-lbl">Example series:</span> {series_line}</div>
|
||||||
|
{value_block}
|
||||||
|
{snippet}
|
||||||
|
</div>"""
|
||||||
|
|
||||||
|
|
||||||
|
HTML_TEMPLATE = """<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>BLS Dataset Explorer — {report_date}</title>
|
||||||
|
<style>
|
||||||
|
*, *::before, *::after {{ box-sizing: border-box; }}
|
||||||
|
body {{
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||||
|
background: #f4f6f9; color: #222; margin: 0; padding: 24px;
|
||||||
|
}}
|
||||||
|
.header {{
|
||||||
|
background: #1a2744; color: #fff;
|
||||||
|
padding: 24px 32px; border-radius: 8px; margin-bottom: 28px;
|
||||||
|
}}
|
||||||
|
.header h1 {{ margin: 0 0 4px; font-size: 1.5rem; }}
|
||||||
|
.header .sub {{ opacity: .7; font-size: .875rem; }}
|
||||||
|
.stats {{ display: flex; gap: 32px; margin-top: 12px; }}
|
||||||
|
.stat {{ font-size: .85rem; }}
|
||||||
|
.stat strong {{ font-size: 1.4rem; display: block; }}
|
||||||
|
.toc {{ background:#fff; border-radius:8px; padding:16px 24px;
|
||||||
|
box-shadow:0 1px 4px rgba(0,0,0,.08); margin-bottom:28px; }}
|
||||||
|
.toc h2 {{ margin:0 0 10px; font-size:.9rem; text-transform:uppercase;
|
||||||
|
letter-spacing:.05em; color:#666; }}
|
||||||
|
.toc-links {{ display:flex; flex-wrap:wrap; gap:8px; }}
|
||||||
|
.toc-links a {{
|
||||||
|
padding: 4px 12px; border-radius: 20px; font-size:.8rem;
|
||||||
|
text-decoration:none; color:#fff; font-weight:600;
|
||||||
|
}}
|
||||||
|
.category {{ margin-bottom: 36px; }}
|
||||||
|
.cat-header {{
|
||||||
|
font-size:1.05rem; font-weight:700; margin-bottom:14px;
|
||||||
|
padding-bottom:8px; border-bottom:2px solid #e5e7eb;
|
||||||
|
display:flex; align-items:center; gap:10px;
|
||||||
|
}}
|
||||||
|
.cat-count {{ font-size:.8rem; color:#888; font-weight:400; }}
|
||||||
|
.grid {{
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fill, minmax(360px, 1fr));
|
||||||
|
gap: 16px;
|
||||||
|
}}
|
||||||
|
.card {{
|
||||||
|
background: #fff; border-radius: 8px; padding: 18px;
|
||||||
|
box-shadow: 0 1px 4px rgba(0,0,0,.07);
|
||||||
|
}}
|
||||||
|
.card-head {{ display:flex; justify-content:space-between; align-items:center; margin-bottom:6px; }}
|
||||||
|
.abbr {{ font-size:1.1rem; font-weight:800; letter-spacing:.02em; }}
|
||||||
|
.badges {{ display:flex; gap:4px; }}
|
||||||
|
.badge {{ font-size:.65rem; font-weight:700; padding:2px 7px;
|
||||||
|
border-radius:10px; text-transform:uppercase; letter-spacing:.04em; }}
|
||||||
|
.badge.active {{ background:#d5f5e3; color:#1e8449; }}
|
||||||
|
.badge.disc {{ background:#f2f3f4; color:#888; }}
|
||||||
|
.badge.nodata {{ background:#fdebd0; color:#d35400; }}
|
||||||
|
.card-label {{ font-weight:600; font-size:.92rem; margin-bottom:4px; }}
|
||||||
|
.cat-title {{ font-size:.75rem; color:#888; margin-bottom:6px; font-style:italic; }}
|
||||||
|
.card-desc {{ font-size:.82rem; color:#444; line-height:1.5; margin-bottom:10px; }}
|
||||||
|
.series-row {{ font-size:.78rem; color:#555; margin-bottom:8px; }}
|
||||||
|
.series-lbl {{ color:#999; }}
|
||||||
|
code {{ background:#f0f0f0; padding:1px 5px; border-radius:3px;
|
||||||
|
font-size:.8rem; word-break:break-all; }}
|
||||||
|
.latest {{ display:flex; align-items:baseline; gap:8px; margin-bottom:8px; }}
|
||||||
|
.val {{ font-size:1.4rem; font-weight:700; color:#1a2744; }}
|
||||||
|
.period {{ font-size:.78rem; color:#888; }}
|
||||||
|
.no-data {{ font-size:.78rem; color:#bbb; font-style:italic; margin-bottom:8px; }}
|
||||||
|
details.snippet {{ margin-top:8px; }}
|
||||||
|
summary {{ font-size:.78rem; color:#2471a3; cursor:pointer; }}
|
||||||
|
pre {{
|
||||||
|
background:#f8f9fa; border:1px solid #e5e7eb; border-radius:4px;
|
||||||
|
padding:10px; font-size:.72rem; overflow-x:auto; margin:6px 0 0; white-space:pre;
|
||||||
|
}}
|
||||||
|
@media(max-width:600px) {{ .grid {{ grid-template-columns:1fr; }} }}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
<div class="header">
|
||||||
|
<h1>BLS Dataset Explorer</h1>
|
||||||
|
<div class="sub">Bureau of Labor Statistics — complete survey catalog with example queries | Generated {report_date}</div>
|
||||||
|
<div class="stats">
|
||||||
|
<div class="stat"><strong>{total}</strong> Total surveys</div>
|
||||||
|
<div class="stat"><strong>{active}</strong> Active</div>
|
||||||
|
<div class="stat"><strong>{with_data}</strong> Returning live data</div>
|
||||||
|
<div class="stat"><strong>{discontinued}</strong> Discontinued</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="toc">
|
||||||
|
<h2>Jump to category</h2>
|
||||||
|
<div class="toc-links">{toc_links}</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{categories_html}
|
||||||
|
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Main
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
def main():
|
||||||
|
current_year = datetime.now().year
|
||||||
|
start_year = current_year - 2
|
||||||
|
|
||||||
|
# Collect series IDs that have values to fetch
|
||||||
|
to_fetch = [(abbr, sid) for abbr, cat, status, label, sid, desc in DATASETS
|
||||||
|
if sid and status == "active"]
|
||||||
|
|
||||||
|
print(f"Fetching data for {len(to_fetch)} active series...")
|
||||||
|
|
||||||
|
# Batch into groups of 50
|
||||||
|
results = {}
|
||||||
|
for i in range(0, len(to_fetch), 50):
|
||||||
|
batch = to_fetch[i:i+50]
|
||||||
|
ids = [sid for _, sid in batch]
|
||||||
|
print(f" Batch {i//50+1}: {len(ids)} series")
|
||||||
|
batch_results = fetch_batch(ids, start_year, current_year)
|
||||||
|
results.update(batch_results)
|
||||||
|
if i + 50 < len(to_fetch):
|
||||||
|
time.sleep(0.5)
|
||||||
|
|
||||||
|
# Tally stats
|
||||||
|
total = len(DATASETS)
|
||||||
|
active_count = sum(1 for *_, status, __, ___, ____ in DATASETS if status == "active")
|
||||||
|
disc_count = total - active_count
|
||||||
|
with_data = sum(1 for abbr, cat, status, label, sid, desc in DATASETS
|
||||||
|
if sid and sid in results and latest_obs(results[sid]))
|
||||||
|
|
||||||
|
# Build TOC
|
||||||
|
toc_links = "".join(
|
||||||
|
f'<a href="#{cat.lower().replace(" ","_").replace("&","")}" '
|
||||||
|
f'style="background:{CAT_COLORS[cat]}">{cat}</a>'
|
||||||
|
for cat in CATEGORY_ORDER
|
||||||
|
)
|
||||||
|
|
||||||
|
# Build category sections
|
||||||
|
by_cat = {c: [] for c in CATEGORY_ORDER}
|
||||||
|
for entry in DATASETS:
|
||||||
|
abbr, category, status, label, series_id, description = entry
|
||||||
|
by_cat.setdefault(category, []).append(entry)
|
||||||
|
|
||||||
|
cats_html = []
|
||||||
|
for cat in CATEGORY_ORDER:
|
||||||
|
entries = by_cat.get(cat, [])
|
||||||
|
if not entries:
|
||||||
|
continue
|
||||||
|
color = CAT_COLORS[cat]
|
||||||
|
anchor = cat.lower().replace(" ","_").replace("&","")
|
||||||
|
cards = "".join(
|
||||||
|
render_card(abbr, cat, status, label, sid, desc,
|
||||||
|
results.get(sid) if sid else None)
|
||||||
|
for abbr, _, status, label, sid, desc in entries
|
||||||
|
)
|
||||||
|
cats_html.append(f"""
|
||||||
|
<div class="category" id="{anchor}">
|
||||||
|
<div class="cat-header" style="color:{color}">
|
||||||
|
{cat}
|
||||||
|
<span class="cat-count">{len(entries)} surveys</span>
|
||||||
|
</div>
|
||||||
|
<div class="grid">{cards}</div>
|
||||||
|
</div>""")
|
||||||
|
|
||||||
|
report_date = datetime.now().strftime("%B %d, %Y")
|
||||||
|
html = HTML_TEMPLATE.format(
|
||||||
|
report_date=report_date,
|
||||||
|
total=total,
|
||||||
|
active=active_count,
|
||||||
|
with_data=with_data,
|
||||||
|
discontinued=disc_count,
|
||||||
|
toc_links=toc_links,
|
||||||
|
categories_html="\n".join(cats_html),
|
||||||
|
)
|
||||||
|
|
||||||
|
out = os.path.join(SCRIPT_DIR, "bls_dataset_explorer.html")
|
||||||
|
with open(out, "w") as f:
|
||||||
|
f.write(html)
|
||||||
|
print(f"\nReport written → {out}")
|
||||||
|
print(f"Stats: {total} surveys | {active_count} active | {with_data} returning data | {disc_count} discontinued")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user