Dave Boyd
eb724aa7ff
Self-contained HTML reference covering all BLS surveys organized by category: Employment, Prices, Wages, Productivity, Safety, Consumer/Household, Other. Each card shows survey description, example series ID, latest data point, collapsible Python query snippet, and active/discontinued status. 19 of 42 active surveys return live data via the API. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
716 lines
26 KiB
Python
716 lines
26 KiB
Python
"""
|
||
BLS Dataset Explorer
|
||
Demonstrates a representative query for every active BLS survey.
|
||
Pulls one example series per dataset, shows the latest value, and
|
||
generates a self-contained HTML reference page.
|
||
|
||
Run: python3 bls_dataset_explorer.py
|
||
Out: bls_dataset_explorer.html
|
||
"""
|
||
|
||
import requests, json, os, time
|
||
from datetime import datetime
|
||
from config import BLS_API_KEY, BLS_API_BASE
|
||
|
||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Dataset catalog
|
||
# Each entry: (survey_abbr, category, status, label, series_id, description)
|
||
# status: "active" | "discontinued"
|
||
# series_id: None for surveys with no API-accessible series
|
||
# ---------------------------------------------------------------------------
|
||
DATASETS = [
|
||
# ── Employment & Labor Force ────────────────────────────────────────────
|
||
("CE", "Employment & Labor Force", "active",
|
||
"Current Employment Statistics — National",
|
||
"CES0000000001",
|
||
"Monthly payroll employment, hours, and earnings by industry. "
|
||
"The primary national jobs count — 'nonfarm payrolls.'"),
|
||
|
||
("SM", "Employment & Labor Force", "active",
|
||
"State & Metro Employment (CES State)",
|
||
"SMS110000000000001",
|
||
"State and MSA-level employment, hours, and earnings. "
|
||
"Same methodology as CES but sub-national."),
|
||
|
||
("LA", "Employment & Labor Force", "active",
|
||
"Local Area Unemployment Statistics (LAUS)",
|
||
"LAUST110000000000003",
|
||
"Monthly unemployment rates, employment, and labor force for all "
|
||
"states, counties, cities, and metro areas."),
|
||
|
||
("LN", "Employment & Labor Force", "active",
|
||
"Labor Force Statistics — CPS (National)",
|
||
"LNS14000000",
|
||
"Household survey: national unemployment rate, labor force participation, "
|
||
"employment-population ratio. The headline U-3 rate."),
|
||
|
||
("JT", "Employment & Labor Force", "active",
|
||
"Job Openings & Labor Turnover (JOLTS)",
|
||
"JTS000000000000JOL",
|
||
"Monthly job openings, hires, quits, layoffs, and total separations "
|
||
"by industry. Key measure of labor demand."),
|
||
|
||
("BD", "Employment & Labor Force", "active",
|
||
"Business Employment Dynamics",
|
||
"BDS0000000000000000110001LQ5",
|
||
"Quarterly gross job gains and losses by establishment size, age, "
|
||
"state, and industry. Tracks business births and deaths."),
|
||
|
||
("EN", "Employment & Labor Force", "active",
|
||
"Quarterly Census of Employment & Wages (QCEW)",
|
||
"ENU0000010510000",
|
||
"Quarterly employment and wages for every county and industry, "
|
||
"derived from UI tax records. Most granular employment dataset BLS produces."),
|
||
|
||
("FM", "Employment & Labor Force", "active",
|
||
"Marital & Family Labor Force — CPS",
|
||
"LNS11000000",
|
||
"CPS supplemental: labor force status by family type, marital status, "
|
||
"and presence of children."),
|
||
|
||
("KV", "Employment & Labor Force", "active",
|
||
"Veterans Labor Force — CPS",
|
||
"LNS14000000",
|
||
"CPS annual supplement: employment and unemployment for veterans "
|
||
"by period of service and disability status."),
|
||
|
||
("GP", "Employment & Labor Force", "active",
|
||
"Geographic Profile of Employment & Unemployment",
|
||
"LAUST110000000000003",
|
||
"Annual CPS-based estimates for states and large metro areas; "
|
||
"includes demographic breakdowns not available in LAUS."),
|
||
|
||
("EP", "Employment & Labor Force", "active",
|
||
"Employment Projections by Industry",
|
||
"EPU001001001001",
|
||
"10-year occupational and industry employment projections; "
|
||
"updated every 2 years."),
|
||
|
||
# ── Prices ──────────────────────────────────────────────────────────────
|
||
("CU", "Prices", "active",
|
||
"Consumer Price Index — All Urban (CPI-U)",
|
||
"CUUR0000SA0",
|
||
"Monthly price change for a market basket of goods and services. "
|
||
"The most widely cited inflation measure."),
|
||
|
||
("CW", "Prices", "active",
|
||
"Consumer Price Index — Urban Wage Earners (CPI-W)",
|
||
"CWUR0000SA0",
|
||
"CPI for urban wage earners and clerical workers. "
|
||
"Used to adjust Social Security benefits."),
|
||
|
||
("SU", "Prices", "active",
|
||
"Chained CPI-U (C-CPI-U)",
|
||
"SUUR0000SA0",
|
||
"CPI that accounts for consumer substitution between goods; "
|
||
"tends to grow slightly slower than standard CPI-U."),
|
||
|
||
("AP", "Prices", "active",
|
||
"Average Price Data",
|
||
"APU0000708111",
|
||
"Monthly average retail prices for specific consumer items "
|
||
"(electricity, gasoline, ground beef, eggs, etc.)."),
|
||
|
||
("WP", "Prices", "active",
|
||
"Producer Price Index — Commodities (PPI)",
|
||
"WPU00000000",
|
||
"Monthly price changes for goods at the producer level, "
|
||
"organized by commodity. Leading indicator for consumer prices."),
|
||
|
||
("PC", "Prices", "active",
|
||
"Producer Price Index — Industry Data",
|
||
"PCU221122211",
|
||
"NAICS-based PPI measuring price change from the seller's perspective "
|
||
"by industry. More granular than WP commodity series."),
|
||
|
||
("EI", "Prices", "active",
|
||
"Import/Export Price Indexes",
|
||
"EIUIR",
|
||
"Monthly price indexes for U.S. imports and exports by category. "
|
||
"Key for trade analysis and inflation forecasting."),
|
||
|
||
# ── Wages & Compensation ────────────────────────────────────────────────
|
||
("OE", "Wages & Compensation", "active",
|
||
"Occupational Employment & Wage Statistics (OEWS)",
|
||
"OEUN000040000000000000001",
|
||
"Annual employment and wage estimates for ~800 occupations "
|
||
"at national, state, and metro levels."),
|
||
|
||
("CI", "Wages & Compensation", "active",
|
||
"Employment Cost Index (ECI)",
|
||
"CIU1010000000000A",
|
||
"Quarterly measure of changes in employer labor costs "
|
||
"(wages + benefits). Watched closely by the Federal Reserve."),
|
||
|
||
("CM", "Wages & Compensation", "active",
|
||
"Employer Costs for Employee Compensation (ECEC)",
|
||
"CMU1010000000000D",
|
||
"Quarterly cost per employee-hour for wages/salaries and each "
|
||
"benefit component (health, retirement, legally required)."),
|
||
|
||
("LE", "Wages & Compensation", "active",
|
||
"Weekly & Hourly Earnings — CPS",
|
||
"LEU0252881600",
|
||
"Median usual weekly earnings by demographic group "
|
||
"from the Current Population Survey."),
|
||
|
||
("LU", "Wages & Compensation", "active",
|
||
"Union Affiliation — CPS",
|
||
"LUU0104469905",
|
||
"Annual union membership and representation rates "
|
||
"by industry, occupation, and state."),
|
||
|
||
("NB", "Wages & Compensation", "active",
|
||
"National Compensation Survey — Benefits",
|
||
"NBU10500000000000I",
|
||
"Incidence and key provisions of employer-sponsored benefit plans "
|
||
"(health, retirement, leave) by establishment size and ownership."),
|
||
|
||
("WM", "Wages & Compensation", "active",
|
||
"Modeled Wage Estimates",
|
||
"WMU0000000000000A",
|
||
"Econometric wage estimates for small geographic areas and "
|
||
"detailed industries where OEWS samples are insufficient."),
|
||
|
||
# ── Productivity ────────────────────────────────────────────────────────
|
||
("PR", "Productivity", "active",
|
||
"Major Sector Productivity & Costs",
|
||
"PRS85006092",
|
||
"Quarterly output per hour, unit labor costs, and compensation "
|
||
"for the business, nonfarm business, and manufacturing sectors."),
|
||
|
||
("IP", "Productivity", "active",
|
||
"Industry Productivity",
|
||
"IPU0000000000",
|
||
"Annual multifactor productivity and related measures for "
|
||
"detailed NAICS industries."),
|
||
|
||
("MP", "Productivity", "active",
|
||
"Major Sector Total Factor Productivity",
|
||
"MPU4900012",
|
||
"Annual total factor productivity for private business sectors; "
|
||
"measures output growth not explained by labor or capital inputs."),
|
||
|
||
("PF", "Productivity", "active",
|
||
"Federal Government Productivity",
|
||
"PFU900000I",
|
||
"Annual output per hour and unit cost indexes for federal "
|
||
"civilian agencies."),
|
||
|
||
("IN", "Productivity", "active",
|
||
"International Labor Comparisons",
|
||
"INP101",
|
||
"Annual manufacturing productivity and labor cost comparisons "
|
||
"across 17 countries. Useful for competitiveness analysis."),
|
||
|
||
# ── Occupational Safety & Health ────────────────────────────────────────
|
||
("II", "Occupational Safety", "active",
|
||
"Occupational Injuries & Illnesses — Industry Data",
|
||
"IIU00_1",
|
||
"Annual nonfatal workplace injury and illness rates and counts "
|
||
"by private industry, NAICS sector."),
|
||
|
||
("FA", "Occupational Safety", "active",
|
||
"Census of Fatal Occupational Injuries (2023+)",
|
||
"FAU001001A01A",
|
||
"Annual count of fatal work injuries by industry, occupation, "
|
||
"event/exposure, and worker characteristics. (2023 forward series)"),
|
||
|
||
("FW", "Occupational Safety", "active",
|
||
"Census of Fatal Occupational Injuries (2011–2022)",
|
||
"FWU00X0X1",
|
||
"Fatal injury data using 2011–2022 series format; "
|
||
"use FA for 2023 onward."),
|
||
|
||
("CD", "Occupational Safety", "active",
|
||
"Nonfatal Cases — Days Away from Work",
|
||
"CDU100XXXXXX_",
|
||
"Detailed characteristics of nonfatal injuries requiring "
|
||
"days away from work (nature, body part, source, event)."),
|
||
|
||
# ── Consumer & Household ────────────────────────────────────────────────
|
||
("CX", "Consumer & Household", "active",
|
||
"Consumer Expenditure Survey (CEX)",
|
||
"CXUTOTALEXPLB0101M",
|
||
"Annual and quarterly household spending patterns by category, "
|
||
"income quintile, age, region. Source for CPI market basket weights."),
|
||
|
||
("TU", "Consumer & Household", "active",
|
||
"American Time Use Survey (ATUS)",
|
||
"TUU10101AA01",
|
||
"Annual time diary survey: how Americans spend their time "
|
||
"(work, leisure, household, caregiving) by demographic."),
|
||
|
||
# ── Other / Specialized ─────────────────────────────────────────────────
|
||
("GG", "Other", "active",
|
||
"Green Goods & Services",
|
||
None,
|
||
"Employment in businesses producing green goods/services. "
|
||
"Series discontinued after 2011 data."),
|
||
|
||
("WS", "Other", "active",
|
||
"Work Stoppage Data",
|
||
"WSU00000001",
|
||
"Annual count of major work stoppages (strikes/lockouts) "
|
||
"involving 1,000+ workers."),
|
||
|
||
("BG", "Other", "active",
|
||
"Collective Bargaining — State & Local Government",
|
||
None,
|
||
"Database of collective bargaining agreements for state and "
|
||
"local government employees; not accessible via time-series API."),
|
||
|
||
("BP", "Other", "active",
|
||
"Collective Bargaining — Private Sector",
|
||
None,
|
||
"Database of private sector collective bargaining agreements; "
|
||
"not accessible via time-series API."),
|
||
|
||
# ── Discontinued ────────────────────────────────────────────────────────
|
||
("ML", "Discontinued", "discontinued",
|
||
"Mass Layoff Statistics",
|
||
None,
|
||
"Discontinued 2013. Tracked layoff events of 50+ workers "
|
||
"from a single employer in a 5-week period."),
|
||
|
||
("EW", "Discontinued", "discontinued",
|
||
"QCEW — SIC-Based",
|
||
None,
|
||
"Discontinued. QCEW data under the older SIC industry classification; "
|
||
"superseded by EN (NAICS-based)."),
|
||
|
||
("SA", "Discontinued", "discontinued",
|
||
"State & Area Employment — SIC",
|
||
None,
|
||
"Discontinued. Pre-NAICS state/metro employment series; "
|
||
"superseded by SM."),
|
||
|
||
("LF", "Discontinued", "discontinued",
|
||
"CPS Labor Force — SIC",
|
||
None,
|
||
"Discontinued. SIC-based CPS supplement; superseded by LN."),
|
||
|
||
("MU", "Discontinued", "discontinued",
|
||
"CPI-U (Old Series)",
|
||
None,
|
||
"Discontinued. Earlier CPI-U methodology series."),
|
||
|
||
("MW", "Discontinued", "discontinued",
|
||
"CPI-W (Old Series)",
|
||
None,
|
||
"Discontinued. Earlier CPI-W methodology series."),
|
||
|
||
("LI", "Discontinued", "discontinued",
|
||
"CPI — Dept Store Inventory Price Index",
|
||
None,
|
||
"Discontinued. Measured price change in department store inventories."),
|
||
|
||
("WD", "Discontinued", "discontinued",
|
||
"PPI Commodities — Discontinued",
|
||
None,
|
||
"Discontinued PPI commodity series; superseded by WP/PC."),
|
||
|
||
("ND", "Discontinued", "discontinued",
|
||
"PPI Industry Data (alternate series)",
|
||
None,
|
||
"Alternate/earlier PPI industry series; use PC for current data."),
|
||
|
||
("PD", "Discontinued", "discontinued",
|
||
"PPI — SIC-Based",
|
||
None,
|
||
"Discontinued. SIC-based PPI; superseded by NAICS-based PC."),
|
||
|
||
("HC", "Discontinued", "discontinued",
|
||
"Nonfatal Days Away — 2002 data",
|
||
None,
|
||
"Single-year series for 2002; use CS for 2011+ data."),
|
||
|
||
("CH", "Discontinued", "discontinued",
|
||
"Nonfatal Days Away — 2003–2010",
|
||
None,
|
||
"Historical series 2003–2010; use CS for 2011+ data."),
|
||
|
||
("CA", "Discontinued", "discontinued",
|
||
"Biennial Nonfatal Cases (version A)",
|
||
None,
|
||
"Biennial supplemental series for nonfatal injury characteristics."),
|
||
|
||
("CB", "Discontinued", "discontinued",
|
||
"Biennial Nonfatal Cases (version B)",
|
||
None,
|
||
"Biennial supplemental series for nonfatal injury characteristics."),
|
||
|
||
("FI", "Discontinued", "discontinued",
|
||
"Fatal Injuries — 2003–2010",
|
||
None,
|
||
"Historical CFOI series 2003–2010; use FW (2011–2022) or FA (2023+)."),
|
||
|
||
("CF", "Discontinued", "discontinued",
|
||
"Fatal Injuries — pre-2003",
|
||
None,
|
||
"Historical CFOI series; use FW or FA for current data."),
|
||
|
||
("SH", "Discontinued", "discontinued",
|
||
"Injuries & Illnesses — 1989–2001",
|
||
None,
|
||
"Historical series; superseded by II."),
|
||
|
||
("HS", "Discontinued", "discontinued",
|
||
"Injuries & Illnesses — pre-1989",
|
||
None,
|
||
"Historical series under pre-1989 methodology."),
|
||
|
||
("SI", "Discontinued", "discontinued",
|
||
"Injuries & Illnesses — 2002",
|
||
None,
|
||
"Single-year 2002 series; use II for current data."),
|
||
|
||
("IS", "Discontinued", "discontinued",
|
||
"Injuries & Illnesses (alternate)",
|
||
None,
|
||
"Alternate injuries series; use II for current data."),
|
||
|
||
("EC", "Discontinued", "discontinued",
|
||
"Employment Cost Index (alternate series)",
|
||
None,
|
||
"Earlier ECI series format; use CI for current data."),
|
||
|
||
("NC", "Discontinued", "discontinued",
|
||
"National Compensation Survey",
|
||
None,
|
||
"Broader NCS series; component data now in CI, CM, NB."),
|
||
|
||
("NW", "Discontinued", "discontinued",
|
||
"National Compensation Survey (alternate)",
|
||
None,
|
||
"Alternate NCS series; component data now in CI, CM, NB."),
|
||
|
||
("EB", "Discontinued", "discontinued",
|
||
"Employee Benefits Survey",
|
||
None,
|
||
"Superseded by NB (National Compensation Survey — Benefits)."),
|
||
|
||
("CC", "Discontinued", "discontinued",
|
||
"Employer Costs for Employee Compensation (alternate)",
|
||
None,
|
||
"Earlier ECEC series; use CM for current data."),
|
||
|
||
("EE", "Discontinued", "discontinued",
|
||
"National Employment, Hours, and Earnings",
|
||
None,
|
||
"Older national CES series; superseded by CE."),
|
||
|
||
("OR", "Wages & Compensation", "active",
|
||
"Occupational Requirements Survey",
|
||
None,
|
||
"Physical demands, environmental conditions, and education/training "
|
||
"requirements for occupations. Supports SSA disability determinations."),
|
||
|
||
("PI", "Productivity", "active",
|
||
"Industry Productivity Index",
|
||
None,
|
||
"Output-per-hour productivity indexes for detailed industries; "
|
||
"companion to IP multifactor productivity measures."),
|
||
]
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# API helpers
|
||
# ---------------------------------------------------------------------------
|
||
def fetch_batch(series_ids, start_year, end_year):
|
||
payload = {
|
||
"seriesid": series_ids,
|
||
"startyear": str(start_year),
|
||
"endyear": str(end_year),
|
||
"registrationkey": API_KEY,
|
||
"catalog": True,
|
||
}
|
||
r = requests.post(f"{BLS_API_BASE}/timeseries/data/", json=payload, timeout=30)
|
||
r.raise_for_status()
|
||
body = r.json()
|
||
return {s["seriesID"]: s for s in body.get("Results", {}).get("series", [])}
|
||
|
||
|
||
API_KEY = BLS_API_KEY
|
||
|
||
def latest_obs(series_data):
|
||
for obs in series_data.get("data", []):
|
||
if obs["value"] != "-":
|
||
return obs
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# HTML generation
|
||
# ---------------------------------------------------------------------------
|
||
CATEGORY_ORDER = [
|
||
"Employment & Labor Force",
|
||
"Prices",
|
||
"Wages & Compensation",
|
||
"Productivity",
|
||
"Occupational Safety",
|
||
"Consumer & Household",
|
||
"Other",
|
||
"Discontinued",
|
||
]
|
||
|
||
CAT_COLORS = {
|
||
"Employment & Labor Force": "#2471a3",
|
||
"Prices": "#c0392b",
|
||
"Wages & Compensation": "#1e8449",
|
||
"Productivity": "#7d3c98",
|
||
"Occupational Safety": "#d35400",
|
||
"Consumer & Household": "#117a65",
|
||
"Other": "#566573",
|
||
"Discontinued": "#aab7b8",
|
||
}
|
||
|
||
def render_card(abbr, category, status, label, series_id, description, series_result):
|
||
color = CAT_COLORS.get(category, "#666")
|
||
disc_badge = '<span class="badge disc">Discontinued</span>' if status == "discontinued" else '<span class="badge active">Active</span>'
|
||
no_data_badge = ''
|
||
|
||
value_block = ""
|
||
catalog_title = ""
|
||
|
||
if series_id and series_result:
|
||
obs = latest_obs(series_result)
|
||
catalog_title = series_result.get("catalog", {}).get("series_title", "")
|
||
if obs:
|
||
value_block = f"""
|
||
<div class="latest">
|
||
<span class="val">{obs['value']}</span>
|
||
<span class="period">{obs.get('periodName','')} {obs.get('year','')}</span>
|
||
</div>"""
|
||
else:
|
||
value_block = '<div class="no-data">No recent data returned</div>'
|
||
no_data_badge = '<span class="badge nodata">No Recent Data</span>'
|
||
elif series_id:
|
||
value_block = '<div class="no-data">Series not found or no data</div>'
|
||
no_data_badge = '<span class="badge nodata">No Recent Data</span>'
|
||
else:
|
||
value_block = '<div class="no-data">Not available via time-series API</div>'
|
||
|
||
series_line = f'<code>{series_id}</code>' if series_id else '<em>No API series</em>'
|
||
cat_title = catalog_title[:80] + "…" if len(catalog_title) > 80 else catalog_title
|
||
cat_subtitle = f'<div class="cat-title">{cat_title}</div>' if cat_title else ''
|
||
|
||
snippet = ""
|
||
if series_id:
|
||
snippet = f"""<details class="snippet">
|
||
<summary>Show query</summary>
|
||
<pre>import requests
|
||
payload = {{
|
||
"seriesid": ["{series_id}"],
|
||
"startyear": "2023",
|
||
"endyear": "2025",
|
||
"registrationkey": "YOUR_KEY",
|
||
}}
|
||
r = requests.post(
|
||
"https://api.bls.gov/publicAPI/v2/timeseries/data/",
|
||
json=payload
|
||
)
|
||
data = r.json()["Results"]["series"][0]["data"]</pre>
|
||
</details>"""
|
||
|
||
return f"""
|
||
<div class="card" style="border-left: 4px solid {color}">
|
||
<div class="card-head">
|
||
<span class="abbr" style="color:{color}">{abbr}</span>
|
||
<div class="badges">{disc_badge}{no_data_badge}</div>
|
||
</div>
|
||
<div class="card-label">{label}</div>
|
||
{cat_subtitle}
|
||
<div class="card-desc">{description}</div>
|
||
<div class="series-row"><span class="series-lbl">Example series:</span> {series_line}</div>
|
||
{value_block}
|
||
{snippet}
|
||
</div>"""
|
||
|
||
|
||
HTML_TEMPLATE = """<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<title>BLS Dataset Explorer — {report_date}</title>
|
||
<style>
|
||
*, *::before, *::after {{ box-sizing: border-box; }}
|
||
body {{
|
||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||
background: #f4f6f9; color: #222; margin: 0; padding: 24px;
|
||
}}
|
||
.header {{
|
||
background: #1a2744; color: #fff;
|
||
padding: 24px 32px; border-radius: 8px; margin-bottom: 28px;
|
||
}}
|
||
.header h1 {{ margin: 0 0 4px; font-size: 1.5rem; }}
|
||
.header .sub {{ opacity: .7; font-size: .875rem; }}
|
||
.stats {{ display: flex; gap: 32px; margin-top: 12px; }}
|
||
.stat {{ font-size: .85rem; }}
|
||
.stat strong {{ font-size: 1.4rem; display: block; }}
|
||
.toc {{ background:#fff; border-radius:8px; padding:16px 24px;
|
||
box-shadow:0 1px 4px rgba(0,0,0,.08); margin-bottom:28px; }}
|
||
.toc h2 {{ margin:0 0 10px; font-size:.9rem; text-transform:uppercase;
|
||
letter-spacing:.05em; color:#666; }}
|
||
.toc-links {{ display:flex; flex-wrap:wrap; gap:8px; }}
|
||
.toc-links a {{
|
||
padding: 4px 12px; border-radius: 20px; font-size:.8rem;
|
||
text-decoration:none; color:#fff; font-weight:600;
|
||
}}
|
||
.category {{ margin-bottom: 36px; }}
|
||
.cat-header {{
|
||
font-size:1.05rem; font-weight:700; margin-bottom:14px;
|
||
padding-bottom:8px; border-bottom:2px solid #e5e7eb;
|
||
display:flex; align-items:center; gap:10px;
|
||
}}
|
||
.cat-count {{ font-size:.8rem; color:#888; font-weight:400; }}
|
||
.grid {{
|
||
display: grid;
|
||
grid-template-columns: repeat(auto-fill, minmax(360px, 1fr));
|
||
gap: 16px;
|
||
}}
|
||
.card {{
|
||
background: #fff; border-radius: 8px; padding: 18px;
|
||
box-shadow: 0 1px 4px rgba(0,0,0,.07);
|
||
}}
|
||
.card-head {{ display:flex; justify-content:space-between; align-items:center; margin-bottom:6px; }}
|
||
.abbr {{ font-size:1.1rem; font-weight:800; letter-spacing:.02em; }}
|
||
.badges {{ display:flex; gap:4px; }}
|
||
.badge {{ font-size:.65rem; font-weight:700; padding:2px 7px;
|
||
border-radius:10px; text-transform:uppercase; letter-spacing:.04em; }}
|
||
.badge.active {{ background:#d5f5e3; color:#1e8449; }}
|
||
.badge.disc {{ background:#f2f3f4; color:#888; }}
|
||
.badge.nodata {{ background:#fdebd0; color:#d35400; }}
|
||
.card-label {{ font-weight:600; font-size:.92rem; margin-bottom:4px; }}
|
||
.cat-title {{ font-size:.75rem; color:#888; margin-bottom:6px; font-style:italic; }}
|
||
.card-desc {{ font-size:.82rem; color:#444; line-height:1.5; margin-bottom:10px; }}
|
||
.series-row {{ font-size:.78rem; color:#555; margin-bottom:8px; }}
|
||
.series-lbl {{ color:#999; }}
|
||
code {{ background:#f0f0f0; padding:1px 5px; border-radius:3px;
|
||
font-size:.8rem; word-break:break-all; }}
|
||
.latest {{ display:flex; align-items:baseline; gap:8px; margin-bottom:8px; }}
|
||
.val {{ font-size:1.4rem; font-weight:700; color:#1a2744; }}
|
||
.period {{ font-size:.78rem; color:#888; }}
|
||
.no-data {{ font-size:.78rem; color:#bbb; font-style:italic; margin-bottom:8px; }}
|
||
details.snippet {{ margin-top:8px; }}
|
||
summary {{ font-size:.78rem; color:#2471a3; cursor:pointer; }}
|
||
pre {{
|
||
background:#f8f9fa; border:1px solid #e5e7eb; border-radius:4px;
|
||
padding:10px; font-size:.72rem; overflow-x:auto; margin:6px 0 0; white-space:pre;
|
||
}}
|
||
@media(max-width:600px) {{ .grid {{ grid-template-columns:1fr; }} }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
|
||
<div class="header">
|
||
<h1>BLS Dataset Explorer</h1>
|
||
<div class="sub">Bureau of Labor Statistics — complete survey catalog with example queries | Generated {report_date}</div>
|
||
<div class="stats">
|
||
<div class="stat"><strong>{total}</strong> Total surveys</div>
|
||
<div class="stat"><strong>{active}</strong> Active</div>
|
||
<div class="stat"><strong>{with_data}</strong> Returning live data</div>
|
||
<div class="stat"><strong>{discontinued}</strong> Discontinued</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="toc">
|
||
<h2>Jump to category</h2>
|
||
<div class="toc-links">{toc_links}</div>
|
||
</div>
|
||
|
||
{categories_html}
|
||
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main
|
||
# ---------------------------------------------------------------------------
|
||
def main():
|
||
current_year = datetime.now().year
|
||
start_year = current_year - 2
|
||
|
||
# Collect series IDs that have values to fetch
|
||
to_fetch = [(abbr, sid) for abbr, cat, status, label, sid, desc in DATASETS
|
||
if sid and status == "active"]
|
||
|
||
print(f"Fetching data for {len(to_fetch)} active series...")
|
||
|
||
# Batch into groups of 50
|
||
results = {}
|
||
for i in range(0, len(to_fetch), 50):
|
||
batch = to_fetch[i:i+50]
|
||
ids = [sid for _, sid in batch]
|
||
print(f" Batch {i//50+1}: {len(ids)} series")
|
||
batch_results = fetch_batch(ids, start_year, current_year)
|
||
results.update(batch_results)
|
||
if i + 50 < len(to_fetch):
|
||
time.sleep(0.5)
|
||
|
||
# Tally stats
|
||
total = len(DATASETS)
|
||
active_count = sum(1 for *_, status, __, ___, ____ in DATASETS if status == "active")
|
||
disc_count = total - active_count
|
||
with_data = sum(1 for abbr, cat, status, label, sid, desc in DATASETS
|
||
if sid and sid in results and latest_obs(results[sid]))
|
||
|
||
# Build TOC
|
||
toc_links = "".join(
|
||
f'<a href="#{cat.lower().replace(" ","_").replace("&","")}" '
|
||
f'style="background:{CAT_COLORS[cat]}">{cat}</a>'
|
||
for cat in CATEGORY_ORDER
|
||
)
|
||
|
||
# Build category sections
|
||
by_cat = {c: [] for c in CATEGORY_ORDER}
|
||
for entry in DATASETS:
|
||
abbr, category, status, label, series_id, description = entry
|
||
by_cat.setdefault(category, []).append(entry)
|
||
|
||
cats_html = []
|
||
for cat in CATEGORY_ORDER:
|
||
entries = by_cat.get(cat, [])
|
||
if not entries:
|
||
continue
|
||
color = CAT_COLORS[cat]
|
||
anchor = cat.lower().replace(" ","_").replace("&","")
|
||
cards = "".join(
|
||
render_card(abbr, cat, status, label, sid, desc,
|
||
results.get(sid) if sid else None)
|
||
for abbr, _, status, label, sid, desc in entries
|
||
)
|
||
cats_html.append(f"""
|
||
<div class="category" id="{anchor}">
|
||
<div class="cat-header" style="color:{color}">
|
||
{cat}
|
||
<span class="cat-count">{len(entries)} surveys</span>
|
||
</div>
|
||
<div class="grid">{cards}</div>
|
||
</div>""")
|
||
|
||
report_date = datetime.now().strftime("%B %d, %Y")
|
||
html = HTML_TEMPLATE.format(
|
||
report_date=report_date,
|
||
total=total,
|
||
active=active_count,
|
||
with_data=with_data,
|
||
discontinued=disc_count,
|
||
toc_links=toc_links,
|
||
categories_html="\n".join(cats_html),
|
||
)
|
||
|
||
out = os.path.join(SCRIPT_DIR, "bls_dataset_explorer.html")
|
||
with open(out, "w") as f:
|
||
f.write(html)
|
||
print(f"\nReport written → {out}")
|
||
print(f"Stats: {total} surveys | {active_count} active | {with_data} returning data | {disc_count} discontinued")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|