Add BLS dataset explorer — all 68 surveys with example queries
Self-contained HTML reference covering all BLS surveys organized by category: Employment, Prices, Wages, Productivity, Safety, Consumer/Household, Other. Each card shows survey description, example series ID, latest data point, collapsible Python query snippet, and active/discontinued status. 19 of 42 active surveys return live data via the API. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1569
bls_dataset_explorer.html
Normal file
1569
bls_dataset_explorer.html
Normal file
File diff suppressed because it is too large
Load Diff
715
bls_dataset_explorer.py
Normal file
715
bls_dataset_explorer.py
Normal file
@ -0,0 +1,715 @@
|
||||
"""
|
||||
BLS Dataset Explorer
|
||||
Demonstrates a representative query for every active BLS survey.
|
||||
Pulls one example series per dataset, shows the latest value, and
|
||||
generates a self-contained HTML reference page.
|
||||
|
||||
Run: python3 bls_dataset_explorer.py
|
||||
Out: bls_dataset_explorer.html
|
||||
"""
|
||||
|
||||
import requests, json, os, time
|
||||
from datetime import datetime
|
||||
from config import BLS_API_KEY, BLS_API_BASE
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataset catalog
|
||||
# Each entry: (survey_abbr, category, status, label, series_id, description)
|
||||
# status: "active" | "discontinued"
|
||||
# series_id: None for surveys with no API-accessible series
|
||||
# ---------------------------------------------------------------------------
|
||||
DATASETS = [
|
||||
# ── Employment & Labor Force ────────────────────────────────────────────
|
||||
("CE", "Employment & Labor Force", "active",
|
||||
"Current Employment Statistics — National",
|
||||
"CES0000000001",
|
||||
"Monthly payroll employment, hours, and earnings by industry. "
|
||||
"The primary national jobs count — 'nonfarm payrolls.'"),
|
||||
|
||||
("SM", "Employment & Labor Force", "active",
|
||||
"State & Metro Employment (CES State)",
|
||||
"SMS110000000000001",
|
||||
"State and MSA-level employment, hours, and earnings. "
|
||||
"Same methodology as CES but sub-national."),
|
||||
|
||||
("LA", "Employment & Labor Force", "active",
|
||||
"Local Area Unemployment Statistics (LAUS)",
|
||||
"LAUST110000000000003",
|
||||
"Monthly unemployment rates, employment, and labor force for all "
|
||||
"states, counties, cities, and metro areas."),
|
||||
|
||||
("LN", "Employment & Labor Force", "active",
|
||||
"Labor Force Statistics — CPS (National)",
|
||||
"LNS14000000",
|
||||
"Household survey: national unemployment rate, labor force participation, "
|
||||
"employment-population ratio. The headline U-3 rate."),
|
||||
|
||||
("JT", "Employment & Labor Force", "active",
|
||||
"Job Openings & Labor Turnover (JOLTS)",
|
||||
"JTS000000000000JOL",
|
||||
"Monthly job openings, hires, quits, layoffs, and total separations "
|
||||
"by industry. Key measure of labor demand."),
|
||||
|
||||
("BD", "Employment & Labor Force", "active",
|
||||
"Business Employment Dynamics",
|
||||
"BDS0000000000000000110001LQ5",
|
||||
"Quarterly gross job gains and losses by establishment size, age, "
|
||||
"state, and industry. Tracks business births and deaths."),
|
||||
|
||||
("EN", "Employment & Labor Force", "active",
|
||||
"Quarterly Census of Employment & Wages (QCEW)",
|
||||
"ENU0000010510000",
|
||||
"Quarterly employment and wages for every county and industry, "
|
||||
"derived from UI tax records. Most granular employment dataset BLS produces."),
|
||||
|
||||
("FM", "Employment & Labor Force", "active",
|
||||
"Marital & Family Labor Force — CPS",
|
||||
"LNS11000000",
|
||||
"CPS supplemental: labor force status by family type, marital status, "
|
||||
"and presence of children."),
|
||||
|
||||
("KV", "Employment & Labor Force", "active",
|
||||
"Veterans Labor Force — CPS",
|
||||
"LNS14000000",
|
||||
"CPS annual supplement: employment and unemployment for veterans "
|
||||
"by period of service and disability status."),
|
||||
|
||||
("GP", "Employment & Labor Force", "active",
|
||||
"Geographic Profile of Employment & Unemployment",
|
||||
"LAUST110000000000003",
|
||||
"Annual CPS-based estimates for states and large metro areas; "
|
||||
"includes demographic breakdowns not available in LAUS."),
|
||||
|
||||
("EP", "Employment & Labor Force", "active",
|
||||
"Employment Projections by Industry",
|
||||
"EPU001001001001",
|
||||
"10-year occupational and industry employment projections; "
|
||||
"updated every 2 years."),
|
||||
|
||||
# ── Prices ──────────────────────────────────────────────────────────────
|
||||
("CU", "Prices", "active",
|
||||
"Consumer Price Index — All Urban (CPI-U)",
|
||||
"CUUR0000SA0",
|
||||
"Monthly price change for a market basket of goods and services. "
|
||||
"The most widely cited inflation measure."),
|
||||
|
||||
("CW", "Prices", "active",
|
||||
"Consumer Price Index — Urban Wage Earners (CPI-W)",
|
||||
"CWUR0000SA0",
|
||||
"CPI for urban wage earners and clerical workers. "
|
||||
"Used to adjust Social Security benefits."),
|
||||
|
||||
("SU", "Prices", "active",
|
||||
"Chained CPI-U (C-CPI-U)",
|
||||
"SUUR0000SA0",
|
||||
"CPI that accounts for consumer substitution between goods; "
|
||||
"tends to grow slightly slower than standard CPI-U."),
|
||||
|
||||
("AP", "Prices", "active",
|
||||
"Average Price Data",
|
||||
"APU0000708111",
|
||||
"Monthly average retail prices for specific consumer items "
|
||||
"(electricity, gasoline, ground beef, eggs, etc.)."),
|
||||
|
||||
("WP", "Prices", "active",
|
||||
"Producer Price Index — Commodities (PPI)",
|
||||
"WPU00000000",
|
||||
"Monthly price changes for goods at the producer level, "
|
||||
"organized by commodity. Leading indicator for consumer prices."),
|
||||
|
||||
("PC", "Prices", "active",
|
||||
"Producer Price Index — Industry Data",
|
||||
"PCU221122211",
|
||||
"NAICS-based PPI measuring price change from the seller's perspective "
|
||||
"by industry. More granular than WP commodity series."),
|
||||
|
||||
("EI", "Prices", "active",
|
||||
"Import/Export Price Indexes",
|
||||
"EIUIR",
|
||||
"Monthly price indexes for U.S. imports and exports by category. "
|
||||
"Key for trade analysis and inflation forecasting."),
|
||||
|
||||
# ── Wages & Compensation ────────────────────────────────────────────────
|
||||
("OE", "Wages & Compensation", "active",
|
||||
"Occupational Employment & Wage Statistics (OEWS)",
|
||||
"OEUN000040000000000000001",
|
||||
"Annual employment and wage estimates for ~800 occupations "
|
||||
"at national, state, and metro levels."),
|
||||
|
||||
("CI", "Wages & Compensation", "active",
|
||||
"Employment Cost Index (ECI)",
|
||||
"CIU1010000000000A",
|
||||
"Quarterly measure of changes in employer labor costs "
|
||||
"(wages + benefits). Watched closely by the Federal Reserve."),
|
||||
|
||||
("CM", "Wages & Compensation", "active",
|
||||
"Employer Costs for Employee Compensation (ECEC)",
|
||||
"CMU1010000000000D",
|
||||
"Quarterly cost per employee-hour for wages/salaries and each "
|
||||
"benefit component (health, retirement, legally required)."),
|
||||
|
||||
("LE", "Wages & Compensation", "active",
|
||||
"Weekly & Hourly Earnings — CPS",
|
||||
"LEU0252881600",
|
||||
"Median usual weekly earnings by demographic group "
|
||||
"from the Current Population Survey."),
|
||||
|
||||
("LU", "Wages & Compensation", "active",
|
||||
"Union Affiliation — CPS",
|
||||
"LUU0104469905",
|
||||
"Annual union membership and representation rates "
|
||||
"by industry, occupation, and state."),
|
||||
|
||||
("NB", "Wages & Compensation", "active",
|
||||
"National Compensation Survey — Benefits",
|
||||
"NBU10500000000000I",
|
||||
"Incidence and key provisions of employer-sponsored benefit plans "
|
||||
"(health, retirement, leave) by establishment size and ownership."),
|
||||
|
||||
("WM", "Wages & Compensation", "active",
|
||||
"Modeled Wage Estimates",
|
||||
"WMU0000000000000A",
|
||||
"Econometric wage estimates for small geographic areas and "
|
||||
"detailed industries where OEWS samples are insufficient."),
|
||||
|
||||
# ── Productivity ────────────────────────────────────────────────────────
|
||||
("PR", "Productivity", "active",
|
||||
"Major Sector Productivity & Costs",
|
||||
"PRS85006092",
|
||||
"Quarterly output per hour, unit labor costs, and compensation "
|
||||
"for the business, nonfarm business, and manufacturing sectors."),
|
||||
|
||||
("IP", "Productivity", "active",
|
||||
"Industry Productivity",
|
||||
"IPU0000000000",
|
||||
"Annual multifactor productivity and related measures for "
|
||||
"detailed NAICS industries."),
|
||||
|
||||
("MP", "Productivity", "active",
|
||||
"Major Sector Total Factor Productivity",
|
||||
"MPU4900012",
|
||||
"Annual total factor productivity for private business sectors; "
|
||||
"measures output growth not explained by labor or capital inputs."),
|
||||
|
||||
("PF", "Productivity", "active",
|
||||
"Federal Government Productivity",
|
||||
"PFU900000I",
|
||||
"Annual output per hour and unit cost indexes for federal "
|
||||
"civilian agencies."),
|
||||
|
||||
("IN", "Productivity", "active",
|
||||
"International Labor Comparisons",
|
||||
"INP101",
|
||||
"Annual manufacturing productivity and labor cost comparisons "
|
||||
"across 17 countries. Useful for competitiveness analysis."),
|
||||
|
||||
# ── Occupational Safety & Health ────────────────────────────────────────
|
||||
("II", "Occupational Safety", "active",
|
||||
"Occupational Injuries & Illnesses — Industry Data",
|
||||
"IIU00_1",
|
||||
"Annual nonfatal workplace injury and illness rates and counts "
|
||||
"by private industry, NAICS sector."),
|
||||
|
||||
("FA", "Occupational Safety", "active",
|
||||
"Census of Fatal Occupational Injuries (2023+)",
|
||||
"FAU001001A01A",
|
||||
"Annual count of fatal work injuries by industry, occupation, "
|
||||
"event/exposure, and worker characteristics. (2023 forward series)"),
|
||||
|
||||
("FW", "Occupational Safety", "active",
|
||||
"Census of Fatal Occupational Injuries (2011–2022)",
|
||||
"FWU00X0X1",
|
||||
"Fatal injury data using 2011–2022 series format; "
|
||||
"use FA for 2023 onward."),
|
||||
|
||||
("CD", "Occupational Safety", "active",
|
||||
"Nonfatal Cases — Days Away from Work",
|
||||
"CDU100XXXXXX_",
|
||||
"Detailed characteristics of nonfatal injuries requiring "
|
||||
"days away from work (nature, body part, source, event)."),
|
||||
|
||||
# ── Consumer & Household ────────────────────────────────────────────────
|
||||
("CX", "Consumer & Household", "active",
|
||||
"Consumer Expenditure Survey (CEX)",
|
||||
"CXUTOTALEXPLB0101M",
|
||||
"Annual and quarterly household spending patterns by category, "
|
||||
"income quintile, age, region. Source for CPI market basket weights."),
|
||||
|
||||
("TU", "Consumer & Household", "active",
|
||||
"American Time Use Survey (ATUS)",
|
||||
"TUU10101AA01",
|
||||
"Annual time diary survey: how Americans spend their time "
|
||||
"(work, leisure, household, caregiving) by demographic."),
|
||||
|
||||
# ── Other / Specialized ─────────────────────────────────────────────────
|
||||
("GG", "Other", "active",
|
||||
"Green Goods & Services",
|
||||
None,
|
||||
"Employment in businesses producing green goods/services. "
|
||||
"Series discontinued after 2011 data."),
|
||||
|
||||
("WS", "Other", "active",
|
||||
"Work Stoppage Data",
|
||||
"WSU00000001",
|
||||
"Annual count of major work stoppages (strikes/lockouts) "
|
||||
"involving 1,000+ workers."),
|
||||
|
||||
("BG", "Other", "active",
|
||||
"Collective Bargaining — State & Local Government",
|
||||
None,
|
||||
"Database of collective bargaining agreements for state and "
|
||||
"local government employees; not accessible via time-series API."),
|
||||
|
||||
("BP", "Other", "active",
|
||||
"Collective Bargaining — Private Sector",
|
||||
None,
|
||||
"Database of private sector collective bargaining agreements; "
|
||||
"not accessible via time-series API."),
|
||||
|
||||
# ── Discontinued ────────────────────────────────────────────────────────
|
||||
("ML", "Discontinued", "discontinued",
|
||||
"Mass Layoff Statistics",
|
||||
None,
|
||||
"Discontinued 2013. Tracked layoff events of 50+ workers "
|
||||
"from a single employer in a 5-week period."),
|
||||
|
||||
("EW", "Discontinued", "discontinued",
|
||||
"QCEW — SIC-Based",
|
||||
None,
|
||||
"Discontinued. QCEW data under the older SIC industry classification; "
|
||||
"superseded by EN (NAICS-based)."),
|
||||
|
||||
("SA", "Discontinued", "discontinued",
|
||||
"State & Area Employment — SIC",
|
||||
None,
|
||||
"Discontinued. Pre-NAICS state/metro employment series; "
|
||||
"superseded by SM."),
|
||||
|
||||
("LF", "Discontinued", "discontinued",
|
||||
"CPS Labor Force — SIC",
|
||||
None,
|
||||
"Discontinued. SIC-based CPS supplement; superseded by LN."),
|
||||
|
||||
("MU", "Discontinued", "discontinued",
|
||||
"CPI-U (Old Series)",
|
||||
None,
|
||||
"Discontinued. Earlier CPI-U methodology series."),
|
||||
|
||||
("MW", "Discontinued", "discontinued",
|
||||
"CPI-W (Old Series)",
|
||||
None,
|
||||
"Discontinued. Earlier CPI-W methodology series."),
|
||||
|
||||
("LI", "Discontinued", "discontinued",
|
||||
"CPI — Dept Store Inventory Price Index",
|
||||
None,
|
||||
"Discontinued. Measured price change in department store inventories."),
|
||||
|
||||
("WD", "Discontinued", "discontinued",
|
||||
"PPI Commodities — Discontinued",
|
||||
None,
|
||||
"Discontinued PPI commodity series; superseded by WP/PC."),
|
||||
|
||||
("ND", "Discontinued", "discontinued",
|
||||
"PPI Industry Data (alternate series)",
|
||||
None,
|
||||
"Alternate/earlier PPI industry series; use PC for current data."),
|
||||
|
||||
("PD", "Discontinued", "discontinued",
|
||||
"PPI — SIC-Based",
|
||||
None,
|
||||
"Discontinued. SIC-based PPI; superseded by NAICS-based PC."),
|
||||
|
||||
("HC", "Discontinued", "discontinued",
|
||||
"Nonfatal Days Away — 2002 data",
|
||||
None,
|
||||
"Single-year series for 2002; use CS for 2011+ data."),
|
||||
|
||||
("CH", "Discontinued", "discontinued",
|
||||
"Nonfatal Days Away — 2003–2010",
|
||||
None,
|
||||
"Historical series 2003–2010; use CS for 2011+ data."),
|
||||
|
||||
("CA", "Discontinued", "discontinued",
|
||||
"Biennial Nonfatal Cases (version A)",
|
||||
None,
|
||||
"Biennial supplemental series for nonfatal injury characteristics."),
|
||||
|
||||
("CB", "Discontinued", "discontinued",
|
||||
"Biennial Nonfatal Cases (version B)",
|
||||
None,
|
||||
"Biennial supplemental series for nonfatal injury characteristics."),
|
||||
|
||||
("FI", "Discontinued", "discontinued",
|
||||
"Fatal Injuries — 2003–2010",
|
||||
None,
|
||||
"Historical CFOI series 2003–2010; use FW (2011–2022) or FA (2023+)."),
|
||||
|
||||
("CF", "Discontinued", "discontinued",
|
||||
"Fatal Injuries — pre-2003",
|
||||
None,
|
||||
"Historical CFOI series; use FW or FA for current data."),
|
||||
|
||||
("SH", "Discontinued", "discontinued",
|
||||
"Injuries & Illnesses — 1989–2001",
|
||||
None,
|
||||
"Historical series; superseded by II."),
|
||||
|
||||
("HS", "Discontinued", "discontinued",
|
||||
"Injuries & Illnesses — pre-1989",
|
||||
None,
|
||||
"Historical series under pre-1989 methodology."),
|
||||
|
||||
("SI", "Discontinued", "discontinued",
|
||||
"Injuries & Illnesses — 2002",
|
||||
None,
|
||||
"Single-year 2002 series; use II for current data."),
|
||||
|
||||
("IS", "Discontinued", "discontinued",
|
||||
"Injuries & Illnesses (alternate)",
|
||||
None,
|
||||
"Alternate injuries series; use II for current data."),
|
||||
|
||||
("EC", "Discontinued", "discontinued",
|
||||
"Employment Cost Index (alternate series)",
|
||||
None,
|
||||
"Earlier ECI series format; use CI for current data."),
|
||||
|
||||
("NC", "Discontinued", "discontinued",
|
||||
"National Compensation Survey",
|
||||
None,
|
||||
"Broader NCS series; component data now in CI, CM, NB."),
|
||||
|
||||
("NW", "Discontinued", "discontinued",
|
||||
"National Compensation Survey (alternate)",
|
||||
None,
|
||||
"Alternate NCS series; component data now in CI, CM, NB."),
|
||||
|
||||
("EB", "Discontinued", "discontinued",
|
||||
"Employee Benefits Survey",
|
||||
None,
|
||||
"Superseded by NB (National Compensation Survey — Benefits)."),
|
||||
|
||||
("CC", "Discontinued", "discontinued",
|
||||
"Employer Costs for Employee Compensation (alternate)",
|
||||
None,
|
||||
"Earlier ECEC series; use CM for current data."),
|
||||
|
||||
("EE", "Discontinued", "discontinued",
|
||||
"National Employment, Hours, and Earnings",
|
||||
None,
|
||||
"Older national CES series; superseded by CE."),
|
||||
|
||||
("OR", "Wages & Compensation", "active",
|
||||
"Occupational Requirements Survey",
|
||||
None,
|
||||
"Physical demands, environmental conditions, and education/training "
|
||||
"requirements for occupations. Supports SSA disability determinations."),
|
||||
|
||||
("PI", "Productivity", "active",
|
||||
"Industry Productivity Index",
|
||||
None,
|
||||
"Output-per-hour productivity indexes for detailed industries; "
|
||||
"companion to IP multifactor productivity measures."),
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def fetch_batch(series_ids, start_year, end_year):
|
||||
payload = {
|
||||
"seriesid": series_ids,
|
||||
"startyear": str(start_year),
|
||||
"endyear": str(end_year),
|
||||
"registrationkey": API_KEY,
|
||||
"catalog": True,
|
||||
}
|
||||
r = requests.post(f"{BLS_API_BASE}/timeseries/data/", json=payload, timeout=30)
|
||||
r.raise_for_status()
|
||||
body = r.json()
|
||||
return {s["seriesID"]: s for s in body.get("Results", {}).get("series", [])}
|
||||
|
||||
|
||||
API_KEY = BLS_API_KEY
|
||||
|
||||
def latest_obs(series_data):
|
||||
for obs in series_data.get("data", []):
|
||||
if obs["value"] != "-":
|
||||
return obs
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTML generation
|
||||
# ---------------------------------------------------------------------------
|
||||
CATEGORY_ORDER = [
|
||||
"Employment & Labor Force",
|
||||
"Prices",
|
||||
"Wages & Compensation",
|
||||
"Productivity",
|
||||
"Occupational Safety",
|
||||
"Consumer & Household",
|
||||
"Other",
|
||||
"Discontinued",
|
||||
]
|
||||
|
||||
CAT_COLORS = {
|
||||
"Employment & Labor Force": "#2471a3",
|
||||
"Prices": "#c0392b",
|
||||
"Wages & Compensation": "#1e8449",
|
||||
"Productivity": "#7d3c98",
|
||||
"Occupational Safety": "#d35400",
|
||||
"Consumer & Household": "#117a65",
|
||||
"Other": "#566573",
|
||||
"Discontinued": "#aab7b8",
|
||||
}
|
||||
|
||||
def render_card(abbr, category, status, label, series_id, description, series_result):
|
||||
color = CAT_COLORS.get(category, "#666")
|
||||
disc_badge = '<span class="badge disc">Discontinued</span>' if status == "discontinued" else '<span class="badge active">Active</span>'
|
||||
no_data_badge = ''
|
||||
|
||||
value_block = ""
|
||||
catalog_title = ""
|
||||
|
||||
if series_id and series_result:
|
||||
obs = latest_obs(series_result)
|
||||
catalog_title = series_result.get("catalog", {}).get("series_title", "")
|
||||
if obs:
|
||||
value_block = f"""
|
||||
<div class="latest">
|
||||
<span class="val">{obs['value']}</span>
|
||||
<span class="period">{obs.get('periodName','')} {obs.get('year','')}</span>
|
||||
</div>"""
|
||||
else:
|
||||
value_block = '<div class="no-data">No recent data returned</div>'
|
||||
no_data_badge = '<span class="badge nodata">No Recent Data</span>'
|
||||
elif series_id:
|
||||
value_block = '<div class="no-data">Series not found or no data</div>'
|
||||
no_data_badge = '<span class="badge nodata">No Recent Data</span>'
|
||||
else:
|
||||
value_block = '<div class="no-data">Not available via time-series API</div>'
|
||||
|
||||
series_line = f'<code>{series_id}</code>' if series_id else '<em>No API series</em>'
|
||||
cat_title = catalog_title[:80] + "…" if len(catalog_title) > 80 else catalog_title
|
||||
cat_subtitle = f'<div class="cat-title">{cat_title}</div>' if cat_title else ''
|
||||
|
||||
snippet = ""
|
||||
if series_id:
|
||||
snippet = f"""<details class="snippet">
|
||||
<summary>Show query</summary>
|
||||
<pre>import requests
|
||||
payload = {{
|
||||
"seriesid": ["{series_id}"],
|
||||
"startyear": "2023",
|
||||
"endyear": "2025",
|
||||
"registrationkey": "YOUR_KEY",
|
||||
}}
|
||||
r = requests.post(
|
||||
"https://api.bls.gov/publicAPI/v2/timeseries/data/",
|
||||
json=payload
|
||||
)
|
||||
data = r.json()["Results"]["series"][0]["data"]</pre>
|
||||
</details>"""
|
||||
|
||||
return f"""
|
||||
<div class="card" style="border-left: 4px solid {color}">
|
||||
<div class="card-head">
|
||||
<span class="abbr" style="color:{color}">{abbr}</span>
|
||||
<div class="badges">{disc_badge}{no_data_badge}</div>
|
||||
</div>
|
||||
<div class="card-label">{label}</div>
|
||||
{cat_subtitle}
|
||||
<div class="card-desc">{description}</div>
|
||||
<div class="series-row"><span class="series-lbl">Example series:</span> {series_line}</div>
|
||||
{value_block}
|
||||
{snippet}
|
||||
</div>"""
|
||||
|
||||
|
||||
HTML_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>BLS Dataset Explorer — {report_date}</title>
|
||||
<style>
|
||||
*, *::before, *::after {{ box-sizing: border-box; }}
|
||||
body {{
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
background: #f4f6f9; color: #222; margin: 0; padding: 24px;
|
||||
}}
|
||||
.header {{
|
||||
background: #1a2744; color: #fff;
|
||||
padding: 24px 32px; border-radius: 8px; margin-bottom: 28px;
|
||||
}}
|
||||
.header h1 {{ margin: 0 0 4px; font-size: 1.5rem; }}
|
||||
.header .sub {{ opacity: .7; font-size: .875rem; }}
|
||||
.stats {{ display: flex; gap: 32px; margin-top: 12px; }}
|
||||
.stat {{ font-size: .85rem; }}
|
||||
.stat strong {{ font-size: 1.4rem; display: block; }}
|
||||
.toc {{ background:#fff; border-radius:8px; padding:16px 24px;
|
||||
box-shadow:0 1px 4px rgba(0,0,0,.08); margin-bottom:28px; }}
|
||||
.toc h2 {{ margin:0 0 10px; font-size:.9rem; text-transform:uppercase;
|
||||
letter-spacing:.05em; color:#666; }}
|
||||
.toc-links {{ display:flex; flex-wrap:wrap; gap:8px; }}
|
||||
.toc-links a {{
|
||||
padding: 4px 12px; border-radius: 20px; font-size:.8rem;
|
||||
text-decoration:none; color:#fff; font-weight:600;
|
||||
}}
|
||||
.category {{ margin-bottom: 36px; }}
|
||||
.cat-header {{
|
||||
font-size:1.05rem; font-weight:700; margin-bottom:14px;
|
||||
padding-bottom:8px; border-bottom:2px solid #e5e7eb;
|
||||
display:flex; align-items:center; gap:10px;
|
||||
}}
|
||||
.cat-count {{ font-size:.8rem; color:#888; font-weight:400; }}
|
||||
.grid {{
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(360px, 1fr));
|
||||
gap: 16px;
|
||||
}}
|
||||
.card {{
|
||||
background: #fff; border-radius: 8px; padding: 18px;
|
||||
box-shadow: 0 1px 4px rgba(0,0,0,.07);
|
||||
}}
|
||||
.card-head {{ display:flex; justify-content:space-between; align-items:center; margin-bottom:6px; }}
|
||||
.abbr {{ font-size:1.1rem; font-weight:800; letter-spacing:.02em; }}
|
||||
.badges {{ display:flex; gap:4px; }}
|
||||
.badge {{ font-size:.65rem; font-weight:700; padding:2px 7px;
|
||||
border-radius:10px; text-transform:uppercase; letter-spacing:.04em; }}
|
||||
.badge.active {{ background:#d5f5e3; color:#1e8449; }}
|
||||
.badge.disc {{ background:#f2f3f4; color:#888; }}
|
||||
.badge.nodata {{ background:#fdebd0; color:#d35400; }}
|
||||
.card-label {{ font-weight:600; font-size:.92rem; margin-bottom:4px; }}
|
||||
.cat-title {{ font-size:.75rem; color:#888; margin-bottom:6px; font-style:italic; }}
|
||||
.card-desc {{ font-size:.82rem; color:#444; line-height:1.5; margin-bottom:10px; }}
|
||||
.series-row {{ font-size:.78rem; color:#555; margin-bottom:8px; }}
|
||||
.series-lbl {{ color:#999; }}
|
||||
code {{ background:#f0f0f0; padding:1px 5px; border-radius:3px;
|
||||
font-size:.8rem; word-break:break-all; }}
|
||||
.latest {{ display:flex; align-items:baseline; gap:8px; margin-bottom:8px; }}
|
||||
.val {{ font-size:1.4rem; font-weight:700; color:#1a2744; }}
|
||||
.period {{ font-size:.78rem; color:#888; }}
|
||||
.no-data {{ font-size:.78rem; color:#bbb; font-style:italic; margin-bottom:8px; }}
|
||||
details.snippet {{ margin-top:8px; }}
|
||||
summary {{ font-size:.78rem; color:#2471a3; cursor:pointer; }}
|
||||
pre {{
|
||||
background:#f8f9fa; border:1px solid #e5e7eb; border-radius:4px;
|
||||
padding:10px; font-size:.72rem; overflow-x:auto; margin:6px 0 0; white-space:pre;
|
||||
}}
|
||||
@media(max-width:600px) {{ .grid {{ grid-template-columns:1fr; }} }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="header">
|
||||
<h1>BLS Dataset Explorer</h1>
|
||||
<div class="sub">Bureau of Labor Statistics — complete survey catalog with example queries | Generated {report_date}</div>
|
||||
<div class="stats">
|
||||
<div class="stat"><strong>{total}</strong> Total surveys</div>
|
||||
<div class="stat"><strong>{active}</strong> Active</div>
|
||||
<div class="stat"><strong>{with_data}</strong> Returning live data</div>
|
||||
<div class="stat"><strong>{discontinued}</strong> Discontinued</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="toc">
|
||||
<h2>Jump to category</h2>
|
||||
<div class="toc-links">{toc_links}</div>
|
||||
</div>
|
||||
|
||||
{categories_html}
|
||||
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
def main():
|
||||
current_year = datetime.now().year
|
||||
start_year = current_year - 2
|
||||
|
||||
# Collect series IDs that have values to fetch
|
||||
to_fetch = [(abbr, sid) for abbr, cat, status, label, sid, desc in DATASETS
|
||||
if sid and status == "active"]
|
||||
|
||||
print(f"Fetching data for {len(to_fetch)} active series...")
|
||||
|
||||
# Batch into groups of 50
|
||||
results = {}
|
||||
for i in range(0, len(to_fetch), 50):
|
||||
batch = to_fetch[i:i+50]
|
||||
ids = [sid for _, sid in batch]
|
||||
print(f" Batch {i//50+1}: {len(ids)} series")
|
||||
batch_results = fetch_batch(ids, start_year, current_year)
|
||||
results.update(batch_results)
|
||||
if i + 50 < len(to_fetch):
|
||||
time.sleep(0.5)
|
||||
|
||||
# Tally stats
|
||||
total = len(DATASETS)
|
||||
active_count = sum(1 for *_, status, __, ___, ____ in DATASETS if status == "active")
|
||||
disc_count = total - active_count
|
||||
with_data = sum(1 for abbr, cat, status, label, sid, desc in DATASETS
|
||||
if sid and sid in results and latest_obs(results[sid]))
|
||||
|
||||
# Build TOC
|
||||
toc_links = "".join(
|
||||
f'<a href="#{cat.lower().replace(" ","_").replace("&","")}" '
|
||||
f'style="background:{CAT_COLORS[cat]}">{cat}</a>'
|
||||
for cat in CATEGORY_ORDER
|
||||
)
|
||||
|
||||
# Build category sections
|
||||
by_cat = {c: [] for c in CATEGORY_ORDER}
|
||||
for entry in DATASETS:
|
||||
abbr, category, status, label, series_id, description = entry
|
||||
by_cat.setdefault(category, []).append(entry)
|
||||
|
||||
cats_html = []
|
||||
for cat in CATEGORY_ORDER:
|
||||
entries = by_cat.get(cat, [])
|
||||
if not entries:
|
||||
continue
|
||||
color = CAT_COLORS[cat]
|
||||
anchor = cat.lower().replace(" ","_").replace("&","")
|
||||
cards = "".join(
|
||||
render_card(abbr, cat, status, label, sid, desc,
|
||||
results.get(sid) if sid else None)
|
||||
for abbr, _, status, label, sid, desc in entries
|
||||
)
|
||||
cats_html.append(f"""
|
||||
<div class="category" id="{anchor}">
|
||||
<div class="cat-header" style="color:{color}">
|
||||
{cat}
|
||||
<span class="cat-count">{len(entries)} surveys</span>
|
||||
</div>
|
||||
<div class="grid">{cards}</div>
|
||||
</div>""")
|
||||
|
||||
report_date = datetime.now().strftime("%B %d, %Y")
|
||||
html = HTML_TEMPLATE.format(
|
||||
report_date=report_date,
|
||||
total=total,
|
||||
active=active_count,
|
||||
with_data=with_data,
|
||||
discontinued=disc_count,
|
||||
toc_links=toc_links,
|
||||
categories_html="\n".join(cats_html),
|
||||
)
|
||||
|
||||
out = os.path.join(SCRIPT_DIR, "bls_dataset_explorer.html")
|
||||
with open(out, "w") as f:
|
||||
f.write(html)
|
||||
print(f"\nReport written → {out}")
|
||||
print(f"Stats: {total} surveys | {active_count} active | {with_data} returning data | {disc_count} discontinued")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user