""" BLS Dataset Explorer Demonstrates a representative query for every active BLS survey. Pulls one example series per dataset, shows the latest value, and generates a self-contained HTML reference page. Run: python3 bls_dataset_explorer.py Out: bls_dataset_explorer.html """ import requests, json, os, time from datetime import datetime from config import BLS_API_KEY, BLS_API_BASE SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) # --------------------------------------------------------------------------- # Dataset catalog # Each entry: (survey_abbr, category, status, label, series_id, description) # status: "active" | "discontinued" # series_id: None for surveys with no API-accessible series # --------------------------------------------------------------------------- DATASETS = [ # ── Employment & Labor Force ──────────────────────────────────────────── ("CE", "Employment & Labor Force", "active", "Current Employment Statistics — National", "CES0000000001", "Monthly payroll employment, hours, and earnings by industry. " "The primary national jobs count — 'nonfarm payrolls.'"), ("SM", "Employment & Labor Force", "active", "State & Metro Employment (CES State)", "SMS110000000000001", "State and MSA-level employment, hours, and earnings. " "Same methodology as CES but sub-national."), ("LA", "Employment & Labor Force", "active", "Local Area Unemployment Statistics (LAUS)", "LAUST110000000000003", "Monthly unemployment rates, employment, and labor force for all " "states, counties, cities, and metro areas."), ("LN", "Employment & Labor Force", "active", "Labor Force Statistics — CPS (National)", "LNS14000000", "Household survey: national unemployment rate, labor force participation, " "employment-population ratio. The headline U-3 rate."), ("JT", "Employment & Labor Force", "active", "Job Openings & Labor Turnover (JOLTS)", "JTS000000000000JOL", "Monthly job openings, hires, quits, layoffs, and total separations " "by industry. Key measure of labor demand."), ("BD", "Employment & Labor Force", "active", "Business Employment Dynamics", "BDS0000000000000000110001LQ5", "Quarterly gross job gains and losses by establishment size, age, " "state, and industry. Tracks business births and deaths."), ("EN", "Employment & Labor Force", "active", "Quarterly Census of Employment & Wages (QCEW)", "ENU0000010510000", "Quarterly employment and wages for every county and industry, " "derived from UI tax records. Most granular employment dataset BLS produces."), ("FM", "Employment & Labor Force", "active", "Marital & Family Labor Force — CPS", "LNS11000000", "CPS supplemental: labor force status by family type, marital status, " "and presence of children."), ("KV", "Employment & Labor Force", "active", "Veterans Labor Force — CPS", "LNS14000000", "CPS annual supplement: employment and unemployment for veterans " "by period of service and disability status."), ("GP", "Employment & Labor Force", "active", "Geographic Profile of Employment & Unemployment", "LAUST110000000000003", "Annual CPS-based estimates for states and large metro areas; " "includes demographic breakdowns not available in LAUS."), ("EP", "Employment & Labor Force", "active", "Employment Projections by Industry", "EPU001001001001", "10-year occupational and industry employment projections; " "updated every 2 years."), # ── Prices ────────────────────────────────────────────────────────────── ("CU", "Prices", "active", "Consumer Price Index — All Urban (CPI-U)", "CUUR0000SA0", "Monthly price change for a market basket of goods and services. " "The most widely cited inflation measure."), ("CW", "Prices", "active", "Consumer Price Index — Urban Wage Earners (CPI-W)", "CWUR0000SA0", "CPI for urban wage earners and clerical workers. " "Used to adjust Social Security benefits."), ("SU", "Prices", "active", "Chained CPI-U (C-CPI-U)", "SUUR0000SA0", "CPI that accounts for consumer substitution between goods; " "tends to grow slightly slower than standard CPI-U."), ("AP", "Prices", "active", "Average Price Data", "APU0000708111", "Monthly average retail prices for specific consumer items " "(electricity, gasoline, ground beef, eggs, etc.)."), ("WP", "Prices", "active", "Producer Price Index — Commodities (PPI)", "WPU00000000", "Monthly price changes for goods at the producer level, " "organized by commodity. Leading indicator for consumer prices."), ("PC", "Prices", "active", "Producer Price Index — Industry Data", "PCU221122211", "NAICS-based PPI measuring price change from the seller's perspective " "by industry. More granular than WP commodity series."), ("EI", "Prices", "active", "Import/Export Price Indexes", "EIUIR", "Monthly price indexes for U.S. imports and exports by category. " "Key for trade analysis and inflation forecasting."), # ── Wages & Compensation ──────────────────────────────────────────────── ("OE", "Wages & Compensation", "active", "Occupational Employment & Wage Statistics (OEWS)", "OEUN000040000000000000001", "Annual employment and wage estimates for ~800 occupations " "at national, state, and metro levels."), ("CI", "Wages & Compensation", "active", "Employment Cost Index (ECI)", "CIU1010000000000A", "Quarterly measure of changes in employer labor costs " "(wages + benefits). Watched closely by the Federal Reserve."), ("CM", "Wages & Compensation", "active", "Employer Costs for Employee Compensation (ECEC)", "CMU1010000000000D", "Quarterly cost per employee-hour for wages/salaries and each " "benefit component (health, retirement, legally required)."), ("LE", "Wages & Compensation", "active", "Weekly & Hourly Earnings — CPS", "LEU0252881600", "Median usual weekly earnings by demographic group " "from the Current Population Survey."), ("LU", "Wages & Compensation", "active", "Union Affiliation — CPS", "LUU0104469905", "Annual union membership and representation rates " "by industry, occupation, and state."), ("NB", "Wages & Compensation", "active", "National Compensation Survey — Benefits", "NBU10500000000000I", "Incidence and key provisions of employer-sponsored benefit plans " "(health, retirement, leave) by establishment size and ownership."), ("WM", "Wages & Compensation", "active", "Modeled Wage Estimates", "WMU0000000000000A", "Econometric wage estimates for small geographic areas and " "detailed industries where OEWS samples are insufficient."), # ── Productivity ──────────────────────────────────────────────────────── ("PR", "Productivity", "active", "Major Sector Productivity & Costs", "PRS85006092", "Quarterly output per hour, unit labor costs, and compensation " "for the business, nonfarm business, and manufacturing sectors."), ("IP", "Productivity", "active", "Industry Productivity", "IPU0000000000", "Annual multifactor productivity and related measures for " "detailed NAICS industries."), ("MP", "Productivity", "active", "Major Sector Total Factor Productivity", "MPU4900012", "Annual total factor productivity for private business sectors; " "measures output growth not explained by labor or capital inputs."), ("PF", "Productivity", "active", "Federal Government Productivity", "PFU900000I", "Annual output per hour and unit cost indexes for federal " "civilian agencies."), ("IN", "Productivity", "active", "International Labor Comparisons", "INP101", "Annual manufacturing productivity and labor cost comparisons " "across 17 countries. Useful for competitiveness analysis."), # ── Occupational Safety & Health ──────────────────────────────────────── ("II", "Occupational Safety", "active", "Occupational Injuries & Illnesses — Industry Data", "IIU00_1", "Annual nonfatal workplace injury and illness rates and counts " "by private industry, NAICS sector."), ("FA", "Occupational Safety", "active", "Census of Fatal Occupational Injuries (2023+)", "FAU001001A01A", "Annual count of fatal work injuries by industry, occupation, " "event/exposure, and worker characteristics. (2023 forward series)"), ("FW", "Occupational Safety", "active", "Census of Fatal Occupational Injuries (2011–2022)", "FWU00X0X1", "Fatal injury data using 2011–2022 series format; " "use FA for 2023 onward."), ("CD", "Occupational Safety", "active", "Nonfatal Cases — Days Away from Work", "CDU100XXXXXX_", "Detailed characteristics of nonfatal injuries requiring " "days away from work (nature, body part, source, event)."), # ── Consumer & Household ──────────────────────────────────────────────── ("CX", "Consumer & Household", "active", "Consumer Expenditure Survey (CEX)", "CXUTOTALEXPLB0101M", "Annual and quarterly household spending patterns by category, " "income quintile, age, region. Source for CPI market basket weights."), ("TU", "Consumer & Household", "active", "American Time Use Survey (ATUS)", "TUU10101AA01", "Annual time diary survey: how Americans spend their time " "(work, leisure, household, caregiving) by demographic."), # ── Other / Specialized ───────────────────────────────────────────────── ("GG", "Other", "active", "Green Goods & Services", None, "Employment in businesses producing green goods/services. " "Series discontinued after 2011 data."), ("WS", "Other", "active", "Work Stoppage Data", "WSU00000001", "Annual count of major work stoppages (strikes/lockouts) " "involving 1,000+ workers."), ("BG", "Other", "active", "Collective Bargaining — State & Local Government", None, "Database of collective bargaining agreements for state and " "local government employees; not accessible via time-series API."), ("BP", "Other", "active", "Collective Bargaining — Private Sector", None, "Database of private sector collective bargaining agreements; " "not accessible via time-series API."), # ── Discontinued ──────────────────────────────────────────────────────── ("ML", "Discontinued", "discontinued", "Mass Layoff Statistics", None, "Discontinued 2013. Tracked layoff events of 50+ workers " "from a single employer in a 5-week period."), ("EW", "Discontinued", "discontinued", "QCEW — SIC-Based", None, "Discontinued. QCEW data under the older SIC industry classification; " "superseded by EN (NAICS-based)."), ("SA", "Discontinued", "discontinued", "State & Area Employment — SIC", None, "Discontinued. Pre-NAICS state/metro employment series; " "superseded by SM."), ("LF", "Discontinued", "discontinued", "CPS Labor Force — SIC", None, "Discontinued. SIC-based CPS supplement; superseded by LN."), ("MU", "Discontinued", "discontinued", "CPI-U (Old Series)", None, "Discontinued. Earlier CPI-U methodology series."), ("MW", "Discontinued", "discontinued", "CPI-W (Old Series)", None, "Discontinued. Earlier CPI-W methodology series."), ("LI", "Discontinued", "discontinued", "CPI — Dept Store Inventory Price Index", None, "Discontinued. Measured price change in department store inventories."), ("WD", "Discontinued", "discontinued", "PPI Commodities — Discontinued", None, "Discontinued PPI commodity series; superseded by WP/PC."), ("ND", "Discontinued", "discontinued", "PPI Industry Data (alternate series)", None, "Alternate/earlier PPI industry series; use PC for current data."), ("PD", "Discontinued", "discontinued", "PPI — SIC-Based", None, "Discontinued. SIC-based PPI; superseded by NAICS-based PC."), ("HC", "Discontinued", "discontinued", "Nonfatal Days Away — 2002 data", None, "Single-year series for 2002; use CS for 2011+ data."), ("CH", "Discontinued", "discontinued", "Nonfatal Days Away — 2003–2010", None, "Historical series 2003–2010; use CS for 2011+ data."), ("CA", "Discontinued", "discontinued", "Biennial Nonfatal Cases (version A)", None, "Biennial supplemental series for nonfatal injury characteristics."), ("CB", "Discontinued", "discontinued", "Biennial Nonfatal Cases (version B)", None, "Biennial supplemental series for nonfatal injury characteristics."), ("FI", "Discontinued", "discontinued", "Fatal Injuries — 2003–2010", None, "Historical CFOI series 2003–2010; use FW (2011–2022) or FA (2023+)."), ("CF", "Discontinued", "discontinued", "Fatal Injuries — pre-2003", None, "Historical CFOI series; use FW or FA for current data."), ("SH", "Discontinued", "discontinued", "Injuries & Illnesses — 1989–2001", None, "Historical series; superseded by II."), ("HS", "Discontinued", "discontinued", "Injuries & Illnesses — pre-1989", None, "Historical series under pre-1989 methodology."), ("SI", "Discontinued", "discontinued", "Injuries & Illnesses — 2002", None, "Single-year 2002 series; use II for current data."), ("IS", "Discontinued", "discontinued", "Injuries & Illnesses (alternate)", None, "Alternate injuries series; use II for current data."), ("EC", "Discontinued", "discontinued", "Employment Cost Index (alternate series)", None, "Earlier ECI series format; use CI for current data."), ("NC", "Discontinued", "discontinued", "National Compensation Survey", None, "Broader NCS series; component data now in CI, CM, NB."), ("NW", "Discontinued", "discontinued", "National Compensation Survey (alternate)", None, "Alternate NCS series; component data now in CI, CM, NB."), ("EB", "Discontinued", "discontinued", "Employee Benefits Survey", None, "Superseded by NB (National Compensation Survey — Benefits)."), ("CC", "Discontinued", "discontinued", "Employer Costs for Employee Compensation (alternate)", None, "Earlier ECEC series; use CM for current data."), ("EE", "Discontinued", "discontinued", "National Employment, Hours, and Earnings", None, "Older national CES series; superseded by CE."), ("OR", "Wages & Compensation", "active", "Occupational Requirements Survey", None, "Physical demands, environmental conditions, and education/training " "requirements for occupations. Supports SSA disability determinations."), ("PI", "Productivity", "active", "Industry Productivity Index", None, "Output-per-hour productivity indexes for detailed industries; " "companion to IP multifactor productivity measures."), ] # --------------------------------------------------------------------------- # API helpers # --------------------------------------------------------------------------- def fetch_batch(series_ids, start_year, end_year): payload = { "seriesid": series_ids, "startyear": str(start_year), "endyear": str(end_year), "registrationkey": API_KEY, "catalog": True, } r = requests.post(f"{BLS_API_BASE}/timeseries/data/", json=payload, timeout=30) r.raise_for_status() body = r.json() return {s["seriesID"]: s for s in body.get("Results", {}).get("series", [])} API_KEY = BLS_API_KEY def latest_obs(series_data): for obs in series_data.get("data", []): if obs["value"] != "-": return obs return None # --------------------------------------------------------------------------- # HTML generation # --------------------------------------------------------------------------- CATEGORY_ORDER = [ "Employment & Labor Force", "Prices", "Wages & Compensation", "Productivity", "Occupational Safety", "Consumer & Household", "Other", "Discontinued", ] CAT_COLORS = { "Employment & Labor Force": "#2471a3", "Prices": "#c0392b", "Wages & Compensation": "#1e8449", "Productivity": "#7d3c98", "Occupational Safety": "#d35400", "Consumer & Household": "#117a65", "Other": "#566573", "Discontinued": "#aab7b8", } def render_card(abbr, category, status, label, series_id, description, series_result): color = CAT_COLORS.get(category, "#666") disc_badge = 'Discontinued' if status == "discontinued" else 'Active' no_data_badge = '' value_block = "" catalog_title = "" if series_id and series_result: obs = latest_obs(series_result) catalog_title = series_result.get("catalog", {}).get("series_title", "") if obs: value_block = f"""
{obs['value']} {obs.get('periodName','')} {obs.get('year','')}
""" else: value_block = '
No recent data returned
' no_data_badge = 'No Recent Data' elif series_id: value_block = '
Series not found or no data
' no_data_badge = 'No Recent Data' else: value_block = '
Not available via time-series API
' series_line = f'{series_id}' if series_id else 'No API series' cat_title = catalog_title[:80] + "…" if len(catalog_title) > 80 else catalog_title cat_subtitle = f'
{cat_title}
' if cat_title else '' snippet = "" if series_id: snippet = f"""
Show query 
import requests
payload = {{
    "seriesid": ["{series_id}"],
    "startyear": "2023",
    "endyear": "2025",
    "registrationkey": "YOUR_KEY",
}}
r = requests.post(
    "https://api.bls.gov/publicAPI/v2/timeseries/data/",
    json=payload
)
data = r.json()["Results"]["series"][0]["data"]
""" return f"""
{abbr}
{disc_badge}{no_data_badge}
{label}
{cat_subtitle}
{description}
Example series: {series_line}
{value_block} {snippet}
""" HTML_TEMPLATE = """ BLS Dataset Explorer — {report_date}

BLS Dataset Explorer

Bureau of Labor Statistics — complete survey catalog with example queries  |  Generated {report_date}
{total} Total surveys
{active} Active
{with_data} Returning live data
{discontinued} Discontinued

Jump to category

{toc_links}
{categories_html} """ # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- def main(): current_year = datetime.now().year start_year = current_year - 2 # Collect series IDs that have values to fetch to_fetch = [(abbr, sid) for abbr, cat, status, label, sid, desc in DATASETS if sid and status == "active"] print(f"Fetching data for {len(to_fetch)} active series...") # Batch into groups of 50 results = {} for i in range(0, len(to_fetch), 50): batch = to_fetch[i:i+50] ids = [sid for _, sid in batch] print(f" Batch {i//50+1}: {len(ids)} series") batch_results = fetch_batch(ids, start_year, current_year) results.update(batch_results) if i + 50 < len(to_fetch): time.sleep(0.5) # Tally stats total = len(DATASETS) active_count = sum(1 for *_, status, __, ___, ____ in DATASETS if status == "active") disc_count = total - active_count with_data = sum(1 for abbr, cat, status, label, sid, desc in DATASETS if sid and sid in results and latest_obs(results[sid])) # Build TOC toc_links = "".join( f'{cat}' for cat in CATEGORY_ORDER ) # Build category sections by_cat = {c: [] for c in CATEGORY_ORDER} for entry in DATASETS: abbr, category, status, label, series_id, description = entry by_cat.setdefault(category, []).append(entry) cats_html = [] for cat in CATEGORY_ORDER: entries = by_cat.get(cat, []) if not entries: continue color = CAT_COLORS[cat] anchor = cat.lower().replace(" ","_").replace("&","") cards = "".join( render_card(abbr, cat, status, label, sid, desc, results.get(sid) if sid else None) for abbr, _, status, label, sid, desc in entries ) cats_html.append(f"""
{cat} {len(entries)} surveys
{cards}
""") report_date = datetime.now().strftime("%B %d, %Y") html = HTML_TEMPLATE.format( report_date=report_date, total=total, active=active_count, with_data=with_data, discontinued=disc_count, toc_links=toc_links, categories_html="\n".join(cats_html), ) out = os.path.join(SCRIPT_DIR, "bls_dataset_explorer.html") with open(out, "w") as f: f.write(html) print(f"\nReport written → {out}") print(f"Stats: {total} surveys | {active_count} active | {with_data} returning data | {disc_count} discontinued") if __name__ == "__main__": main()