Dave Boyd
0d0d6dabe2
- bls_client/retry.py: with_retries() retries timeouts/connection drops and 429/5xx with exponential backoff; honors Retry-After; never retries 4xx or BLSQuotaError (those won't fix themselves) - BLSClient(retries=2, backoff=0.5) wraps the data POST and discovery GETs; qcew CSV fetch wrapped too. retries=0 disables. - tests/test_retry.py: 6 offline tests (injectable sleep, no real delays) - README: retry behavior documented; drop the now-resolved limitation Offline suite 125 passing; live smoke green. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
78 lines
3.0 KiB
Python
78 lines
3.0 KiB
Python
"""QCEW Open Data Access — county/industry employment & wages.
|
||
|
||
The BLS *timeseries* API only carries QCEW national/state totals. The full
|
||
QCEW detail (every county, every NAICS industry, establishment counts and
|
||
wages) lives behind a separate CSV service that needs no API key and counts
|
||
against no quota:
|
||
|
||
https://data.bls.gov/cew/data/api/{year}/{qtr}/{slice}/{code}.csv
|
||
|
||
This module is a thin wrapper over that service. It is deliberately separate
|
||
from BLSClient (different API, CSV not JSON, no key). Each call returns a list
|
||
of plain dict rows (the CSV columns), so it composes with csv/pandas directly.
|
||
|
||
Quarter argument: 1-4 for a specific quarter, or "a" for annual averages.
|
||
Area FIPS examples: "US000" national, "11000" DC (state), "11001" a county,
|
||
metro codes start with "C". Industry codes are NAICS (e.g. "10" = all
|
||
industries, "722" = food services, "622" = hospitals).
|
||
"""
|
||
import csv
|
||
import io
|
||
|
||
import requests
|
||
|
||
from .retry import with_retries
|
||
|
||
QCEW_BASE = "https://data.bls.gov/cew/data/api"
|
||
_HEADERS = {"User-Agent": "bls-data-library (https://gitea.doesworks.net/giteaadmin/bls-data)"}
|
||
|
||
# A few common columns, for reference / convenience:
|
||
# area_fips, own_code, industry_code, agglvl_code, size_code, year, qtr,
|
||
# annual_avg_estabs, annual_avg_emplvl, total_annual_wages, annual_avg_wkly_wage,
|
||
# avg_annual_pay (quarterly slices use month1_emplvl/qtrly_estabs/total_qtrly_wages, etc.)
|
||
|
||
OWNERSHIP = {
|
||
"total": "0", "federal": "1", "state": "2", "local": "3",
|
||
"private": "5", "government": "8",
|
||
}
|
||
|
||
|
||
def _fetch_csv(url: str, retries=2, backoff=0.5) -> list[dict]:
|
||
r = with_retries(lambda: requests.get(url, timeout=60, headers=_HEADERS),
|
||
retries=retries, backoff=backoff)
|
||
r.raise_for_status()
|
||
return parse_csv(r.text)
|
||
|
||
|
||
def parse_csv(text: str) -> list[dict]:
|
||
"""Parse QCEW CSV text into a list of row dicts (BLS quotes every field)."""
|
||
return list(csv.DictReader(io.StringIO(text)))
|
||
|
||
|
||
def area(area_fips: str, year: int, qtr="a") -> list[dict]:
|
||
"""Every industry × ownership for one area (county/state/metro/national)."""
|
||
return _fetch_csv(f"{QCEW_BASE}/{year}/{qtr}/area/{area_fips}.csv")
|
||
|
||
|
||
def industry(industry_code: str, year: int, qtr="a") -> list[dict]:
|
||
"""One NAICS industry across all areas."""
|
||
return _fetch_csv(f"{QCEW_BASE}/{year}/{qtr}/industry/{industry_code}.csv")
|
||
|
||
|
||
def size(size_code: str, year: int) -> list[dict]:
|
||
"""Establishment-size breakdown (size data is published for Q1 only)."""
|
||
return _fetch_csv(f"{QCEW_BASE}/{year}/1/size/{size_code}.csv")
|
||
|
||
|
||
def filter_rows(rows, own_code=None, industry_code=None, agglvl_code=None) -> list[dict]:
|
||
"""Filter QCEW rows by ownership / industry / aggregation-level code."""
|
||
out = rows
|
||
if own_code is not None:
|
||
own = OWNERSHIP.get(own_code, own_code)
|
||
out = [r for r in out if r.get("own_code") == own]
|
||
if industry_code is not None:
|
||
out = [r for r in out if r.get("industry_code") == industry_code]
|
||
if agglvl_code is not None:
|
||
out = [r for r in out if r.get("agglvl_code") == agglvl_code]
|
||
return out
|