Add QCEW CSV module, ECEC breakdown, response caching + typed errors
Item 1 — QCEW county/industry detail: - new bls_client/qcew.py: thin client over the QCEW Open Data CSV service (area/industry/size slices; no API key, no quota), with filter_rows helper - covers the county/industry detail the timeseries API can't reach Item 2 — ECEC benefit breakdown: - new series.ecec() builder from the authoritative cm.estimate decode table - real helpers: wages, total benefits, paid leave, supplemental pay, health insurance, retirement & savings, legally required + ecec_dashboard() - FIX: ecec_total_benefits was CMU1036... (education/health industries only); correct all-civilian total benefits is CMU1030000000000D Item 3 — caching + typed errors: - bls_client/cache.py FileCache (on-disk, TTL); BLSClient(cache=True), queries_used counter (cache hits don't spend quota) - bls_client/errors.py: BLSError / BLSQuotaError / BLSRequestError; quota exhaustion now raises a clear BLSQuotaError instead of a generic RuntimeError Tests: 119 offline (+24) incl. ECEC goldens, QCEW fixture parse, cache/quota behavior; live smoke extended to ECEC + QCEW. Helper sweep: 96/96 live. README: caching/QCEW usage, updated coverage + limitations. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@ -1,2 +1,3 @@
|
||||
from .client import BLSClient
|
||||
from . import queries, series
|
||||
from .errors import BLSError, BLSRequestError, BLSQuotaError
|
||||
from . import queries, series, qcew
|
||||
|
||||
49
bls_client/cache.py
Normal file
49
bls_client/cache.py
Normal file
@ -0,0 +1,49 @@
|
||||
"""A tiny on-disk response cache for BLS requests.
|
||||
|
||||
BLS data updates monthly/quarterly, so caching identical requests avoids
|
||||
re-spending the 500/day quota on reruns (reports, the explorer, test sweeps).
|
||||
Pure stdlib — no extra dependency.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class FileCache:
|
||||
"""JSON file cache keyed by request payload, with a TTL (seconds)."""
|
||||
|
||||
def __init__(self, cache_dir=None, ttl=86400):
|
||||
self.dir = Path(cache_dir or os.path.expanduser("~/.cache/bls"))
|
||||
self.ttl = ttl
|
||||
self.dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@staticmethod
|
||||
def key(payload: dict) -> str:
|
||||
"""Deterministic key from a request payload (API key excluded, series sorted)."""
|
||||
relevant = {k: v for k, v in payload.items() if k != "registrationkey"}
|
||||
if isinstance(relevant.get("seriesid"), list):
|
||||
relevant["seriesid"] = sorted(relevant["seriesid"])
|
||||
blob = json.dumps(relevant, sort_keys=True, default=str)
|
||||
return hashlib.sha256(blob.encode()).hexdigest()
|
||||
|
||||
def _path(self, key: str) -> Path:
|
||||
return self.dir / f"{key}.json"
|
||||
|
||||
def get(self, key: str):
|
||||
p = self._path(key)
|
||||
if not p.exists():
|
||||
return None
|
||||
if self.ttl is not None and (time.time() - p.stat().st_mtime) > self.ttl:
|
||||
return None
|
||||
try:
|
||||
return json.loads(p.read_text())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
def set(self, key: str, value) -> None:
|
||||
try:
|
||||
self._path(key).write_text(json.dumps(value))
|
||||
except OSError:
|
||||
pass # caching is best-effort; never fail a fetch over it
|
||||
@ -10,14 +10,32 @@ Usage:
|
||||
import requests
|
||||
from datetime import datetime
|
||||
|
||||
from .cache import FileCache
|
||||
from .errors import BLSError, BLSRequestError, BLSQuotaError
|
||||
|
||||
|
||||
class BLSClient:
|
||||
BASE_URL = "https://api.bls.gov/publicAPI/v2"
|
||||
MAX_SERIES_PER_CALL = 50
|
||||
MAX_YEARS_PER_CALL = 20
|
||||
|
||||
def __init__(self, api_key: str):
|
||||
def __init__(self, api_key: str, cache=False, cache_dir=None, cache_ttl=86400):
|
||||
"""
|
||||
Args:
|
||||
api_key: BLS registration key.
|
||||
cache: True to enable the default on-disk cache (~/.cache/bls),
|
||||
or pass a custom cache object with get(key)/set(key, value).
|
||||
cache_dir: Override the default cache directory.
|
||||
cache_ttl: Cache entry lifetime in seconds (default 1 day).
|
||||
"""
|
||||
self.api_key = api_key
|
||||
if cache is True:
|
||||
self.cache = FileCache(cache_dir, cache_ttl)
|
||||
elif cache:
|
||||
self.cache = cache # caller-supplied cache object
|
||||
else:
|
||||
self.cache = None
|
||||
self.queries_used = 0 # network POSTs spent this session (cache hits don't count)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Core fetch
|
||||
@ -64,23 +82,44 @@ class BLSClient:
|
||||
"calculations": calculations,
|
||||
"annualaverage": annual_average,
|
||||
}
|
||||
r = requests.post(
|
||||
f"{self.BASE_URL}/timeseries/data/",
|
||||
json=payload,
|
||||
timeout=30,
|
||||
)
|
||||
r.raise_for_status()
|
||||
body = r.json()
|
||||
if body["status"] != "REQUEST_SUCCEEDED":
|
||||
raise RuntimeError(f"BLS API error: {body['message']}")
|
||||
for s in body["Results"]["series"]:
|
||||
results[s["seriesID"]] = {
|
||||
"data": s["data"],
|
||||
"catalog": s.get("catalog", {}),
|
||||
}
|
||||
|
||||
cache_key = self.cache.key(payload) if self.cache else None
|
||||
batch_result = self.cache.get(cache_key) if cache_key else None
|
||||
|
||||
if batch_result is None:
|
||||
batch_result = self._fetch_batch(payload)
|
||||
if cache_key:
|
||||
self.cache.set(cache_key, batch_result)
|
||||
|
||||
results.update(batch_result)
|
||||
|
||||
return results
|
||||
|
||||
def _fetch_batch(self, payload: dict) -> dict:
|
||||
"""Execute one (uncached) API call and return {series_id: {data, catalog}}."""
|
||||
r = requests.post(f"{self.BASE_URL}/timeseries/data/", json=payload, timeout=30)
|
||||
r.raise_for_status()
|
||||
self.queries_used += 1
|
||||
body = r.json()
|
||||
|
||||
status = body.get("status")
|
||||
if status != "REQUEST_SUCCEEDED":
|
||||
msgs = body.get("message", [])
|
||||
text = " ".join(msgs) if isinstance(msgs, list) else str(msgs)
|
||||
low = text.lower()
|
||||
if status == "REQUEST_NOT_PROCESSED" and ("threshold" in low or "exceed" in low):
|
||||
raise BLSQuotaError(
|
||||
f"BLS daily query limit reached: {text}", status=status, messages=msgs
|
||||
)
|
||||
raise BLSRequestError(
|
||||
f"BLS API request failed ({status}): {text}", status=status, messages=msgs
|
||||
)
|
||||
|
||||
return {
|
||||
s["seriesID"]: {"data": s["data"], "catalog": s.get("catalog", {})}
|
||||
for s in body["Results"]["series"]
|
||||
}
|
||||
|
||||
def fetch_latest(
|
||||
self,
|
||||
series_ids: list[str] | str,
|
||||
|
||||
23
bls_client/errors.py
Normal file
23
bls_client/errors.py
Normal file
@ -0,0 +1,23 @@
|
||||
"""Typed exceptions for the BLS client.
|
||||
|
||||
Lets callers distinguish "you're throttled" from "your request was bad" — the
|
||||
two failure modes the old generic RuntimeError conflated.
|
||||
"""
|
||||
|
||||
|
||||
class BLSError(RuntimeError):
|
||||
"""Base class for all BLS API errors."""
|
||||
|
||||
def __init__(self, message, *, status=None, messages=None):
|
||||
super().__init__(message)
|
||||
self.status = status # the API "status" field, e.g. REQUEST_FAILED
|
||||
self.messages = messages or [] # the API "message" list, verbatim
|
||||
|
||||
|
||||
class BLSRequestError(BLSError):
|
||||
"""The API rejected the request (malformed series ID, bad year range, etc.)."""
|
||||
|
||||
|
||||
class BLSQuotaError(BLSError):
|
||||
"""The daily query threshold was exceeded (500/day for a registered key,
|
||||
25/day unregistered). Resets at midnight Eastern."""
|
||||
74
bls_client/qcew.py
Normal file
74
bls_client/qcew.py
Normal file
@ -0,0 +1,74 @@
|
||||
"""QCEW Open Data Access — county/industry employment & wages.
|
||||
|
||||
The BLS *timeseries* API only carries QCEW national/state totals. The full
|
||||
QCEW detail (every county, every NAICS industry, establishment counts and
|
||||
wages) lives behind a separate CSV service that needs no API key and counts
|
||||
against no quota:
|
||||
|
||||
https://data.bls.gov/cew/data/api/{year}/{qtr}/{slice}/{code}.csv
|
||||
|
||||
This module is a thin wrapper over that service. It is deliberately separate
|
||||
from BLSClient (different API, CSV not JSON, no key). Each call returns a list
|
||||
of plain dict rows (the CSV columns), so it composes with csv/pandas directly.
|
||||
|
||||
Quarter argument: 1-4 for a specific quarter, or "a" for annual averages.
|
||||
Area FIPS examples: "US000" national, "11000" DC (state), "11001" a county,
|
||||
metro codes start with "C". Industry codes are NAICS (e.g. "10" = all
|
||||
industries, "722" = food services, "622" = hospitals).
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
|
||||
import requests
|
||||
|
||||
QCEW_BASE = "https://data.bls.gov/cew/data/api"
|
||||
_HEADERS = {"User-Agent": "bls-data-library (https://gitea.doesworks.net/giteaadmin/bls-data)"}
|
||||
|
||||
# A few common columns, for reference / convenience:
|
||||
# area_fips, own_code, industry_code, agglvl_code, size_code, year, qtr,
|
||||
# annual_avg_estabs, annual_avg_emplvl, total_annual_wages, annual_avg_wkly_wage,
|
||||
# avg_annual_pay (quarterly slices use month1_emplvl/qtrly_estabs/total_qtrly_wages, etc.)
|
||||
|
||||
OWNERSHIP = {
|
||||
"total": "0", "federal": "1", "state": "2", "local": "3",
|
||||
"private": "5", "government": "8",
|
||||
}
|
||||
|
||||
|
||||
def _fetch_csv(url: str) -> list[dict]:
|
||||
r = requests.get(url, timeout=60, headers=_HEADERS)
|
||||
r.raise_for_status()
|
||||
return parse_csv(r.text)
|
||||
|
||||
|
||||
def parse_csv(text: str) -> list[dict]:
|
||||
"""Parse QCEW CSV text into a list of row dicts (BLS quotes every field)."""
|
||||
return list(csv.DictReader(io.StringIO(text)))
|
||||
|
||||
|
||||
def area(area_fips: str, year: int, qtr="a") -> list[dict]:
|
||||
"""Every industry × ownership for one area (county/state/metro/national)."""
|
||||
return _fetch_csv(f"{QCEW_BASE}/{year}/{qtr}/area/{area_fips}.csv")
|
||||
|
||||
|
||||
def industry(industry_code: str, year: int, qtr="a") -> list[dict]:
|
||||
"""One NAICS industry across all areas."""
|
||||
return _fetch_csv(f"{QCEW_BASE}/{year}/{qtr}/industry/{industry_code}.csv")
|
||||
|
||||
|
||||
def size(size_code: str, year: int) -> list[dict]:
|
||||
"""Establishment-size breakdown (size data is published for Q1 only)."""
|
||||
return _fetch_csv(f"{QCEW_BASE}/{year}/1/size/{size_code}.csv")
|
||||
|
||||
|
||||
def filter_rows(rows, own_code=None, industry_code=None, agglvl_code=None) -> list[dict]:
|
||||
"""Filter QCEW rows by ownership / industry / aggregation-level code."""
|
||||
out = rows
|
||||
if own_code is not None:
|
||||
own = OWNERSHIP.get(own_code, own_code)
|
||||
out = [r for r in out if r.get("own_code") == own]
|
||||
if industry_code is not None:
|
||||
out = [r for r in out if r.get("industry_code") == industry_code]
|
||||
if agglvl_code is not None:
|
||||
out = [r for r in out if r.get("agglvl_code") == agglvl_code]
|
||||
return out
|
||||
@ -2,7 +2,7 @@
|
||||
Pre-built wage and compensation series IDs — OES, ECI, ECEC, CPS earnings.
|
||||
"""
|
||||
|
||||
from ..series import oes_national, eci
|
||||
from ..series import oes_national, eci, ecec
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OES — Occupational Employment & Wage Statistics
|
||||
@ -89,19 +89,58 @@ def eci_dashboard() -> dict:
|
||||
# ---------------------------------------------------------------------------
|
||||
# ECEC — Employer Costs for Employee Compensation
|
||||
# ---------------------------------------------------------------------------
|
||||
def ecec_total_compensation() -> str:
|
||||
"""ECEC — civilian workers, total compensation cost per hour worked."""
|
||||
return "CMU1010000000000D"
|
||||
def ecec_total_compensation(owner: str = "civilian") -> str:
|
||||
"""ECEC — total compensation cost per hour worked."""
|
||||
return ecec("total_compensation", owner)
|
||||
|
||||
|
||||
def ecec_total_benefits() -> str:
|
||||
"""ECEC — civilian workers, total benefits cost per hour worked."""
|
||||
return "CMU1036000000000D"
|
||||
def ecec_wages_and_salaries(owner: str = "civilian") -> str:
|
||||
"""ECEC — wages and salaries cost per hour worked."""
|
||||
return ecec("wages_and_salaries", owner)
|
||||
|
||||
# Note: ECEC benefit subcomponents (health insurance, retirement & savings, etc.)
|
||||
# are encoded as specific benefit-subcell codes in the series ID (suffix stays "D").
|
||||
# Look them up against the ECEC component list before adding helpers — do not
|
||||
# fabricate them with a letter suffix (the old "...H"/"...R" forms were invalid).
|
||||
|
||||
def ecec_total_benefits(owner: str = "civilian") -> str:
|
||||
"""ECEC — total benefits cost per hour worked."""
|
||||
return ecec("total_benefits", owner)
|
||||
|
||||
|
||||
def ecec_paid_leave(owner: str = "civilian") -> str:
|
||||
"""ECEC — paid leave cost per hour worked (vacation, holiday, sick, personal)."""
|
||||
return ecec("paid_leave", owner)
|
||||
|
||||
|
||||
def ecec_supplemental_pay(owner: str = "civilian") -> str:
|
||||
"""ECEC — supplemental pay cost per hour worked (overtime, bonuses, shift diff)."""
|
||||
return ecec("supplemental_pay", owner)
|
||||
|
||||
|
||||
def ecec_health_insurance(owner: str = "civilian") -> str:
|
||||
"""ECEC — health insurance cost per hour worked."""
|
||||
return ecec("health_insurance", owner)
|
||||
|
||||
|
||||
def ecec_retirement(owner: str = "civilian") -> str:
|
||||
"""ECEC — retirement & savings cost per hour worked (defined benefit + contribution)."""
|
||||
return ecec("retirement_and_savings", owner)
|
||||
|
||||
|
||||
def ecec_legally_required(owner: str = "civilian") -> str:
|
||||
"""ECEC — legally required benefits per hour (Social Security, Medicare, UI, workers' comp)."""
|
||||
return ecec("legally_required", owner)
|
||||
|
||||
|
||||
def ecec_dashboard(owner: str = "civilian") -> dict:
|
||||
"""The compensation cost breakdown — what an hour of labor costs an employer."""
|
||||
return {
|
||||
"Total Compensation": ecec_total_compensation(owner),
|
||||
"Wages & Salaries": ecec_wages_and_salaries(owner),
|
||||
"Total Benefits": ecec_total_benefits(owner),
|
||||
"Paid Leave": ecec_paid_leave(owner),
|
||||
"Supplemental Pay": ecec_supplemental_pay(owner),
|
||||
"Health Insurance": ecec_health_insurance(owner),
|
||||
"Retirement & Savings": ecec_retirement(owner),
|
||||
"Legally Required": ecec_legally_required(owner),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@ -336,6 +336,55 @@ def eci(
|
||||
return f"CI{adj}{owner}{component}{'0'*9}{estimate}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ECEC — Employer Costs for Employee Compensation
|
||||
# Component ("estimate") codes from the BLS cm.estimate decode table.
|
||||
# ---------------------------------------------------------------------------
|
||||
_ECEC_COMPONENT = {
|
||||
"total_compensation": "01",
|
||||
"wages_and_salaries": "02",
|
||||
"total_benefits": "03",
|
||||
"paid_leave": "04",
|
||||
"supplemental_pay": "09",
|
||||
"insurance": "13",
|
||||
"health_insurance": "15",
|
||||
"retirement_and_savings": "18",
|
||||
"legally_required": "21",
|
||||
"workers_compensation": "27",
|
||||
}
|
||||
|
||||
_ECEC_OWNER = {
|
||||
"civilian": "1",
|
||||
"private": "2",
|
||||
"state_local_gov": "3",
|
||||
}
|
||||
|
||||
def ecec(
|
||||
component: str = "total_compensation",
|
||||
owner: str = "civilian",
|
||||
datatype: str = "D", # D = cost per hour worked (dollars)
|
||||
) -> str:
|
||||
"""
|
||||
ECEC series — employer cost per hour worked, all industries / all workers.
|
||||
|
||||
Format: CM + U + owner(1) + component(2) + 10 zeros + datatype(1) = 17 chars.
|
||||
|
||||
Args:
|
||||
component: key in _ECEC_COMPONENT (e.g. "health_insurance") or a raw 2-digit code
|
||||
owner: "civilian" | "private" | "state_local_gov" (or raw "1"/"2"/"3")
|
||||
datatype: "D" = cost per hour in dollars (default); "P" = percent of total comp
|
||||
|
||||
Examples:
|
||||
ecec() → "CMU1010000000000D" (civilian total compensation)
|
||||
ecec("health_insurance") → "CMU1150000000000D"
|
||||
ecec("retirement_and_savings") → "CMU1180000000000D"
|
||||
ecec("total_benefits", "private") → "CMU2030000000000D"
|
||||
"""
|
||||
own = _ECEC_OWNER.get(owner, owner)
|
||||
code = _ECEC_COMPONENT.get(component, component)
|
||||
return f"CMU{own}{code}{'0'*10}{datatype}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Productivity (Major Sector)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user