Add QCEW CSV module, ECEC breakdown, response caching + typed errors

Item 1 — QCEW county/industry detail:
- new bls_client/qcew.py: thin client over the QCEW Open Data CSV service
  (area/industry/size slices; no API key, no quota), with filter_rows helper
- covers the county/industry detail the timeseries API can't reach

Item 2 — ECEC benefit breakdown:
- new series.ecec() builder from the authoritative cm.estimate decode table
- real helpers: wages, total benefits, paid leave, supplemental pay,
  health insurance, retirement & savings, legally required + ecec_dashboard()
- FIX: ecec_total_benefits was CMU1036... (education/health industries only);
  correct all-civilian total benefits is CMU1030000000000D

Item 3 — caching + typed errors:
- bls_client/cache.py FileCache (on-disk, TTL); BLSClient(cache=True),
  queries_used counter (cache hits don't spend quota)
- bls_client/errors.py: BLSError / BLSQuotaError / BLSRequestError; quota
  exhaustion now raises a clear BLSQuotaError instead of a generic RuntimeError

Tests: 119 offline (+24) incl. ECEC goldens, QCEW fixture parse, cache/quota
behavior; live smoke extended to ECEC + QCEW. Helper sweep: 96/96 live.
README: caching/QCEW usage, updated coverage + limitations.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-22 10:38:12 -04:00
parent 881d62cf32
commit 9e2c55c568
14 changed files with 499 additions and 44 deletions

7
tests/fixtures/qcew_dc_sample.csv vendored Normal file
View File

@ -0,0 +1,7 @@
area_fips,own_code,industry_code,agglvl_code,size_code,year,qtr,disclosure_code,annual_avg_estabs,annual_avg_emplvl,total_annual_wages,taxable_annual_wages,annual_contributions,annual_avg_wkly_wage,avg_annual_pay,lq_disclosure_code,lq_annual_avg_estabs,lq_annual_avg_emplvl,lq_total_annual_wages,lq_taxable_annual_wages,lq_annual_contributions,lq_annual_avg_wkly_wage,lq_avg_annual_pay,oty_disclosure_code,oty_annual_avg_estabs_chg,oty_annual_avg_estabs_pct_chg,oty_annual_avg_emplvl_chg,oty_annual_avg_emplvl_pct_chg,oty_total_annual_wages_chg,oty_total_annual_wages_pct_chg,oty_taxable_annual_wages_chg,oty_taxable_annual_wages_pct_chg,oty_annual_contributions_chg,oty_annual_contributions_pct_chg,oty_annual_avg_wkly_wage_chg,oty_annual_avg_wkly_wage_pct_chg,oty_avg_annual_pay_chg,oty_avg_annual_pay_pct_chg
11000,0,10,50,0,2024,A,,52240,759666,93430310874,5400587099,119288884,2365,122989,,1.00,1.00,1.00,1.00,1.00,1.00,1.00,,737,1.4,1912,0.3,4099845160,4.6,-94377645,-1.7,-3132981,-2.6,98,4.3,5101,4.3
11000,1,10,51,0,2024,A,,341,193145,27067241499,0,0,2695,140140,,1.29,13.13,11.09,0.00,0.00,0.85,0.84,,1,0.3,2528,1.3,1098653313,4.2,0,0.0,0,0.0,75,2.9,3905,2.9
11000,2,10,51,0,2024,A,,14,38244,3958604832,16139000,243,1991,103508,,0.04,1.62,1.27,0.52,0.00,0.79,0.79,,0,0.0,733,2.0,15426132,0.4,3893850,31.8,104,74.8,-31,-1.5,-1612,-1.5
11000,3,10,51,0,2024,A,,25,3993,538270989,249180,4736,2592,134804,,0.03,0.06,0.07,0.00,0.01,1.25,1.25,,-1,-3.8,-75,-1.8,7573080,1.4,-94668,-27.5,-1797,-27.5,83,3.3,4331,3.3
11000,5,10,51,0,2024,A,,51860,524284,61866193554,5384198919,119283905,2269,118001,,1.02,0.81,0.77,1.02,1.01,0.96,0.96,,736,1.4,-1275,-0.2,2978192635,5.1,-98176827,-1.8,-3131288,-2.6,114,5.3,5953,5.3
11000,8,10,96,0,2024,A,,380,235382,31564117320,16388180,4979,2579,134097,,0.29,2.14,2.38,0.15,0.01,1.11,1.11,,1,0.3,3186,1.4,1121652525,3.7,3799182,30.2,-1693,-25.4,58,2.3,2990,2.3
1 area_fips own_code industry_code agglvl_code size_code year qtr disclosure_code annual_avg_estabs annual_avg_emplvl total_annual_wages taxable_annual_wages annual_contributions annual_avg_wkly_wage avg_annual_pay lq_disclosure_code lq_annual_avg_estabs lq_annual_avg_emplvl lq_total_annual_wages lq_taxable_annual_wages lq_annual_contributions lq_annual_avg_wkly_wage lq_avg_annual_pay oty_disclosure_code oty_annual_avg_estabs_chg oty_annual_avg_estabs_pct_chg oty_annual_avg_emplvl_chg oty_annual_avg_emplvl_pct_chg oty_total_annual_wages_chg oty_total_annual_wages_pct_chg oty_taxable_annual_wages_chg oty_taxable_annual_wages_pct_chg oty_annual_contributions_chg oty_annual_contributions_pct_chg oty_annual_avg_wkly_wage_chg oty_annual_avg_wkly_wage_pct_chg oty_avg_annual_pay_chg oty_avg_annual_pay_pct_chg
2 11000 0 10 50 0 2024 A 52240 759666 93430310874 5400587099 119288884 2365 122989 1.00 1.00 1.00 1.00 1.00 1.00 1.00 737 1.4 1912 0.3 4099845160 4.6 -94377645 -1.7 -3132981 -2.6 98 4.3 5101 4.3
3 11000 1 10 51 0 2024 A 341 193145 27067241499 0 0 2695 140140 1.29 13.13 11.09 0.00 0.00 0.85 0.84 1 0.3 2528 1.3 1098653313 4.2 0 0.0 0 0.0 75 2.9 3905 2.9
4 11000 2 10 51 0 2024 A 14 38244 3958604832 16139000 243 1991 103508 0.04 1.62 1.27 0.52 0.00 0.79 0.79 0 0.0 733 2.0 15426132 0.4 3893850 31.8 104 74.8 -31 -1.5 -1612 -1.5
5 11000 3 10 51 0 2024 A 25 3993 538270989 249180 4736 2592 134804 0.03 0.06 0.07 0.00 0.01 1.25 1.25 -1 -3.8 -75 -1.8 7573080 1.4 -94668 -27.5 -1797 -27.5 83 3.3 4331 3.3
6 11000 5 10 51 0 2024 A 51860 524284 61866193554 5384198919 119283905 2269 118001 1.02 0.81 0.77 1.02 1.01 0.96 0.96 736 1.4 -1275 -0.2 2978192635 5.1 -98176827 -1.8 -3131288 -2.6 114 5.3 5953 5.3
7 11000 8 10 96 0 2024 A 380 235382 31564117320 16388180 4979 2579 134097 0.29 2.14 2.38 0.15 0.01 1.11 1.11 1 0.3 3186 1.4 1121652525 3.7 3799182 30.2 -1693 -25.4 58 2.3 2990 2.3

View File

@ -0,0 +1,83 @@
"""Offline tests for caching, the query counter, and typed errors.
A fake transport stands in for requests.post so nothing touches the network.
"""
import pytest
import bls_client.client as client_mod
from bls_client import BLSClient, BLSQuotaError, BLSRequestError
from bls_client.cache import FileCache
class FakeResp:
def __init__(self, payload):
self._payload = payload
def raise_for_status(self):
pass
def json(self):
return self._payload
def _ok(series_id):
return {
"status": "REQUEST_SUCCEEDED",
"Results": {"series": [{"seriesID": series_id, "data": [{"year": "2024", "period": "M01", "value": "1.0"}]}]},
}
def test_query_counter_and_cache(tmp_path, monkeypatch):
calls = {"n": 0}
def fake_post(url, json=None, timeout=None):
calls["n"] += 1
return FakeResp(_ok(json["seriesid"][0]))
monkeypatch.setattr(client_mod.requests, "post", fake_post)
c = BLSClient("key", cache=FileCache(cache_dir=tmp_path, ttl=86400))
c.fetch(["CES0000000001"], 2024, 2024)
assert c.queries_used == 1 and calls["n"] == 1
# identical request -> served from cache, no new network call
c.fetch(["CES0000000001"], 2024, 2024)
assert c.queries_used == 1 and calls["n"] == 1
# different request -> one more call
c.fetch(["CES0000000001"], 2023, 2024)
assert c.queries_used == 2 and calls["n"] == 2
def test_no_cache_by_default(monkeypatch):
def fake_post(url, json=None, timeout=None):
return FakeResp(_ok(json["seriesid"][0]))
monkeypatch.setattr(client_mod.requests, "post", fake_post)
c = BLSClient("key")
assert c.cache is None
c.fetch(["CES0000000001"], 2024, 2024)
c.fetch(["CES0000000001"], 2024, 2024)
assert c.queries_used == 2 # no caching -> both hit the (fake) network
def test_quota_error(monkeypatch):
def fake_post(url, json=None, timeout=None):
return FakeResp({
"status": "REQUEST_NOT_PROCESSED",
"message": ["Query exceeds the threshold of 500 queries per day."],
})
monkeypatch.setattr(client_mod.requests, "post", fake_post)
with pytest.raises(BLSQuotaError) as exc:
BLSClient("key").fetch(["CES0000000001"], 2024, 2024)
assert exc.value.status == "REQUEST_NOT_PROCESSED"
def test_request_error(monkeypatch):
def fake_post(url, json=None, timeout=None):
return FakeResp({"status": "REQUEST_FAILED", "message": ["invalid series"]})
monkeypatch.setattr(client_mod.requests, "post", fake_post)
with pytest.raises(BLSRequestError):
BLSClient("key").fetch(["BOGUS"], 2024, 2024)

View File

@ -48,7 +48,8 @@ REPAIRED = {
"Productivity": PR.business_output_per_hour(),
"QCEW": E.qcew_national_private(),
"PPI final": P.ppi_final_demand(),
"ECEC": W.ecec_total_benefits(),
"ECEC health": W.ecec_health_insurance(),
"ECEC retire": W.ecec_retirement(),
}

51
tests/test_qcew.py Normal file
View File

@ -0,0 +1,51 @@
"""Tests for the QCEW Open Data CSV module.
Offline tests parse a committed fixture (no network). The live test (opt-in via
`-m live`) hits the real CSV service, which needs no API key and no quota.
"""
import os
import pytest
from bls_client import qcew
FIXTURE = os.path.join(os.path.dirname(__file__), "fixtures", "qcew_dc_sample.csv")
@pytest.fixture(scope="module")
def dc_rows():
with open(FIXTURE) as f:
return qcew.parse_csv(f.read())
def test_parse_returns_dict_rows(dc_rows):
assert dc_rows and isinstance(dc_rows[0], dict)
# core QCEW columns present
for col in ("area_fips", "own_code", "industry_code", "annual_avg_emplvl", "annual_avg_wkly_wage"):
assert col in dc_rows[0]
def test_all_rows_are_dc_all_industries(dc_rows):
assert all(r["area_fips"] == "11000" for r in dc_rows)
assert all(r["industry_code"] == "10" for r in dc_rows)
def test_filter_by_ownership(dc_rows):
# ownership name resolves to its code, and total (own_code 0) exists
total = qcew.filter_rows(dc_rows, own_code="total")
assert len(total) == 1
assert int(total[0]["annual_avg_emplvl"]) > 0
def test_ownership_name_mapping():
assert qcew.OWNERSHIP["private"] == "5"
assert qcew.OWNERSHIP["federal"] == "1"
@pytest.mark.live
def test_live_area_fetch():
rows = qcew.area("11000", 2024, "a") # DC annual, no key / no quota
assert len(rows) > 100
private_total = qcew.filter_rows(rows, own_code="private", industry_code="10", agglvl_code="51")
assert len(private_total) == 1
assert int(private_total[0]["annual_avg_emplvl"]) > 100_000 # DC private ~525k

View File

@ -18,7 +18,10 @@ GOLDEN = [
(P.ppi_finished_goods, "WPUFD4"), # back-compat alias
(W.eci_wages, "CIU1020000000000A"),
(W.eci_total_compensation, "CIU1010000000000A"),
(W.ecec_total_benefits, "CMU1036000000000D"),
(W.ecec_total_compensation, "CMU1010000000000D"),
(W.ecec_total_benefits, "CMU1030000000000D"),
(W.ecec_health_insurance, "CMU1150000000000D"),
(W.ecec_retirement, "CMU1180000000000D"),
(PR.business_output_per_hour, "PRS84006092"),
]

View File

@ -42,6 +42,12 @@ GOLDEN = [
(S.productivity, dict(), "PRS85006092"),
(S.productivity, dict(sector="manufacturing", measure="unit_labor_cost"), "PRS30006112"),
(S.productivity, dict(sector="business"), "PRS84006092"),
# ECEC — 17 chars, CMU + owner(1) + component(2) + 10 zeros + datatype
(S.ecec, dict(), "CMU1010000000000D"),
(S.ecec, dict(component="health_insurance"), "CMU1150000000000D"),
(S.ecec, dict(component="retirement_and_savings"), "CMU1180000000000D"),
(S.ecec, dict(component="total_benefits", owner="private"), "CMU2030000000000D"),
(S.ecec, dict(component="total_compensation", owner="state_local_gov"), "CMU3010000000000D"),
]
@ -77,6 +83,13 @@ def test_productivity_is_11_chars():
assert sid.startswith("PRS")
def test_ecec_is_17_chars_cost_per_hour():
for comp in ("total_compensation", "health_insurance", "retirement_and_savings"):
sid = S.ecec(comp)
assert len(sid) == 17
assert sid.startswith("CMU") and sid.endswith("D")
def test_laus_state_is_20_chars():
assert len(S.laus_state(48, "rate")) == 20