Add QCEW CSV module, ECEC breakdown, response caching + typed errors
Item 1 — QCEW county/industry detail: - new bls_client/qcew.py: thin client over the QCEW Open Data CSV service (area/industry/size slices; no API key, no quota), with filter_rows helper - covers the county/industry detail the timeseries API can't reach Item 2 — ECEC benefit breakdown: - new series.ecec() builder from the authoritative cm.estimate decode table - real helpers: wages, total benefits, paid leave, supplemental pay, health insurance, retirement & savings, legally required + ecec_dashboard() - FIX: ecec_total_benefits was CMU1036... (education/health industries only); correct all-civilian total benefits is CMU1030000000000D Item 3 — caching + typed errors: - bls_client/cache.py FileCache (on-disk, TTL); BLSClient(cache=True), queries_used counter (cache hits don't spend quota) - bls_client/errors.py: BLSError / BLSQuotaError / BLSRequestError; quota exhaustion now raises a clear BLSQuotaError instead of a generic RuntimeError Tests: 119 offline (+24) incl. ECEC goldens, QCEW fixture parse, cache/quota behavior; live smoke extended to ECEC + QCEW. Helper sweep: 96/96 live. README: caching/QCEW usage, updated coverage + limitations. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
7
tests/fixtures/qcew_dc_sample.csv
vendored
Normal file
7
tests/fixtures/qcew_dc_sample.csv
vendored
Normal file
@ -0,0 +1,7 @@
|
||||
area_fips,own_code,industry_code,agglvl_code,size_code,year,qtr,disclosure_code,annual_avg_estabs,annual_avg_emplvl,total_annual_wages,taxable_annual_wages,annual_contributions,annual_avg_wkly_wage,avg_annual_pay,lq_disclosure_code,lq_annual_avg_estabs,lq_annual_avg_emplvl,lq_total_annual_wages,lq_taxable_annual_wages,lq_annual_contributions,lq_annual_avg_wkly_wage,lq_avg_annual_pay,oty_disclosure_code,oty_annual_avg_estabs_chg,oty_annual_avg_estabs_pct_chg,oty_annual_avg_emplvl_chg,oty_annual_avg_emplvl_pct_chg,oty_total_annual_wages_chg,oty_total_annual_wages_pct_chg,oty_taxable_annual_wages_chg,oty_taxable_annual_wages_pct_chg,oty_annual_contributions_chg,oty_annual_contributions_pct_chg,oty_annual_avg_wkly_wage_chg,oty_annual_avg_wkly_wage_pct_chg,oty_avg_annual_pay_chg,oty_avg_annual_pay_pct_chg
|
||||
11000,0,10,50,0,2024,A,,52240,759666,93430310874,5400587099,119288884,2365,122989,,1.00,1.00,1.00,1.00,1.00,1.00,1.00,,737,1.4,1912,0.3,4099845160,4.6,-94377645,-1.7,-3132981,-2.6,98,4.3,5101,4.3
|
||||
11000,1,10,51,0,2024,A,,341,193145,27067241499,0,0,2695,140140,,1.29,13.13,11.09,0.00,0.00,0.85,0.84,,1,0.3,2528,1.3,1098653313,4.2,0,0.0,0,0.0,75,2.9,3905,2.9
|
||||
11000,2,10,51,0,2024,A,,14,38244,3958604832,16139000,243,1991,103508,,0.04,1.62,1.27,0.52,0.00,0.79,0.79,,0,0.0,733,2.0,15426132,0.4,3893850,31.8,104,74.8,-31,-1.5,-1612,-1.5
|
||||
11000,3,10,51,0,2024,A,,25,3993,538270989,249180,4736,2592,134804,,0.03,0.06,0.07,0.00,0.01,1.25,1.25,,-1,-3.8,-75,-1.8,7573080,1.4,-94668,-27.5,-1797,-27.5,83,3.3,4331,3.3
|
||||
11000,5,10,51,0,2024,A,,51860,524284,61866193554,5384198919,119283905,2269,118001,,1.02,0.81,0.77,1.02,1.01,0.96,0.96,,736,1.4,-1275,-0.2,2978192635,5.1,-98176827,-1.8,-3131288,-2.6,114,5.3,5953,5.3
|
||||
11000,8,10,96,0,2024,A,,380,235382,31564117320,16388180,4979,2579,134097,,0.29,2.14,2.38,0.15,0.01,1.11,1.11,,1,0.3,3186,1.4,1121652525,3.7,3799182,30.2,-1693,-25.4,58,2.3,2990,2.3
|
||||
|
83
tests/test_client_cache.py
Normal file
83
tests/test_client_cache.py
Normal file
@ -0,0 +1,83 @@
|
||||
"""Offline tests for caching, the query counter, and typed errors.
|
||||
|
||||
A fake transport stands in for requests.post so nothing touches the network.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
import bls_client.client as client_mod
|
||||
from bls_client import BLSClient, BLSQuotaError, BLSRequestError
|
||||
from bls_client.cache import FileCache
|
||||
|
||||
|
||||
class FakeResp:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
def _ok(series_id):
|
||||
return {
|
||||
"status": "REQUEST_SUCCEEDED",
|
||||
"Results": {"series": [{"seriesID": series_id, "data": [{"year": "2024", "period": "M01", "value": "1.0"}]}]},
|
||||
}
|
||||
|
||||
|
||||
def test_query_counter_and_cache(tmp_path, monkeypatch):
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_post(url, json=None, timeout=None):
|
||||
calls["n"] += 1
|
||||
return FakeResp(_ok(json["seriesid"][0]))
|
||||
|
||||
monkeypatch.setattr(client_mod.requests, "post", fake_post)
|
||||
|
||||
c = BLSClient("key", cache=FileCache(cache_dir=tmp_path, ttl=86400))
|
||||
c.fetch(["CES0000000001"], 2024, 2024)
|
||||
assert c.queries_used == 1 and calls["n"] == 1
|
||||
|
||||
# identical request -> served from cache, no new network call
|
||||
c.fetch(["CES0000000001"], 2024, 2024)
|
||||
assert c.queries_used == 1 and calls["n"] == 1
|
||||
|
||||
# different request -> one more call
|
||||
c.fetch(["CES0000000001"], 2023, 2024)
|
||||
assert c.queries_used == 2 and calls["n"] == 2
|
||||
|
||||
|
||||
def test_no_cache_by_default(monkeypatch):
|
||||
def fake_post(url, json=None, timeout=None):
|
||||
return FakeResp(_ok(json["seriesid"][0]))
|
||||
|
||||
monkeypatch.setattr(client_mod.requests, "post", fake_post)
|
||||
c = BLSClient("key")
|
||||
assert c.cache is None
|
||||
c.fetch(["CES0000000001"], 2024, 2024)
|
||||
c.fetch(["CES0000000001"], 2024, 2024)
|
||||
assert c.queries_used == 2 # no caching -> both hit the (fake) network
|
||||
|
||||
|
||||
def test_quota_error(monkeypatch):
|
||||
def fake_post(url, json=None, timeout=None):
|
||||
return FakeResp({
|
||||
"status": "REQUEST_NOT_PROCESSED",
|
||||
"message": ["Query exceeds the threshold of 500 queries per day."],
|
||||
})
|
||||
|
||||
monkeypatch.setattr(client_mod.requests, "post", fake_post)
|
||||
with pytest.raises(BLSQuotaError) as exc:
|
||||
BLSClient("key").fetch(["CES0000000001"], 2024, 2024)
|
||||
assert exc.value.status == "REQUEST_NOT_PROCESSED"
|
||||
|
||||
|
||||
def test_request_error(monkeypatch):
|
||||
def fake_post(url, json=None, timeout=None):
|
||||
return FakeResp({"status": "REQUEST_FAILED", "message": ["invalid series"]})
|
||||
|
||||
monkeypatch.setattr(client_mod.requests, "post", fake_post)
|
||||
with pytest.raises(BLSRequestError):
|
||||
BLSClient("key").fetch(["BOGUS"], 2024, 2024)
|
||||
@ -48,7 +48,8 @@ REPAIRED = {
|
||||
"Productivity": PR.business_output_per_hour(),
|
||||
"QCEW": E.qcew_national_private(),
|
||||
"PPI final": P.ppi_final_demand(),
|
||||
"ECEC": W.ecec_total_benefits(),
|
||||
"ECEC health": W.ecec_health_insurance(),
|
||||
"ECEC retire": W.ecec_retirement(),
|
||||
}
|
||||
|
||||
|
||||
|
||||
51
tests/test_qcew.py
Normal file
51
tests/test_qcew.py
Normal file
@ -0,0 +1,51 @@
|
||||
"""Tests for the QCEW Open Data CSV module.
|
||||
|
||||
Offline tests parse a committed fixture (no network). The live test (opt-in via
|
||||
`-m live`) hits the real CSV service, which needs no API key and no quota.
|
||||
"""
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from bls_client import qcew
|
||||
|
||||
FIXTURE = os.path.join(os.path.dirname(__file__), "fixtures", "qcew_dc_sample.csv")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def dc_rows():
|
||||
with open(FIXTURE) as f:
|
||||
return qcew.parse_csv(f.read())
|
||||
|
||||
|
||||
def test_parse_returns_dict_rows(dc_rows):
|
||||
assert dc_rows and isinstance(dc_rows[0], dict)
|
||||
# core QCEW columns present
|
||||
for col in ("area_fips", "own_code", "industry_code", "annual_avg_emplvl", "annual_avg_wkly_wage"):
|
||||
assert col in dc_rows[0]
|
||||
|
||||
|
||||
def test_all_rows_are_dc_all_industries(dc_rows):
|
||||
assert all(r["area_fips"] == "11000" for r in dc_rows)
|
||||
assert all(r["industry_code"] == "10" for r in dc_rows)
|
||||
|
||||
|
||||
def test_filter_by_ownership(dc_rows):
|
||||
# ownership name resolves to its code, and total (own_code 0) exists
|
||||
total = qcew.filter_rows(dc_rows, own_code="total")
|
||||
assert len(total) == 1
|
||||
assert int(total[0]["annual_avg_emplvl"]) > 0
|
||||
|
||||
|
||||
def test_ownership_name_mapping():
|
||||
assert qcew.OWNERSHIP["private"] == "5"
|
||||
assert qcew.OWNERSHIP["federal"] == "1"
|
||||
|
||||
|
||||
@pytest.mark.live
|
||||
def test_live_area_fetch():
|
||||
rows = qcew.area("11000", 2024, "a") # DC annual, no key / no quota
|
||||
assert len(rows) > 100
|
||||
private_total = qcew.filter_rows(rows, own_code="private", industry_code="10", agglvl_code="51")
|
||||
assert len(private_total) == 1
|
||||
assert int(private_total[0]["annual_avg_emplvl"]) > 100_000 # DC private ~525k
|
||||
@ -18,7 +18,10 @@ GOLDEN = [
|
||||
(P.ppi_finished_goods, "WPUFD4"), # back-compat alias
|
||||
(W.eci_wages, "CIU1020000000000A"),
|
||||
(W.eci_total_compensation, "CIU1010000000000A"),
|
||||
(W.ecec_total_benefits, "CMU1036000000000D"),
|
||||
(W.ecec_total_compensation, "CMU1010000000000D"),
|
||||
(W.ecec_total_benefits, "CMU1030000000000D"),
|
||||
(W.ecec_health_insurance, "CMU1150000000000D"),
|
||||
(W.ecec_retirement, "CMU1180000000000D"),
|
||||
(PR.business_output_per_hour, "PRS84006092"),
|
||||
]
|
||||
|
||||
|
||||
@ -42,6 +42,12 @@ GOLDEN = [
|
||||
(S.productivity, dict(), "PRS85006092"),
|
||||
(S.productivity, dict(sector="manufacturing", measure="unit_labor_cost"), "PRS30006112"),
|
||||
(S.productivity, dict(sector="business"), "PRS84006092"),
|
||||
# ECEC — 17 chars, CMU + owner(1) + component(2) + 10 zeros + datatype
|
||||
(S.ecec, dict(), "CMU1010000000000D"),
|
||||
(S.ecec, dict(component="health_insurance"), "CMU1150000000000D"),
|
||||
(S.ecec, dict(component="retirement_and_savings"), "CMU1180000000000D"),
|
||||
(S.ecec, dict(component="total_benefits", owner="private"), "CMU2030000000000D"),
|
||||
(S.ecec, dict(component="total_compensation", owner="state_local_gov"), "CMU3010000000000D"),
|
||||
]
|
||||
|
||||
|
||||
@ -77,6 +83,13 @@ def test_productivity_is_11_chars():
|
||||
assert sid.startswith("PRS")
|
||||
|
||||
|
||||
def test_ecec_is_17_chars_cost_per_hour():
|
||||
for comp in ("total_compensation", "health_insurance", "retirement_and_savings"):
|
||||
sid = S.ecec(comp)
|
||||
assert len(sid) == 17
|
||||
assert sid.startswith("CMU") and sid.endswith("D")
|
||||
|
||||
|
||||
def test_laus_state_is_20_chars():
|
||||
assert len(S.laus_state(48, "rate")) == 20
|
||||
|
||||
|
||||
Reference in New Issue
Block a user