diff --git a/README.md b/README.md index 53c770c..8de460c 100644 --- a/README.md +++ b/README.md @@ -64,12 +64,15 @@ pytest -m live # also hit the live API (needs network + ``` bls_client/ -├── client.py BLSClient — batching, named fetches, catalog metadata, row flattening -├── series.py low-level series-ID builders (LAUS, CES, CPI, PPI, OES, JOLTS, ECI, QCEW, productivity) +├── client.py BLSClient — batching, caching, named fetches, catalog metadata, row flattening +├── series.py low-level series-ID builders (LAUS, CES, CPI, PPI, OES, JOLTS, ECI, ECEC, QCEW, productivity) +├── qcew.py QCEW Open Data CSV client (county/industry detail; no key, no quota) +├── cache.py on-disk response cache (FileCache) +├── errors.py BLSError / BLSQuotaError / BLSRequestError └── queries/ - ├── employment.py payrolls, unemployment (LAUS), JOLTS, QCEW + ├── employment.py payrolls, unemployment (LAUS), JOLTS, QCEW totals ├── prices.py CPI, PPI, average prices, import/export prices - ├── wages.py OES occupational wages, ECI, ECEC + ├── wages.py OES occupational wages, ECI, ECEC breakdown └── productivity.py major-sector productivity & costs ``` @@ -96,20 +99,40 @@ for the series-ID decode tables. | PPI — producer prices | all commodities, final demand, food, energy | ✅ | | OES — occupational wages | employment + wage percentiles by SOC; 15 common occupations | ✅ | | ECI — employment cost index | total comp / wages / benefits × civilian/private/gov | ✅ | -| ECEC — employer cost levels | total compensation, total benefits | ✅ (totals only) | +| ECEC — employer cost levels | full breakdown: comp, wages, benefits, paid leave, supplemental, health insurance, retirement, legally-required | ✅ | | Productivity & costs | output/hr, ULC, comp, hours × business/nonfarm/manufacturing | ✅ | -| QCEW — quarterly census | national & state private totals | ⚠️ totals only via this API | +| QCEW — quarterly census | national/state totals (timeseries) **plus** full county/industry detail via the CSV module (`bls_client.qcew`) | ✅ | -## Known limitations / what "complete" would add +## Caching & quota handling -- **QCEW** is only partially served by the BLS *timeseries* API used here (national and - state-level totals work). County- and industry-level QCEW detail requires the separate - **QCEW Open Data API** (CSV: `https://data.bls.gov/cew/data/api/...`). Not yet wired up. -- **ECEC benefit subcomponents** (health insurance, retirement & savings, etc.) need - specific benefit-subcell codes from the ECEC component list; only the compensation and - benefits totals are currently exposed. -- **No caching / rate-limit handling.** Repeated runs spend against the 500/day quota; a - small on-disk cache and a friendly error on `REQUEST_NOT_PROCESSED` (quota hit) would help. +```python +client = BLSClient(API_KEY, cache=True) # on-disk cache at ~/.cache/bls (1-day TTL) +client.fetch(...) # repeated identical requests don't re-spend quota +client.queries_used # network calls made this session (cache hits excluded) +``` + +A blown daily quota raises `BLSQuotaError` (vs `BLSRequestError` for a bad request), so +"am I throttled or is my series ID wrong?" is no longer ambiguous. + +## QCEW county/industry detail + +The timeseries helpers give QCEW national/state totals; the `qcew` module reaches the full +county- and industry-level detail via BLS's separate CSV service (no key, no quota): + +```python +from bls_client import qcew +rows = qcew.area("11000", 2024, "a") # everything for DC, annual +dc_private = qcew.filter_rows(rows, own_code="private", industry_code="10", agglvl_code="51") +hospitals = qcew.industry("622", 2024, "a") # one NAICS across all areas +``` + +## Known limitations + +- **No automatic retry/backoff** on transient network errors (a `requests` failure surfaces + directly). Caching mitigates repeat load but there's no rate-limit pacing. +- **Coverage is the headline cut** of each survey, not an exhaustive mirror — e.g. CES is + national supersectors + a couple of states; CPI is the common items; OES ships 15 named + occupations (any SOC works via `occupation_*`). Broaden the helper dicts as needed. --- diff --git a/bls_client/__init__.py b/bls_client/__init__.py index b425f46..3725ec5 100644 --- a/bls_client/__init__.py +++ b/bls_client/__init__.py @@ -1,2 +1,3 @@ from .client import BLSClient -from . import queries, series +from .errors import BLSError, BLSRequestError, BLSQuotaError +from . import queries, series, qcew diff --git a/bls_client/cache.py b/bls_client/cache.py new file mode 100644 index 0000000..1d2fe2e --- /dev/null +++ b/bls_client/cache.py @@ -0,0 +1,49 @@ +"""A tiny on-disk response cache for BLS requests. + +BLS data updates monthly/quarterly, so caching identical requests avoids +re-spending the 500/day quota on reruns (reports, the explorer, test sweeps). +Pure stdlib — no extra dependency. +""" +import hashlib +import json +import os +import time +from pathlib import Path + + +class FileCache: + """JSON file cache keyed by request payload, with a TTL (seconds).""" + + def __init__(self, cache_dir=None, ttl=86400): + self.dir = Path(cache_dir or os.path.expanduser("~/.cache/bls")) + self.ttl = ttl + self.dir.mkdir(parents=True, exist_ok=True) + + @staticmethod + def key(payload: dict) -> str: + """Deterministic key from a request payload (API key excluded, series sorted).""" + relevant = {k: v for k, v in payload.items() if k != "registrationkey"} + if isinstance(relevant.get("seriesid"), list): + relevant["seriesid"] = sorted(relevant["seriesid"]) + blob = json.dumps(relevant, sort_keys=True, default=str) + return hashlib.sha256(blob.encode()).hexdigest() + + def _path(self, key: str) -> Path: + return self.dir / f"{key}.json" + + def get(self, key: str): + p = self._path(key) + if not p.exists(): + return None + if self.ttl is not None and (time.time() - p.stat().st_mtime) > self.ttl: + return None + try: + return json.loads(p.read_text()) + except (OSError, ValueError): + return None + + def set(self, key: str, value) -> None: + try: + self._path(key).write_text(json.dumps(value)) + except OSError: + pass # caching is best-effort; never fail a fetch over it diff --git a/bls_client/client.py b/bls_client/client.py index 458c223..deee7b6 100644 --- a/bls_client/client.py +++ b/bls_client/client.py @@ -10,14 +10,32 @@ Usage: import requests from datetime import datetime +from .cache import FileCache +from .errors import BLSError, BLSRequestError, BLSQuotaError + class BLSClient: BASE_URL = "https://api.bls.gov/publicAPI/v2" MAX_SERIES_PER_CALL = 50 MAX_YEARS_PER_CALL = 20 - def __init__(self, api_key: str): + def __init__(self, api_key: str, cache=False, cache_dir=None, cache_ttl=86400): + """ + Args: + api_key: BLS registration key. + cache: True to enable the default on-disk cache (~/.cache/bls), + or pass a custom cache object with get(key)/set(key, value). + cache_dir: Override the default cache directory. + cache_ttl: Cache entry lifetime in seconds (default 1 day). + """ self.api_key = api_key + if cache is True: + self.cache = FileCache(cache_dir, cache_ttl) + elif cache: + self.cache = cache # caller-supplied cache object + else: + self.cache = None + self.queries_used = 0 # network POSTs spent this session (cache hits don't count) # ------------------------------------------------------------------ # Core fetch @@ -64,23 +82,44 @@ class BLSClient: "calculations": calculations, "annualaverage": annual_average, } - r = requests.post( - f"{self.BASE_URL}/timeseries/data/", - json=payload, - timeout=30, - ) - r.raise_for_status() - body = r.json() - if body["status"] != "REQUEST_SUCCEEDED": - raise RuntimeError(f"BLS API error: {body['message']}") - for s in body["Results"]["series"]: - results[s["seriesID"]] = { - "data": s["data"], - "catalog": s.get("catalog", {}), - } + + cache_key = self.cache.key(payload) if self.cache else None + batch_result = self.cache.get(cache_key) if cache_key else None + + if batch_result is None: + batch_result = self._fetch_batch(payload) + if cache_key: + self.cache.set(cache_key, batch_result) + + results.update(batch_result) return results + def _fetch_batch(self, payload: dict) -> dict: + """Execute one (uncached) API call and return {series_id: {data, catalog}}.""" + r = requests.post(f"{self.BASE_URL}/timeseries/data/", json=payload, timeout=30) + r.raise_for_status() + self.queries_used += 1 + body = r.json() + + status = body.get("status") + if status != "REQUEST_SUCCEEDED": + msgs = body.get("message", []) + text = " ".join(msgs) if isinstance(msgs, list) else str(msgs) + low = text.lower() + if status == "REQUEST_NOT_PROCESSED" and ("threshold" in low or "exceed" in low): + raise BLSQuotaError( + f"BLS daily query limit reached: {text}", status=status, messages=msgs + ) + raise BLSRequestError( + f"BLS API request failed ({status}): {text}", status=status, messages=msgs + ) + + return { + s["seriesID"]: {"data": s["data"], "catalog": s.get("catalog", {})} + for s in body["Results"]["series"] + } + def fetch_latest( self, series_ids: list[str] | str, diff --git a/bls_client/errors.py b/bls_client/errors.py new file mode 100644 index 0000000..a164c93 --- /dev/null +++ b/bls_client/errors.py @@ -0,0 +1,23 @@ +"""Typed exceptions for the BLS client. + +Lets callers distinguish "you're throttled" from "your request was bad" — the +two failure modes the old generic RuntimeError conflated. +""" + + +class BLSError(RuntimeError): + """Base class for all BLS API errors.""" + + def __init__(self, message, *, status=None, messages=None): + super().__init__(message) + self.status = status # the API "status" field, e.g. REQUEST_FAILED + self.messages = messages or [] # the API "message" list, verbatim + + +class BLSRequestError(BLSError): + """The API rejected the request (malformed series ID, bad year range, etc.).""" + + +class BLSQuotaError(BLSError): + """The daily query threshold was exceeded (500/day for a registered key, + 25/day unregistered). Resets at midnight Eastern.""" diff --git a/bls_client/qcew.py b/bls_client/qcew.py new file mode 100644 index 0000000..7d5f8f6 --- /dev/null +++ b/bls_client/qcew.py @@ -0,0 +1,74 @@ +"""QCEW Open Data Access — county/industry employment & wages. + +The BLS *timeseries* API only carries QCEW national/state totals. The full +QCEW detail (every county, every NAICS industry, establishment counts and +wages) lives behind a separate CSV service that needs no API key and counts +against no quota: + + https://data.bls.gov/cew/data/api/{year}/{qtr}/{slice}/{code}.csv + +This module is a thin wrapper over that service. It is deliberately separate +from BLSClient (different API, CSV not JSON, no key). Each call returns a list +of plain dict rows (the CSV columns), so it composes with csv/pandas directly. + +Quarter argument: 1-4 for a specific quarter, or "a" for annual averages. +Area FIPS examples: "US000" national, "11000" DC (state), "11001" a county, +metro codes start with "C". Industry codes are NAICS (e.g. "10" = all +industries, "722" = food services, "622" = hospitals). +""" +import csv +import io + +import requests + +QCEW_BASE = "https://data.bls.gov/cew/data/api" +_HEADERS = {"User-Agent": "bls-data-library (https://gitea.doesworks.net/giteaadmin/bls-data)"} + +# A few common columns, for reference / convenience: +# area_fips, own_code, industry_code, agglvl_code, size_code, year, qtr, +# annual_avg_estabs, annual_avg_emplvl, total_annual_wages, annual_avg_wkly_wage, +# avg_annual_pay (quarterly slices use month1_emplvl/qtrly_estabs/total_qtrly_wages, etc.) + +OWNERSHIP = { + "total": "0", "federal": "1", "state": "2", "local": "3", + "private": "5", "government": "8", +} + + +def _fetch_csv(url: str) -> list[dict]: + r = requests.get(url, timeout=60, headers=_HEADERS) + r.raise_for_status() + return parse_csv(r.text) + + +def parse_csv(text: str) -> list[dict]: + """Parse QCEW CSV text into a list of row dicts (BLS quotes every field).""" + return list(csv.DictReader(io.StringIO(text))) + + +def area(area_fips: str, year: int, qtr="a") -> list[dict]: + """Every industry × ownership for one area (county/state/metro/national).""" + return _fetch_csv(f"{QCEW_BASE}/{year}/{qtr}/area/{area_fips}.csv") + + +def industry(industry_code: str, year: int, qtr="a") -> list[dict]: + """One NAICS industry across all areas.""" + return _fetch_csv(f"{QCEW_BASE}/{year}/{qtr}/industry/{industry_code}.csv") + + +def size(size_code: str, year: int) -> list[dict]: + """Establishment-size breakdown (size data is published for Q1 only).""" + return _fetch_csv(f"{QCEW_BASE}/{year}/1/size/{size_code}.csv") + + +def filter_rows(rows, own_code=None, industry_code=None, agglvl_code=None) -> list[dict]: + """Filter QCEW rows by ownership / industry / aggregation-level code.""" + out = rows + if own_code is not None: + own = OWNERSHIP.get(own_code, own_code) + out = [r for r in out if r.get("own_code") == own] + if industry_code is not None: + out = [r for r in out if r.get("industry_code") == industry_code] + if agglvl_code is not None: + out = [r for r in out if r.get("agglvl_code") == agglvl_code] + return out diff --git a/bls_client/queries/wages.py b/bls_client/queries/wages.py index 275ac42..0baaaa2 100644 --- a/bls_client/queries/wages.py +++ b/bls_client/queries/wages.py @@ -2,7 +2,7 @@ Pre-built wage and compensation series IDs — OES, ECI, ECEC, CPS earnings. """ -from ..series import oes_national, eci +from ..series import oes_national, eci, ecec # --------------------------------------------------------------------------- # OES — Occupational Employment & Wage Statistics @@ -89,19 +89,58 @@ def eci_dashboard() -> dict: # --------------------------------------------------------------------------- # ECEC — Employer Costs for Employee Compensation # --------------------------------------------------------------------------- -def ecec_total_compensation() -> str: - """ECEC — civilian workers, total compensation cost per hour worked.""" - return "CMU1010000000000D" +def ecec_total_compensation(owner: str = "civilian") -> str: + """ECEC — total compensation cost per hour worked.""" + return ecec("total_compensation", owner) -def ecec_total_benefits() -> str: - """ECEC — civilian workers, total benefits cost per hour worked.""" - return "CMU1036000000000D" +def ecec_wages_and_salaries(owner: str = "civilian") -> str: + """ECEC — wages and salaries cost per hour worked.""" + return ecec("wages_and_salaries", owner) -# Note: ECEC benefit subcomponents (health insurance, retirement & savings, etc.) -# are encoded as specific benefit-subcell codes in the series ID (suffix stays "D"). -# Look them up against the ECEC component list before adding helpers — do not -# fabricate them with a letter suffix (the old "...H"/"...R" forms were invalid). + +def ecec_total_benefits(owner: str = "civilian") -> str: + """ECEC — total benefits cost per hour worked.""" + return ecec("total_benefits", owner) + + +def ecec_paid_leave(owner: str = "civilian") -> str: + """ECEC — paid leave cost per hour worked (vacation, holiday, sick, personal).""" + return ecec("paid_leave", owner) + + +def ecec_supplemental_pay(owner: str = "civilian") -> str: + """ECEC — supplemental pay cost per hour worked (overtime, bonuses, shift diff).""" + return ecec("supplemental_pay", owner) + + +def ecec_health_insurance(owner: str = "civilian") -> str: + """ECEC — health insurance cost per hour worked.""" + return ecec("health_insurance", owner) + + +def ecec_retirement(owner: str = "civilian") -> str: + """ECEC — retirement & savings cost per hour worked (defined benefit + contribution).""" + return ecec("retirement_and_savings", owner) + + +def ecec_legally_required(owner: str = "civilian") -> str: + """ECEC — legally required benefits per hour (Social Security, Medicare, UI, workers' comp).""" + return ecec("legally_required", owner) + + +def ecec_dashboard(owner: str = "civilian") -> dict: + """The compensation cost breakdown — what an hour of labor costs an employer.""" + return { + "Total Compensation": ecec_total_compensation(owner), + "Wages & Salaries": ecec_wages_and_salaries(owner), + "Total Benefits": ecec_total_benefits(owner), + "Paid Leave": ecec_paid_leave(owner), + "Supplemental Pay": ecec_supplemental_pay(owner), + "Health Insurance": ecec_health_insurance(owner), + "Retirement & Savings": ecec_retirement(owner), + "Legally Required": ecec_legally_required(owner), + } # --------------------------------------------------------------------------- diff --git a/bls_client/series.py b/bls_client/series.py index 21d2aaf..c4efb31 100644 --- a/bls_client/series.py +++ b/bls_client/series.py @@ -336,6 +336,55 @@ def eci( return f"CI{adj}{owner}{component}{'0'*9}{estimate}" +# --------------------------------------------------------------------------- +# ECEC — Employer Costs for Employee Compensation +# Component ("estimate") codes from the BLS cm.estimate decode table. +# --------------------------------------------------------------------------- +_ECEC_COMPONENT = { + "total_compensation": "01", + "wages_and_salaries": "02", + "total_benefits": "03", + "paid_leave": "04", + "supplemental_pay": "09", + "insurance": "13", + "health_insurance": "15", + "retirement_and_savings": "18", + "legally_required": "21", + "workers_compensation": "27", +} + +_ECEC_OWNER = { + "civilian": "1", + "private": "2", + "state_local_gov": "3", +} + +def ecec( + component: str = "total_compensation", + owner: str = "civilian", + datatype: str = "D", # D = cost per hour worked (dollars) +) -> str: + """ + ECEC series — employer cost per hour worked, all industries / all workers. + + Format: CM + U + owner(1) + component(2) + 10 zeros + datatype(1) = 17 chars. + + Args: + component: key in _ECEC_COMPONENT (e.g. "health_insurance") or a raw 2-digit code + owner: "civilian" | "private" | "state_local_gov" (or raw "1"/"2"/"3") + datatype: "D" = cost per hour in dollars (default); "P" = percent of total comp + + Examples: + ecec() → "CMU1010000000000D" (civilian total compensation) + ecec("health_insurance") → "CMU1150000000000D" + ecec("retirement_and_savings") → "CMU1180000000000D" + ecec("total_benefits", "private") → "CMU2030000000000D" + """ + own = _ECEC_OWNER.get(owner, owner) + code = _ECEC_COMPONENT.get(component, component) + return f"CMU{own}{code}{'0'*10}{datatype}" + + # --------------------------------------------------------------------------- # Productivity (Major Sector) # --------------------------------------------------------------------------- diff --git a/tests/fixtures/qcew_dc_sample.csv b/tests/fixtures/qcew_dc_sample.csv new file mode 100644 index 0000000..966478a --- /dev/null +++ b/tests/fixtures/qcew_dc_sample.csv @@ -0,0 +1,7 @@ +area_fips,own_code,industry_code,agglvl_code,size_code,year,qtr,disclosure_code,annual_avg_estabs,annual_avg_emplvl,total_annual_wages,taxable_annual_wages,annual_contributions,annual_avg_wkly_wage,avg_annual_pay,lq_disclosure_code,lq_annual_avg_estabs,lq_annual_avg_emplvl,lq_total_annual_wages,lq_taxable_annual_wages,lq_annual_contributions,lq_annual_avg_wkly_wage,lq_avg_annual_pay,oty_disclosure_code,oty_annual_avg_estabs_chg,oty_annual_avg_estabs_pct_chg,oty_annual_avg_emplvl_chg,oty_annual_avg_emplvl_pct_chg,oty_total_annual_wages_chg,oty_total_annual_wages_pct_chg,oty_taxable_annual_wages_chg,oty_taxable_annual_wages_pct_chg,oty_annual_contributions_chg,oty_annual_contributions_pct_chg,oty_annual_avg_wkly_wage_chg,oty_annual_avg_wkly_wage_pct_chg,oty_avg_annual_pay_chg,oty_avg_annual_pay_pct_chg +11000,0,10,50,0,2024,A,,52240,759666,93430310874,5400587099,119288884,2365,122989,,1.00,1.00,1.00,1.00,1.00,1.00,1.00,,737,1.4,1912,0.3,4099845160,4.6,-94377645,-1.7,-3132981,-2.6,98,4.3,5101,4.3 +11000,1,10,51,0,2024,A,,341,193145,27067241499,0,0,2695,140140,,1.29,13.13,11.09,0.00,0.00,0.85,0.84,,1,0.3,2528,1.3,1098653313,4.2,0,0.0,0,0.0,75,2.9,3905,2.9 +11000,2,10,51,0,2024,A,,14,38244,3958604832,16139000,243,1991,103508,,0.04,1.62,1.27,0.52,0.00,0.79,0.79,,0,0.0,733,2.0,15426132,0.4,3893850,31.8,104,74.8,-31,-1.5,-1612,-1.5 +11000,3,10,51,0,2024,A,,25,3993,538270989,249180,4736,2592,134804,,0.03,0.06,0.07,0.00,0.01,1.25,1.25,,-1,-3.8,-75,-1.8,7573080,1.4,-94668,-27.5,-1797,-27.5,83,3.3,4331,3.3 +11000,5,10,51,0,2024,A,,51860,524284,61866193554,5384198919,119283905,2269,118001,,1.02,0.81,0.77,1.02,1.01,0.96,0.96,,736,1.4,-1275,-0.2,2978192635,5.1,-98176827,-1.8,-3131288,-2.6,114,5.3,5953,5.3 +11000,8,10,96,0,2024,A,,380,235382,31564117320,16388180,4979,2579,134097,,0.29,2.14,2.38,0.15,0.01,1.11,1.11,,1,0.3,3186,1.4,1121652525,3.7,3799182,30.2,-1693,-25.4,58,2.3,2990,2.3 diff --git a/tests/test_client_cache.py b/tests/test_client_cache.py new file mode 100644 index 0000000..80c82a4 --- /dev/null +++ b/tests/test_client_cache.py @@ -0,0 +1,83 @@ +"""Offline tests for caching, the query counter, and typed errors. + +A fake transport stands in for requests.post so nothing touches the network. +""" +import pytest + +import bls_client.client as client_mod +from bls_client import BLSClient, BLSQuotaError, BLSRequestError +from bls_client.cache import FileCache + + +class FakeResp: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + pass + + def json(self): + return self._payload + + +def _ok(series_id): + return { + "status": "REQUEST_SUCCEEDED", + "Results": {"series": [{"seriesID": series_id, "data": [{"year": "2024", "period": "M01", "value": "1.0"}]}]}, + } + + +def test_query_counter_and_cache(tmp_path, monkeypatch): + calls = {"n": 0} + + def fake_post(url, json=None, timeout=None): + calls["n"] += 1 + return FakeResp(_ok(json["seriesid"][0])) + + monkeypatch.setattr(client_mod.requests, "post", fake_post) + + c = BLSClient("key", cache=FileCache(cache_dir=tmp_path, ttl=86400)) + c.fetch(["CES0000000001"], 2024, 2024) + assert c.queries_used == 1 and calls["n"] == 1 + + # identical request -> served from cache, no new network call + c.fetch(["CES0000000001"], 2024, 2024) + assert c.queries_used == 1 and calls["n"] == 1 + + # different request -> one more call + c.fetch(["CES0000000001"], 2023, 2024) + assert c.queries_used == 2 and calls["n"] == 2 + + +def test_no_cache_by_default(monkeypatch): + def fake_post(url, json=None, timeout=None): + return FakeResp(_ok(json["seriesid"][0])) + + monkeypatch.setattr(client_mod.requests, "post", fake_post) + c = BLSClient("key") + assert c.cache is None + c.fetch(["CES0000000001"], 2024, 2024) + c.fetch(["CES0000000001"], 2024, 2024) + assert c.queries_used == 2 # no caching -> both hit the (fake) network + + +def test_quota_error(monkeypatch): + def fake_post(url, json=None, timeout=None): + return FakeResp({ + "status": "REQUEST_NOT_PROCESSED", + "message": ["Query exceeds the threshold of 500 queries per day."], + }) + + monkeypatch.setattr(client_mod.requests, "post", fake_post) + with pytest.raises(BLSQuotaError) as exc: + BLSClient("key").fetch(["CES0000000001"], 2024, 2024) + assert exc.value.status == "REQUEST_NOT_PROCESSED" + + +def test_request_error(monkeypatch): + def fake_post(url, json=None, timeout=None): + return FakeResp({"status": "REQUEST_FAILED", "message": ["invalid series"]}) + + monkeypatch.setattr(client_mod.requests, "post", fake_post) + with pytest.raises(BLSRequestError): + BLSClient("key").fetch(["BOGUS"], 2024, 2024) diff --git a/tests/test_live_smoke.py b/tests/test_live_smoke.py index 237cacd..2ddc5f4 100644 --- a/tests/test_live_smoke.py +++ b/tests/test_live_smoke.py @@ -48,7 +48,8 @@ REPAIRED = { "Productivity": PR.business_output_per_hour(), "QCEW": E.qcew_national_private(), "PPI final": P.ppi_final_demand(), - "ECEC": W.ecec_total_benefits(), + "ECEC health": W.ecec_health_insurance(), + "ECEC retire": W.ecec_retirement(), } diff --git a/tests/test_qcew.py b/tests/test_qcew.py new file mode 100644 index 0000000..27864b6 --- /dev/null +++ b/tests/test_qcew.py @@ -0,0 +1,51 @@ +"""Tests for the QCEW Open Data CSV module. + +Offline tests parse a committed fixture (no network). The live test (opt-in via +`-m live`) hits the real CSV service, which needs no API key and no quota. +""" +import os + +import pytest + +from bls_client import qcew + +FIXTURE = os.path.join(os.path.dirname(__file__), "fixtures", "qcew_dc_sample.csv") + + +@pytest.fixture(scope="module") +def dc_rows(): + with open(FIXTURE) as f: + return qcew.parse_csv(f.read()) + + +def test_parse_returns_dict_rows(dc_rows): + assert dc_rows and isinstance(dc_rows[0], dict) + # core QCEW columns present + for col in ("area_fips", "own_code", "industry_code", "annual_avg_emplvl", "annual_avg_wkly_wage"): + assert col in dc_rows[0] + + +def test_all_rows_are_dc_all_industries(dc_rows): + assert all(r["area_fips"] == "11000" for r in dc_rows) + assert all(r["industry_code"] == "10" for r in dc_rows) + + +def test_filter_by_ownership(dc_rows): + # ownership name resolves to its code, and total (own_code 0) exists + total = qcew.filter_rows(dc_rows, own_code="total") + assert len(total) == 1 + assert int(total[0]["annual_avg_emplvl"]) > 0 + + +def test_ownership_name_mapping(): + assert qcew.OWNERSHIP["private"] == "5" + assert qcew.OWNERSHIP["federal"] == "1" + + +@pytest.mark.live +def test_live_area_fetch(): + rows = qcew.area("11000", 2024, "a") # DC annual, no key / no quota + assert len(rows) > 100 + private_total = qcew.filter_rows(rows, own_code="private", industry_code="10", agglvl_code="51") + assert len(private_total) == 1 + assert int(private_total[0]["annual_avg_emplvl"]) > 100_000 # DC private ~525k diff --git a/tests/test_query_helpers.py b/tests/test_query_helpers.py index a995b36..9ac53cc 100644 --- a/tests/test_query_helpers.py +++ b/tests/test_query_helpers.py @@ -18,7 +18,10 @@ GOLDEN = [ (P.ppi_finished_goods, "WPUFD4"), # back-compat alias (W.eci_wages, "CIU1020000000000A"), (W.eci_total_compensation, "CIU1010000000000A"), - (W.ecec_total_benefits, "CMU1036000000000D"), + (W.ecec_total_compensation, "CMU1010000000000D"), + (W.ecec_total_benefits, "CMU1030000000000D"), + (W.ecec_health_insurance, "CMU1150000000000D"), + (W.ecec_retirement, "CMU1180000000000D"), (PR.business_output_per_hour, "PRS84006092"), ] diff --git a/tests/test_series_builders.py b/tests/test_series_builders.py index 4b0d596..53bb047 100644 --- a/tests/test_series_builders.py +++ b/tests/test_series_builders.py @@ -42,6 +42,12 @@ GOLDEN = [ (S.productivity, dict(), "PRS85006092"), (S.productivity, dict(sector="manufacturing", measure="unit_labor_cost"), "PRS30006112"), (S.productivity, dict(sector="business"), "PRS84006092"), + # ECEC — 17 chars, CMU + owner(1) + component(2) + 10 zeros + datatype + (S.ecec, dict(), "CMU1010000000000D"), + (S.ecec, dict(component="health_insurance"), "CMU1150000000000D"), + (S.ecec, dict(component="retirement_and_savings"), "CMU1180000000000D"), + (S.ecec, dict(component="total_benefits", owner="private"), "CMU2030000000000D"), + (S.ecec, dict(component="total_compensation", owner="state_local_gov"), "CMU3010000000000D"), ] @@ -77,6 +83,13 @@ def test_productivity_is_11_chars(): assert sid.startswith("PRS") +def test_ecec_is_17_chars_cost_per_hour(): + for comp in ("total_compensation", "health_insurance", "retirement_and_savings"): + sid = S.ecec(comp) + assert len(sid) == 17 + assert sid.startswith("CMU") and sid.endswith("D") + + def test_laus_state_is_20_chars(): assert len(S.laus_state(48, "rate")) == 20