Add transient-network retry with exponential backoff
- bls_client/retry.py: with_retries() retries timeouts/connection drops and 429/5xx with exponential backoff; honors Retry-After; never retries 4xx or BLSQuotaError (those won't fix themselves) - BLSClient(retries=2, backoff=0.5) wraps the data POST and discovery GETs; qcew CSV fetch wrapped too. retries=0 disables. - tests/test_retry.py: 6 offline tests (injectable sleep, no real delays) - README: retry behavior documented; drop the now-resolved limitation Offline suite 125 passing; live smoke green. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@ -12,6 +12,7 @@ from datetime import datetime
|
||||
|
||||
from .cache import FileCache
|
||||
from .errors import BLSError, BLSRequestError, BLSQuotaError
|
||||
from .retry import with_retries
|
||||
|
||||
|
||||
class BLSClient:
|
||||
@ -19,7 +20,8 @@ class BLSClient:
|
||||
MAX_SERIES_PER_CALL = 50
|
||||
MAX_YEARS_PER_CALL = 20
|
||||
|
||||
def __init__(self, api_key: str, cache=False, cache_dir=None, cache_ttl=86400):
|
||||
def __init__(self, api_key: str, cache=False, cache_dir=None, cache_ttl=86400,
|
||||
retries=2, backoff=0.5):
|
||||
"""
|
||||
Args:
|
||||
api_key: BLS registration key.
|
||||
@ -27,6 +29,9 @@ class BLSClient:
|
||||
or pass a custom cache object with get(key)/set(key, value).
|
||||
cache_dir: Override the default cache directory.
|
||||
cache_ttl: Cache entry lifetime in seconds (default 1 day).
|
||||
retries: retries after the first attempt on transient network errors
|
||||
(timeouts, connection drops, 429/5xx). 0 disables retrying.
|
||||
backoff: base exponential-backoff interval in seconds.
|
||||
"""
|
||||
self.api_key = api_key
|
||||
if cache is True:
|
||||
@ -35,8 +40,15 @@ class BLSClient:
|
||||
self.cache = cache # caller-supplied cache object
|
||||
else:
|
||||
self.cache = None
|
||||
self.retries = retries
|
||||
self.backoff = backoff
|
||||
self.queries_used = 0 # network POSTs spent this session (cache hits don't count)
|
||||
|
||||
def _get(self, url, **kwargs):
|
||||
"""GET with retry/backoff."""
|
||||
return with_retries(lambda: requests.get(url, **kwargs),
|
||||
retries=self.retries, backoff=self.backoff)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Core fetch
|
||||
# ------------------------------------------------------------------
|
||||
@ -97,7 +109,9 @@ class BLSClient:
|
||||
|
||||
def _fetch_batch(self, payload: dict) -> dict:
|
||||
"""Execute one (uncached) API call and return {series_id: {data, catalog}}."""
|
||||
r = requests.post(f"{self.BASE_URL}/timeseries/data/", json=payload, timeout=30)
|
||||
url = f"{self.BASE_URL}/timeseries/data/"
|
||||
r = with_retries(lambda: requests.post(url, json=payload, timeout=30),
|
||||
retries=self.retries, backoff=self.backoff)
|
||||
r.raise_for_status()
|
||||
self.queries_used += 1
|
||||
body = r.json()
|
||||
@ -155,13 +169,13 @@ class BLSClient:
|
||||
# ------------------------------------------------------------------
|
||||
def surveys(self) -> list[dict]:
|
||||
"""Return all BLS surveys with abbreviation and name."""
|
||||
r = requests.get(f"{self.BASE_URL}/surveys", timeout=15)
|
||||
r = self._get(f"{self.BASE_URL}/surveys", timeout=15)
|
||||
r.raise_for_status()
|
||||
return r.json()["Results"]["survey"]
|
||||
|
||||
def popular(self, survey: str) -> list[str]:
|
||||
"""Return popular series IDs for a given survey abbreviation."""
|
||||
r = requests.get(
|
||||
r = self._get(
|
||||
f"{self.BASE_URL}/timeseries/popular",
|
||||
params={"survey": survey},
|
||||
timeout=15,
|
||||
|
||||
Reference in New Issue
Block a user