Dave Boyd
0d0d6dabe2
- bls_client/retry.py: with_retries() retries timeouts/connection drops and 429/5xx with exponential backoff; honors Retry-After; never retries 4xx or BLSQuotaError (those won't fix themselves) - BLSClient(retries=2, backoff=0.5) wraps the data POST and discovery GETs; qcew CSV fetch wrapped too. retries=0 disables. - tests/test_retry.py: 6 offline tests (injectable sleep, no real delays) - README: retry behavior documented; drop the now-resolved limitation Offline suite 125 passing; live smoke green. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
59 lines
2.0 KiB
Python
59 lines
2.0 KiB
Python
"""Retry transient network failures with exponential backoff.
|
|
|
|
Shared by the JSON client (POST) and the QCEW CSV client (GET). Retries only
|
|
genuinely transient conditions — connection drops, timeouts, and 429/5xx — never
|
|
4xx client errors or a BLS quota rejection (those won't fix themselves).
|
|
"""
|
|
import time
|
|
|
|
import requests
|
|
|
|
# HTTP statuses worth retrying (429 = throttled, 5xx = server-side transient).
|
|
RETRY_STATUS = {429, 500, 502, 503, 504}
|
|
|
|
# Network-level exceptions worth retrying.
|
|
RETRY_EXCEPTIONS = (
|
|
requests.exceptions.ConnectionError,
|
|
requests.exceptions.Timeout,
|
|
requests.exceptions.ChunkedEncodingError,
|
|
)
|
|
|
|
|
|
def _retry_after(resp) -> float | None:
|
|
"""Honor a numeric Retry-After header if the server sent one."""
|
|
ra = getattr(resp, "headers", {}).get("Retry-After") if resp is not None else None
|
|
if ra and str(ra).isdigit():
|
|
return float(ra)
|
|
return None
|
|
|
|
|
|
def with_retries(send, *, retries=2, backoff=0.5, backoff_max=8.0, sleep=time.sleep):
|
|
"""
|
|
Call ``send()`` (which returns a requests.Response), retrying transient
|
|
failures. Backoff is ``backoff * 2**attempt`` seconds, capped at ``backoff_max``.
|
|
|
|
Args:
|
|
send: zero-arg callable issuing the request and returning a Response.
|
|
retries: max retries after the first attempt (so retries=2 -> 3 tries).
|
|
backoff: base backoff in seconds.
|
|
backoff_max: cap on a single backoff interval.
|
|
sleep: sleep function (injectable for tests).
|
|
"""
|
|
attempt = 0
|
|
while True:
|
|
try:
|
|
resp = send()
|
|
except RETRY_EXCEPTIONS:
|
|
if attempt >= retries:
|
|
raise
|
|
sleep(min(backoff * (2 ** attempt), backoff_max))
|
|
attempt += 1
|
|
continue
|
|
|
|
if getattr(resp, "status_code", None) in RETRY_STATUS and attempt < retries:
|
|
sleep(_retry_after(resp) or min(backoff * (2 ** attempt), backoff_max))
|
|
attempt += 1
|
|
continue
|
|
|
|
return resp
|