"""Retention scaffold — policy type + nightly job (milestone 0.7; YPMS addition). ARCHITECTURE.md §6.4: every entity declares a retention policy; a nightly job scans for records past their envelope and applies the policy. NIST AU-11, SI-12; source RFP `C.5.7.42`, `C.5.21.34`. **Phase 0 is report-only.** The job never modifies data. For each mapped table it resolves the policy and logs one of: - `retain` — kept by design (configuration tables); - `unset` — no period yet; named so the gap stays visible; - `interim` — PIA §9 interim period, not yet signed off. Rows past the interim envelope are *counted* (`interim_eligible`) but never purged; - `approved` — signed-off period. Rows past it are counted as `to_purge`. Disposal execution lands in Phase 5 with the ingest pipelines; until then the job still only reports. Every run writes one `retention_run` audit row with the summary. Policy catalog: `ypms/retention_classes.py`. Entity models declare `retention_policy: ClassVar[RetentionPolicy]`; vendored framework tables are mapped by table name in `retention_classes.FRAMEWORK_TABLE_POLICIES` so the vendored files stay unchanged. Run: ``python -m ypms.framework.retention`` (nightly schedule is wired at deployment — Azure Container Apps job / cron; see ARCH §9). """ from __future__ import annotations import enum import json from dataclasses import asdict, dataclass, field from datetime import UTC, datetime, timedelta from typing import Any from sqlalchemy import func, select from sqlalchemy.orm import Session from ypms.audit import service as audit from ypms.logging import get_logger RETENTION_RUN_ACTION = "retention_run" RETENTION_ENTITY = "Retention" class Disposal(enum.StrEnum): TOMBSTONE = "tombstone" # soft-delete + scrub, keep the row shell HARD_DELETE = "hard_delete" # delete with audit ARCHIVE = "archive" # move to cold storage, then delete online copy ARCHIVE_SCRUB = "archive_scrub" # archive, scrub PII from the online copy RETAIN = "retain" # never purged (configuration) class Cadence(enum.StrEnum): NIGHTLY = "nightly" ANNUAL = "annual" @dataclass(frozen=True) class RetentionPolicy: data_class: str disposal: Disposal period_days: int | None anchor: str = "created_at" cadence: Cadence = Cadence.NIGHTLY approved: bool = False authority: str = "" @property def status(self) -> str: if self.disposal is Disposal.RETAIN: return "retain" if self.period_days is None: return "unset" return "approved" if self.approved else "interim" @dataclass class TableResult: table: str data_class: str status: str period_days: int | None authority: str interim_eligible: int = 0 to_purge: int = 0 @dataclass class RetentionReport: ran_at: str dry_run: bool = True tables: list[TableResult] = field(default_factory=list) @property def to_purge(self) -> int: return sum(t.to_purge for t in self.tables) @property def interim_eligible(self) -> int: return sum(t.interim_eligible for t in self.tables) def tables_with(self, status: str) -> list[str]: return sorted(t.table for t in self.tables if t.status == status) def summary(self) -> dict[str, Any]: return { "ran_at": self.ran_at, "dry_run": self.dry_run, "to_purge": self.to_purge, "interim_eligible": self.interim_eligible, "unset": self.tables_with("unset"), "tables": len(self.tables), } def resolve_policy(model: type[Any]) -> RetentionPolicy | None: from ypms.retention_classes import FRAMEWORK_TABLE_POLICIES policy: RetentionPolicy | None = getattr(model, "retention_policy", None) if policy is not None: return policy return FRAMEWORK_TABLE_POLICIES.get(model.__tablename__) def mapped_models() -> list[type[Any]]: import ypms.models # noqa: F401 — registers every model on Base from ypms.db import Base return sorted((m.class_ for m in Base.registry.mappers), key=lambda c: c.__tablename__) def _count_past(db: Session, model: type[Any], policy: RetentionPolicy, now: datetime) -> int: if policy.period_days is None: raise ValueError(f"policy {policy.data_class!r} has no period") anchor = getattr(model, policy.anchor) cutoff = now - timedelta(days=policy.period_days) stmt = select(func.count()).select_from(model).where(anchor < cutoff) return int(db.execute(stmt).scalar_one()) def run_retention(db: Session, *, now: datetime | None = None) -> RetentionReport: """Report-only retention pass over every mapped table. Never mutates data other than the single `retention_run` audit row; caller commits.""" log = get_logger(__name__) now = now or datetime.now(UTC) report = RetentionReport(ran_at=now.isoformat()) for model in mapped_models(): policy = resolve_policy(model) if policy is None: raise LookupError(f"no retention policy for table {model.__tablename__}") result = TableResult( table=model.__tablename__, data_class=policy.data_class, status=policy.status, period_days=policy.period_days, authority=policy.authority, ) if result.status in {"interim", "approved"}: n = _count_past(db, model, policy, now) if result.status == "approved": result.to_purge = n else: result.interim_eligible = n report.tables.append(result) log.info("retention.table", **asdict(result)) for result in report.tables: if result.status == "unset": log.warning("retention.policy_unset", table=result.table, authority=result.authority) log.info( "retention.run", message=f"{report.to_purge} records to purge", **report.summary(), ) audit.record( db, actor=None, action=RETENTION_RUN_ACTION, entity=RETENTION_ENTITY, after=report.summary(), ) return report def main() -> int: from ypms.db import session_factory from ypms.logging import configure_logging configure_logging() with session_factory()() as db: report = run_retention(db) db.commit() print(json.dumps(report.summary())) return 0 if __name__ == "__main__": raise SystemExit(main())