Phase 0 milestone 0.7: retention scaffold

Every mapped table resolves a retention policy; a report-only job runs
over all of them and logs "N records to purge". ARCH §6.4; NIST AU-11,
SI-12; source RFP C.5.7.42.

- framework/retention.py: RetentionPolicy (data class, disposal, period,
  anchor column, cadence, approved, authority), run_retention(), and a
  `python -m ypms.framework.retention` entrypoint. Phase 0 never mutates
  data: interim-period rows are counted (interim_eligible), approved-period
  rows are counted (to_purge), nothing is purged. One retention_run audit
  row per run.
- retention_classes.py: PIA §9 data classes at their interim periods, all
  approved=False pending YPMS-POAM-054. Classes without a PIA basis
  (hosts, operational logs, accounts, consent-derived) are unset and logged
  by name with their governing POA&M item. Vendored framework tables are
  mapped by table name so vendored files stay unchanged.
- stub_models.py + migration 0004: id + mixin models for the 26 stub
  entities so each can carry a policy. Migration uses a literal table list.
- ypms/models.py: single import-all module used by Alembic env, the job,
  and the test fixtures.
- Drift guard: test upgrades a scratch SQLite DB to head and compares to
  Base.metadata. Found pre-existing drift in 0002 (named unique
  constraints duplicating the unique indexes on roles.code, users.username,
  users.email); allow-listed, follow-up noted in PHASE0_STATUS.

156 tests passing; ruff + mypy clean.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/ypms/framework/retention.py

@@ -0,0 +1,196 @@
+"""Retention scaffold — policy type + nightly job (milestone 0.7; YPMS addition).
+
+ARCHITECTURE.md §6.4: every entity declares a retention policy; a nightly job
+scans for records past their envelope and applies the policy. NIST AU-11,
+SI-12; source RFP `C.5.7.42`, `C.5.21.34`.
+
+**Phase 0 is report-only.** The job never modifies data. For each mapped
+table it resolves the policy and logs one of:
+
+- `retain`       — kept by design (configuration tables);
+- `unset`        — no period yet; named so the gap stays visible;
+- `interim`      — PIA §9 interim period, not yet signed off. Rows past the
+                   interim envelope are *counted* (`interim_eligible`) but
+                   never purged;
+- `approved`     — signed-off period. Rows past it are counted as
+                   `to_purge`. Disposal execution lands in Phase 5 with the
+                   ingest pipelines; until then the job still only reports.
+
+Every run writes one `retention_run` audit row with the summary.
+
+Policy catalog: `ypms/retention_classes.py`. Entity models declare
+`retention_policy: ClassVar[RetentionPolicy]`; vendored framework tables are
+mapped by table name in `retention_classes.FRAMEWORK_TABLE_POLICIES` so the
+vendored files stay unchanged.
+
+Run: ``python -m ypms.framework.retention`` (nightly schedule is wired at
+deployment — Azure Container Apps job / cron; see ARCH §9).
+"""
+
+from __future__ import annotations
+
+import enum
+import json
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from ypms.audit import service as audit
+from ypms.logging import get_logger
+
+RETENTION_RUN_ACTION = "retention_run"
+RETENTION_ENTITY = "Retention"
+
+
+class Disposal(enum.StrEnum):
+    TOMBSTONE = "tombstone"            # soft-delete + scrub, keep the row shell
+    HARD_DELETE = "hard_delete"        # delete with audit
+    ARCHIVE = "archive"                # move to cold storage, then delete online copy
+    ARCHIVE_SCRUB = "archive_scrub"    # archive, scrub PII from the online copy
+    RETAIN = "retain"                  # never purged (configuration)
+
+
+class Cadence(enum.StrEnum):
+    NIGHTLY = "nightly"
+    ANNUAL = "annual"
+
+
+@dataclass(frozen=True)
+class RetentionPolicy:
+    data_class: str
+    disposal: Disposal
+    period_days: int | None
+    anchor: str = "created_at"
+    cadence: Cadence = Cadence.NIGHTLY
+    approved: bool = False
+    authority: str = ""
+
+    @property
+    def status(self) -> str:
+        if self.disposal is Disposal.RETAIN:
+            return "retain"
+        if self.period_days is None:
+            return "unset"
+        return "approved" if self.approved else "interim"
+
+
+@dataclass
+class TableResult:
+    table: str
+    data_class: str
+    status: str
+    period_days: int | None
+    authority: str
+    interim_eligible: int = 0
+    to_purge: int = 0
+
+
+@dataclass
+class RetentionReport:
+    ran_at: str
+    dry_run: bool = True
+    tables: list[TableResult] = field(default_factory=list)
+
+    @property
+    def to_purge(self) -> int:
+        return sum(t.to_purge for t in self.tables)
+
+    @property
+    def interim_eligible(self) -> int:
+        return sum(t.interim_eligible for t in self.tables)
+
+    def tables_with(self, status: str) -> list[str]:
+        return sorted(t.table for t in self.tables if t.status == status)
+
+    def summary(self) -> dict[str, Any]:
+        return {
+            "ran_at": self.ran_at,
+            "dry_run": self.dry_run,
+            "to_purge": self.to_purge,
+            "interim_eligible": self.interim_eligible,
+            "unset": self.tables_with("unset"),
+            "tables": len(self.tables),
+        }
+
+
+def resolve_policy(model: type[Any]) -> RetentionPolicy | None:
+    from ypms.retention_classes import FRAMEWORK_TABLE_POLICIES
+
+    policy: RetentionPolicy | None = getattr(model, "retention_policy", None)
+    if policy is not None:
+        return policy
+    return FRAMEWORK_TABLE_POLICIES.get(model.__tablename__)
+
+
+def mapped_models() -> list[type[Any]]:
+    import ypms.models  # noqa: F401 — registers every model on Base
+    from ypms.db import Base
+
+    return sorted((m.class_ for m in Base.registry.mappers), key=lambda c: c.__tablename__)
+
+
+def _count_past(db: Session, model: type[Any], policy: RetentionPolicy, now: datetime) -> int:
+    assert policy.period_days is not None
+    anchor = getattr(model, policy.anchor)
+    cutoff = now - timedelta(days=policy.period_days)
+    stmt = select(func.count()).select_from(model).where(anchor < cutoff)
+    return int(db.execute(stmt).scalar_one())
+
+
+def run_retention(db: Session, *, now: datetime | None = None) -> RetentionReport:
+    """Report-only retention pass over every mapped table. Never mutates data
+    other than the single `retention_run` audit row; caller commits."""
+    log = get_logger(__name__)
+    now = now or datetime.now(UTC)
+    report = RetentionReport(ran_at=now.isoformat())
+
+    for model in mapped_models():
+        policy = resolve_policy(model)
+        if policy is None:
+            raise LookupError(f"no retention policy for table {model.__tablename__}")
+        result = TableResult(
+            table=model.__tablename__, data_class=policy.data_class,
+            status=policy.status, period_days=policy.period_days,
+            authority=policy.authority,
+        )
+        if result.status in {"interim", "approved"}:
+            n = _count_past(db, model, policy, now)
+            if result.status == "approved":
+                result.to_purge = n
+            else:
+                result.interim_eligible = n
+        report.tables.append(result)
+        log.info("retention.table", **asdict(result))
+
+    for result in report.tables:
+        if result.status == "unset":
+            log.warning("retention.policy_unset", table=result.table, authority=result.authority)
+    log.info(
+        "retention.run",
+        message=f"{report.to_purge} records to purge",
+        **report.summary(),
+    )
+    audit.record(
+        db, actor=None, action=RETENTION_RUN_ACTION, entity=RETENTION_ENTITY,
+        after=report.summary(),
+    )
+    return report
+
+
+def main() -> int:
+    from ypms.db import session_factory
+    from ypms.logging import configure_logging
+
+    configure_logging()
+    with session_factory()() as db:
+        report = run_retention(db)
+        db.commit()
+    print(json.dumps(report.summary()))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())