Phase 0 milestone 0.7: retention scaffold

Every mapped table resolves a retention policy; a report-only job runs
over all of them and logs "N records to purge". ARCH §6.4; NIST AU-11,
SI-12; source RFP C.5.7.42.

- framework/retention.py: RetentionPolicy (data class, disposal, period,
  anchor column, cadence, approved, authority), run_retention(), and a
  `python -m ypms.framework.retention` entrypoint. Phase 0 never mutates
  data: interim-period rows are counted (interim_eligible), approved-period
  rows are counted (to_purge), nothing is purged. One retention_run audit
  row per run.
- retention_classes.py: PIA §9 data classes at their interim periods, all
  approved=False pending YPMS-POAM-054. Classes without a PIA basis
  (hosts, operational logs, accounts, consent-derived) are unset and logged
  by name with their governing POA&M item. Vendored framework tables are
  mapped by table name so vendored files stay unchanged.
- stub_models.py + migration 0004: id + mixin models for the 26 stub
  entities so each can carry a policy. Migration uses a literal table list.
- ypms/models.py: single import-all module used by Alembic env, the job,
  and the test fixtures.
- Drift guard: test upgrades a scratch SQLite DB to head and compares to
  Base.metadata. Found pre-existing drift in 0002 (named unique
  constraints duplicating the unique indexes on roles.code, users.username,
  users.email); allow-listed, follow-up noted in PHASE0_STATUS.

156 tests passing; ruff + mypy clean.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/framework/_fixtures.py

@@ -5,16 +5,23 @@ slice; until then, this Thing model is enough to prove CRUD + audit + versioning
 
 from __future__ import annotations
 
+from typing import ClassVar
+
 from pydantic import BaseModel
 from sqlalchemy import String
 from sqlalchemy.orm import Mapped, mapped_column
 
 from ypms.db import Base
 from ypms.framework.base import SoftDeleteMixin, TimestampMixin, VersionedMixin
+from ypms.framework.retention import Disposal, RetentionPolicy
 
 
 class Thing(Base, TimestampMixin, SoftDeleteMixin, VersionedMixin):
     __tablename__ = "things"
+    # Every mapped table needs a policy, test-only ones included (milestone 0.7).
+    retention_policy: ClassVar[RetentionPolicy] = RetentionPolicy(
+        "test_fixture", Disposal.RETAIN, None, authority="test-only model"
+    )
 
     id: Mapped[int] = mapped_column(primary_key=True)
     name: Mapped[str] = mapped_column(String(64), nullable=False)

tests/framework/test_retention.py

@@ -0,0 +1,240 @@
+"""Milestone 0.7 — retention scaffold and schema-drift guard."""
+
+from __future__ import annotations
+
+import dataclasses
+import importlib.util
+import json
+from collections.abc import Iterator
+from datetime import UTC, date, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import pytest
+from alembic import command
+from alembic.autogenerate import compare_metadata
+from alembic.config import Config
+from alembic.runtime.migration import MigrationContext
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session, sessionmaker
+
+import ypms.retention_classes as catalog
+from ypms.audit.models import AuditLog
+from ypms.db import Base
+from ypms.framework import retention
+from ypms.framework.retention import (
+    RETENTION_RUN_ACTION,
+    Disposal,
+    RetentionPolicy,
+    mapped_models,
+    resolve_policy,
+    run_retention,
+)
+from ypms.settings import get_settings
+from ypms.stub_models import FosterIndicator
+from ypms.youth.models import Registrant
+
+ROOT = Path(__file__).resolve().parents[2]
+NOW = datetime(2026, 10, 1, tzinfo=UTC)
+
+# ---------------- policy coverage ----------------
+
+
+def test_every_mapped_table_has_a_policy() -> None:
+    missing = [m.__tablename__ for m in mapped_models() if resolve_policy(m) is None]
+    assert missing == []
+
+
+def test_entity_models_declare_policy_on_the_class() -> None:
+    """Framework tables may use the name map; YPMS entities must not."""
+    framework = set(catalog.FRAMEWORK_TABLE_POLICIES)
+    for m in mapped_models():
+        if m.__tablename__ not in framework:
+            assert "retention_policy" in vars(m), m.__tablename__
+
+
+def test_unset_policies_cite_poam() -> None:
+    for m in mapped_models():
+        p = resolve_policy(m)
+        assert p is not None
+        if p.status == "unset":
+            assert "POAM" in p.authority, m.__tablename__
+
+
+def test_nothing_is_approved_before_records_officer_signoff() -> None:
+    # Flip deliberately when YPMS-POAM-054 closes; update this test with it.
+    assert not any(p.approved for p in _all_policies())
+
+
+def test_anchor_columns_exist() -> None:
+    for m in mapped_models():
+        p = resolve_policy(m)
+        assert p is not None
+        if p.period_days is not None:
+            assert p.anchor in m.__table__.columns, (m.__tablename__, p.anchor)
+
+
+@pytest.mark.parametrize(
+    ("policy", "status"),
+    [
+        (RetentionPolicy("x", Disposal.RETAIN, None), "retain"),
+        (RetentionPolicy("x", Disposal.RETAIN, 10), "retain"),
+        (RetentionPolicy("x", Disposal.TOMBSTONE, None), "unset"),
+        (RetentionPolicy("x", Disposal.TOMBSTONE, 10), "interim"),
+        (RetentionPolicy("x", Disposal.TOMBSTONE, 10, approved=True), "approved"),
+    ],
+)
+def test_policy_status(policy: RetentionPolicy, status: str) -> None:
+    assert policy.status == status
+
+
+def _all_policies() -> list[RetentionPolicy]:
+    return [v for v in vars(catalog).values() if isinstance(v, RetentionPolicy)]
+
+
+# ---------------- job ----------------
+
+
+def _registrant(db: Session, *, updated_at: datetime) -> Registrant:
+    r = Registrant(first_name="A", last_name="B", dob=date(2009, 1, 1))
+    db.add(r)
+    db.flush()
+    r.updated_at = updated_at
+    db.flush()
+    return r
+
+
+def test_empty_run_reports_zero_and_audits(db: Session) -> None:
+    report = run_retention(db, now=NOW)
+    assert report.to_purge == 0
+    assert report.dry_run is True
+    assert {t.table for t in report.tables} == {m.__tablename__ for m in mapped_models()}
+    assert "consent_records" in report.tables_with("unset")
+    assert "permissions" in report.tables_with("retain")
+
+    (row,) = db.execute(select(AuditLog).where(AuditLog.action == RETENTION_RUN_ACTION)).scalars()
+    assert row.actor_id is None
+    assert row.after == report.summary()
+
+
+def test_interim_policy_counts_but_never_purges(db: Session) -> None:
+    old = _registrant(db, updated_at=NOW - timedelta(days=8 * 365))
+    _registrant(db, updated_at=NOW - timedelta(days=30))
+
+    report = run_retention(db, now=NOW)
+
+    reg = next(t for t in report.tables if t.table == "registrants")
+    assert (reg.status, reg.interim_eligible, reg.to_purge) == ("interim", 1, 0)
+    assert report.to_purge == 0
+    assert db.get(Registrant, old.id) is not None
+    assert old.deleted_at is None
+
+
+def test_approved_policy_counts_to_purge_but_phase0_still_reports_only(
+    db: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(
+        Registrant, "retention_policy",
+        dataclasses.replace(catalog.PARTICIPATION, approved=True),
+    )
+    old = _registrant(db, updated_at=NOW - timedelta(days=8 * 365))
+
+    report = run_retention(db, now=NOW)
+
+    reg = next(t for t in report.tables if t.table == "registrants")
+    assert (reg.status, reg.to_purge) == ("approved", 1)
+    assert report.to_purge == 1
+    assert db.get(Registrant, old.id) is not None and old.deleted_at is None
+
+
+def test_annual_ingest_classes_use_two_year_window(db: Session) -> None:
+    for days in (800, 100):
+        f = FosterIndicator()
+        db.add(f)
+        db.flush()
+        f.created_at = NOW - timedelta(days=days)
+    db.flush()
+    report = run_retention(db, now=NOW)
+    foster = next(t for t in report.tables if t.table == "foster_indicators")
+    assert foster.interim_eligible == 1
+
+
+def test_missing_policy_is_an_error(db: Session, monkeypatch: pytest.MonkeyPatch) -> None:
+    trimmed = {k: v for k, v in catalog.FRAMEWORK_TABLE_POLICIES.items() if k != "roles"}
+    monkeypatch.setattr(catalog, "FRAMEWORK_TABLE_POLICIES", trimmed)
+    with pytest.raises(LookupError, match="roles"):
+        run_retention(db, now=NOW)
+
+
+def test_main_runs_commits_and_prints_summary(
+    db: Session, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    import ypms.db
+
+    bind = db.get_bind()
+    monkeypatch.setattr(ypms.db, "session_factory", lambda: sessionmaker(bind=bind))
+    assert retention.main() == 0
+    out = json.loads(capsys.readouterr().out.strip().splitlines()[-1])
+    assert out["to_purge"] == 0 and out["dry_run"] is True
+    assert db.execute(select(AuditLog).where(AuditLog.action == RETENTION_RUN_ACTION)).first()
+
+
+# ---------------- migrations ----------------
+
+
+def _load_migration(name: str) -> Any:
+    spec = importlib.util.spec_from_file_location(name, ROOT / "migrations" / "versions" / f"{name}.py")
+    assert spec and spec.loader
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+
+def test_stub_migration_matches_stub_models() -> None:
+    import ypms.stub_models as stubs
+
+    tables = {
+        v.__tablename__ for v in vars(stubs).values()
+        if isinstance(v, type) and issubclass(v, Base) and v is not Base
+    }
+    assert set(_load_migration("0004_stub_entity_tables").TABLES) == tables
+
+
+# Pre-existing in 0002 (immutable): named unique constraints duplicated by the
+# unique indexes the models declare. Follow-up tracked in PHASE0_STATUS.md.
+KNOWN_0002_DRIFT = {("roles", "code"), ("users", "email"), ("users", "username")}
+
+# Mapped only when tests/framework/_fixtures.py is imported; never migrated.
+TEST_ONLY_TABLES = {"things"}
+
+
+@pytest.fixture
+def migrated_url(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
+    url = f"sqlite:///{tmp_path / 'drift.db'}"
+    monkeypatch.setenv("DATABASE_URL", url)
+    get_settings.cache_clear()
+    try:
+        command.upgrade(Config(str(ROOT / "alembic.ini")), "head")
+        yield url
+    finally:
+        monkeypatch.undo()
+        get_settings.cache_clear()
+
+
+def test_migrations_match_models(migrated_url: str) -> None:
+    engine = create_engine(migrated_url)
+    with engine.connect() as conn:
+        diff = compare_metadata(MigrationContext.configure(conn), Base.metadata)
+    engine.dispose()
+
+    unexpected = []
+    for op in diff:
+        if op[0] == "add_table" and op[1].name in TEST_ONLY_TABLES:
+            continue
+        if op[0] == "remove_constraint":
+            uc = op[1]
+            cols = [c.name for c in uc.columns]
+            if len(cols) == 1 and (uc.table.name, cols[0]) in KNOWN_0002_DRIFT:
+                continue
+        unexpected.append(op)
+    assert unexpected == []