"""Milestone 0.7 — retention scaffold and schema-drift guard.""" from __future__ import annotations import dataclasses import importlib.util import json from collections.abc import Iterator from datetime import UTC, date, datetime, timedelta from pathlib import Path from typing import Any import pytest from alembic import command from alembic.autogenerate import compare_metadata from alembic.config import Config from alembic.runtime.migration import MigrationContext from sqlalchemy import create_engine, select from sqlalchemy.orm import Session, sessionmaker import ypms.retention_classes as catalog from ypms.audit.models import AuditLog from ypms.db import Base from ypms.framework import retention from ypms.framework.retention import ( RETENTION_RUN_ACTION, Disposal, RetentionPolicy, mapped_models, resolve_policy, run_retention, ) from ypms.settings import get_settings from ypms.stub_models import FosterIndicator from ypms.youth.models import Registrant ROOT = Path(__file__).resolve().parents[2] NOW = datetime(2026, 10, 1, tzinfo=UTC) # ---------------- policy coverage ---------------- def test_every_mapped_table_has_a_policy() -> None: missing = [m.__tablename__ for m in mapped_models() if resolve_policy(m) is None] assert missing == [] def test_entity_models_declare_policy_on_the_class() -> None: """Framework tables may use the name map; YPMS entities must not.""" framework = set(catalog.FRAMEWORK_TABLE_POLICIES) for m in mapped_models(): if m.__tablename__ not in framework: assert "retention_policy" in vars(m), m.__tablename__ def test_unset_policies_cite_poam() -> None: for m in mapped_models(): p = resolve_policy(m) assert p is not None if p.status == "unset": assert "POAM" in p.authority, m.__tablename__ def test_nothing_is_approved_before_records_officer_signoff() -> None: # Flip deliberately when YPMS-POAM-054 closes; update this test with it. assert not any(p.approved for p in _all_policies()) def test_anchor_columns_exist() -> None: for m in mapped_models(): p = resolve_policy(m) assert p is not None if p.period_days is not None: assert p.anchor in m.__table__.columns, (m.__tablename__, p.anchor) @pytest.mark.parametrize( ("policy", "status"), [ (RetentionPolicy("x", Disposal.RETAIN, None), "retain"), (RetentionPolicy("x", Disposal.RETAIN, 10), "retain"), (RetentionPolicy("x", Disposal.TOMBSTONE, None), "unset"), (RetentionPolicy("x", Disposal.TOMBSTONE, 10), "interim"), (RetentionPolicy("x", Disposal.TOMBSTONE, 10, approved=True), "approved"), ], ) def test_policy_status(policy: RetentionPolicy, status: str) -> None: assert policy.status == status def _all_policies() -> list[RetentionPolicy]: return [v for v in vars(catalog).values() if isinstance(v, RetentionPolicy)] # ---------------- job ---------------- def _registrant(db: Session, *, updated_at: datetime) -> Registrant: r = Registrant(first_name="A", last_name="B", dob=date(2009, 1, 1)) db.add(r) db.flush() r.updated_at = updated_at db.flush() return r def test_empty_run_reports_zero_and_audits(db: Session) -> None: report = run_retention(db, now=NOW) assert report.to_purge == 0 assert report.dry_run is True assert {t.table for t in report.tables} == {m.__tablename__ for m in mapped_models()} assert "consent_records" in report.tables_with("unset") assert "permissions" in report.tables_with("retain") (row,) = db.execute(select(AuditLog).where(AuditLog.action == RETENTION_RUN_ACTION)).scalars() assert row.actor_id is None assert row.after == report.summary() def test_interim_policy_counts_but_never_purges(db: Session) -> None: old = _registrant(db, updated_at=NOW - timedelta(days=8 * 365)) _registrant(db, updated_at=NOW - timedelta(days=30)) report = run_retention(db, now=NOW) reg = next(t for t in report.tables if t.table == "registrants") assert (reg.status, reg.interim_eligible, reg.to_purge) == ("interim", 1, 0) assert report.to_purge == 0 assert db.get(Registrant, old.id) is not None assert old.deleted_at is None def test_approved_policy_counts_to_purge_but_phase0_still_reports_only( db: Session, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( Registrant, "retention_policy", dataclasses.replace(catalog.PARTICIPATION, approved=True), ) old = _registrant(db, updated_at=NOW - timedelta(days=8 * 365)) report = run_retention(db, now=NOW) reg = next(t for t in report.tables if t.table == "registrants") assert (reg.status, reg.to_purge) == ("approved", 1) assert report.to_purge == 1 assert db.get(Registrant, old.id) is not None and old.deleted_at is None def test_annual_ingest_classes_use_two_year_window(db: Session) -> None: for days in (800, 100): f = FosterIndicator() db.add(f) db.flush() f.created_at = NOW - timedelta(days=days) db.flush() report = run_retention(db, now=NOW) foster = next(t for t in report.tables if t.table == "foster_indicators") assert foster.interim_eligible == 1 def test_missing_policy_is_an_error(db: Session, monkeypatch: pytest.MonkeyPatch) -> None: trimmed = {k: v for k, v in catalog.FRAMEWORK_TABLE_POLICIES.items() if k != "roles"} monkeypatch.setattr(catalog, "FRAMEWORK_TABLE_POLICIES", trimmed) with pytest.raises(LookupError, match="roles"): run_retention(db, now=NOW) def test_main_runs_commits_and_prints_summary( db: Session, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: import ypms.db bind = db.get_bind() monkeypatch.setattr(ypms.db, "session_factory", lambda: sessionmaker(bind=bind)) assert retention.main() == 0 out = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) assert out["to_purge"] == 0 and out["dry_run"] is True assert db.execute(select(AuditLog).where(AuditLog.action == RETENTION_RUN_ACTION)).first() # ---------------- migrations ---------------- def _load_migration(name: str) -> Any: spec = importlib.util.spec_from_file_location(name, ROOT / "migrations" / "versions" / f"{name}.py") assert spec and spec.loader mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod def test_stub_migration_matches_stub_models() -> None: import ypms.stub_models as stubs tables = { v.__tablename__ for v in vars(stubs).values() if isinstance(v, type) and issubclass(v, Base) and v is not Base } assert set(_load_migration("0004_stub_entity_tables").TABLES) == tables # Pre-existing in 0002 (immutable): named unique constraints duplicated by the # unique indexes the models declare. Follow-up tracked in PHASE0_STATUS.md. KNOWN_0002_DRIFT = {("roles", "code"), ("users", "email"), ("users", "username")} # Mapped only when tests/framework/_fixtures.py is imported; never migrated. TEST_ONLY_TABLES = {"things"} @pytest.fixture def migrated_url(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: url = f"sqlite:///{tmp_path / 'drift.db'}" monkeypatch.setenv("DATABASE_URL", url) get_settings.cache_clear() try: command.upgrade(Config(str(ROOT / "alembic.ini")), "head") yield url finally: monkeypatch.undo() get_settings.cache_clear() def test_migrations_match_models(migrated_url: str) -> None: engine = create_engine(migrated_url) with engine.connect() as conn: diff = compare_metadata(MigrationContext.configure(conn), Base.metadata) engine.dispose() unexpected = [] for op in diff: if op[0] == "add_table" and op[1].name in TEST_ONLY_TABLES: continue if op[0] == "remove_constraint": uc = op[1] cols = [c.name for c in uc.columns] if len(cols) == 1 and (uc.table.name, cols[0]) in KNOWN_0002_DRIFT: continue unexpected.append(op) assert unexpected == []