YPMS/src/ypms/framework/retention.py

"""Retention scaffold — policy type + nightly job (milestone 0.7; YPMS addition).

ARCHITECTURE.md §6.4: every entity declares a retention policy; a nightly job
scans for records past their envelope and applies the policy. NIST AU-11,
SI-12; source RFP `C.5.7.42`, `C.5.21.34`.

**Phase 0 is report-only.** The job never modifies data. For each mapped
table it resolves the policy and logs one of:

- `retain`       — kept by design (configuration tables);
- `unset`        — no period yet; named so the gap stays visible;
- `interim`      — PIA §9 interim period, not yet signed off. Rows past the
                   interim envelope are *counted* (`interim_eligible`) but
                   never purged;
- `approved`     — signed-off period. Rows past it are counted as
                   `to_purge`. Disposal execution lands in Phase 5 with the
                   ingest pipelines; until then the job still only reports.

Every run writes one `retention_run` audit row with the summary.

Policy catalog: `ypms/retention_classes.py`. Entity models declare
`retention_policy: ClassVar[RetentionPolicy]`; vendored framework tables are
mapped by table name in `retention_classes.FRAMEWORK_TABLE_POLICIES` so the
vendored files stay unchanged.

Run: ``python -m ypms.framework.retention`` (nightly schedule is wired at
deployment — Azure Container Apps job / cron; see ARCH §9).
"""

from __future__ import annotations

import enum
import json
from dataclasses import asdict, dataclass, field
from datetime import UTC, datetime, timedelta
from typing import Any

from sqlalchemy import func, select
from sqlalchemy.orm import Session

from ypms.audit import service as audit
from ypms.logging import get_logger

RETENTION_RUN_ACTION = "retention_run"
RETENTION_ENTITY = "Retention"


class Disposal(enum.StrEnum):
    TOMBSTONE = "tombstone"            # soft-delete + scrub, keep the row shell
    HARD_DELETE = "hard_delete"        # delete with audit
    ARCHIVE = "archive"                # move to cold storage, then delete online copy
    ARCHIVE_SCRUB = "archive_scrub"    # archive, scrub PII from the online copy
    RETAIN = "retain"                  # never purged (configuration)


class Cadence(enum.StrEnum):
    NIGHTLY = "nightly"
    ANNUAL = "annual"


@dataclass(frozen=True)
class RetentionPolicy:
    data_class: str
    disposal: Disposal
    period_days: int | None
    anchor: str = "created_at"
    cadence: Cadence = Cadence.NIGHTLY
    approved: bool = False
    authority: str = ""

    @property
    def status(self) -> str:
        if self.disposal is Disposal.RETAIN:
            return "retain"
        if self.period_days is None:
            return "unset"
        return "approved" if self.approved else "interim"


@dataclass
class TableResult:
    table: str
    data_class: str
    status: str
    period_days: int | None
    authority: str
    interim_eligible: int = 0
    to_purge: int = 0


@dataclass
class RetentionReport:
    ran_at: str
    dry_run: bool = True
    tables: list[TableResult] = field(default_factory=list)

    @property
    def to_purge(self) -> int:
        return sum(t.to_purge for t in self.tables)

    @property
    def interim_eligible(self) -> int:
        return sum(t.interim_eligible for t in self.tables)

    def tables_with(self, status: str) -> list[str]:
        return sorted(t.table for t in self.tables if t.status == status)

    def summary(self) -> dict[str, Any]:
        return {
            "ran_at": self.ran_at,
            "dry_run": self.dry_run,
            "to_purge": self.to_purge,
            "interim_eligible": self.interim_eligible,
            "unset": self.tables_with("unset"),
            "tables": len(self.tables),
        }


def resolve_policy(model: type[Any]) -> RetentionPolicy | None:
    from ypms.retention_classes import FRAMEWORK_TABLE_POLICIES

    policy: RetentionPolicy | None = getattr(model, "retention_policy", None)
    if policy is not None:
        return policy
    return FRAMEWORK_TABLE_POLICIES.get(model.__tablename__)


def mapped_models() -> list[type[Any]]:
    import ypms.models  # noqa: F401 — registers every model on Base
    from ypms.db import Base

    return sorted((m.class_ for m in Base.registry.mappers), key=lambda c: c.__tablename__)


def _count_past(db: Session, model: type[Any], policy: RetentionPolicy, now: datetime) -> int:
    if policy.period_days is None:
        raise ValueError(f"policy {policy.data_class!r} has no period")
    anchor = getattr(model, policy.anchor)
    cutoff = now - timedelta(days=policy.period_days)
    stmt = select(func.count()).select_from(model).where(anchor < cutoff)
    return int(db.execute(stmt).scalar_one())


def run_retention(db: Session, *, now: datetime | None = None) -> RetentionReport:
    """Report-only retention pass over every mapped table. Never mutates data
    other than the single `retention_run` audit row; caller commits."""
    log = get_logger(__name__)
    now = now or datetime.now(UTC)
    report = RetentionReport(ran_at=now.isoformat())

    for model in mapped_models():
        policy = resolve_policy(model)
        if policy is None:
            raise LookupError(f"no retention policy for table {model.__tablename__}")
        result = TableResult(
            table=model.__tablename__, data_class=policy.data_class,
            status=policy.status, period_days=policy.period_days,
            authority=policy.authority,
        )
        if result.status in {"interim", "approved"}:
            n = _count_past(db, model, policy, now)
            if result.status == "approved":
                result.to_purge = n
            else:
                result.interim_eligible = n
        report.tables.append(result)
        log.info("retention.table", **asdict(result))

    for result in report.tables:
        if result.status == "unset":
            log.warning("retention.policy_unset", table=result.table, authority=result.authority)
    log.info(
        "retention.run",
        message=f"{report.to_purge} records to purge",
        **report.summary(),
    )
    audit.record(
        db, actor=None, action=RETENTION_RUN_ACTION, entity=RETENTION_ENTITY,
        after=report.summary(),
    )
    return report


def main() -> int:
    from ypms.db import session_factory
    from ypms.logging import configure_logging

    configure_logging()
    with session_factory()() as db:
        report = run_retention(db)
        db.commit()
    print(json.dumps(report.summary()))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())