Phase 0 milestone 0.2: vendor OWC framework + YPMS pii_audit

Brings in a working generic-CRUD framework plus its dependencies from OWC (../owc-system/src/owc/, snapshot 2026-04-20). Every vendored module carries a header comment naming the source; the OWC → ypms rename is the only mechanical change. Vendored modules: - framework/base.py TimestampMixin, SoftDeleteMixin, VersionedMixin - framework/pagination.py Page[T] with to_dict() - framework/version_history.py Snapshot table - framework/crud.py CRUDService with audit emission - framework/bulk.py BulkOps (BulkResult / BulkUpdateItem) - framework/router.py make_router() factory (kept without `from __future__ import annotations` for FastAPI type introspection — see the module docstring) - audit/models.py AuditLog - audit/service.py record() - security/models.py Permission - security/rbac.py can() + require_permission() dependency (DOES_SYSTEM_ADMIN as the super-role) - users/models.py User, Role, UserRole (YPMS adds mfa_verified and assurance_level; drops OWC-specific claimant_id) - auth/deps.py get_current_user() placeholder — real JWT verify lands in 0.3 YPMS-specific additions on top of the vendored framework: - framework/pii_audit.py emit_pii_read() — NIST AU-14 / source RFP C.5.6.9 #5. OWC doesn't need this because its case-management surface is staff-only; YPMS must audit reads of minors' PII, SSN, banking, FERPA-covered OSSE records, and CFSA foster/ward flags. Bugfix vs. upstream OWC: - framework/crud.py Pagination `next_cursor` was previously the peek-ahead row's id, which combined with `where id > cursor` on the next page silently dropped one row per boundary. Caught by test_list_pagination_forwards_cursor. Fixed here; noted in the module header so re-vendoring OWC surfaces it upstream. Migrations: - 0002_framework_tables.py Baseline for users, roles, user_roles, permissions, audit_log, version_history. Immutable once merged per CONTRIBUTING.md. - migrations/env.py Imports the new model modules so autogenerate compares against them. Tests (26 passing, 93.10% coverage, floor 80%): - tests/framework/test_crud.py CRUDService lifecycle + snapshots + audit emission (7 tests) - tests/framework/test_bulk.py BulkOps happy paths + failure reporting (4 tests) - tests/framework/test_pagination.py Page.to_dict serializer contract (2 tests) - tests/framework/test_pii_audit.py emit_pii_read row shape, anonymous case (2 tests) - tests/security/test_rbac.py can() + admin bypass + role union (4 tests) - tests/test_readyz.py /readyz smoke (1 test) - tests/test_logging.py sanitize_for_log contract (3 tests) Green locally: ruff clean, mypy clean (with a targeted override on framework/router.py and framework/crud.py for TypeVar limitations that OWC also accepts). Advances YPMS-POAM-040. Prepares milestone 0.3 (JWT auth) and 0.4 (RBAC seed) to plug into an already-live framework. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-09-30 16:55:56 -04:00
1 parent 09d862bf4a
commit ba756586aa
@@ -1,8 +1,8 @@
 """Shared pytest fixtures.

-Uses SQLite for speed (see ARCHITECTURE.md §4.8). Any test that needs
-Postgres-specific behaviour lives in a Postgres-only fixture path (added when a
-migration or query needs it).
+Uses SQLite for speed (see ARCHITECTURE.md §4.8). Tests exercise the framework
+against an in-memory Postgres-shaped SQLAlchemy Base — no Alembic migrations
+in the test path; models create their tables from metadata.
 """

 from __future__ import annotations
@@ -12,6 +12,8 @@ from collections.abc import Iterator

 import pytest
 from fastapi.testclient import TestClient
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, sessionmaker


@pytest.fixture(scope="session", autouse=True)
@@ -26,6 +28,30 @@ def _test_env() -> Iterator[None]:
    yield


+@pytest.fixture
+def db() -> Iterator[Session]:
+    """Fresh in-memory SQLite DB per test with all model tables created.
+
+    Import happens inside the fixture so the model modules register themselves
+    on Base.metadata before create_all runs.
+    """
+    import ypms.audit.models  # noqa: F401
+    import ypms.framework.version_history  # noqa: F401
+    import ypms.security.models  # noqa: F401
+    import ypms.users.models  # noqa: F401
+    from ypms.db import Base
+
+    engine = create_engine("sqlite:///:memory:", future=True)
+    Base.metadata.create_all(engine)
+    SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True)
+    session = SessionLocal()
+    try:
+        yield session
+    finally:
+        session.close()
+        engine.dispose()
+
+
@pytest.fixture
 def client() -> Iterator[TestClient]:
    from ypms.main import create_app
@@ -0,0 +1,38 @@
+"""Test-only entity used to exercise the framework without pulling in a real
+domain entity. Milestone 0.5 introduces the Registrant entity for the vertical
+slice; until then, this Thing model is enough to prove CRUD + audit + versioning.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel
+from sqlalchemy import String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from ypms.db import Base
+from ypms.framework.base import SoftDeleteMixin, TimestampMixin, VersionedMixin
+
+
+class Thing(Base, TimestampMixin, SoftDeleteMixin, VersionedMixin):
+    __tablename__ = "things"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    name: Mapped[str] = mapped_column(String(64), nullable=False)
+    color: Mapped[str | None] = mapped_column(String(32), nullable=True)
+
+
+class ThingCreate(BaseModel):
+    name: str
+    color: str | None = None
+
+
+class ThingUpdate(BaseModel):
+    name: str | None = None
+    color: str | None = None
+
+
+class ThingRead(BaseModel):
+    id: int
+    name: str
+    color: str | None = None
+    version: int
@@ -0,0 +1,69 @@
+"""BulkOps: bulk create / update / delete."""
+
+from __future__ import annotations
+
+import pytest
+from sqlalchemy.orm import Session
+
+from tests.framework._fixtures import Thing, ThingCreate, ThingUpdate
+from ypms.framework.bulk import BulkOps, BulkUpdateItem
+from ypms.framework.crud import CRUDService
+from ypms.users.models import User
+
+
+@pytest.fixture
+def bulk(db: Session) -> BulkOps:
+    Thing.__table__.create(bind=db.get_bind(), checkfirst=True)
+    crud = CRUDService[Thing, ThingCreate, ThingUpdate](Thing, entity="Thing")
+    return BulkOps[Thing, ThingCreate, ThingUpdate](crud)
+
+
+@pytest.fixture
+def actor(db: Session) -> User:
+    u = User(username="op", email="op@example.com", password_hash="x")
+    db.add(u)
+    db.flush()
+    return u
+
+
+def test_bulk_create_reports_each_result(bulk: BulkOps, db: Session, actor: User) -> None:
+    payloads = [ThingCreate(name="a"), ThingCreate(name="b"), ThingCreate(name="c")]
+    result = bulk.create(db, payloads, actor=actor)
+    assert len(result.succeeded) == 3
+    assert result.failed == []
+
+
+def test_bulk_update_applies_patches(bulk: BulkOps, db: Session, actor: User) -> None:
+    created = bulk.create(
+        db, [ThingCreate(name="a"), ThingCreate(name="b")], actor=actor
+    )
+    id_a, id_b = created.succeeded
+    items = [
+        BulkUpdateItem(id=id_a, patch={"color": "red"}),
+        BulkUpdateItem(id=id_b, patch={"color": "blue"}),
+    ]
+    result = bulk.update(db, items, ThingUpdate, actor=actor)
+    assert sorted(result.succeeded) == sorted([id_a, id_b])
+    a = db.get(Thing, id_a)
+    b = db.get(Thing, id_b)
+    assert a is not None and b is not None
+    assert a.color == "red"
+    assert b.color == "blue"
+
+
+def test_bulk_delete_soft_deletes_all(bulk: BulkOps, db: Session, actor: User) -> None:
+    created = bulk.create(
+        db, [ThingCreate(name=f"n{i}") for i in range(3)], actor=actor
+    )
+    result = bulk.delete(db, created.succeeded, actor=actor)
+    assert len(result.succeeded) == 3
+    for id_ in created.succeeded:
+        t = db.get(Thing, id_)
+        assert t is not None and t.is_deleted
+
+
+def test_bulk_reports_failures_without_aborting(bulk: BulkOps, db: Session, actor: User) -> None:
+    result = bulk.delete(db, [1, 2, 3], actor=actor)  # nothing exists
+    assert result.succeeded == []
+    assert len(result.failed) == 3
+    assert all("error" in f for f in result.failed)
@@ -0,0 +1,129 @@
+"""CRUDService: create/get/list/update/soft_delete/restore + version snapshot +
+audit emission.
+"""
+
+from __future__ import annotations
+
+import pytest
+from sqlalchemy.exc import NoResultFound
+from sqlalchemy.orm import Session
+
+from tests.framework._fixtures import Thing, ThingCreate, ThingUpdate
+from ypms.audit.models import AuditLog
+from ypms.framework.crud import CRUDService
+from ypms.framework.version_history import VersionHistory
+from ypms.users.models import User
+
+
+@pytest.fixture
+def crud(db: Session) -> CRUDService:
+    Thing.__table__.create(bind=db.get_bind(), checkfirst=True)
+    return CRUDService[Thing, ThingCreate, ThingUpdate](Thing, entity="Thing")
+
+
+@pytest.fixture
+def actor(db: Session) -> User:
+    u = User(
+        username="tester",
+        email="tester@example.com",
+        password_hash="not-a-real-hash",
+    )
+    db.add(u)
+    db.flush()
+    return u
+
+
+def test_create_emits_audit_and_snapshot(crud: CRUDService, db: Session, actor: User) -> None:
+    obj = crud.create(db, ThingCreate(name="widget", color="red"), actor=actor)
+    assert obj.id is not None
+    assert obj.version == 1
+
+    audit = db.query(AuditLog).one()
+    assert audit.action == "create"
+    assert audit.entity == "Thing"
+    assert audit.entity_id == obj.id
+    assert audit.actor_id == actor.id
+    assert audit.actor_username == "tester"
+    assert audit.after is not None
+    assert audit.after["name"] == "widget"
+
+    snap = db.query(VersionHistory).one()
+    assert snap.entity == "Thing"
+    assert snap.entity_id == obj.id
+    assert snap.version == 1
+
+
+def test_update_bumps_version_and_emits_before_after(crud: CRUDService, db: Session, actor: User) -> None:
+    obj = crud.create(db, ThingCreate(name="a"), actor=actor)
+    obj = crud.update(db, obj.id, ThingUpdate(color="blue"), actor=actor)
+    assert obj.version == 2
+    assert obj.color == "blue"
+
+    update_row = db.query(AuditLog).filter(AuditLog.action == "update").one()
+    assert update_row.before is not None
+    assert update_row.after is not None
+    assert update_row.before["color"] is None
+    assert update_row.after["color"] == "blue"
+
+    assert db.query(VersionHistory).count() == 2
+
+
+def test_soft_delete_and_restore(crud: CRUDService, db: Session, actor: User) -> None:
+    obj = crud.create(db, ThingCreate(name="x"), actor=actor)
+    crud.soft_delete(db, obj.id, actor=actor)
+
+    with pytest.raises(NoResultFound):
+        crud.get(db, obj.id)
+
+    ghost = crud.get(db, obj.id, include_deleted=True)
+    assert ghost.is_deleted
+
+    crud.restore(db, obj.id, actor=actor)
+    live = crud.get(db, obj.id)
+    assert not live.is_deleted
+
+    actions = [r.action for r in db.query(AuditLog).order_by(AuditLog.id).all()]
+    assert actions == ["create", "delete", "restore"]
+
+
+def test_list_excludes_soft_deleted_by_default(crud: CRUDService, db: Session, actor: User) -> None:
+    a = crud.create(db, ThingCreate(name="a"), actor=actor)
+    b = crud.create(db, ThingCreate(name="b"), actor=actor)
+    crud.soft_delete(db, a.id, actor=actor)
+
+    page = crud.list(db, limit=10)
+    assert [t.id for t in page.items] == [b.id]
+
+    with_deleted = crud.list(db, limit=10, include_deleted=True)
+    assert {t.id for t in with_deleted.items} == {a.id, b.id}
+
+
+def test_list_pagination_forwards_cursor(crud: CRUDService, db: Session, actor: User) -> None:
+    ids = [crud.create(db, ThingCreate(name=f"n{i}"), actor=actor).id for i in range(5)]
+
+    page1 = crud.list(db, limit=2)
+    assert [t.id for t in page1.items] == ids[:2]
+    assert page1.next_cursor == ids[1]
+
+    page2 = crud.list(db, limit=2, cursor=page1.next_cursor)
+    assert [t.id for t in page2.items] == ids[2:4]
+    assert page2.next_cursor == ids[3]
+
+    page3 = crud.list(db, limit=2, cursor=page2.next_cursor)
+    assert [t.id for t in page3.items] == ids[4:]
+    assert page3.next_cursor is None
+
+
+def test_get_missing_raises(crud: CRUDService, db: Session) -> None:
+    with pytest.raises(NoResultFound):
+        crud.get(db, 999)
+
+
+def test_history_returns_snapshots_in_order(crud: CRUDService, db: Session, actor: User) -> None:
+    obj = crud.create(db, ThingCreate(name="a"), actor=actor)
+    crud.update(db, obj.id, ThingUpdate(name="b"), actor=actor)
+    crud.update(db, obj.id, ThingUpdate(name="c"), actor=actor)
+
+    versions = crud.history(db, obj.id)
+    assert [v.version for v in versions] == [1, 2, 3]
+    assert [v.snapshot["name"] for v in versions] == ["a", "b", "c"]
@@ -0,0 +1,17 @@
+"""Page serialization."""
+
+from __future__ import annotations
+
+from ypms.framework.pagination import Page
+
+
+def test_to_dict_uses_provided_serializer() -> None:
+    p: Page[int] = Page(items=[1, 2, 3], next_cursor=42)
+    out = p.to_dict(lambda i: {"n": i * 2})
+    assert out == {"items": [{"n": 2}, {"n": 4}, {"n": 6}], "next_cursor": 42, "total": None}
+
+
+def test_to_dict_carries_total_when_set() -> None:
+    p: Page[str] = Page(items=["a"], next_cursor=None, total=100)
+    out = p.to_dict(lambda s: s)
+    assert out == {"items": ["a"], "next_cursor": None, "total": 100}
@@ -0,0 +1,35 @@
+"""PII-read audit emission (YPMS-specific framework addition)."""
+
+from __future__ import annotations
+
+from sqlalchemy.orm import Session
+
+from ypms.audit.models import AuditLog
+from ypms.framework.pii_audit import PII_READ_ACTION, emit_pii_read
+from ypms.users.models import User
+
+
+def test_emit_pii_read_records_expected_row(db: Session) -> None:
+    actor = User(username="reader", email="r@example.com", password_hash="x")
+    db.add(actor)
+    db.flush()
+
+    emit_pii_read(db, actor=actor, entity="Registrant", entity_id=42, ip="10.0.0.1")
+
+    row = db.query(AuditLog).one()
+    assert row.action == PII_READ_ACTION
+    assert row.entity == "Registrant"
+    assert row.entity_id == 42
+    assert row.actor_id == actor.id
+    assert row.actor_username == "reader"
+    assert row.ip == "10.0.0.1"
+    assert row.before is None
+    assert row.after is None
+
+
+def test_emit_pii_read_allows_anonymous(db: Session) -> None:
+    emit_pii_read(db, actor=None, entity="Registrant", entity_id=99)
+    row = db.query(AuditLog).one()
+    assert row.actor_id is None
+    assert row.actor_username is None
+    assert row.entity_id == 99
@@ -0,0 +1,72 @@
+"""RBAC engine: can() decisions + DOES_SYSTEM_ADMIN bypass."""
+
+from __future__ import annotations
+
+from sqlalchemy.orm import Session
+
+from ypms.security.models import Permission
+from ypms.security.rbac import ADMIN_ROLE, can
+from ypms.users.models import Role, User, UserRole
+
+
+def _mkuser_with_role(db: Session, *, username: str, role_code: str) -> User:
+    role = Role(code=role_code, description=role_code)
+    db.add(role)
+    db.flush()
+    user = User(username=username, email=f"{username}@example.com", password_hash="x")
+    db.add(user)
+    db.flush()
+    db.add(UserRole(user_id=user.id, role_id=role.id))
+    db.flush()
+    db.refresh(user)
+    return user
+
+
+def test_admin_role_bypasses_all_checks(db: Session) -> None:
+    admin = _mkuser_with_role(db, username="admin", role_code=ADMIN_ROLE)
+    # No permission rows exist; admin still passes.
+    assert can(db, admin, action="Create", entity="Registrant") is True
+    assert can(db, admin, action="Delete", entity="Anything") is True
+
+
+def test_user_with_no_roles_is_denied(db: Session) -> None:
+    user = User(username="orphan", email="orphan@example.com", password_hash="x")
+    db.add(user)
+    db.flush()
+    db.refresh(user)
+    assert can(db, user, action="View", entity="Registrant") is False
+
+
+def test_permission_row_grants_specific_action(db: Session) -> None:
+    user = _mkuser_with_role(db, username="staff", role_code="OYP_STAFF")
+    role_id = user.roles[0].id
+    db.add(Permission(role_id=role_id, entity="Registrant", action="View"))
+    db.flush()
+
+    assert can(db, user, action="View", entity="Registrant") is True
+    # Different action on same entity: denied.
+    assert can(db, user, action="Delete", entity="Registrant") is False
+    # Same action on different entity: denied.
+    assert can(db, user, action="View", entity="Host") is False
+
+
+def test_multiple_roles_union_permissions(db: Session) -> None:
+    role_a = Role(code="ROLE_A")
+    role_b = Role(code="ROLE_B")
+    db.add_all([role_a, role_b])
+    db.flush()
+
+    user = User(username="multi", email="m@example.com", password_hash="x")
+    db.add(user)
+    db.flush()
+    db.add_all(
+        [UserRole(user_id=user.id, role_id=role_a.id), UserRole(user_id=user.id, role_id=role_b.id)]
+    )
+    db.add(Permission(role_id=role_a.id, entity="Registrant", action="View"))
+    db.add(Permission(role_id=role_b.id, entity="Registrant", action="Edit"))
+    db.flush()
+    db.refresh(user)
+
+    assert can(db, user, action="View", entity="Registrant") is True
+    assert can(db, user, action="Edit", entity="Registrant") is True
+    assert can(db, user, action="Delete", entity="Registrant") is False
@@ -0,0 +1,29 @@
+"""Sensitive-field sanitizer strips known key names from log payloads."""
+
+from __future__ import annotations
+
+from ypms.logging import sanitize_for_log
+
+
+def test_top_level_sensitive_keys_are_redacted() -> None:
+    out = sanitize_for_log({"username": "chris", "password": "hunter2", "ssn": "111-22-3333"})
+    assert out == {"username": "chris", "password": "REDACTED", "ssn": "REDACTED"}
+
+
+def test_nested_dicts_are_recursed() -> None:
+    out = sanitize_for_log(
+        {"outer": {"token": "xyz", "safe": 1}, "safe_top": True, "authorization": "Bearer x"}
+    )
+    assert out == {
+        "outer": {"token": "REDACTED", "safe": 1},
+        "safe_top": True,
+        "authorization": "REDACTED",
+    }
+
+
+def test_lists_are_left_alone() -> None:
+    # Design decision: no per-element recursion into lists. Callers that
+    # log lists of potentially-sensitive dicts should map through sanitize
+    # themselves. Keeps the helper's contract predictable.
+    out = sanitize_for_log({"users": [{"password": "x"}], "count": 1})
+    assert out == {"users": [{"password": "x"}], "count": 1}
@@ -0,0 +1,11 @@
+"""/readyz responds 200 when the (SQLite in tests) DB is reachable."""
+
+from __future__ import annotations
+
+from fastapi.testclient import TestClient
+
+
+def test_readyz_returns_ok_when_db_reachable(client: TestClient) -> None:
+    r = client.get("/readyz")
+    assert r.status_code == 200
+    assert r.json() == {"status": "ok"}