Operationalize RC10 data retention

2026-07-18 06:58:10 +02:00
parent bf867f8f4f
commit 7b96037853
22 changed files with 1306 additions and 2 deletions
@@ -0,0 +1,259 @@
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import json
+import sys
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPTS = ROOT / "scripts"
+
+
+def load_script(name: str):
+    path = SCRIPTS / name
+    spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha256") -> None:
+    root.mkdir(parents=True)
+    manifest = {
+        "schema_version": 1,
+        "release_id": "rc10-test",
+        "created_at": created_at.isoformat(),
+        "read_only_source": True,
+        "database_password_secure": True,
+        "inventory_mode": inventory_mode,
+        "storage_inventory_requested": True,
+        "git_commit": "0123456789abcdef",
+    }
+    files = {
+        "manifest.json": json.dumps(manifest),
+        "database.dump": "database",
+        "database.list": "list",
+        "database-metadata.tsv": "alembic_head\t202607160001",
+        "table-counts.tsv": "datasets\t1",
+        "storage-manifest.tsv": "relative_path\tsize_bytes\tmtime_ns\tsha256",
+    }
+    for name, content in files.items():
+        (root / name).write_text(content, encoding="utf-8")
+    checksums = []
+    for name in sorted(files):
+        digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
+        checksums.append(f"{digest}  {name}")
+    (root / "CHECKSUMS.sha256").write_text("\n".join(checksums) + "\n", encoding="utf-8")
+
+
+def test_backup_guard_requires_recent_complete_sha256_storage_backup(tmp_path: Path) -> None:
+    guard = load_script("release_backup_guard.py")
+    now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
+    backup = tmp_path / "backup"
+    write_backup(backup, created_at=now - timedelta(hours=2))
+
+    verified = guard.verify_current_backup(backup, now=now)
+
+    assert verified.release_id == "rc10-test"
+    assert verified.age_hours == pytest.approx(2)
+
+
+def test_backup_guard_rejects_stale_or_tampered_backup(tmp_path: Path) -> None:
+    guard = load_script("release_backup_guard.py")
+    now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
+    stale = tmp_path / "stale"
+    write_backup(stale, created_at=now - timedelta(hours=30))
+    with pytest.raises(RuntimeError, match="maximum allowed age"):
+        guard.verify_current_backup(stale, now=now)
+
+    current = tmp_path / "tampered"
+    write_backup(current, created_at=now)
+    (current / "database.dump").write_text("tampered", encoding="utf-8")
+    with pytest.raises(RuntimeError, match="checksum mismatch"):
+        guard.verify_current_backup(current, now=now)
+
+
+def test_storage_lifecycle_is_fail_closed_and_protects_release_evidence() -> None:
+    audit = load_script("audit_data_operations.py")
+
+    assert audit.classify_relative_path("release-evidence/rc11/manifest.json") == (
+        "release-evidence",
+        True,
+        False,
+    )
+    assert audit.classify_relative_path("operator-evidence/source/raw.json")[1:] == (True, False)
+    assert audit.classify_relative_path("uploads/project/data.geojson")[1:] == (True, False)
+    assert audit.classify_relative_path("exports/project/old.json")[1:] == (False, True)
+    assert audit.classify_relative_path("unknown/value.bin")[1:] == (True, False)
+
+
+def test_storage_audit_only_selects_old_unreferenced_allowlisted_files(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    audit = load_script("audit_data_operations.py")
+    storage = tmp_path / "storage"
+    old_orphan = storage / "exports" / "project" / "old.json"
+    referenced = storage / "exports" / "project" / "kept.json"
+    protected = storage / "release-evidence" / "rc" / "manifest.json"
+    unknown = storage / "misc" / "unknown.bin"
+    for path in (old_orphan, referenced, protected, unknown):
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(path.name, encoding="utf-8")
+    old_timestamp = (datetime.now(timezone.utc) - timedelta(days=30)).timestamp()
+    for path in (old_orphan, referenced, protected, unknown):
+        path.touch()
+        Path(path).chmod(0o644)
+        import os
+
+        os.utime(path, (old_timestamp, old_timestamp))
+
+    monkeypatch.setattr(
+        audit,
+        "collect_database_state",
+        lambda _db, _root: {
+            "references": {referenced.resolve()},
+            "counts": {},
+            "source_families": {"national": [], "regional": [], "maritime": []},
+        },
+    )
+    monkeypatch.setattr(
+        audit,
+        "disk_pressure",
+        lambda _root: {
+            "status": "ok",
+            "total_bytes": 100,
+            "used_bytes": 50,
+            "free_bytes": 50,
+            "free_percent": 50.0,
+            "acquisition_allowed": True,
+        },
+    )
+
+    report, candidates = audit.build_report(storage, SimpleNamespace(), minimum_age_days=7)
+
+    assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"]
+    assert report["cleanup"]["candidate_count"] == 1
+    assert "release-evidence" in report["cleanup"]["protected_prefixes"]
+
+
+def test_source_family_report_covers_national_regional_and_maritime(tmp_path: Path) -> None:
+    audit = load_script("audit_data_operations.py")
+    national_id = uuid4()
+    regional_id = uuid4()
+    now = datetime.now(timezone.utc)
+    rows = {
+        audit.Project: [
+            SimpleNamespace(id=national_id, name=audit.NATIONAL_PROJECT_NAME, status="active"),
+            SimpleNamespace(id=regional_id, name="Wallonia operator", status="active"),
+        ],
+        audit.Dataset: [
+            SimpleNamespace(
+                project_id=national_id,
+                source_name="ngi_adminvector",
+                source="ngi",
+                name="Belgium boundary",
+                source_metadata={"coverage_zones": ["belgium"]},
+                source_version="2026",
+                imported_at=now,
+                status="ready",
+                storage_path=None,
+                metadata_json=None,
+                provenance_metadata=None,
+            ),
+            SimpleNamespace(
+                project_id=national_id,
+                source_name="rbins_marine_reporting_units",
+                source="rbins",
+                name="Belgian North Sea",
+                source_metadata={"coverage_zones": ["belgian_north_sea"]},
+                source_version="2024",
+                imported_at=now,
+                status="ready",
+                storage_path=None,
+                metadata_json=None,
+                provenance_metadata=None,
+            ),
+            SimpleNamespace(
+                project_id=regional_id,
+                source_name="wallonia_manual",
+                source="manual",
+                name="Wallonia source",
+                source_metadata={},
+                source_version="1",
+                imported_at=now,
+                status="ready",
+                storage_path=None,
+                metadata_json=None,
+                provenance_metadata=None,
+            ),
+        ],
+        audit.DatasetVersion: [],
+        audit.Export: [],
+        audit.Detection: [],
+        audit.Segmentation: [],
+        audit.Job: [],
+        audit.AnalysisRun: [],
+    }
+
+    class Query:
+        def __init__(self, values):
+            self.values = values
+
+        def all(self):
+            return self.values
+
+    class Session:
+        def query(self, model):
+            return Query(rows[model])
+
+    state = audit.collect_database_state(Session(), tmp_path)
+
+    assert {item["source_name"] for item in state["source_families"]["national"]} == {
+        "ngi_adminvector",
+        "rbins_marine_reporting_units",
+    }
+    assert {item["source_name"] for item in state["source_families"]["maritime"]} == {
+        "rbins_marine_reporting_units"
+    }
+    assert {item["source_name"] for item in state["source_families"]["regional"]} == {
+        "wallonia_manual"
+    }
+
+
+def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> None:
+    generic = (SCRIPTS / "cleanup_storage_artifacts.py").read_text(encoding="utf-8")
+    demo = (ROOT / "backend/scripts/cleanup_demo_artifacts.py").read_text(encoding="utf-8")
+    compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
+    dockerman = (ROOT / "deploy/unraid/run-dockerman-container.sh").read_text(encoding="utf-8")
+    dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
+    readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
+    live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8")
+
+    assert "DELETE_STORAGE_ARTIFACTS" in generic
+    assert "verify_current_backup" in generic
+    assert "DELETE_DEMO_EXPORTS" in demo
+    assert "verify_current_backup" in demo
+    assert "/app/backups:ro" in compose
+    assert '/app/backups:ro"' in dockerman
+    for name in (
+        "release_backup_guard.py",
+        "audit_data_operations.py",
+        "cleanup_storage_artifacts.py",
+    ):
+        assert f"COPY scripts/{name}" in dockerfile
+        assert f"py_compile scripts/{name}" in readiness
+    assert "bash -n scripts/run_rc10_data_operations_audit.sh" in readiness
+    assert "--apply" not in live_audit
+    assert "table-counts-before.tsv" in live_audit
+    assert "table-counts-after.tsv" in live_audit
+    assert "deleted_count" in live_audit
@@ -65,6 +65,7 @@ def test_unraid_template_exposes_every_operator_owned_runtime_setting() -> None:
    template_variables = set(re.findall(r'Target="([A-Z][A-Z0-9_]+)"', template))
    bridged_or_internal = {
        "GEOINTEL_FRONTEND_PORT",
+        "GEOINTEL_BACKUPS_PATH",
        "GEOINTEL_IMAGE",
        "GEOINTEL_MODELS_PATH",
        "GEOINTEL_POSTGIS_DATA_PATH",