Operationalize RC10 data retention

backend/README.md

@@ -1920,3 +1920,17 @@ file. It never imports provider data or writes directly to database tables.
 Explicit demo seeding also reactivates its own archived technical project.
 This keeps the opt-in fixture workflow selectable without changing the normal
 active-project lifecycle.
+
+## Data operations and retention
+
+The runtime packages `audit_data_operations.py`,
+`cleanup_storage_artifacts.py` and the shared release-backup guard. The audit
+is read-only and combines disk pressure, storage lifecycle, persisted path
+integrity, failed-work counts and national/regional/maritime source-family
+inventory. Cleanup is limited to old unreferenced derived/cache/export files.
+
+Unknown paths, official source material, uploads, models and release/operator
+evidence are protected by default. Apply mode requires an exact confirmation,
+an explicit candidate ceiling and a recent checksum-verified database plus
+SHA-256 storage backup mounted read-only under `/app/backups`. See
+`docs/DATA_OPERATIONS_RUNBOOK.md`.

backend/scripts/cleanup_demo_artifacts.py

@@ -9,13 +9,19 @@ from typing import Any
 
 BACKEND_ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(BACKEND_ROOT))
+REPOSITORY_ROOT = BACKEND_ROOT.parent
+SCRIPTS_ROOT = REPOSITORY_ROOT / "scripts"
+if SCRIPTS_ROOT.is_dir():
+    sys.path.insert(0, str(SCRIPTS_ROOT))
 
 from app.core.config import get_settings
 from app.db.session import SessionLocal
 from app.models import Export, Project
+from release_backup_guard import require_confirmation, verify_current_backup
 
 
 DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
+DELETE_CONFIRMATION = "DELETE_DEMO_EXPORTS"
 
 
 def is_within_storage_root(path: Path, storage_root: Path) -> bool:
@@ -189,6 +195,21 @@ def build_parser() -> argparse.ArgumentParser:
         help="Restrict cleanup to an export_type. Repeat for multiple types.",
     )
     parser.add_argument("--apply", action="store_true", help="Delete selected export rows and files.")
+    parser.add_argument(
+        "--backup-dir",
+        type=Path,
+        help="Recent checksum-verified release backup mounted read-only in the runtime.",
+    )
+    parser.add_argument(
+        "--backup-max-age-hours",
+        type=float,
+        default=24.0,
+        help="Maximum age accepted for the required release backup.",
+    )
+    parser.add_argument(
+        "--confirm",
+        help=f"Exact destructive-maintenance confirmation token: {DELETE_CONFIRMATION}",
+    )
     return parser
 
 
@@ -199,6 +220,17 @@ def main() -> int:
         parser.error("--keep-latest must be greater than or equal to zero")
     if args.max_delete < 0:
         parser.error("--max-delete must be greater than or equal to zero")
+    if args.apply:
+        try:
+            require_confirmation(args.confirm, DELETE_CONFIRMATION)
+            if args.backup_dir is None:
+                raise RuntimeError("--backup-dir is required with --apply")
+            verify_current_backup(
+                args.backup_dir,
+                max_age_hours=args.backup_max_age_hours,
+            )
+        except (RuntimeError, ValueError) as exc:
+            parser.error(str(exc))
 
     summary = cleanup_demo_exports(
         project_name=args.project_name,

backend/tests/test_rc10_data_operations.py

@@ -0,0 +1,259 @@
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import json
+import sys
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPTS = ROOT / "scripts"
+
+
+def load_script(name: str):
+    path = SCRIPTS / name
+    spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha256") -> None:
+    root.mkdir(parents=True)
+    manifest = {
+        "schema_version": 1,
+        "release_id": "rc10-test",
+        "created_at": created_at.isoformat(),
+        "read_only_source": True,
+        "database_password_secure": True,
+        "inventory_mode": inventory_mode,
+        "storage_inventory_requested": True,
+        "git_commit": "0123456789abcdef",
+    }
+    files = {
+        "manifest.json": json.dumps(manifest),
+        "database.dump": "database",
+        "database.list": "list",
+        "database-metadata.tsv": "alembic_head\t202607160001",
+        "table-counts.tsv": "datasets\t1",
+        "storage-manifest.tsv": "relative_path\tsize_bytes\tmtime_ns\tsha256",
+    }
+    for name, content in files.items():
+        (root / name).write_text(content, encoding="utf-8")
+    checksums = []
+    for name in sorted(files):
+        digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
+        checksums.append(f"{digest}  {name}")
+    (root / "CHECKSUMS.sha256").write_text("\n".join(checksums) + "\n", encoding="utf-8")
+
+
+def test_backup_guard_requires_recent_complete_sha256_storage_backup(tmp_path: Path) -> None:
+    guard = load_script("release_backup_guard.py")
+    now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
+    backup = tmp_path / "backup"
+    write_backup(backup, created_at=now - timedelta(hours=2))
+
+    verified = guard.verify_current_backup(backup, now=now)
+
+    assert verified.release_id == "rc10-test"
+    assert verified.age_hours == pytest.approx(2)
+
+
+def test_backup_guard_rejects_stale_or_tampered_backup(tmp_path: Path) -> None:
+    guard = load_script("release_backup_guard.py")
+    now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
+    stale = tmp_path / "stale"
+    write_backup(stale, created_at=now - timedelta(hours=30))
+    with pytest.raises(RuntimeError, match="maximum allowed age"):
+        guard.verify_current_backup(stale, now=now)
+
+    current = tmp_path / "tampered"
+    write_backup(current, created_at=now)
+    (current / "database.dump").write_text("tampered", encoding="utf-8")
+    with pytest.raises(RuntimeError, match="checksum mismatch"):
+        guard.verify_current_backup(current, now=now)
+
+
+def test_storage_lifecycle_is_fail_closed_and_protects_release_evidence() -> None:
+    audit = load_script("audit_data_operations.py")
+
+    assert audit.classify_relative_path("release-evidence/rc11/manifest.json") == (
+        "release-evidence",
+        True,
+        False,
+    )
+    assert audit.classify_relative_path("operator-evidence/source/raw.json")[1:] == (True, False)
+    assert audit.classify_relative_path("uploads/project/data.geojson")[1:] == (True, False)
+    assert audit.classify_relative_path("exports/project/old.json")[1:] == (False, True)
+    assert audit.classify_relative_path("unknown/value.bin")[1:] == (True, False)
+
+
+def test_storage_audit_only_selects_old_unreferenced_allowlisted_files(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    audit = load_script("audit_data_operations.py")
+    storage = tmp_path / "storage"
+    old_orphan = storage / "exports" / "project" / "old.json"
+    referenced = storage / "exports" / "project" / "kept.json"
+    protected = storage / "release-evidence" / "rc" / "manifest.json"
+    unknown = storage / "misc" / "unknown.bin"
+    for path in (old_orphan, referenced, protected, unknown):
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(path.name, encoding="utf-8")
+    old_timestamp = (datetime.now(timezone.utc) - timedelta(days=30)).timestamp()
+    for path in (old_orphan, referenced, protected, unknown):
+        path.touch()
+        Path(path).chmod(0o644)
+        import os
+
+        os.utime(path, (old_timestamp, old_timestamp))
+
+    monkeypatch.setattr(
+        audit,
+        "collect_database_state",
+        lambda _db, _root: {
+            "references": {referenced.resolve()},
+            "counts": {},
+            "source_families": {"national": [], "regional": [], "maritime": []},
+        },
+    )
+    monkeypatch.setattr(
+        audit,
+        "disk_pressure",
+        lambda _root: {
+            "status": "ok",
+            "total_bytes": 100,
+            "used_bytes": 50,
+            "free_bytes": 50,
+            "free_percent": 50.0,
+            "acquisition_allowed": True,
+        },
+    )
+
+    report, candidates = audit.build_report(storage, SimpleNamespace(), minimum_age_days=7)
+
+    assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"]
+    assert report["cleanup"]["candidate_count"] == 1
+    assert "release-evidence" in report["cleanup"]["protected_prefixes"]
+
+
+def test_source_family_report_covers_national_regional_and_maritime(tmp_path: Path) -> None:
+    audit = load_script("audit_data_operations.py")
+    national_id = uuid4()
+    regional_id = uuid4()
+    now = datetime.now(timezone.utc)
+    rows = {
+        audit.Project: [
+            SimpleNamespace(id=national_id, name=audit.NATIONAL_PROJECT_NAME, status="active"),
+            SimpleNamespace(id=regional_id, name="Wallonia operator", status="active"),
+        ],
+        audit.Dataset: [
+            SimpleNamespace(
+                project_id=national_id,
+                source_name="ngi_adminvector",
+                source="ngi",
+                name="Belgium boundary",
+                source_metadata={"coverage_zones": ["belgium"]},
+                source_version="2026",
+                imported_at=now,
+                status="ready",
+                storage_path=None,
+                metadata_json=None,
+                provenance_metadata=None,
+            ),
+            SimpleNamespace(
+                project_id=national_id,
+                source_name="rbins_marine_reporting_units",
+                source="rbins",
+                name="Belgian North Sea",
+                source_metadata={"coverage_zones": ["belgian_north_sea"]},
+                source_version="2024",
+                imported_at=now,
+                status="ready",
+                storage_path=None,
+                metadata_json=None,
+                provenance_metadata=None,
+            ),
+            SimpleNamespace(
+                project_id=regional_id,
+                source_name="wallonia_manual",
+                source="manual",
+                name="Wallonia source",
+                source_metadata={},
+                source_version="1",
+                imported_at=now,
+                status="ready",
+                storage_path=None,
+                metadata_json=None,
+                provenance_metadata=None,
+            ),
+        ],
+        audit.DatasetVersion: [],
+        audit.Export: [],
+        audit.Detection: [],
+        audit.Segmentation: [],
+        audit.Job: [],
+        audit.AnalysisRun: [],
+    }
+
+    class Query:
+        def __init__(self, values):
+            self.values = values
+
+        def all(self):
+            return self.values
+
+    class Session:
+        def query(self, model):
+            return Query(rows[model])
+
+    state = audit.collect_database_state(Session(), tmp_path)
+
+    assert {item["source_name"] for item in state["source_families"]["national"]} == {
+        "ngi_adminvector",
+        "rbins_marine_reporting_units",
+    }
+    assert {item["source_name"] for item in state["source_families"]["maritime"]} == {
+        "rbins_marine_reporting_units"
+    }
+    assert {item["source_name"] for item in state["source_families"]["regional"]} == {
+        "wallonia_manual"
+    }
+
+
+def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> None:
+    generic = (SCRIPTS / "cleanup_storage_artifacts.py").read_text(encoding="utf-8")
+    demo = (ROOT / "backend/scripts/cleanup_demo_artifacts.py").read_text(encoding="utf-8")
+    compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
+    dockerman = (ROOT / "deploy/unraid/run-dockerman-container.sh").read_text(encoding="utf-8")
+    dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
+    readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
+    live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8")
+
+    assert "DELETE_STORAGE_ARTIFACTS" in generic
+    assert "verify_current_backup" in generic
+    assert "DELETE_DEMO_EXPORTS" in demo
+    assert "verify_current_backup" in demo
+    assert "/app/backups:ro" in compose
+    assert '/app/backups:ro"' in dockerman
+    for name in (
+        "release_backup_guard.py",
+        "audit_data_operations.py",
+        "cleanup_storage_artifacts.py",
+    ):
+        assert f"COPY scripts/{name}" in dockerfile
+        assert f"py_compile scripts/{name}" in readiness
+    assert "bash -n scripts/run_rc10_data_operations_audit.sh" in readiness
+    assert "--apply" not in live_audit
+    assert "table-counts-before.tsv" in live_audit
+    assert "table-counts-after.tsv" in live_audit
+    assert "deleted_count" in live_audit

backend/tests/test_sprint31_unraid_template.py

@@ -65,6 +65,7 @@ def test_unraid_template_exposes_every_operator_owned_runtime_setting() -> None:
     template_variables = set(re.findall(r'Target="([A-Z][A-Z0-9_]+)"', template))
     bridged_or_internal = {
         "GEOINTEL_FRONTEND_PORT",
+        "GEOINTEL_BACKUPS_PATH",
         "GEOINTEL_IMAGE",
         "GEOINTEL_MODELS_PATH",
         "GEOINTEL_POSTGIS_DATA_PATH",