from __future__ import annotations import hashlib import importlib.util import json import sys from datetime import datetime, timedelta, timezone from pathlib import Path from types import SimpleNamespace from uuid import uuid4 import pytest ROOT = Path(__file__).resolve().parents[2] SCRIPTS = ROOT / "scripts" def load_script(name: str): path = SCRIPTS / name spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha256") -> None: root.mkdir(parents=True) manifest = { "schema_version": 1, "release_id": "rc10-test", "created_at": created_at.isoformat(), "read_only_source": True, "database_password_secure": True, "inventory_mode": inventory_mode, "storage_inventory_requested": True, "git_commit": "0123456789abcdef", } files = { "manifest.json": json.dumps(manifest), "database.dump": "database", "database.list": "list", "database-metadata.tsv": "alembic_head\t202607160001", "table-counts.tsv": "datasets\t1", "storage-manifest.tsv": "relative_path\tsize_bytes\tmtime_ns\tsha256", } for name, content in files.items(): (root / name).write_text(content, encoding="utf-8") checksums = [] for name in sorted(files): digest = hashlib.sha256((root / name).read_bytes()).hexdigest() checksums.append(f"{digest} {name}") (root / "CHECKSUMS.sha256").write_text("\n".join(checksums) + "\n", encoding="utf-8") def test_backup_guard_requires_recent_complete_sha256_storage_backup(tmp_path: Path) -> None: guard = load_script("release_backup_guard.py") now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc) backup = tmp_path / "backup" write_backup(backup, created_at=now - timedelta(hours=2)) verified = guard.verify_current_backup(backup, now=now) assert verified.release_id == "rc10-test" assert verified.age_hours == pytest.approx(2) def test_backup_guard_rejects_stale_or_tampered_backup(tmp_path: Path) -> None: guard = load_script("release_backup_guard.py") now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc) stale = tmp_path / "stale" write_backup(stale, created_at=now - timedelta(hours=30)) with pytest.raises(RuntimeError, match="maximum allowed age"): guard.verify_current_backup(stale, now=now) current = tmp_path / "tampered" write_backup(current, created_at=now) (current / "database.dump").write_text("tampered", encoding="utf-8") with pytest.raises(RuntimeError, match="checksum mismatch"): guard.verify_current_backup(current, now=now) def test_storage_lifecycle_is_fail_closed_and_protects_release_evidence() -> None: audit = load_script("audit_data_operations.py") assert audit.classify_relative_path("release-evidence/rc11/manifest.json") == ( "release-evidence", True, False, ) assert audit.classify_relative_path("operator-evidence/source/raw.json")[1:] == (True, False) assert audit.classify_relative_path("uploads/project/data.geojson")[1:] == (True, False) assert audit.classify_relative_path("exports/project/old.json")[1:] == (False, True) assert audit.classify_relative_path("unknown/value.bin")[1:] == (True, False) def test_storage_audit_only_selects_old_unreferenced_allowlisted_files( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: audit = load_script("audit_data_operations.py") storage = tmp_path / "storage" old_orphan = storage / "exports" / "project" / "old.json" referenced = storage / "exports" / "project" / "kept.json" protected = storage / "release-evidence" / "rc" / "manifest.json" unknown = storage / "misc" / "unknown.bin" for path in (old_orphan, referenced, protected, unknown): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(path.name, encoding="utf-8") old_timestamp = (datetime.now(timezone.utc) - timedelta(days=30)).timestamp() for path in (old_orphan, referenced, protected, unknown): path.touch() Path(path).chmod(0o644) import os os.utime(path, (old_timestamp, old_timestamp)) monkeypatch.setattr( audit, "collect_database_state", lambda _db, _root: { "references": {referenced.resolve()}, "counts": {}, "source_families": {"national": [], "regional": [], "maritime": []}, }, ) monkeypatch.setattr( audit, "disk_pressure", lambda _root: { "status": "ok", "total_bytes": 100, "used_bytes": 50, "free_bytes": 50, "free_percent": 50.0, "acquisition_allowed": True, }, ) report, candidates = audit.build_report(storage, SimpleNamespace(), minimum_age_days=7) assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"] assert report["cleanup"]["candidate_count"] == 1 assert "release-evidence" in report["cleanup"]["protected_prefixes"] def test_source_family_report_covers_national_regional_and_maritime(tmp_path: Path) -> None: audit = load_script("audit_data_operations.py") national_id = uuid4() regional_id = uuid4() now = datetime.now(timezone.utc) rows = { audit.Project: [ SimpleNamespace(id=national_id, name=audit.NATIONAL_PROJECT_NAME, status="active"), SimpleNamespace(id=regional_id, name="Wallonia operator", status="active"), ], audit.Dataset: [ SimpleNamespace( project_id=national_id, source_name="ngi_adminvector", source="ngi", name="Belgium boundary", source_metadata={"coverage_zones": ["belgium"]}, source_version="2026", imported_at=now, status="ready", storage_path=None, metadata_json=None, provenance_metadata=None, ), SimpleNamespace( project_id=national_id, source_name="rbins_marine_reporting_units", source="rbins", name="Belgian North Sea", source_metadata={"coverage_zones": ["belgian_north_sea"]}, source_version="2024", imported_at=now, status="ready", storage_path=None, metadata_json=None, provenance_metadata=None, ), SimpleNamespace( project_id=regional_id, source_name="wallonia_manual", source="manual", name="Wallonia source", source_metadata={}, source_version="1", imported_at=now, status="ready", storage_path=None, metadata_json=None, provenance_metadata=None, ), ], audit.DatasetVersion: [], audit.Export: [], audit.Detection: [], audit.Segmentation: [], audit.Job: [], audit.AnalysisRun: [], } class Query: def __init__(self, values): self.values = values def all(self): return self.values class Session: def query(self, model, *_fields): if model in rows: return Query(rows[model]) owner = getattr(model, "class_", None) if owner in rows: return Query(rows[owner]) raise AssertionError(f"Unexpected query entity: {model!r}") state = audit.collect_database_state(Session(), tmp_path) assert {item["source_name"] for item in state["source_families"]["national"]} == { "ngi_adminvector", "rbins_marine_reporting_units", } assert {item["source_name"] for item in state["source_families"]["maritime"]} == { "rbins_marine_reporting_units" } assert {item["source_name"] for item in state["source_families"]["regional"]} == { "wallonia_manual" } def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> None: generic = (SCRIPTS / "cleanup_storage_artifacts.py").read_text(encoding="utf-8") demo = (ROOT / "backend/scripts/cleanup_demo_artifacts.py").read_text(encoding="utf-8") compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8") dockerman = (ROOT / "deploy/unraid/run-dockerman-container.sh").read_text(encoding="utf-8") dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8") readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8") live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8") assert "DELETE_STORAGE_ARTIFACTS" in generic assert "verify_current_backup" in generic assert "DELETE_DEMO_EXPORTS" in demo assert "verify_current_backup" in demo assert "/app/backups:ro" in compose assert '/app/backups:ro"' in dockerman for name in ( "release_backup_guard.py", "audit_data_operations.py", "cleanup_storage_artifacts.py", ): assert f"COPY scripts/{name}" in dockerfile assert f"py_compile scripts/{name}" in readiness assert "bash -n scripts/run_rc10_data_operations_audit.sh" in readiness assert "--apply" not in live_audit assert "table-counts-before.tsv" in live_audit assert "table-counts-after.tsv" in live_audit assert "deleted_count" in live_audit