359 lines
13 KiB
Python
359 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPTS = ROOT / "scripts"
|
|
|
|
|
|
def load_script(name: str):
|
|
path = SCRIPTS / name
|
|
spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha256") -> None:
|
|
root.mkdir(parents=True)
|
|
manifest = {
|
|
"schema_version": 1,
|
|
"release_id": "rc10-test",
|
|
"created_at": created_at.isoformat(),
|
|
"read_only_source": True,
|
|
"database_password_secure": True,
|
|
"inventory_mode": inventory_mode,
|
|
"storage_inventory_requested": True,
|
|
"git_commit": "0123456789abcdef",
|
|
}
|
|
files = {
|
|
"manifest.json": json.dumps(manifest),
|
|
"database.dump": "database",
|
|
"database.list": "list",
|
|
"database-metadata.tsv": "alembic_head\t202607160001",
|
|
"table-counts.tsv": "datasets\t1",
|
|
"storage-manifest.tsv": "relative_path\tsize_bytes\tmtime_ns\tsha256",
|
|
}
|
|
for name, content in files.items():
|
|
(root / name).write_text(content, encoding="utf-8")
|
|
checksums = []
|
|
for name in sorted(files):
|
|
digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
|
|
checksums.append(f"{digest} {name}")
|
|
(root / "CHECKSUMS.sha256").write_text("\n".join(checksums) + "\n", encoding="utf-8")
|
|
|
|
|
|
def test_backup_guard_requires_recent_complete_sha256_storage_backup(tmp_path: Path) -> None:
|
|
guard = load_script("release_backup_guard.py")
|
|
now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
|
|
backup = tmp_path / "backup"
|
|
write_backup(backup, created_at=now - timedelta(hours=2))
|
|
|
|
verified = guard.verify_current_backup(backup, now=now)
|
|
|
|
assert verified.release_id == "rc10-test"
|
|
assert verified.age_hours == pytest.approx(2)
|
|
|
|
|
|
def test_backup_guard_rejects_stale_or_tampered_backup(tmp_path: Path) -> None:
|
|
guard = load_script("release_backup_guard.py")
|
|
now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
|
|
stale = tmp_path / "stale"
|
|
write_backup(stale, created_at=now - timedelta(hours=30))
|
|
with pytest.raises(RuntimeError, match="maximum allowed age"):
|
|
guard.verify_current_backup(stale, now=now)
|
|
|
|
current = tmp_path / "tampered"
|
|
write_backup(current, created_at=now)
|
|
(current / "database.dump").write_text("tampered", encoding="utf-8")
|
|
with pytest.raises(RuntimeError, match="checksum mismatch"):
|
|
guard.verify_current_backup(current, now=now)
|
|
|
|
|
|
def test_storage_lifecycle_is_fail_closed_and_protects_release_evidence() -> None:
|
|
audit = load_script("audit_data_operations.py")
|
|
|
|
assert audit.classify_relative_path("release-evidence/rc11/manifest.json") == (
|
|
"release-evidence",
|
|
True,
|
|
False,
|
|
)
|
|
assert audit.classify_relative_path("operator-evidence/source/raw.json")[1:] == (True, False)
|
|
assert audit.classify_relative_path("uploads/project/data.geojson")[1:] == (True, False)
|
|
assert audit.classify_relative_path("exports/project/old.json")[1:] == (False, True)
|
|
assert audit.classify_relative_path("unknown/value.bin")[1:] == (True, False)
|
|
|
|
|
|
def test_storage_audit_only_selects_old_unreferenced_allowlisted_files(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
audit = load_script("audit_data_operations.py")
|
|
storage = tmp_path / "storage"
|
|
old_orphan = storage / "exports" / "project" / "old.json"
|
|
referenced = storage / "exports" / "project" / "kept.json"
|
|
protected = storage / "release-evidence" / "rc" / "manifest.json"
|
|
unknown = storage / "misc" / "unknown.bin"
|
|
for path in (old_orphan, referenced, protected, unknown):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(path.name, encoding="utf-8")
|
|
old_timestamp = (datetime.now(timezone.utc) - timedelta(days=30)).timestamp()
|
|
for path in (old_orphan, referenced, protected, unknown):
|
|
path.touch()
|
|
Path(path).chmod(0o644)
|
|
import os
|
|
|
|
os.utime(path, (old_timestamp, old_timestamp))
|
|
|
|
monkeypatch.setattr(
|
|
audit,
|
|
"collect_database_state",
|
|
lambda _db, _root: {
|
|
"references": {referenced.resolve()},
|
|
"counts": {},
|
|
"source_families": {"national": [], "regional": [], "maritime": []},
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
audit,
|
|
"disk_pressure",
|
|
lambda _root: {
|
|
"status": "ok",
|
|
"total_bytes": 100,
|
|
"used_bytes": 50,
|
|
"free_bytes": 50,
|
|
"free_percent": 50.0,
|
|
"acquisition_allowed": True,
|
|
},
|
|
)
|
|
|
|
report, candidates = audit.build_report(storage, SimpleNamespace(), minimum_age_days=7)
|
|
|
|
assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"]
|
|
assert report["cleanup"]["candidate_count"] == 1
|
|
assert "release-evidence" in report["cleanup"]["protected_prefixes"]
|
|
assert report["integrity"]["missing_referenced_path_count"] == 0
|
|
assert report["integrity"]["missing_manifest_artifact_count"] == 0
|
|
|
|
|
|
def test_referenced_tile_manifest_protects_its_tiles(tmp_path: Path) -> None:
|
|
audit = load_script("audit_data_operations.py")
|
|
storage = tmp_path / "storage"
|
|
manifest = storage / "tiles" / "dataset" / "set" / "manifest.json"
|
|
tile = manifest.parent / "tile_0000.tif"
|
|
tile.parent.mkdir(parents=True)
|
|
tile.write_bytes(b"tile")
|
|
manifest.write_text(json.dumps({"tiles": [{"path": "tile_0000.tif"}]}), encoding="utf-8")
|
|
|
|
expanded = audit.expand_manifest_references({manifest.resolve()}, storage)
|
|
|
|
assert manifest.resolve() in expanded
|
|
assert tile.resolve() in expanded
|
|
|
|
|
|
def test_ordinary_json_export_is_not_treated_as_an_artifact_manifest(tmp_path: Path) -> None:
|
|
audit = load_script("audit_data_operations.py")
|
|
storage = tmp_path / "storage"
|
|
export = storage / "exports" / "project" / "report.json"
|
|
export.parent.mkdir(parents=True)
|
|
export.write_text(json.dumps({"dataset_id": "not-a-file"}), encoding="utf-8")
|
|
|
|
expanded = audit.expand_manifest_references({export.resolve()}, storage)
|
|
|
|
assert expanded == {export.resolve()}
|
|
|
|
|
|
def test_manifest_ids_dates_and_labels_are_not_treated_as_paths(tmp_path: Path) -> None:
|
|
audit = load_script("audit_data_operations.py")
|
|
storage = tmp_path / "storage"
|
|
manifest = storage / "operator-data" / "scope" / "scope.manifest.json"
|
|
actual = manifest.parent / "scope.geojson"
|
|
manifest.parent.mkdir(parents=True)
|
|
actual.write_text("{}", encoding="utf-8")
|
|
manifest.write_text(
|
|
json.dumps(
|
|
{
|
|
"municipality_ids": ["13025", "11001"],
|
|
"generated_at": "2026-07-18T00:00:00Z",
|
|
"label": "Belgium",
|
|
"output_path": "scope.geojson",
|
|
"output_checksum_sha256": "a" * 64,
|
|
"output_crs": "EPSG:4326",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
expanded = audit.expand_manifest_references({manifest.resolve()}, storage)
|
|
|
|
assert expanded == {manifest.resolve(), actual.resolve()}
|
|
|
|
|
|
def test_disk_pressure_uses_absolute_headroom_for_large_arrays(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
audit = load_script("audit_data_operations.py")
|
|
monkeypatch.setattr(
|
|
audit.shutil,
|
|
"disk_usage",
|
|
lambda _path: SimpleNamespace(
|
|
total=56 * 1024**4,
|
|
used=(56 * 1024**4) - (700 * 1024**3),
|
|
free=700 * 1024**3,
|
|
),
|
|
)
|
|
|
|
pressure = audit.disk_pressure(tmp_path)
|
|
|
|
assert pressure["free_percent"] < 2
|
|
assert pressure["status"] == "ok"
|
|
assert pressure["acquisition_allowed"] is True
|
|
|
|
|
|
def test_source_family_report_covers_national_regional_and_maritime(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
audit = load_script("audit_data_operations.py")
|
|
national_id = uuid4()
|
|
regional_id = uuid4()
|
|
now = datetime.now(timezone.utc)
|
|
rows = {
|
|
audit.Project: [
|
|
SimpleNamespace(id=national_id, name=audit.NATIONAL_PROJECT_NAME, status="active"),
|
|
SimpleNamespace(id=regional_id, name="Wallonia operator", status="active"),
|
|
],
|
|
audit.Dataset: [
|
|
SimpleNamespace(
|
|
project_id=national_id,
|
|
source_name="ngi_adminvector",
|
|
source="ngi",
|
|
name="Belgium boundary",
|
|
source_metadata={"coverage_zones": ["belgium"]},
|
|
source_version="2026",
|
|
imported_at=now,
|
|
status="ready",
|
|
storage_path=None,
|
|
metadata_json=None,
|
|
provenance_metadata=None,
|
|
),
|
|
SimpleNamespace(
|
|
project_id=national_id,
|
|
source_name="rbins_marine_reporting_units",
|
|
source="rbins",
|
|
name="Belgian North Sea",
|
|
source_metadata={"coverage_zones": ["belgian_north_sea"]},
|
|
source_version="2024",
|
|
imported_at=now,
|
|
status="ready",
|
|
storage_path=None,
|
|
metadata_json=None,
|
|
provenance_metadata=None,
|
|
),
|
|
SimpleNamespace(
|
|
project_id=regional_id,
|
|
source_name="wallonia_manual",
|
|
source="manual",
|
|
name="Wallonia source",
|
|
source_metadata={},
|
|
source_version="1",
|
|
imported_at=now,
|
|
status="ready",
|
|
storage_path=None,
|
|
metadata_json=None,
|
|
provenance_metadata=None,
|
|
),
|
|
],
|
|
audit.DatasetVersion: [],
|
|
audit.Export: [],
|
|
audit.Detection: [],
|
|
audit.Segmentation: [],
|
|
audit.Job: [],
|
|
audit.AnalysisRun: [],
|
|
}
|
|
|
|
class Query:
|
|
def __init__(self, values):
|
|
self.values = values
|
|
|
|
def all(self):
|
|
return self.values
|
|
|
|
class Session:
|
|
def query(self, model, *_fields):
|
|
if model in rows:
|
|
return Query(rows[model])
|
|
owner = getattr(model, "class_", None)
|
|
if owner in rows:
|
|
return Query(rows[owner])
|
|
raise AssertionError(f"Unexpected query entity: {model!r}")
|
|
|
|
monkeypatch.setattr(audit, "query_count", lambda _db, model, *_conditions: len(rows[model]))
|
|
monkeypatch.setattr(audit, "query_distinct_nonnull", lambda _db, _column: [])
|
|
state = audit.collect_database_state(Session(), tmp_path)
|
|
|
|
assert {item["source_name"] for item in state["source_families"]["national"]} == {
|
|
"ngi_adminvector",
|
|
"rbins_marine_reporting_units",
|
|
}
|
|
assert {item["source_name"] for item in state["source_families"]["maritime"]} == {
|
|
"rbins_marine_reporting_units"
|
|
}
|
|
assert {item["source_name"] for item in state["source_families"]["regional"]} == {
|
|
"wallonia_manual"
|
|
}
|
|
|
|
|
|
def test_national_and_maritime_sources_have_explicit_freshness_policies() -> None:
|
|
from app.services.source_freshness_service import SOURCE_POLICIES
|
|
|
|
for source_name in (
|
|
"ngi_adminvector",
|
|
"rbins_marine_reporting_units",
|
|
"rbins_msp_2026",
|
|
):
|
|
assert SOURCE_POLICIES[source_name].refresh_policy == "edition"
|
|
|
|
|
|
def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> None:
|
|
generic = (SCRIPTS / "cleanup_storage_artifacts.py").read_text(encoding="utf-8")
|
|
demo = (ROOT / "backend/scripts/cleanup_demo_artifacts.py").read_text(encoding="utf-8")
|
|
compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
|
|
dockerman = (ROOT / "deploy/unraid/run-dockerman-container.sh").read_text(encoding="utf-8")
|
|
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
|
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
|
|
live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8")
|
|
|
|
assert "DELETE_STORAGE_ARTIFACTS" in generic
|
|
assert "verify_current_backup" in generic
|
|
assert "DELETE_DEMO_EXPORTS" in demo
|
|
assert "verify_current_backup" in demo
|
|
assert "/app/backups:ro" in compose
|
|
assert '/app/backups:ro"' in dockerman
|
|
for name in (
|
|
"release_backup_guard.py",
|
|
"audit_data_operations.py",
|
|
"cleanup_storage_artifacts.py",
|
|
):
|
|
assert f"COPY scripts/{name}" in dockerfile
|
|
assert f"py_compile scripts/{name}" in readiness
|
|
assert "bash -n scripts/run_rc10_data_operations_audit.sh" in readiness
|
|
assert "--apply" not in live_audit
|
|
assert "table-counts-before.tsv" in live_audit
|
|
assert "table-counts-after.tsv" in live_audit
|
|
assert "deleted_count" in live_audit
|
|
assert "missing_manifest_artifact_count" in live_audit
|