Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(release): make deployment backup and rollback immutable

Browse Source
This commit is contained in:
Jens
2026-08-30 06:00:43 +02:00
parent a0884d64c9
commit c272220277
47 changed files with 3035 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_rc10_data_operations.py
+212 -1
View File
@@ -3,6 +3,8 @@ from __future__ import annotations
import hashlib
import importlib.util
import json
import os
import subprocess
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
@@ -18,6 +20,8 @@ SCRIPTS = ROOT / "scripts"
def load_script(name: str):
path = SCRIPTS / name
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
@@ -36,6 +40,9 @@ def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha
"database_password_secure": True,
"inventory_mode": inventory_mode,
"storage_inventory_requested": True,
"storage_snapshot_requested": True,
"models_inventory_requested": False,
"models_snapshot_requested": False,
"git_commit": "0123456789abcdef",
}
files = {
@@ -48,6 +55,7 @@ def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha
}
for name, content in files.items():
(root / name).write_text(content, encoding="utf-8")
(root / "storage-snapshot").mkdir()
checksums = []
for name in sorted(files):
digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
@@ -337,8 +345,11 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8")
assert "DELETE_STORAGE_ARTIFACTS" in generic
assert "QUARANTINE_STORAGE_ARTIFACTS" in generic
assert "verify_current_backup" in generic
assert "os.link" in generic
assert 'entry["status"] = "linked"' in generic
assert "cleanup-quarantine" in generic
assert "DELETE_DEMO_EXPORTS" in demo
assert "verify_current_backup" in demo
assert "/app/backups:ro" in compose
@@ -347,6 +358,8 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
"release_backup_guard.py",
"audit_data_operations.py",
"cleanup_storage_artifacts.py",
"restore_storage_quarantine.py",
"release_backup_snapshot.py",
):
assert f"COPY scripts/{name}" in dockerfile
assert f"py_compile scripts/{name}" in readiness
@@ -356,3 +369,201 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
assert "table-counts-after.tsv" in live_audit
assert "deleted_count" in live_audit
assert "missing_manifest_artifact_count" in live_audit
def test_cleanup_apply_moves_bytes_to_protected_traceable_quarantine(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
monkeypatch.syspath_prepend(str(SCRIPTS))
cleanup = load_script("cleanup_storage_artifacts.py")
storage = tmp_path / "storage"
source = storage / "derived" / "orphan.bin"
source.parent.mkdir(parents=True)
source.write_bytes(b"recoverable-derived-artifact")
candidate = SimpleNamespace(
path=source.resolve(),
relative_path="derived/orphan.bin",
size_bytes=source.stat().st_size,
)
now = datetime.now(timezone.utc)
class SessionContext:
def __enter__(self):
return SimpleNamespace()
def __exit__(self, *_args):
return False
monkeypatch.setattr(
cleanup,
"parse_args",
lambda: SimpleNamespace(
storage_root=storage,
minimum_age_days=7,
max_delete=1,
apply=True,
confirm="QUARANTINE_STORAGE_ARTIFACTS",
backup_dir=tmp_path / "backup",
backup_max_age_hours=24.0,
quarantine_root=None,
),
)
monkeypatch.setattr(cleanup, "SessionLocal", lambda: SessionContext())
monkeypatch.setattr(
cleanup,
"build_report",
lambda *_args, **_kwargs: ({"cleanup": {"protected_prefixes": []}}, [candidate]),
)
monkeypatch.setattr(
cleanup,
"verify_current_backup",
lambda *_args, **_kwargs: SimpleNamespace(
release_id="predeploy-test",
created_at=now,
age_hours=0.1,
backup_tool_revision="0123456789abcdef",
),
)
assert cleanup.main() == 0
payload = json.loads(capsys.readouterr().out)
manifest_path = Path(payload["quarantine_manifest"])
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
quarantined_path = storage / payload["quarantined"][0]["quarantine_relative_path"]
assert not source.exists()
assert quarantined_path.read_bytes() == b"recoverable-derived-artifact"
assert manifest["state"] == "complete"
assert manifest["backup_release_id"] == "predeploy-test"
assert manifest["entries"][0]["status"] == "quarantined"
assert payload["deleted_count"] == 0
restore = subprocess.run(
[
sys.executable,
str(SCRIPTS / "restore_storage_quarantine.py"),
"--storage-root",
str(storage),
"--manifest",
str(manifest_path),
"--confirm",
"RESTORE_QUARANTINED_ARTIFACTS",
],
cwd=ROOT,
capture_output=True,
text=True,
check=False,
)
assert restore.returncode == 0, restore.stderr
assert source.read_bytes() == b"recoverable-derived-artifact"
assert not quarantined_path.exists()
restored_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
assert restored_manifest["state"] == "restored"
assert restored_manifest["entries"][0]["status"] == "restored"
def _write_interrupted_quarantine(
storage: Path,
*,
original_exists: bool,
quarantine_exists: bool,
hard_linked: bool = False,
) -> tuple[Path, Path, Path]:
original = storage / "derived" / "interrupted.bin"
operation = storage / "operator-evidence" / "cleanup-quarantine" / "cleanup-interrupted"
quarantined = operation / "files" / "derived" / "interrupted.bin"
original.parent.mkdir(parents=True, exist_ok=True)
quarantined.parent.mkdir(parents=True, exist_ok=True)
retained = b"interrupted-retained-bytes"
if original_exists:
original.write_bytes(retained)
if quarantine_exists:
if hard_linked:
os.link(original, quarantined)
else:
quarantined.write_bytes(retained)
manifest = operation / "manifest.json"
manifest.write_text(
json.dumps(
{
"schema_version": 1,
"state": "in_progress",
"entries": [
{
"relative_path": "derived/interrupted.bin",
"quarantine_relative_path": quarantined.relative_to(storage).as_posix(),
"size_bytes": len(retained),
"sha256": hashlib.sha256(retained).hexdigest(),
"status": "linked" if hard_linked else "planned",
}
],
}
),
encoding="utf-8",
)
return manifest, original, quarantined
@pytest.mark.parametrize(
("original_exists", "quarantine_exists", "hard_linked"),
((False, True, False), (True, True, True)),
)
def test_quarantine_restore_recovers_each_interrupted_move_window(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
original_exists: bool,
quarantine_exists: bool,
hard_linked: bool,
) -> None:
restore = load_script("restore_storage_quarantine.py")
storage = tmp_path / "storage"
manifest, original, quarantined = _write_interrupted_quarantine(
storage,
original_exists=original_exists,
quarantine_exists=quarantine_exists,
hard_linked=hard_linked,
)
monkeypatch.setattr(
restore,
"parse_args",
lambda: SimpleNamespace(
storage_root=storage,
manifest=manifest,
confirm="RESTORE_QUARANTINED_ARTIFACTS",
),
)
assert restore.main() == 0
assert original.read_bytes() == b"interrupted-retained-bytes"
assert not quarantined.exists()
assert json.loads(manifest.read_text(encoding="utf-8"))["state"] == "restored"
def test_quarantine_restore_never_clobbers_recreated_destination(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
restore = load_script("restore_storage_quarantine.py")
storage = tmp_path / "storage"
manifest, original, quarantined = _write_interrupted_quarantine(
storage,
original_exists=False,
quarantine_exists=True,
)
original.write_bytes(b"new-runtime-bytes")
monkeypatch.setattr(
restore,
"parse_args",
lambda: SimpleNamespace(
storage_root=storage,
manifest=manifest,
confirm="RESTORE_QUARANTINED_ARTIFACTS",
),
)
with pytest.raises(RuntimeError, match="different bytes"):
restore.main()
assert original.read_bytes() == b"new-runtime-bytes"
assert quarantined.read_bytes() == b"interrupted-retained-bytes"