Operationalize RC10 data retention
This commit is contained in:
@@ -109,6 +109,7 @@ GEOINTEL_FRONTEND_PORT=1202
|
||||
GEOINTEL_BACKEND_PORT=8000
|
||||
GEOINTEL_INSTALL_AI=false
|
||||
GEOINTEL_STORAGE_PATH=./storage
|
||||
GEOINTEL_BACKUPS_PATH=./backups
|
||||
GEOINTEL_MODELS_PATH=./models
|
||||
GEOINTEL_POSTGIS_DATA_PATH=./postgres-data
|
||||
GEOINTEL_POSTGRES_DB=geointel
|
||||
|
||||
@@ -66,6 +66,16 @@
|
||||
the live three-viewport audit passed with zero horizontal overflow,
|
||||
console errors or failed requests and verified truthful delayed loading,
|
||||
keyboard behavior, accessible control names and visible timing feedback.
|
||||
- Added an RC-10 read-only data-operations audit covering storage lifecycle,
|
||||
disk pressure, persisted-path integrity, old failed work and
|
||||
national/regional/maritime source families.
|
||||
- Added dry-run-first orphan/cache cleanup with fail-closed path categories,
|
||||
an explicit delete ceiling, exact confirmation token and a recent
|
||||
checksum-verified database plus SHA-256 storage-backup requirement.
|
||||
- Mounted release backups read-only at `/app/backups` and applied the same
|
||||
backup/confirmation guard to the older demo-export cleanup path.
|
||||
- Added a live RC-10 audit that proves critical table counts remain unchanged
|
||||
across the storage report and cleanup dry run.
|
||||
- Added a read-only release-evidence manifest command with Git, migration,
|
||||
dependency, configuration checksum and optional live endpoint evidence.
|
||||
- Replaced the obsolete pre-build status with the current implemented
|
||||
|
||||
@@ -1920,3 +1920,17 @@ file. It never imports provider data or writes directly to database tables.
|
||||
Explicit demo seeding also reactivates its own archived technical project.
|
||||
This keeps the opt-in fixture workflow selectable without changing the normal
|
||||
active-project lifecycle.
|
||||
|
||||
## Data operations and retention
|
||||
|
||||
The runtime packages `audit_data_operations.py`,
|
||||
`cleanup_storage_artifacts.py` and the shared release-backup guard. The audit
|
||||
is read-only and combines disk pressure, storage lifecycle, persisted path
|
||||
integrity, failed-work counts and national/regional/maritime source-family
|
||||
inventory. Cleanup is limited to old unreferenced derived/cache/export files.
|
||||
|
||||
Unknown paths, official source material, uploads, models and release/operator
|
||||
evidence are protected by default. Apply mode requires an exact confirmation,
|
||||
an explicit candidate ceiling and a recent checksum-verified database plus
|
||||
SHA-256 storage backup mounted read-only under `/app/backups`. See
|
||||
`docs/DATA_OPERATIONS_RUNBOOK.md`.
|
||||
|
||||
@@ -9,13 +9,19 @@ from typing import Any
|
||||
|
||||
BACKEND_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(BACKEND_ROOT))
|
||||
REPOSITORY_ROOT = BACKEND_ROOT.parent
|
||||
SCRIPTS_ROOT = REPOSITORY_ROOT / "scripts"
|
||||
if SCRIPTS_ROOT.is_dir():
|
||||
sys.path.insert(0, str(SCRIPTS_ROOT))
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.session import SessionLocal
|
||||
from app.models import Export, Project
|
||||
from release_backup_guard import require_confirmation, verify_current_backup
|
||||
|
||||
|
||||
DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
|
||||
DELETE_CONFIRMATION = "DELETE_DEMO_EXPORTS"
|
||||
|
||||
|
||||
def is_within_storage_root(path: Path, storage_root: Path) -> bool:
|
||||
@@ -189,6 +195,21 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
help="Restrict cleanup to an export_type. Repeat for multiple types.",
|
||||
)
|
||||
parser.add_argument("--apply", action="store_true", help="Delete selected export rows and files.")
|
||||
parser.add_argument(
|
||||
"--backup-dir",
|
||||
type=Path,
|
||||
help="Recent checksum-verified release backup mounted read-only in the runtime.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backup-max-age-hours",
|
||||
type=float,
|
||||
default=24.0,
|
||||
help="Maximum age accepted for the required release backup.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--confirm",
|
||||
help=f"Exact destructive-maintenance confirmation token: {DELETE_CONFIRMATION}",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
@@ -199,6 +220,17 @@ def main() -> int:
|
||||
parser.error("--keep-latest must be greater than or equal to zero")
|
||||
if args.max_delete < 0:
|
||||
parser.error("--max-delete must be greater than or equal to zero")
|
||||
if args.apply:
|
||||
try:
|
||||
require_confirmation(args.confirm, DELETE_CONFIRMATION)
|
||||
if args.backup_dir is None:
|
||||
raise RuntimeError("--backup-dir is required with --apply")
|
||||
verify_current_backup(
|
||||
args.backup_dir,
|
||||
max_age_hours=args.backup_max_age_hours,
|
||||
)
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
parser.error(str(exc))
|
||||
|
||||
summary = cleanup_demo_exports(
|
||||
project_name=args.project_name,
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
|
||||
|
||||
def load_script(name: str):
|
||||
path = SCRIPTS / name
|
||||
spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha256") -> None:
|
||||
root.mkdir(parents=True)
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"release_id": "rc10-test",
|
||||
"created_at": created_at.isoformat(),
|
||||
"read_only_source": True,
|
||||
"database_password_secure": True,
|
||||
"inventory_mode": inventory_mode,
|
||||
"storage_inventory_requested": True,
|
||||
"git_commit": "0123456789abcdef",
|
||||
}
|
||||
files = {
|
||||
"manifest.json": json.dumps(manifest),
|
||||
"database.dump": "database",
|
||||
"database.list": "list",
|
||||
"database-metadata.tsv": "alembic_head\t202607160001",
|
||||
"table-counts.tsv": "datasets\t1",
|
||||
"storage-manifest.tsv": "relative_path\tsize_bytes\tmtime_ns\tsha256",
|
||||
}
|
||||
for name, content in files.items():
|
||||
(root / name).write_text(content, encoding="utf-8")
|
||||
checksums = []
|
||||
for name in sorted(files):
|
||||
digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
|
||||
checksums.append(f"{digest} {name}")
|
||||
(root / "CHECKSUMS.sha256").write_text("\n".join(checksums) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def test_backup_guard_requires_recent_complete_sha256_storage_backup(tmp_path: Path) -> None:
|
||||
guard = load_script("release_backup_guard.py")
|
||||
now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
|
||||
backup = tmp_path / "backup"
|
||||
write_backup(backup, created_at=now - timedelta(hours=2))
|
||||
|
||||
verified = guard.verify_current_backup(backup, now=now)
|
||||
|
||||
assert verified.release_id == "rc10-test"
|
||||
assert verified.age_hours == pytest.approx(2)
|
||||
|
||||
|
||||
def test_backup_guard_rejects_stale_or_tampered_backup(tmp_path: Path) -> None:
|
||||
guard = load_script("release_backup_guard.py")
|
||||
now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
|
||||
stale = tmp_path / "stale"
|
||||
write_backup(stale, created_at=now - timedelta(hours=30))
|
||||
with pytest.raises(RuntimeError, match="maximum allowed age"):
|
||||
guard.verify_current_backup(stale, now=now)
|
||||
|
||||
current = tmp_path / "tampered"
|
||||
write_backup(current, created_at=now)
|
||||
(current / "database.dump").write_text("tampered", encoding="utf-8")
|
||||
with pytest.raises(RuntimeError, match="checksum mismatch"):
|
||||
guard.verify_current_backup(current, now=now)
|
||||
|
||||
|
||||
def test_storage_lifecycle_is_fail_closed_and_protects_release_evidence() -> None:
|
||||
audit = load_script("audit_data_operations.py")
|
||||
|
||||
assert audit.classify_relative_path("release-evidence/rc11/manifest.json") == (
|
||||
"release-evidence",
|
||||
True,
|
||||
False,
|
||||
)
|
||||
assert audit.classify_relative_path("operator-evidence/source/raw.json")[1:] == (True, False)
|
||||
assert audit.classify_relative_path("uploads/project/data.geojson")[1:] == (True, False)
|
||||
assert audit.classify_relative_path("exports/project/old.json")[1:] == (False, True)
|
||||
assert audit.classify_relative_path("unknown/value.bin")[1:] == (True, False)
|
||||
|
||||
|
||||
def test_storage_audit_only_selects_old_unreferenced_allowlisted_files(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
audit = load_script("audit_data_operations.py")
|
||||
storage = tmp_path / "storage"
|
||||
old_orphan = storage / "exports" / "project" / "old.json"
|
||||
referenced = storage / "exports" / "project" / "kept.json"
|
||||
protected = storage / "release-evidence" / "rc" / "manifest.json"
|
||||
unknown = storage / "misc" / "unknown.bin"
|
||||
for path in (old_orphan, referenced, protected, unknown):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(path.name, encoding="utf-8")
|
||||
old_timestamp = (datetime.now(timezone.utc) - timedelta(days=30)).timestamp()
|
||||
for path in (old_orphan, referenced, protected, unknown):
|
||||
path.touch()
|
||||
Path(path).chmod(0o644)
|
||||
import os
|
||||
|
||||
os.utime(path, (old_timestamp, old_timestamp))
|
||||
|
||||
monkeypatch.setattr(
|
||||
audit,
|
||||
"collect_database_state",
|
||||
lambda _db, _root: {
|
||||
"references": {referenced.resolve()},
|
||||
"counts": {},
|
||||
"source_families": {"national": [], "regional": [], "maritime": []},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
audit,
|
||||
"disk_pressure",
|
||||
lambda _root: {
|
||||
"status": "ok",
|
||||
"total_bytes": 100,
|
||||
"used_bytes": 50,
|
||||
"free_bytes": 50,
|
||||
"free_percent": 50.0,
|
||||
"acquisition_allowed": True,
|
||||
},
|
||||
)
|
||||
|
||||
report, candidates = audit.build_report(storage, SimpleNamespace(), minimum_age_days=7)
|
||||
|
||||
assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"]
|
||||
assert report["cleanup"]["candidate_count"] == 1
|
||||
assert "release-evidence" in report["cleanup"]["protected_prefixes"]
|
||||
|
||||
|
||||
def test_source_family_report_covers_national_regional_and_maritime(tmp_path: Path) -> None:
|
||||
audit = load_script("audit_data_operations.py")
|
||||
national_id = uuid4()
|
||||
regional_id = uuid4()
|
||||
now = datetime.now(timezone.utc)
|
||||
rows = {
|
||||
audit.Project: [
|
||||
SimpleNamespace(id=national_id, name=audit.NATIONAL_PROJECT_NAME, status="active"),
|
||||
SimpleNamespace(id=regional_id, name="Wallonia operator", status="active"),
|
||||
],
|
||||
audit.Dataset: [
|
||||
SimpleNamespace(
|
||||
project_id=national_id,
|
||||
source_name="ngi_adminvector",
|
||||
source="ngi",
|
||||
name="Belgium boundary",
|
||||
source_metadata={"coverage_zones": ["belgium"]},
|
||||
source_version="2026",
|
||||
imported_at=now,
|
||||
status="ready",
|
||||
storage_path=None,
|
||||
metadata_json=None,
|
||||
provenance_metadata=None,
|
||||
),
|
||||
SimpleNamespace(
|
||||
project_id=national_id,
|
||||
source_name="rbins_marine_reporting_units",
|
||||
source="rbins",
|
||||
name="Belgian North Sea",
|
||||
source_metadata={"coverage_zones": ["belgian_north_sea"]},
|
||||
source_version="2024",
|
||||
imported_at=now,
|
||||
status="ready",
|
||||
storage_path=None,
|
||||
metadata_json=None,
|
||||
provenance_metadata=None,
|
||||
),
|
||||
SimpleNamespace(
|
||||
project_id=regional_id,
|
||||
source_name="wallonia_manual",
|
||||
source="manual",
|
||||
name="Wallonia source",
|
||||
source_metadata={},
|
||||
source_version="1",
|
||||
imported_at=now,
|
||||
status="ready",
|
||||
storage_path=None,
|
||||
metadata_json=None,
|
||||
provenance_metadata=None,
|
||||
),
|
||||
],
|
||||
audit.DatasetVersion: [],
|
||||
audit.Export: [],
|
||||
audit.Detection: [],
|
||||
audit.Segmentation: [],
|
||||
audit.Job: [],
|
||||
audit.AnalysisRun: [],
|
||||
}
|
||||
|
||||
class Query:
|
||||
def __init__(self, values):
|
||||
self.values = values
|
||||
|
||||
def all(self):
|
||||
return self.values
|
||||
|
||||
class Session:
|
||||
def query(self, model):
|
||||
return Query(rows[model])
|
||||
|
||||
state = audit.collect_database_state(Session(), tmp_path)
|
||||
|
||||
assert {item["source_name"] for item in state["source_families"]["national"]} == {
|
||||
"ngi_adminvector",
|
||||
"rbins_marine_reporting_units",
|
||||
}
|
||||
assert {item["source_name"] for item in state["source_families"]["maritime"]} == {
|
||||
"rbins_marine_reporting_units"
|
||||
}
|
||||
assert {item["source_name"] for item in state["source_families"]["regional"]} == {
|
||||
"wallonia_manual"
|
||||
}
|
||||
|
||||
|
||||
def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> None:
|
||||
generic = (SCRIPTS / "cleanup_storage_artifacts.py").read_text(encoding="utf-8")
|
||||
demo = (ROOT / "backend/scripts/cleanup_demo_artifacts.py").read_text(encoding="utf-8")
|
||||
compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
|
||||
dockerman = (ROOT / "deploy/unraid/run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "DELETE_STORAGE_ARTIFACTS" in generic
|
||||
assert "verify_current_backup" in generic
|
||||
assert "DELETE_DEMO_EXPORTS" in demo
|
||||
assert "verify_current_backup" in demo
|
||||
assert "/app/backups:ro" in compose
|
||||
assert '/app/backups:ro"' in dockerman
|
||||
for name in (
|
||||
"release_backup_guard.py",
|
||||
"audit_data_operations.py",
|
||||
"cleanup_storage_artifacts.py",
|
||||
):
|
||||
assert f"COPY scripts/{name}" in dockerfile
|
||||
assert f"py_compile scripts/{name}" in readiness
|
||||
assert "bash -n scripts/run_rc10_data_operations_audit.sh" in readiness
|
||||
assert "--apply" not in live_audit
|
||||
assert "table-counts-before.tsv" in live_audit
|
||||
assert "table-counts-after.tsv" in live_audit
|
||||
assert "deleted_count" in live_audit
|
||||
@@ -65,6 +65,7 @@ def test_unraid_template_exposes_every_operator_owned_runtime_setting() -> None:
|
||||
template_variables = set(re.findall(r'Target="([A-Z][A-Z0-9_]+)"', template))
|
||||
bridged_or_internal = {
|
||||
"GEOINTEL_FRONTEND_PORT",
|
||||
"GEOINTEL_BACKUPS_PATH",
|
||||
"GEOINTEL_IMAGE",
|
||||
"GEOINTEL_MODELS_PATH",
|
||||
"GEOINTEL_POSTGIS_DATA_PATH",
|
||||
|
||||
@@ -142,6 +142,9 @@ COPY scripts/run_split_background_promotion_workflow.sh /app/scripts/run_split_b
|
||||
COPY scripts/activate_promoted_yolo_candidate.py /app/scripts/activate_promoted_yolo_candidate.py
|
||||
COPY scripts/archive_technical_projects.py /app/scripts/archive_technical_projects.py
|
||||
COPY scripts/runtime_state_report.py /app/scripts/runtime_state_report.py
|
||||
COPY scripts/release_backup_guard.py /app/scripts/release_backup_guard.py
|
||||
COPY scripts/audit_data_operations.py /app/scripts/audit_data_operations.py
|
||||
COPY scripts/cleanup_storage_artifacts.py /app/scripts/cleanup_storage_artifacts.py
|
||||
COPY deploy/unraid/nginx-all-in-one.conf /etc/nginx/conf.d/default.conf
|
||||
COPY deploy/unraid/all-in-one-start.sh /usr/local/bin/geointel-all-in-one-start
|
||||
COPY --from=frontend-build /frontend/dist/ /usr/share/nginx/html/
|
||||
@@ -171,7 +174,7 @@ LABEL org.opencontainers.image.title="GeoIntel" \
|
||||
org.opencontainers.image.created="${GEOINTEL_BUILD_TIME}" \
|
||||
io.geointel.ai.enabled="${GEOINTEL_INSTALL_AI}"
|
||||
|
||||
VOLUME ["/var/lib/postgresql/data", "/app/storage"]
|
||||
VOLUME ["/var/lib/postgresql/data", "/app/storage", "/app/backups"]
|
||||
|
||||
EXPOSE 80
|
||||
|
||||
|
||||
@@ -293,6 +293,24 @@ The configured upload limit is shared by FastAPI and the generated nginx
|
||||
runtime configuration. Values outside `1..2048` MiB are rejected before the
|
||||
active application is replaced.
|
||||
|
||||
## Data operations
|
||||
|
||||
`GEOINTEL_BACKUPS_PATH` defaults to
|
||||
`/mnt/user/appdata/geointel/backups` and is mounted read-only at
|
||||
`/app/backups`. Explicit cleanup commands can therefore verify a recent
|
||||
backup without permission to alter it.
|
||||
|
||||
These commands are non-mutating:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/audit_data_operations.py
|
||||
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py
|
||||
```
|
||||
|
||||
The full backup, confirmation, candidate-limit and apply sequence is in
|
||||
`docs/DATA_OPERATIONS_RUNBOOK.md`. GeoIntel installs no automatic cleanup
|
||||
schedule.
|
||||
|
||||
## Safe cleanup
|
||||
|
||||
Safe cache cleanup if Docker build cache fills the Unraid Docker image:
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
<Config Name="Web UI Port" Target="80" Default="1202" Mode="tcp" Description="Host port mapped to the GeoIntel all-in-one web UI. Change this to edit the browser port." Type="Port" Display="always" Required="true" Mask="false">1202</Config>
|
||||
<Config Name="Storage Path" Target="/app/storage" Default="/mnt/user/appdata/geointel/storage" Mode="rw" Description="Persistent GeoIntel artifact storage for uploads, tiles, masks, reports and exports." Type="Path" Display="always" Required="true" Mask="false">/mnt/user/appdata/geointel/storage</Config>
|
||||
<Config Name="AI Models Path" Target="/app/models" Default="/mnt/user/appdata/geointel/models" Mode="rw" Description="Persistent local model files mounted into the container. GeoIntel never downloads weights automatically." Type="Path" Display="always" Required="true" Mask="false">/mnt/user/appdata/geointel/models</Config>
|
||||
<Config Name="Release Backups Path" Target="/app/backups" Default="/mnt/user/appdata/geointel/backups" Mode="ro" Description="Read-only release backups used to guard explicitly confirmed cleanup operations." Type="Path" Display="advanced" Required="true" Mask="false">/mnt/user/appdata/geointel/backups</Config>
|
||||
<Config Name="PostGIS Data Path" Target="/var/lib/postgresql/data" Default="/mnt/user/appdata/geointel/postgres-data" Mode="rw" Description="Persistent embedded PostGIS data directory for the all-in-one container." Type="Path" Display="always" Required="true" Mask="false">/mnt/user/appdata/geointel/postgres-data</Config>
|
||||
<Config Name="Postgres Database" Target="GEOINTEL_POSTGRES_DB" Default="geointel" Mode="" Description="Embedded PostGIS database name." Type="Variable" Display="advanced" Required="true" Mask="false">geointel</Config>
|
||||
<Config Name="Postgres User" Target="GEOINTEL_POSTGRES_USER" Default="geointel" Mode="" Description="Embedded PostGIS database user." Type="Variable" Display="advanced" Required="true" Mask="false">geointel</Config>
|
||||
|
||||
@@ -10,6 +10,9 @@ GEOINTEL_STORAGE_PATH=/mnt/user/appdata/geointel/storage
|
||||
# Local AI model files mounted into the container as /app/models.
|
||||
GEOINTEL_MODELS_PATH=/mnt/user/appdata/geointel/models
|
||||
|
||||
# Checksum-verified release backups mounted read-only for cleanup guards.
|
||||
GEOINTEL_BACKUPS_PATH=/mnt/user/appdata/geointel/backups
|
||||
|
||||
# Embedded PostGIS data directory for the all-in-one container.
|
||||
GEOINTEL_POSTGIS_DATA_PATH=/mnt/user/appdata/geointel/postgres-data
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ GEOINTEL_FRONTEND_PORT="${GEOINTEL_FRONTEND_PORT:-1202}"
|
||||
GEOINTEL_IMAGE="${GEOINTEL_IMAGE:-geointel-all-in-one:latest}"
|
||||
GEOINTEL_STORAGE_PATH="${GEOINTEL_STORAGE_PATH:-/mnt/user/appdata/geointel/storage}"
|
||||
GEOINTEL_MODELS_PATH="${GEOINTEL_MODELS_PATH:-/mnt/user/appdata/geointel/models}"
|
||||
GEOINTEL_BACKUPS_PATH="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
|
||||
GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}"
|
||||
GEOINTEL_POSTGRES_DB="${GEOINTEL_POSTGRES_DB:-geointel}"
|
||||
GEOINTEL_POSTGRES_USER="${GEOINTEL_POSTGRES_USER:-geointel}"
|
||||
@@ -168,7 +169,7 @@ if docker ps -a --format '{{.Names}}' | grep -qx geointel; then
|
||||
docker rm -f geointel
|
||||
fi
|
||||
|
||||
mkdir -p "$GEOINTEL_STORAGE_PATH" "$GEOINTEL_MODELS_PATH" "$GEOINTEL_POSTGIS_DATA_PATH"
|
||||
mkdir -p "$GEOINTEL_STORAGE_PATH" "$GEOINTEL_MODELS_PATH" "$GEOINTEL_BACKUPS_PATH" "$GEOINTEL_POSTGIS_DATA_PATH"
|
||||
migrate_compose_volume_if_needed
|
||||
|
||||
docker run -d \
|
||||
@@ -268,6 +269,7 @@ docker run -d \
|
||||
-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data" \
|
||||
-v "${GEOINTEL_STORAGE_PATH}:/app/storage" \
|
||||
-v "${GEOINTEL_MODELS_PATH}:/app/models" \
|
||||
-v "${GEOINTEL_BACKUPS_PATH}:/app/backups:ro" \
|
||||
"$GEOINTEL_IMAGE"
|
||||
|
||||
docker ps --filter name=geointel
|
||||
|
||||
@@ -89,6 +89,7 @@ services:
|
||||
- ${GEOINTEL_POSTGIS_DATA_PATH:-geointel_postgis}:/var/lib/postgresql/data
|
||||
- ${GEOINTEL_STORAGE_PATH:-./storage}:/app/storage
|
||||
- ${GEOINTEL_MODELS_PATH:-./models}:/app/models
|
||||
- ${GEOINTEL_BACKUPS_PATH:-./backups}:/app/backups:ro
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
restart: unless-stopped
|
||||
|
||||
@@ -119,6 +119,7 @@ services:
|
||||
volumes:
|
||||
- ${GEOINTEL_STORAGE_PATH:-./storage}:/app/storage
|
||||
- ${GEOINTEL_MODELS_PATH:-./models}:/app/models
|
||||
- ${GEOINTEL_BACKUPS_PATH:-./backups}:/app/backups:ro
|
||||
- ./fixtures:/app/fixtures:ro
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
# Data Operations and Retention Runbook
|
||||
|
||||
## Purpose
|
||||
|
||||
This runbook governs persistent GeoIntel data for Belgium and the Belgian
|
||||
North Sea. It covers raw, normalized, derived, export, AI and release-evidence
|
||||
artifacts. No command in this runbook schedules itself and no source is
|
||||
downloaded implicitly.
|
||||
|
||||
## Lifecycle classes
|
||||
|
||||
| Class | Paths | Retention |
|
||||
|---|---|---|
|
||||
| Raw and normalized source | `originals/`, `uploads/`, `operator-data/` | Protected; replace no official edition in place |
|
||||
| Derived and cache | `derived/`, `rasters/derived/`, `previews/`, `tiles/`, `masks/` | Eligible only when old and not referenced by persisted provenance |
|
||||
| Exports | `exports/` | Retain persisted exports; unreferenced old files may be candidates |
|
||||
| AI models | `/app/models` and `storage/models/` | Protected; activation and promotion evidence govern removal |
|
||||
| Immutable evidence | `release-evidence/`, `operator-evidence/` | Never a cleanup candidate |
|
||||
| Unknown | every unclassified path | Protected by default |
|
||||
|
||||
Dataset, DatasetVersion, Export, Detection, Segmentation, Job and AnalysisRun
|
||||
paths and nested provenance are treated as references. Failed work remains
|
||||
audit provenance. Project archival remains reversible and does not physically
|
||||
delete its datasets or files.
|
||||
|
||||
## Read-only audit
|
||||
|
||||
Run the complete local storage, database-reference, disk-pressure and
|
||||
national/regional/maritime source-family report:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/audit_data_operations.py \
|
||||
--minimum-age-days 7 \
|
||||
--output /app/storage/release-evidence/rc-current/data-operations.json
|
||||
```
|
||||
|
||||
The command:
|
||||
|
||||
- inventories storage without following symlinks;
|
||||
- reports bytes and file counts per lifecycle class;
|
||||
- reports free disk bytes and percentage before new acquisition;
|
||||
- reports missing persisted path references;
|
||||
- identifies only old, unreferenced files inside the explicit cleanup
|
||||
allowlist;
|
||||
- summarizes national, regional and maritime source names, versions and latest
|
||||
import dates;
|
||||
- reports old failed Jobs and AnalysisRuns without deleting them.
|
||||
|
||||
Use `--fail-on-pressure warning` for a stricter acquisition preflight and
|
||||
`--fail-on-missing-reference` for release gates. `critical` disk pressure
|
||||
blocks acquisition by default. The thresholds are 10 GiB or 5% free for
|
||||
critical and 25 GiB or 10% free for warning.
|
||||
|
||||
The existing project-level freshness policy remains authoritative:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/audit_source_freshness.py \
|
||||
--project-id <belgium-and-north-sea-project-id> \
|
||||
--api-url http://127.0.0.1/api/v1 \
|
||||
--json \
|
||||
--output /app/storage/release-evidence/rc-current/source-freshness.json
|
||||
```
|
||||
|
||||
Together, these reports cover the national workbench, regional source
|
||||
families and Belgian North Sea datasets without performing an external
|
||||
refresh. Catalog probes remain separate, explicit and read-only.
|
||||
|
||||
## Cleanup dry run
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
|
||||
--minimum-age-days 7 \
|
||||
--max-delete 25
|
||||
```
|
||||
|
||||
This is always a dry run unless `--apply` is present. Review every candidate.
|
||||
Release evidence, operator evidence, official source material, uploads,
|
||||
models, unknown paths and every persisted path reference are excluded.
|
||||
|
||||
Technical benchmark projects can be archived reversibly:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/archive_technical_projects.py --show-names
|
||||
```
|
||||
|
||||
The normal project lifecycle archive path does not need destructive
|
||||
confirmation because it changes only `status=archived` and preserves all
|
||||
data.
|
||||
|
||||
## Destructive apply gate
|
||||
|
||||
First create a fresh backup with a SHA-256 storage inventory on the host:
|
||||
|
||||
```bash
|
||||
bash scripts/backup_release_state.sh \
|
||||
--container geointel \
|
||||
--release-id rc10-before-cleanup-$(date -u +%Y%m%dT%H%M%SZ) \
|
||||
--output-root /mnt/user/appdata/geointel/backups \
|
||||
--storage-path /mnt/user/appdata/geointel/storage \
|
||||
--models-path /mnt/user/appdata/geointel/models \
|
||||
--inventory-mode sha256
|
||||
```
|
||||
|
||||
Verify it read-only:
|
||||
|
||||
```bash
|
||||
bash scripts/verify_release_backup.sh \
|
||||
--backup-dir /mnt/user/appdata/geointel/backups/<release-id> \
|
||||
--container geointel
|
||||
```
|
||||
|
||||
The Unraid runtime mounts `GEOINTEL_BACKUPS_PATH` read-only at `/app/backups`.
|
||||
Only after reviewing the dry run may an operator execute:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
|
||||
--minimum-age-days 7 \
|
||||
--max-delete <reviewed-candidate-count> \
|
||||
--backup-dir /app/backups/<release-id> \
|
||||
--backup-max-age-hours 24 \
|
||||
--confirm DELETE_STORAGE_ARTIFACTS \
|
||||
--apply
|
||||
```
|
||||
|
||||
The command re-runs the audit immediately before deletion. It refuses the
|
||||
operation when the exact token is absent, the candidate count exceeds the
|
||||
operator limit, the backup is stale/incomplete, checksums differ, the storage
|
||||
inventory is not SHA-256, or the path is outside the cleanup allowlist.
|
||||
|
||||
The older demo-export cleanup has the same gate and uses confirmation token
|
||||
`DELETE_DEMO_EXPORTS`.
|
||||
|
||||
## Failure meaning
|
||||
|
||||
- `disk_pressure=critical`: stop new source acquisition and resolve capacity.
|
||||
- `missing_referenced_path_count>0`: persisted provenance points to absent
|
||||
files; investigate before cleanup or release.
|
||||
- checksum or backup-age failure: create and verify a new backup.
|
||||
- candidate count above `--max-delete`: keep dry-run mode and review; never
|
||||
increase the limit blindly.
|
||||
- symlink listed as skipped: inspect manually; the audit will not traverse or
|
||||
delete it.
|
||||
|
||||
No automatic cron or background cleanup is installed. Operators may schedule
|
||||
the read-only audit externally, but destructive commands must remain manual.
|
||||
@@ -321,6 +321,21 @@ The smoke never passes `--apply`. It fails if the cleanup summary is not a
|
||||
dry-run, if any export/file deletion is reported, or if the dry-run candidate
|
||||
fields are missing.
|
||||
|
||||
## RC-10 data operations
|
||||
|
||||
`docs/DATA_OPERATIONS_RUNBOOK.md` is the executable retention source of truth.
|
||||
`scripts/audit_data_operations.py` reports storage growth, disk pressure,
|
||||
persisted path integrity, old failed work and national/regional/maritime
|
||||
source families without mutation. `scripts/cleanup_storage_artifacts.py` can
|
||||
only remove old unreferenced artifacts from the explicit derived/cache/export
|
||||
allowlist.
|
||||
|
||||
Every unknown category, upload, original, model, operator evidence and release
|
||||
evidence path is protected by default. Apply mode requires an exact
|
||||
confirmation token and a recent checksum-verified release backup containing a
|
||||
SHA-256 storage inventory. The backup root is mounted read-only at
|
||||
`/app/backups`; no cleanup is scheduled implicitly.
|
||||
|
||||
## Governed cross-domain raster and soil evidence
|
||||
|
||||
MercatorNet thematic products use the ordinary raster Dataset and immutable
|
||||
|
||||
@@ -2037,6 +2037,30 @@ The command atomically updates the operator-owned `.env`, changes the matching
|
||||
PostgreSQL role and recreates the container. A failed role change restores the
|
||||
previous environment file. The generated secret is never printed.
|
||||
|
||||
## RC-10 data operations and retention
|
||||
|
||||
Run the read-only storage, provenance, disk-pressure and source-family audit:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/audit_data_operations.py \
|
||||
--minimum-age-days 7 \
|
||||
--output /app/storage/release-evidence/rc-current/data-operations.json
|
||||
```
|
||||
|
||||
Preview old unreferenced derived/cache/export candidates without deletion:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
|
||||
--minimum-age-days 7 \
|
||||
--max-delete 25
|
||||
```
|
||||
|
||||
Apply requires a reviewed candidate count, the exact
|
||||
`DELETE_STORAGE_ARTIFACTS` token and a backup no older than 24 hours with a
|
||||
checksum-verified database dump and SHA-256 storage inventory. The host backup
|
||||
root is mounted read-only at `/app/backups`. See
|
||||
`docs/DATA_OPERATIONS_RUNBOOK.md`. No cleanup is scheduled by GeoIntel.
|
||||
|
||||
## RC-8 Belgium/North Sea release journeys
|
||||
|
||||
Preview the seven release areas without mutating GeoIntel:
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only storage, provenance and source-family audit for GeoIntel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BACKEND_ROOT = ROOT / "backend" if (ROOT / "backend" / "app").is_dir() else ROOT
|
||||
if str(BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_ROOT))
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.session import SessionLocal
|
||||
from app.models import AnalysisRun, Dataset, DatasetVersion, Detection, Export, Job, Project, Segmentation
|
||||
|
||||
|
||||
NATIONAL_PROJECT_NAME = "Belgium and North Sea Workbench"
|
||||
PROTECTED_PREFIXES = (
|
||||
"release-evidence",
|
||||
"operator-evidence",
|
||||
"operator-data",
|
||||
"originals",
|
||||
"uploads",
|
||||
"models",
|
||||
)
|
||||
CLEANUP_PREFIXES = (
|
||||
"exports",
|
||||
"previews",
|
||||
"tiles",
|
||||
"masks",
|
||||
"derived",
|
||||
"rasters/derived",
|
||||
)
|
||||
IGNORED_FILENAMES = frozenset({".gitkeep", "README.md"})
|
||||
PATH_METADATA_FIELDS = (
|
||||
"metadata_json",
|
||||
"source_metadata",
|
||||
"provenance_metadata",
|
||||
"parameters_json",
|
||||
"result_json",
|
||||
"properties_json",
|
||||
"provenance_json",
|
||||
"bbox_json",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FileRecord:
|
||||
path: Path
|
||||
relative_path: str
|
||||
category: str
|
||||
size_bytes: int
|
||||
modified_at: datetime
|
||||
protected: bool
|
||||
cleanup_eligible: bool
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _prefix_match(relative_path: str, prefixes: Iterable[str]) -> str | None:
|
||||
normalized = relative_path.strip("/")
|
||||
matches = [
|
||||
prefix
|
||||
for prefix in prefixes
|
||||
if normalized == prefix or normalized.startswith(f"{prefix}/")
|
||||
]
|
||||
return max(matches, key=len) if matches else None
|
||||
|
||||
|
||||
def classify_relative_path(relative_path: str) -> tuple[str, bool, bool]:
|
||||
protected = _prefix_match(relative_path, PROTECTED_PREFIXES)
|
||||
if protected:
|
||||
return protected, True, False
|
||||
cleanup = _prefix_match(relative_path, CLEANUP_PREFIXES)
|
||||
if cleanup:
|
||||
return cleanup, False, True
|
||||
category = relative_path.split("/", 1)[0] if relative_path else "."
|
||||
return category, True, False
|
||||
|
||||
|
||||
def inventory_storage(storage_root: Path) -> tuple[list[FileRecord], list[str]]:
|
||||
root = storage_root.resolve()
|
||||
records: list[FileRecord] = []
|
||||
skipped_symlinks: list[str] = []
|
||||
for current, directories, filenames in os.walk(root, followlinks=False):
|
||||
current_path = Path(current)
|
||||
retained_directories: list[str] = []
|
||||
for directory in directories:
|
||||
child = current_path / directory
|
||||
if child.is_symlink():
|
||||
skipped_symlinks.append(child.relative_to(root).as_posix())
|
||||
else:
|
||||
retained_directories.append(directory)
|
||||
directories[:] = retained_directories
|
||||
for filename in filenames:
|
||||
path = current_path / filename
|
||||
if path.is_symlink():
|
||||
skipped_symlinks.append(path.relative_to(root).as_posix())
|
||||
continue
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
continue
|
||||
relative = path.relative_to(root).as_posix()
|
||||
category, protected, cleanup_eligible = classify_relative_path(relative)
|
||||
records.append(
|
||||
FileRecord(
|
||||
path=path.resolve(),
|
||||
relative_path=relative,
|
||||
category=category,
|
||||
size_bytes=stat.st_size,
|
||||
modified_at=datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc),
|
||||
protected=protected,
|
||||
cleanup_eligible=cleanup_eligible,
|
||||
)
|
||||
)
|
||||
return records, sorted(skipped_symlinks)
|
||||
|
||||
|
||||
def _iter_strings(value: Any) -> Iterable[str]:
|
||||
if isinstance(value, str):
|
||||
yield value
|
||||
elif isinstance(value, dict):
|
||||
for nested in value.values():
|
||||
yield from _iter_strings(nested)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
for nested in value:
|
||||
yield from _iter_strings(nested)
|
||||
|
||||
|
||||
def normalize_storage_reference(value: str | None, storage_root: Path) -> Path | None:
|
||||
if not value:
|
||||
return None
|
||||
stripped = value.strip()
|
||||
if not stripped or "://" in stripped or stripped.startswith("/vsi"):
|
||||
return None
|
||||
candidate = Path(stripped)
|
||||
if not candidate.is_absolute():
|
||||
normalized = stripped.replace("\\", "/")
|
||||
if normalized.startswith("storage/"):
|
||||
normalized = normalized.removeprefix("storage/")
|
||||
if _prefix_match(normalized, PROTECTED_PREFIXES + CLEANUP_PREFIXES) is None:
|
||||
return None
|
||||
candidate = storage_root / normalized
|
||||
try:
|
||||
resolved = candidate.resolve()
|
||||
resolved.relative_to(storage_root.resolve())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
return resolved
|
||||
|
||||
|
||||
def _add_row_references(target: set[Path], row: Any, storage_root: Path, direct_fields: Iterable[str]) -> None:
|
||||
for field in direct_fields:
|
||||
normalized = normalize_storage_reference(getattr(row, field, None), storage_root)
|
||||
if normalized:
|
||||
target.add(normalized)
|
||||
for field in PATH_METADATA_FIELDS:
|
||||
for value in _iter_strings(getattr(row, field, None)):
|
||||
normalized = normalize_storage_reference(value, storage_root)
|
||||
if normalized:
|
||||
target.add(normalized)
|
||||
|
||||
|
||||
def collect_database_state(db: Any, storage_root: Path) -> dict[str, Any]:
|
||||
projects = db.query(Project).all()
|
||||
project_names = {project.id: project.name for project in projects}
|
||||
datasets = db.query(Dataset).all()
|
||||
versions = db.query(DatasetVersion).all()
|
||||
exports = db.query(Export).all()
|
||||
detections = db.query(Detection).all()
|
||||
segmentations = db.query(Segmentation).all()
|
||||
jobs = db.query(Job).all()
|
||||
analysis_runs = db.query(AnalysisRun).all()
|
||||
|
||||
references: set[Path] = set()
|
||||
for row in datasets:
|
||||
_add_row_references(references, row, storage_root, ("storage_path",))
|
||||
for row in versions:
|
||||
_add_row_references(references, row, storage_root, ("storage_path",))
|
||||
for row in exports:
|
||||
_add_row_references(references, row, storage_root, ("storage_path",))
|
||||
for row in detections:
|
||||
_add_row_references(references, row, storage_root, ("source_tile_path",))
|
||||
for row in segmentations:
|
||||
_add_row_references(references, row, storage_root, ("mask_path", "source_tile_path"))
|
||||
for row in jobs:
|
||||
_add_row_references(references, row, storage_root, ())
|
||||
for row in analysis_runs:
|
||||
_add_row_references(references, row, storage_root, ())
|
||||
|
||||
source_families: dict[str, dict[str, dict[str, Any]]] = {
|
||||
"national": {},
|
||||
"regional": {},
|
||||
"maritime": {},
|
||||
}
|
||||
maritime_tokens = ("marine", "maritime", "north_sea", "bathymetry", "rbin", "mdk", "msp")
|
||||
for dataset in datasets:
|
||||
source_name = (dataset.source_name or dataset.source or "unknown").strip().lower()
|
||||
metadata = dataset.source_metadata or {}
|
||||
zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or []
|
||||
if isinstance(zones, str):
|
||||
zones = [zones]
|
||||
families: set[str] = set()
|
||||
project_name = project_names.get(dataset.project_id, "")
|
||||
if project_name == NATIONAL_PROJECT_NAME or "belgium" in zones:
|
||||
families.add("national")
|
||||
else:
|
||||
families.add("regional")
|
||||
searchable = " ".join([source_name, dataset.name.lower(), *(str(zone).lower() for zone in zones)])
|
||||
if "belgian_north_sea" in zones or any(token in searchable for token in maritime_tokens):
|
||||
families.add("maritime")
|
||||
for family in families:
|
||||
item = source_families[family].setdefault(
|
||||
source_name,
|
||||
{
|
||||
"source_name": source_name,
|
||||
"dataset_count": 0,
|
||||
"ready_count": 0,
|
||||
"latest_imported_at": None,
|
||||
"source_versions": set(),
|
||||
},
|
||||
)
|
||||
item["dataset_count"] += 1
|
||||
item["ready_count"] += int(dataset.status == "ready")
|
||||
if dataset.source_version:
|
||||
item["source_versions"].add(dataset.source_version)
|
||||
if dataset.imported_at and (
|
||||
item["latest_imported_at"] is None or dataset.imported_at > item["latest_imported_at"]
|
||||
):
|
||||
item["latest_imported_at"] = dataset.imported_at
|
||||
|
||||
serialized_families: dict[str, list[dict[str, Any]]] = {}
|
||||
for family, sources in source_families.items():
|
||||
serialized_families[family] = []
|
||||
for source in sorted(sources.values(), key=lambda item: item["source_name"]):
|
||||
serialized_families[family].append(
|
||||
{
|
||||
**source,
|
||||
"latest_imported_at": (
|
||||
source["latest_imported_at"].isoformat()
|
||||
if source["latest_imported_at"] is not None
|
||||
else None
|
||||
),
|
||||
"source_versions": sorted(source["source_versions"]),
|
||||
}
|
||||
)
|
||||
|
||||
failed_cutoff = utc_now() - timedelta(days=7)
|
||||
return {
|
||||
"references": references,
|
||||
"counts": {
|
||||
"projects": len(projects),
|
||||
"active_projects": sum(project.status == "active" for project in projects),
|
||||
"archived_projects": sum(project.status == "archived" for project in projects),
|
||||
"datasets": len(datasets),
|
||||
"dataset_versions": len(versions),
|
||||
"exports": len(exports),
|
||||
"detections": len(detections),
|
||||
"segmentations": len(segmentations),
|
||||
"jobs": len(jobs),
|
||||
"analysis_runs": len(analysis_runs),
|
||||
"failed_jobs_older_than_7d": sum(
|
||||
job.status == "failed" and job.created_at and job.created_at < failed_cutoff for job in jobs
|
||||
),
|
||||
"failed_runs_older_than_7d": sum(
|
||||
run.status == "failed" and run.created_at and run.created_at < failed_cutoff
|
||||
for run in analysis_runs
|
||||
),
|
||||
},
|
||||
"source_families": serialized_families,
|
||||
}
|
||||
|
||||
|
||||
def disk_pressure(storage_root: Path) -> dict[str, Any]:
|
||||
usage = shutil.disk_usage(storage_root)
|
||||
free_percent = (usage.free / usage.total * 100) if usage.total else 0.0
|
||||
if usage.free < 10 * 1024**3 or free_percent < 5:
|
||||
status = "critical"
|
||||
elif usage.free < 25 * 1024**3 or free_percent < 10:
|
||||
status = "warning"
|
||||
else:
|
||||
status = "ok"
|
||||
return {
|
||||
"status": status,
|
||||
"total_bytes": usage.total,
|
||||
"used_bytes": usage.used,
|
||||
"free_bytes": usage.free,
|
||||
"free_percent": round(free_percent, 2),
|
||||
"acquisition_allowed": status != "critical",
|
||||
}
|
||||
|
||||
|
||||
def build_report(
|
||||
storage_root: Path,
|
||||
db: Any,
|
||||
*,
|
||||
minimum_age_days: int = 7,
|
||||
max_candidate_records: int = 500,
|
||||
) -> tuple[dict[str, Any], list[FileRecord]]:
|
||||
if minimum_age_days < 1:
|
||||
raise ValueError("minimum_age_days must be at least one")
|
||||
root = storage_root.expanduser().resolve()
|
||||
if not root.is_dir():
|
||||
raise RuntimeError(f"Storage root is not a directory: {root}")
|
||||
records, skipped_symlinks = inventory_storage(root)
|
||||
database = collect_database_state(db, root)
|
||||
references: set[Path] = database.pop("references")
|
||||
cutoff = utc_now() - timedelta(days=minimum_age_days)
|
||||
|
||||
categories: dict[str, dict[str, Any]] = defaultdict(
|
||||
lambda: {"file_count": 0, "size_bytes": 0, "protected": False, "cleanup_eligible": False}
|
||||
)
|
||||
for record in records:
|
||||
item = categories[record.category]
|
||||
item["file_count"] += 1
|
||||
item["size_bytes"] += record.size_bytes
|
||||
item["protected"] = item["protected"] or record.protected
|
||||
item["cleanup_eligible"] = item["cleanup_eligible"] or record.cleanup_eligible
|
||||
|
||||
existing_paths = {record.path for record in records}
|
||||
missing_references = sorted(
|
||||
path.relative_to(root).as_posix()
|
||||
for path in references
|
||||
if path not in existing_paths and not path.is_dir()
|
||||
)
|
||||
candidates = sorted(
|
||||
(
|
||||
record
|
||||
for record in records
|
||||
if record.cleanup_eligible
|
||||
and record.path not in references
|
||||
and record.modified_at <= cutoff
|
||||
and record.path.name not in IGNORED_FILENAMES
|
||||
),
|
||||
key=lambda item: (item.modified_at, item.relative_path),
|
||||
)
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"generated_at": utc_now().isoformat(),
|
||||
"mode": "read-only",
|
||||
"storage_root": str(root),
|
||||
"minimum_age_days": minimum_age_days,
|
||||
"disk_pressure": disk_pressure(root),
|
||||
"lifecycle": {
|
||||
"raw_and_normalized": ["originals", "uploads", "operator-data"],
|
||||
"derived_and_cache": ["derived", "rasters/derived", "previews", "tiles", "masks"],
|
||||
"exports": ["exports"],
|
||||
"ai_models": ["models"],
|
||||
"immutable_evidence": ["release-evidence", "operator-evidence"],
|
||||
},
|
||||
"categories": [
|
||||
{"category": category, **values}
|
||||
for category, values in sorted(categories.items())
|
||||
],
|
||||
"database": database,
|
||||
"integrity": {
|
||||
"referenced_path_count": len(references),
|
||||
"missing_referenced_path_count": len(missing_references),
|
||||
"missing_referenced_paths": missing_references[:max_candidate_records],
|
||||
"skipped_symlinks": skipped_symlinks[:max_candidate_records],
|
||||
},
|
||||
"cleanup": {
|
||||
"candidate_count": len(candidates),
|
||||
"candidate_bytes": sum(item.size_bytes for item in candidates),
|
||||
"candidate_records_truncated": len(candidates) > max_candidate_records,
|
||||
"candidates": [
|
||||
{
|
||||
"relative_path": item.relative_path,
|
||||
"category": item.category,
|
||||
"size_bytes": item.size_bytes,
|
||||
"modified_at": item.modified_at.isoformat(),
|
||||
}
|
||||
for item in candidates[:max_candidate_records]
|
||||
],
|
||||
"protected_prefixes": list(PROTECTED_PREFIXES),
|
||||
"eligible_prefixes": list(CLEANUP_PREFIXES),
|
||||
"apply_requires": [
|
||||
"exact confirmation token",
|
||||
"recent checksum-verified database and SHA-256 storage backup",
|
||||
"explicit maximum delete count",
|
||||
],
|
||||
},
|
||||
"limitations": [
|
||||
"The audit never downloads or refreshes an official source.",
|
||||
"Failed jobs and analysis runs remain provenance and are reported, not deleted.",
|
||||
"Unknown storage categories are protected by default.",
|
||||
"No cleanup is scheduled implicitly.",
|
||||
],
|
||||
}
|
||||
return report, candidates
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--storage-root", type=Path)
|
||||
parser.add_argument("--minimum-age-days", type=int, default=7)
|
||||
parser.add_argument("--max-candidate-records", type=int, default=500)
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--fail-on-pressure", choices=("never", "critical", "warning"), default="critical")
|
||||
parser.add_argument("--fail-on-missing-reference", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
storage_root = args.storage_root or Path(get_settings().storage_root)
|
||||
with SessionLocal() as db:
|
||||
report, _ = build_report(
|
||||
storage_root,
|
||||
db,
|
||||
minimum_age_days=args.minimum_age_days,
|
||||
max_candidate_records=args.max_candidate_records,
|
||||
)
|
||||
serialized = json.dumps(report, indent=2, sort_keys=True)
|
||||
print(serialized)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = args.output.with_suffix(f"{args.output.suffix}.partial")
|
||||
temporary.write_text(serialized + "\n", encoding="utf-8")
|
||||
temporary.replace(args.output)
|
||||
pressure = report["disk_pressure"]["status"]
|
||||
if args.fail_on_pressure == "critical" and pressure == "critical":
|
||||
return 1
|
||||
if args.fail_on_pressure == "warning" and pressure in {"warning", "critical"}:
|
||||
return 1
|
||||
if args.fail_on_missing_reference and report["integrity"]["missing_referenced_path_count"]:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -18,6 +18,7 @@ def load_backend_script() -> ModuleType:
|
||||
_impl = load_backend_script()
|
||||
|
||||
DEMO_PROJECT_NAME = _impl.DEMO_PROJECT_NAME
|
||||
DELETE_CONFIRMATION = _impl.DELETE_CONFIRMATION
|
||||
is_within_storage_root = _impl.is_within_storage_root
|
||||
export_created_at = _impl.export_created_at
|
||||
select_cleanup_candidates = _impl.select_cleanup_candidates
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dry-run-first cleanup for old unreferenced derived/cache artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from audit_data_operations import build_report
|
||||
from release_backup_guard import require_confirmation, verify_current_backup
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.session import SessionLocal
|
||||
|
||||
|
||||
CONFIRMATION = "DELETE_STORAGE_ARTIFACTS"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--storage-root", type=Path)
|
||||
parser.add_argument("--minimum-age-days", type=int, default=7)
|
||||
parser.add_argument("--max-delete", type=int, default=25)
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
parser.add_argument("--confirm")
|
||||
parser.add_argument("--backup-dir", type=Path)
|
||||
parser.add_argument("--backup-max-age-hours", type=float, default=24.0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if args.max_delete < 0:
|
||||
raise SystemExit("--max-delete must be greater than or equal to zero")
|
||||
storage_root = (args.storage_root or Path(get_settings().storage_root)).resolve()
|
||||
with SessionLocal() as db:
|
||||
report, candidates = build_report(
|
||||
storage_root,
|
||||
db,
|
||||
minimum_age_days=args.minimum_age_days,
|
||||
max_candidate_records=max(args.max_delete, 500),
|
||||
)
|
||||
|
||||
blocked_reason = None
|
||||
backup = None
|
||||
deleted: list[str] = []
|
||||
if args.apply:
|
||||
require_confirmation(args.confirm, CONFIRMATION)
|
||||
if args.backup_dir is None:
|
||||
raise RuntimeError("--backup-dir is required with --apply")
|
||||
backup = verify_current_backup(
|
||||
args.backup_dir,
|
||||
max_age_hours=args.backup_max_age_hours,
|
||||
)
|
||||
if len(candidates) > args.max_delete:
|
||||
blocked_reason = (
|
||||
f"candidate_count {len(candidates)} exceeds --max-delete {args.max_delete}; "
|
||||
"review the dry run and raise the explicit limit"
|
||||
)
|
||||
else:
|
||||
for candidate in candidates:
|
||||
candidate.path.unlink()
|
||||
deleted.append(candidate.relative_path)
|
||||
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"mode": "apply" if args.apply else "dry-run",
|
||||
"storage_root": str(storage_root),
|
||||
"minimum_age_days": args.minimum_age_days,
|
||||
"max_delete": args.max_delete,
|
||||
"candidate_count": len(candidates),
|
||||
"candidate_bytes": sum(item.size_bytes for item in candidates),
|
||||
"candidates": [item.relative_path for item in candidates],
|
||||
"deleted_count": len(deleted),
|
||||
"deleted": deleted,
|
||||
"blocked_reason": blocked_reason,
|
||||
"protected_prefixes": report["cleanup"]["protected_prefixes"],
|
||||
"backup": (
|
||||
{
|
||||
"release_id": backup.release_id,
|
||||
"created_at": backup.created_at.isoformat(),
|
||||
"age_hours": round(backup.age_hours, 3),
|
||||
"git_commit": backup.git_commit,
|
||||
}
|
||||
if backup
|
||||
else None
|
||||
),
|
||||
}
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
return 1 if blocked_reason else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verification guard shared by destructive GeoIntel operator commands."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VerifiedBackup:
|
||||
backup_dir: Path
|
||||
release_id: str
|
||||
created_at: datetime
|
||||
age_hours: float
|
||||
git_commit: str
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _created_at(value: object) -> datetime:
|
||||
if not isinstance(value, str):
|
||||
raise RuntimeError("Backup manifest does not contain created_at")
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def verify_current_backup(
|
||||
backup_dir: str | Path,
|
||||
*,
|
||||
max_age_hours: float = 24.0,
|
||||
now: datetime | None = None,
|
||||
) -> VerifiedBackup:
|
||||
"""Verify checksums and release metadata without mutating the backup."""
|
||||
if max_age_hours <= 0:
|
||||
raise ValueError("max_age_hours must be greater than zero")
|
||||
|
||||
root = Path(backup_dir).expanduser().resolve()
|
||||
if not root.is_dir():
|
||||
raise RuntimeError(f"Backup directory does not exist: {root}")
|
||||
|
||||
required = {
|
||||
"manifest.json",
|
||||
"CHECKSUMS.sha256",
|
||||
"database.dump",
|
||||
"database.list",
|
||||
"database-metadata.tsv",
|
||||
"table-counts.tsv",
|
||||
"storage-manifest.tsv",
|
||||
}
|
||||
missing = sorted(name for name in required if not (root / name).is_file())
|
||||
if missing:
|
||||
raise RuntimeError(f"Backup is incomplete; missing: {', '.join(missing)}")
|
||||
|
||||
checksum_lines = (root / "CHECKSUMS.sha256").read_text(encoding="utf-8").splitlines()
|
||||
checked: set[str] = set()
|
||||
for line in checksum_lines:
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
expected, name = line.split(maxsplit=1)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("Backup checksum file has an invalid line") from exc
|
||||
name = name.lstrip("*")
|
||||
if "/" in name or "\\" in name or name in {".", ".."}:
|
||||
raise RuntimeError(f"Backup checksum contains an unsafe path: {name}")
|
||||
target = root / name
|
||||
if not target.is_file():
|
||||
raise RuntimeError(f"Backup checksum target is missing: {name}")
|
||||
if _sha256(target) != expected.lower():
|
||||
raise RuntimeError(f"Backup checksum mismatch: {name}")
|
||||
checked.add(name)
|
||||
|
||||
unchecked = sorted((required - {"CHECKSUMS.sha256"}) - checked)
|
||||
if unchecked:
|
||||
raise RuntimeError(f"Backup checksum coverage is incomplete: {', '.join(unchecked)}")
|
||||
|
||||
manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8"))
|
||||
if manifest.get("schema_version") != 1 or manifest.get("read_only_source") is not True:
|
||||
raise RuntimeError("Backup manifest schema or read-only marker is invalid")
|
||||
if manifest.get("database_password_secure") is not True:
|
||||
raise RuntimeError("Backup was made from an insecure database configuration")
|
||||
if manifest.get("inventory_mode") != "sha256" or manifest.get("storage_inventory_requested") is not True:
|
||||
raise RuntimeError("Destructive maintenance requires a SHA-256 storage inventory backup")
|
||||
|
||||
created = _created_at(manifest.get("created_at"))
|
||||
current = now or datetime.now(timezone.utc)
|
||||
if current.tzinfo is None:
|
||||
current = current.replace(tzinfo=timezone.utc)
|
||||
age_hours = (current.astimezone(timezone.utc) - created).total_seconds() / 3600
|
||||
if age_hours < -0.1:
|
||||
raise RuntimeError("Backup timestamp is in the future")
|
||||
if age_hours > max_age_hours:
|
||||
raise RuntimeError(
|
||||
f"Backup is {age_hours:.1f} hours old; maximum allowed age is {max_age_hours:.1f} hours"
|
||||
)
|
||||
|
||||
release_id = manifest.get("release_id")
|
||||
git_commit = manifest.get("git_commit")
|
||||
if not isinstance(release_id, str) or not release_id:
|
||||
raise RuntimeError("Backup release id is missing")
|
||||
if not isinstance(git_commit, str) or len(git_commit) < 7:
|
||||
raise RuntimeError("Backup Git commit is missing")
|
||||
return VerifiedBackup(
|
||||
backup_dir=root,
|
||||
release_id=release_id,
|
||||
created_at=created,
|
||||
age_hours=age_hours,
|
||||
git_commit=git_commit,
|
||||
)
|
||||
|
||||
|
||||
def require_confirmation(actual: str | None, expected: str) -> None:
|
||||
if actual != expected:
|
||||
raise RuntimeError(f"Refusing destructive maintenance; pass --confirm {expected}")
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
CONTAINER="${RC10_CONTAINER:-geointel}"
|
||||
OUTPUT_DIR="${1:-artifacts/rc10-data-operations}"
|
||||
MINIMUM_AGE_DAYS="${RC10_MINIMUM_AGE_DAYS:-7}"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
||||
echo "Container '$CONTAINER' is not running." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
read_counts() {
|
||||
docker exec "$CONTAINER" sh -c '
|
||||
db="${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"
|
||||
user="${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"
|
||||
psql -X -v ON_ERROR_STOP=1 -U "$user" -d "$db" -AtF $'"'"'\t'"'"' \
|
||||
-c "SELECT table_name,
|
||||
CASE table_name
|
||||
WHEN '\''projects'\'' THEN (SELECT count(*) FROM projects)
|
||||
WHEN '\''datasets'\'' THEN (SELECT count(*) FROM datasets)
|
||||
WHEN '\''dataset_versions'\'' THEN (SELECT count(*) FROM dataset_versions)
|
||||
WHEN '\''jobs'\'' THEN (SELECT count(*) FROM jobs)
|
||||
WHEN '\''analysis_runs'\'' THEN (SELECT count(*) FROM analysis_runs)
|
||||
WHEN '\''exports'\'' THEN (SELECT count(*) FROM exports)
|
||||
WHEN '\''detections'\'' THEN (SELECT count(*) FROM detections)
|
||||
WHEN '\''segmentations'\'' THEN (SELECT count(*) FROM segmentations)
|
||||
WHEN '\''quality_checks'\'' THEN (SELECT count(*) FROM quality_checks)
|
||||
END
|
||||
FROM (VALUES
|
||||
('\''projects'\''), ('\''datasets'\''), ('\''dataset_versions'\''),
|
||||
('\''jobs'\''), ('\''analysis_runs'\''), ('\''exports'\''),
|
||||
('\''detections'\''), ('\''segmentations'\''), ('\''quality_checks'\'')
|
||||
) AS critical(table_name)
|
||||
ORDER BY table_name;"
|
||||
'
|
||||
}
|
||||
|
||||
read_counts > "$OUTPUT_DIR/table-counts-before.tsv"
|
||||
docker exec "$CONTAINER" python /app/scripts/audit_data_operations.py \
|
||||
--minimum-age-days "$MINIMUM_AGE_DAYS" \
|
||||
--fail-on-pressure never \
|
||||
> "$OUTPUT_DIR/data-operations.json"
|
||||
docker exec "$CONTAINER" python /app/scripts/cleanup_storage_artifacts.py \
|
||||
--minimum-age-days "$MINIMUM_AGE_DAYS" \
|
||||
--max-delete 25 \
|
||||
> "$OUTPUT_DIR/cleanup-dry-run.json"
|
||||
read_counts > "$OUTPUT_DIR/table-counts-after.tsv"
|
||||
|
||||
python3 - "$OUTPUT_DIR" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
root = pathlib.Path(sys.argv[1])
|
||||
audit = json.loads((root / "data-operations.json").read_text(encoding="utf-8"))
|
||||
cleanup = json.loads((root / "cleanup-dry-run.json").read_text(encoding="utf-8"))
|
||||
before = (root / "table-counts-before.tsv").read_text(encoding="utf-8")
|
||||
after = (root / "table-counts-after.tsv").read_text(encoding="utf-8")
|
||||
|
||||
if before != after:
|
||||
raise SystemExit("RC10 read-only audit changed one or more critical table counts")
|
||||
if audit.get("mode") != "read-only":
|
||||
raise SystemExit("Data operations audit did not report read-only mode")
|
||||
if cleanup.get("mode") != "dry-run" or cleanup.get("deleted_count") != 0:
|
||||
raise SystemExit("Storage cleanup audit was not a zero-delete dry run")
|
||||
if "release-evidence" not in audit.get("cleanup", {}).get("protected_prefixes", []):
|
||||
raise SystemExit("Release evidence is not protected")
|
||||
families = audit.get("database", {}).get("source_families", {})
|
||||
if set(families) != {"national", "regional", "maritime"}:
|
||||
raise SystemExit("National/regional/maritime source-family report is incomplete")
|
||||
if audit.get("integrity", {}).get("missing_referenced_path_count", 0):
|
||||
raise SystemExit("Persisted storage references are missing")
|
||||
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"status": "passed",
|
||||
"disk_pressure": audit.get("disk_pressure"),
|
||||
"storage_category_count": len(audit.get("categories", [])),
|
||||
"cleanup_candidate_count": cleanup.get("candidate_count", 0),
|
||||
"cleanup_candidate_bytes": cleanup.get("candidate_bytes", 0),
|
||||
"critical_table_counts_unchanged": True,
|
||||
"source_family_counts": {
|
||||
family: len(items) for family, items in families.items()
|
||||
},
|
||||
"missing_referenced_path_count": 0,
|
||||
}
|
||||
(root / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(
|
||||
"RC10 data operations audit passed: "
|
||||
f"pressure={manifest['disk_pressure']['status']}, "
|
||||
f"candidates={manifest['cleanup_candidate_count']}, "
|
||||
f"evidence={root / 'manifest.json'}"
|
||||
)
|
||||
PY
|
||||
@@ -119,6 +119,9 @@ ${PYTHON_BIN} -m py_compile scripts/validate_detection_false_negative_review_dec
|
||||
${PYTHON_BIN} -m py_compile scripts/activate_promoted_yolo_candidate.py
|
||||
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
||||
${PYTHON_BIN} -m py_compile scripts/archive_technical_projects.py
|
||||
${PYTHON_BIN} -m py_compile scripts/release_backup_guard.py
|
||||
${PYTHON_BIN} -m py_compile scripts/audit_data_operations.py
|
||||
${PYTHON_BIN} -m py_compile scripts/cleanup_storage_artifacts.py
|
||||
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
||||
${PYTHON_BIN} -m compileall backend/app
|
||||
(cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning)
|
||||
@@ -162,4 +165,5 @@ bash -n scripts/verify_demo_cleanup_dry_run.sh
|
||||
bash -n scripts/capture_workbench_screenshots.sh
|
||||
bash -n scripts/run_rc8_release_journeys.sh
|
||||
bash -n scripts/run_rc9_ux_audit.sh
|
||||
bash -n scripts/run_rc10_data_operations_audit.sh
|
||||
echo "== Run readiness check passed =="
|
||||
|
||||
Reference in New Issue
Block a user