Operationalize RC10 data retention
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-18 06:58:10 +02:00
parent bf867f8f4f
commit 7b96037853
22 changed files with 1306 additions and 2 deletions
+1
View File
@@ -109,6 +109,7 @@ GEOINTEL_FRONTEND_PORT=1202
GEOINTEL_BACKEND_PORT=8000
GEOINTEL_INSTALL_AI=false
GEOINTEL_STORAGE_PATH=./storage
GEOINTEL_BACKUPS_PATH=./backups
GEOINTEL_MODELS_PATH=./models
GEOINTEL_POSTGIS_DATA_PATH=./postgres-data
GEOINTEL_POSTGRES_DB=geointel
+10
View File
@@ -66,6 +66,16 @@
the live three-viewport audit passed with zero horizontal overflow,
console errors or failed requests and verified truthful delayed loading,
keyboard behavior, accessible control names and visible timing feedback.
- Added an RC-10 read-only data-operations audit covering storage lifecycle,
disk pressure, persisted-path integrity, old failed work and
national/regional/maritime source families.
- Added dry-run-first orphan/cache cleanup with fail-closed path categories,
an explicit delete ceiling, exact confirmation token and a recent
checksum-verified database plus SHA-256 storage-backup requirement.
- Mounted release backups read-only at `/app/backups` and applied the same
backup/confirmation guard to the older demo-export cleanup path.
- Added a live RC-10 audit that proves critical table counts remain unchanged
across the storage report and cleanup dry run.
- Added a read-only release-evidence manifest command with Git, migration,
dependency, configuration checksum and optional live endpoint evidence.
- Replaced the obsolete pre-build status with the current implemented
+14
View File
@@ -1920,3 +1920,17 @@ file. It never imports provider data or writes directly to database tables.
Explicit demo seeding also reactivates its own archived technical project.
This keeps the opt-in fixture workflow selectable without changing the normal
active-project lifecycle.
## Data operations and retention
The runtime packages `audit_data_operations.py`,
`cleanup_storage_artifacts.py` and the shared release-backup guard. The audit
is read-only and combines disk pressure, storage lifecycle, persisted path
integrity, failed-work counts and national/regional/maritime source-family
inventory. Cleanup is limited to old unreferenced derived/cache/export files.
Unknown paths, official source material, uploads, models and release/operator
evidence are protected by default. Apply mode requires an exact confirmation,
an explicit candidate ceiling and a recent checksum-verified database plus
SHA-256 storage backup mounted read-only under `/app/backups`. See
`docs/DATA_OPERATIONS_RUNBOOK.md`.
+32
View File
@@ -9,13 +9,19 @@ from typing import Any
BACKEND_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(BACKEND_ROOT))
REPOSITORY_ROOT = BACKEND_ROOT.parent
SCRIPTS_ROOT = REPOSITORY_ROOT / "scripts"
if SCRIPTS_ROOT.is_dir():
sys.path.insert(0, str(SCRIPTS_ROOT))
from app.core.config import get_settings
from app.db.session import SessionLocal
from app.models import Export, Project
from release_backup_guard import require_confirmation, verify_current_backup
DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
DELETE_CONFIRMATION = "DELETE_DEMO_EXPORTS"
def is_within_storage_root(path: Path, storage_root: Path) -> bool:
@@ -189,6 +195,21 @@ def build_parser() -> argparse.ArgumentParser:
help="Restrict cleanup to an export_type. Repeat for multiple types.",
)
parser.add_argument("--apply", action="store_true", help="Delete selected export rows and files.")
parser.add_argument(
"--backup-dir",
type=Path,
help="Recent checksum-verified release backup mounted read-only in the runtime.",
)
parser.add_argument(
"--backup-max-age-hours",
type=float,
default=24.0,
help="Maximum age accepted for the required release backup.",
)
parser.add_argument(
"--confirm",
help=f"Exact destructive-maintenance confirmation token: {DELETE_CONFIRMATION}",
)
return parser
@@ -199,6 +220,17 @@ def main() -> int:
parser.error("--keep-latest must be greater than or equal to zero")
if args.max_delete < 0:
parser.error("--max-delete must be greater than or equal to zero")
if args.apply:
try:
require_confirmation(args.confirm, DELETE_CONFIRMATION)
if args.backup_dir is None:
raise RuntimeError("--backup-dir is required with --apply")
verify_current_backup(
args.backup_dir,
max_age_hours=args.backup_max_age_hours,
)
except (RuntimeError, ValueError) as exc:
parser.error(str(exc))
summary = cleanup_demo_exports(
project_name=args.project_name,
+259
View File
@@ -0,0 +1,259 @@
from __future__ import annotations
import hashlib
import importlib.util
import json
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
import pytest
ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
def load_script(name: str):
path = SCRIPTS / name
spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha256") -> None:
root.mkdir(parents=True)
manifest = {
"schema_version": 1,
"release_id": "rc10-test",
"created_at": created_at.isoformat(),
"read_only_source": True,
"database_password_secure": True,
"inventory_mode": inventory_mode,
"storage_inventory_requested": True,
"git_commit": "0123456789abcdef",
}
files = {
"manifest.json": json.dumps(manifest),
"database.dump": "database",
"database.list": "list",
"database-metadata.tsv": "alembic_head\t202607160001",
"table-counts.tsv": "datasets\t1",
"storage-manifest.tsv": "relative_path\tsize_bytes\tmtime_ns\tsha256",
}
for name, content in files.items():
(root / name).write_text(content, encoding="utf-8")
checksums = []
for name in sorted(files):
digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
checksums.append(f"{digest} {name}")
(root / "CHECKSUMS.sha256").write_text("\n".join(checksums) + "\n", encoding="utf-8")
def test_backup_guard_requires_recent_complete_sha256_storage_backup(tmp_path: Path) -> None:
guard = load_script("release_backup_guard.py")
now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
backup = tmp_path / "backup"
write_backup(backup, created_at=now - timedelta(hours=2))
verified = guard.verify_current_backup(backup, now=now)
assert verified.release_id == "rc10-test"
assert verified.age_hours == pytest.approx(2)
def test_backup_guard_rejects_stale_or_tampered_backup(tmp_path: Path) -> None:
guard = load_script("release_backup_guard.py")
now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
stale = tmp_path / "stale"
write_backup(stale, created_at=now - timedelta(hours=30))
with pytest.raises(RuntimeError, match="maximum allowed age"):
guard.verify_current_backup(stale, now=now)
current = tmp_path / "tampered"
write_backup(current, created_at=now)
(current / "database.dump").write_text("tampered", encoding="utf-8")
with pytest.raises(RuntimeError, match="checksum mismatch"):
guard.verify_current_backup(current, now=now)
def test_storage_lifecycle_is_fail_closed_and_protects_release_evidence() -> None:
audit = load_script("audit_data_operations.py")
assert audit.classify_relative_path("release-evidence/rc11/manifest.json") == (
"release-evidence",
True,
False,
)
assert audit.classify_relative_path("operator-evidence/source/raw.json")[1:] == (True, False)
assert audit.classify_relative_path("uploads/project/data.geojson")[1:] == (True, False)
assert audit.classify_relative_path("exports/project/old.json")[1:] == (False, True)
assert audit.classify_relative_path("unknown/value.bin")[1:] == (True, False)
def test_storage_audit_only_selects_old_unreferenced_allowlisted_files(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
audit = load_script("audit_data_operations.py")
storage = tmp_path / "storage"
old_orphan = storage / "exports" / "project" / "old.json"
referenced = storage / "exports" / "project" / "kept.json"
protected = storage / "release-evidence" / "rc" / "manifest.json"
unknown = storage / "misc" / "unknown.bin"
for path in (old_orphan, referenced, protected, unknown):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(path.name, encoding="utf-8")
old_timestamp = (datetime.now(timezone.utc) - timedelta(days=30)).timestamp()
for path in (old_orphan, referenced, protected, unknown):
path.touch()
Path(path).chmod(0o644)
import os
os.utime(path, (old_timestamp, old_timestamp))
monkeypatch.setattr(
audit,
"collect_database_state",
lambda _db, _root: {
"references": {referenced.resolve()},
"counts": {},
"source_families": {"national": [], "regional": [], "maritime": []},
},
)
monkeypatch.setattr(
audit,
"disk_pressure",
lambda _root: {
"status": "ok",
"total_bytes": 100,
"used_bytes": 50,
"free_bytes": 50,
"free_percent": 50.0,
"acquisition_allowed": True,
},
)
report, candidates = audit.build_report(storage, SimpleNamespace(), minimum_age_days=7)
assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"]
assert report["cleanup"]["candidate_count"] == 1
assert "release-evidence" in report["cleanup"]["protected_prefixes"]
def test_source_family_report_covers_national_regional_and_maritime(tmp_path: Path) -> None:
audit = load_script("audit_data_operations.py")
national_id = uuid4()
regional_id = uuid4()
now = datetime.now(timezone.utc)
rows = {
audit.Project: [
SimpleNamespace(id=national_id, name=audit.NATIONAL_PROJECT_NAME, status="active"),
SimpleNamespace(id=regional_id, name="Wallonia operator", status="active"),
],
audit.Dataset: [
SimpleNamespace(
project_id=national_id,
source_name="ngi_adminvector",
source="ngi",
name="Belgium boundary",
source_metadata={"coverage_zones": ["belgium"]},
source_version="2026",
imported_at=now,
status="ready",
storage_path=None,
metadata_json=None,
provenance_metadata=None,
),
SimpleNamespace(
project_id=national_id,
source_name="rbins_marine_reporting_units",
source="rbins",
name="Belgian North Sea",
source_metadata={"coverage_zones": ["belgian_north_sea"]},
source_version="2024",
imported_at=now,
status="ready",
storage_path=None,
metadata_json=None,
provenance_metadata=None,
),
SimpleNamespace(
project_id=regional_id,
source_name="wallonia_manual",
source="manual",
name="Wallonia source",
source_metadata={},
source_version="1",
imported_at=now,
status="ready",
storage_path=None,
metadata_json=None,
provenance_metadata=None,
),
],
audit.DatasetVersion: [],
audit.Export: [],
audit.Detection: [],
audit.Segmentation: [],
audit.Job: [],
audit.AnalysisRun: [],
}
class Query:
def __init__(self, values):
self.values = values
def all(self):
return self.values
class Session:
def query(self, model):
return Query(rows[model])
state = audit.collect_database_state(Session(), tmp_path)
assert {item["source_name"] for item in state["source_families"]["national"]} == {
"ngi_adminvector",
"rbins_marine_reporting_units",
}
assert {item["source_name"] for item in state["source_families"]["maritime"]} == {
"rbins_marine_reporting_units"
}
assert {item["source_name"] for item in state["source_families"]["regional"]} == {
"wallonia_manual"
}
def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> None:
generic = (SCRIPTS / "cleanup_storage_artifacts.py").read_text(encoding="utf-8")
demo = (ROOT / "backend/scripts/cleanup_demo_artifacts.py").read_text(encoding="utf-8")
compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
dockerman = (ROOT / "deploy/unraid/run-dockerman-container.sh").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8")
assert "DELETE_STORAGE_ARTIFACTS" in generic
assert "verify_current_backup" in generic
assert "DELETE_DEMO_EXPORTS" in demo
assert "verify_current_backup" in demo
assert "/app/backups:ro" in compose
assert '/app/backups:ro"' in dockerman
for name in (
"release_backup_guard.py",
"audit_data_operations.py",
"cleanup_storage_artifacts.py",
):
assert f"COPY scripts/{name}" in dockerfile
assert f"py_compile scripts/{name}" in readiness
assert "bash -n scripts/run_rc10_data_operations_audit.sh" in readiness
assert "--apply" not in live_audit
assert "table-counts-before.tsv" in live_audit
assert "table-counts-after.tsv" in live_audit
assert "deleted_count" in live_audit
@@ -65,6 +65,7 @@ def test_unraid_template_exposes_every_operator_owned_runtime_setting() -> None:
template_variables = set(re.findall(r'Target="([A-Z][A-Z0-9_]+)"', template))
bridged_or_internal = {
"GEOINTEL_FRONTEND_PORT",
"GEOINTEL_BACKUPS_PATH",
"GEOINTEL_IMAGE",
"GEOINTEL_MODELS_PATH",
"GEOINTEL_POSTGIS_DATA_PATH",
+4 -1
View File
@@ -142,6 +142,9 @@ COPY scripts/run_split_background_promotion_workflow.sh /app/scripts/run_split_b
COPY scripts/activate_promoted_yolo_candidate.py /app/scripts/activate_promoted_yolo_candidate.py
COPY scripts/archive_technical_projects.py /app/scripts/archive_technical_projects.py
COPY scripts/runtime_state_report.py /app/scripts/runtime_state_report.py
COPY scripts/release_backup_guard.py /app/scripts/release_backup_guard.py
COPY scripts/audit_data_operations.py /app/scripts/audit_data_operations.py
COPY scripts/cleanup_storage_artifacts.py /app/scripts/cleanup_storage_artifacts.py
COPY deploy/unraid/nginx-all-in-one.conf /etc/nginx/conf.d/default.conf
COPY deploy/unraid/all-in-one-start.sh /usr/local/bin/geointel-all-in-one-start
COPY --from=frontend-build /frontend/dist/ /usr/share/nginx/html/
@@ -171,7 +174,7 @@ LABEL org.opencontainers.image.title="GeoIntel" \
org.opencontainers.image.created="${GEOINTEL_BUILD_TIME}" \
io.geointel.ai.enabled="${GEOINTEL_INSTALL_AI}"
VOLUME ["/var/lib/postgresql/data", "/app/storage"]
VOLUME ["/var/lib/postgresql/data", "/app/storage", "/app/backups"]
EXPOSE 80
+18
View File
@@ -293,6 +293,24 @@ The configured upload limit is shared by FastAPI and the generated nginx
runtime configuration. Values outside `1..2048` MiB are rejected before the
active application is replaced.
## Data operations
`GEOINTEL_BACKUPS_PATH` defaults to
`/mnt/user/appdata/geointel/backups` and is mounted read-only at
`/app/backups`. Explicit cleanup commands can therefore verify a recent
backup without permission to alter it.
These commands are non-mutating:
```bash
docker exec geointel python /app/scripts/audit_data_operations.py
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py
```
The full backup, confirmation, candidate-limit and apply sequence is in
`docs/DATA_OPERATIONS_RUNBOOK.md`. GeoIntel installs no automatic cleanup
schedule.
## Safe cleanup
Safe cache cleanup if Docker build cache fills the Unraid Docker image:
@@ -25,6 +25,7 @@
<Config Name="Web UI Port" Target="80" Default="1202" Mode="tcp" Description="Host port mapped to the GeoIntel all-in-one web UI. Change this to edit the browser port." Type="Port" Display="always" Required="true" Mask="false">1202</Config>
<Config Name="Storage Path" Target="/app/storage" Default="/mnt/user/appdata/geointel/storage" Mode="rw" Description="Persistent GeoIntel artifact storage for uploads, tiles, masks, reports and exports." Type="Path" Display="always" Required="true" Mask="false">/mnt/user/appdata/geointel/storage</Config>
<Config Name="AI Models Path" Target="/app/models" Default="/mnt/user/appdata/geointel/models" Mode="rw" Description="Persistent local model files mounted into the container. GeoIntel never downloads weights automatically." Type="Path" Display="always" Required="true" Mask="false">/mnt/user/appdata/geointel/models</Config>
<Config Name="Release Backups Path" Target="/app/backups" Default="/mnt/user/appdata/geointel/backups" Mode="ro" Description="Read-only release backups used to guard explicitly confirmed cleanup operations." Type="Path" Display="advanced" Required="true" Mask="false">/mnt/user/appdata/geointel/backups</Config>
<Config Name="PostGIS Data Path" Target="/var/lib/postgresql/data" Default="/mnt/user/appdata/geointel/postgres-data" Mode="rw" Description="Persistent embedded PostGIS data directory for the all-in-one container." Type="Path" Display="always" Required="true" Mask="false">/mnt/user/appdata/geointel/postgres-data</Config>
<Config Name="Postgres Database" Target="GEOINTEL_POSTGRES_DB" Default="geointel" Mode="" Description="Embedded PostGIS database name." Type="Variable" Display="advanced" Required="true" Mask="false">geointel</Config>
<Config Name="Postgres User" Target="GEOINTEL_POSTGRES_USER" Default="geointel" Mode="" Description="Embedded PostGIS database user." Type="Variable" Display="advanced" Required="true" Mask="false">geointel</Config>
+3
View File
@@ -10,6 +10,9 @@ GEOINTEL_STORAGE_PATH=/mnt/user/appdata/geointel/storage
# Local AI model files mounted into the container as /app/models.
GEOINTEL_MODELS_PATH=/mnt/user/appdata/geointel/models
# Checksum-verified release backups mounted read-only for cleanup guards.
GEOINTEL_BACKUPS_PATH=/mnt/user/appdata/geointel/backups
# Embedded PostGIS data directory for the all-in-one container.
GEOINTEL_POSTGIS_DATA_PATH=/mnt/user/appdata/geointel/postgres-data
+3 -1
View File
@@ -15,6 +15,7 @@ GEOINTEL_FRONTEND_PORT="${GEOINTEL_FRONTEND_PORT:-1202}"
GEOINTEL_IMAGE="${GEOINTEL_IMAGE:-geointel-all-in-one:latest}"
GEOINTEL_STORAGE_PATH="${GEOINTEL_STORAGE_PATH:-/mnt/user/appdata/geointel/storage}"
GEOINTEL_MODELS_PATH="${GEOINTEL_MODELS_PATH:-/mnt/user/appdata/geointel/models}"
GEOINTEL_BACKUPS_PATH="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}"
GEOINTEL_POSTGRES_DB="${GEOINTEL_POSTGRES_DB:-geointel}"
GEOINTEL_POSTGRES_USER="${GEOINTEL_POSTGRES_USER:-geointel}"
@@ -168,7 +169,7 @@ if docker ps -a --format '{{.Names}}' | grep -qx geointel; then
docker rm -f geointel
fi
mkdir -p "$GEOINTEL_STORAGE_PATH" "$GEOINTEL_MODELS_PATH" "$GEOINTEL_POSTGIS_DATA_PATH"
mkdir -p "$GEOINTEL_STORAGE_PATH" "$GEOINTEL_MODELS_PATH" "$GEOINTEL_BACKUPS_PATH" "$GEOINTEL_POSTGIS_DATA_PATH"
migrate_compose_volume_if_needed
docker run -d \
@@ -268,6 +269,7 @@ docker run -d \
-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data" \
-v "${GEOINTEL_STORAGE_PATH}:/app/storage" \
-v "${GEOINTEL_MODELS_PATH}:/app/models" \
-v "${GEOINTEL_BACKUPS_PATH}:/app/backups:ro" \
"$GEOINTEL_IMAGE"
docker ps --filter name=geointel
+1
View File
@@ -89,6 +89,7 @@ services:
- ${GEOINTEL_POSTGIS_DATA_PATH:-geointel_postgis}:/var/lib/postgresql/data
- ${GEOINTEL_STORAGE_PATH:-./storage}:/app/storage
- ${GEOINTEL_MODELS_PATH:-./models}:/app/models
- ${GEOINTEL_BACKUPS_PATH:-./backups}:/app/backups:ro
extra_hosts:
- "host.docker.internal:host-gateway"
restart: unless-stopped
+1
View File
@@ -119,6 +119,7 @@ services:
volumes:
- ${GEOINTEL_STORAGE_PATH:-./storage}:/app/storage
- ${GEOINTEL_MODELS_PATH:-./models}:/app/models
- ${GEOINTEL_BACKUPS_PATH:-./backups}:/app/backups:ro
- ./fixtures:/app/fixtures:ro
extra_hosts:
- "host.docker.internal:host-gateway"
+145
View File
@@ -0,0 +1,145 @@
# Data Operations and Retention Runbook
## Purpose
This runbook governs persistent GeoIntel data for Belgium and the Belgian
North Sea. It covers raw, normalized, derived, export, AI and release-evidence
artifacts. No command in this runbook schedules itself and no source is
downloaded implicitly.
## Lifecycle classes
| Class | Paths | Retention |
|---|---|---|
| Raw and normalized source | `originals/`, `uploads/`, `operator-data/` | Protected; replace no official edition in place |
| Derived and cache | `derived/`, `rasters/derived/`, `previews/`, `tiles/`, `masks/` | Eligible only when old and not referenced by persisted provenance |
| Exports | `exports/` | Retain persisted exports; unreferenced old files may be candidates |
| AI models | `/app/models` and `storage/models/` | Protected; activation and promotion evidence govern removal |
| Immutable evidence | `release-evidence/`, `operator-evidence/` | Never a cleanup candidate |
| Unknown | every unclassified path | Protected by default |
Dataset, DatasetVersion, Export, Detection, Segmentation, Job and AnalysisRun
paths and nested provenance are treated as references. Failed work remains
audit provenance. Project archival remains reversible and does not physically
delete its datasets or files.
## Read-only audit
Run the complete local storage, database-reference, disk-pressure and
national/regional/maritime source-family report:
```bash
docker exec geointel python /app/scripts/audit_data_operations.py \
--minimum-age-days 7 \
--output /app/storage/release-evidence/rc-current/data-operations.json
```
The command:
- inventories storage without following symlinks;
- reports bytes and file counts per lifecycle class;
- reports free disk bytes and percentage before new acquisition;
- reports missing persisted path references;
- identifies only old, unreferenced files inside the explicit cleanup
allowlist;
- summarizes national, regional and maritime source names, versions and latest
import dates;
- reports old failed Jobs and AnalysisRuns without deleting them.
Use `--fail-on-pressure warning` for a stricter acquisition preflight and
`--fail-on-missing-reference` for release gates. `critical` disk pressure
blocks acquisition by default. The thresholds are 10 GiB or 5% free for
critical and 25 GiB or 10% free for warning.
The existing project-level freshness policy remains authoritative:
```bash
docker exec geointel python /app/scripts/audit_source_freshness.py \
--project-id <belgium-and-north-sea-project-id> \
--api-url http://127.0.0.1/api/v1 \
--json \
--output /app/storage/release-evidence/rc-current/source-freshness.json
```
Together, these reports cover the national workbench, regional source
families and Belgian North Sea datasets without performing an external
refresh. Catalog probes remain separate, explicit and read-only.
## Cleanup dry run
```bash
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
--minimum-age-days 7 \
--max-delete 25
```
This is always a dry run unless `--apply` is present. Review every candidate.
Release evidence, operator evidence, official source material, uploads,
models, unknown paths and every persisted path reference are excluded.
Technical benchmark projects can be archived reversibly:
```bash
docker exec geointel python /app/scripts/archive_technical_projects.py --show-names
```
The normal project lifecycle archive path does not need destructive
confirmation because it changes only `status=archived` and preserves all
data.
## Destructive apply gate
First create a fresh backup with a SHA-256 storage inventory on the host:
```bash
bash scripts/backup_release_state.sh \
--container geointel \
--release-id rc10-before-cleanup-$(date -u +%Y%m%dT%H%M%SZ) \
--output-root /mnt/user/appdata/geointel/backups \
--storage-path /mnt/user/appdata/geointel/storage \
--models-path /mnt/user/appdata/geointel/models \
--inventory-mode sha256
```
Verify it read-only:
```bash
bash scripts/verify_release_backup.sh \
--backup-dir /mnt/user/appdata/geointel/backups/<release-id> \
--container geointel
```
The Unraid runtime mounts `GEOINTEL_BACKUPS_PATH` read-only at `/app/backups`.
Only after reviewing the dry run may an operator execute:
```bash
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
--minimum-age-days 7 \
--max-delete <reviewed-candidate-count> \
--backup-dir /app/backups/<release-id> \
--backup-max-age-hours 24 \
--confirm DELETE_STORAGE_ARTIFACTS \
--apply
```
The command re-runs the audit immediately before deletion. It refuses the
operation when the exact token is absent, the candidate count exceeds the
operator limit, the backup is stale/incomplete, checksums differ, the storage
inventory is not SHA-256, or the path is outside the cleanup allowlist.
The older demo-export cleanup has the same gate and uses confirmation token
`DELETE_DEMO_EXPORTS`.
## Failure meaning
- `disk_pressure=critical`: stop new source acquisition and resolve capacity.
- `missing_referenced_path_count>0`: persisted provenance points to absent
files; investigate before cleanup or release.
- checksum or backup-age failure: create and verify a new backup.
- candidate count above `--max-delete`: keep dry-run mode and review; never
increase the limit blindly.
- symlink listed as skipped: inspect manually; the audit will not traverse or
delete it.
No automatic cron or background cleanup is installed. Operators may schedule
the read-only audit externally, but destructive commands must remain manual.
+15
View File
@@ -321,6 +321,21 @@ The smoke never passes `--apply`. It fails if the cleanup summary is not a
dry-run, if any export/file deletion is reported, or if the dry-run candidate
fields are missing.
## RC-10 data operations
`docs/DATA_OPERATIONS_RUNBOOK.md` is the executable retention source of truth.
`scripts/audit_data_operations.py` reports storage growth, disk pressure,
persisted path integrity, old failed work and national/regional/maritime
source families without mutation. `scripts/cleanup_storage_artifacts.py` can
only remove old unreferenced artifacts from the explicit derived/cache/export
allowlist.
Every unknown category, upload, original, model, operator evidence and release
evidence path is protected by default. Apply mode requires an exact
confirmation token and a recent checksum-verified release backup containing a
SHA-256 storage inventory. The backup root is mounted read-only at
`/app/backups`; no cleanup is scheduled implicitly.
## Governed cross-domain raster and soil evidence
MercatorNet thematic products use the ordinary raster Dataset and immutable
+24
View File
@@ -2037,6 +2037,30 @@ The command atomically updates the operator-owned `.env`, changes the matching
PostgreSQL role and recreates the container. A failed role change restores the
previous environment file. The generated secret is never printed.
## RC-10 data operations and retention
Run the read-only storage, provenance, disk-pressure and source-family audit:
```bash
docker exec geointel python /app/scripts/audit_data_operations.py \
--minimum-age-days 7 \
--output /app/storage/release-evidence/rc-current/data-operations.json
```
Preview old unreferenced derived/cache/export candidates without deletion:
```bash
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
--minimum-age-days 7 \
--max-delete 25
```
Apply requires a reviewed candidate count, the exact
`DELETE_STORAGE_ARTIFACTS` token and a backup no older than 24 hours with a
checksum-verified database dump and SHA-256 storage inventory. The host backup
root is mounted read-only at `/app/backups`. See
`docs/DATA_OPERATIONS_RUNBOOK.md`. No cleanup is scheduled by GeoIntel.
## RC-8 Belgium/North Sea release journeys
Preview the seven release areas without mutating GeoIntel:
+446
View File
@@ -0,0 +1,446 @@
#!/usr/bin/env python3
"""Read-only storage, provenance and source-family audit for GeoIntel."""
from __future__ import annotations
import argparse
import json
import os
import shutil
import sys
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Iterable
ROOT = Path(__file__).resolve().parents[1]
BACKEND_ROOT = ROOT / "backend" if (ROOT / "backend" / "app").is_dir() else ROOT
if str(BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(BACKEND_ROOT))
from app.core.config import get_settings
from app.db.session import SessionLocal
from app.models import AnalysisRun, Dataset, DatasetVersion, Detection, Export, Job, Project, Segmentation
NATIONAL_PROJECT_NAME = "Belgium and North Sea Workbench"
PROTECTED_PREFIXES = (
"release-evidence",
"operator-evidence",
"operator-data",
"originals",
"uploads",
"models",
)
CLEANUP_PREFIXES = (
"exports",
"previews",
"tiles",
"masks",
"derived",
"rasters/derived",
)
IGNORED_FILENAMES = frozenset({".gitkeep", "README.md"})
PATH_METADATA_FIELDS = (
"metadata_json",
"source_metadata",
"provenance_metadata",
"parameters_json",
"result_json",
"properties_json",
"provenance_json",
"bbox_json",
)
@dataclass(frozen=True)
class FileRecord:
path: Path
relative_path: str
category: str
size_bytes: int
modified_at: datetime
protected: bool
cleanup_eligible: bool
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def _prefix_match(relative_path: str, prefixes: Iterable[str]) -> str | None:
normalized = relative_path.strip("/")
matches = [
prefix
for prefix in prefixes
if normalized == prefix or normalized.startswith(f"{prefix}/")
]
return max(matches, key=len) if matches else None
def classify_relative_path(relative_path: str) -> tuple[str, bool, bool]:
protected = _prefix_match(relative_path, PROTECTED_PREFIXES)
if protected:
return protected, True, False
cleanup = _prefix_match(relative_path, CLEANUP_PREFIXES)
if cleanup:
return cleanup, False, True
category = relative_path.split("/", 1)[0] if relative_path else "."
return category, True, False
def inventory_storage(storage_root: Path) -> tuple[list[FileRecord], list[str]]:
root = storage_root.resolve()
records: list[FileRecord] = []
skipped_symlinks: list[str] = []
for current, directories, filenames in os.walk(root, followlinks=False):
current_path = Path(current)
retained_directories: list[str] = []
for directory in directories:
child = current_path / directory
if child.is_symlink():
skipped_symlinks.append(child.relative_to(root).as_posix())
else:
retained_directories.append(directory)
directories[:] = retained_directories
for filename in filenames:
path = current_path / filename
if path.is_symlink():
skipped_symlinks.append(path.relative_to(root).as_posix())
continue
try:
stat = path.stat()
except OSError:
continue
relative = path.relative_to(root).as_posix()
category, protected, cleanup_eligible = classify_relative_path(relative)
records.append(
FileRecord(
path=path.resolve(),
relative_path=relative,
category=category,
size_bytes=stat.st_size,
modified_at=datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc),
protected=protected,
cleanup_eligible=cleanup_eligible,
)
)
return records, sorted(skipped_symlinks)
def _iter_strings(value: Any) -> Iterable[str]:
if isinstance(value, str):
yield value
elif isinstance(value, dict):
for nested in value.values():
yield from _iter_strings(nested)
elif isinstance(value, (list, tuple)):
for nested in value:
yield from _iter_strings(nested)
def normalize_storage_reference(value: str | None, storage_root: Path) -> Path | None:
if not value:
return None
stripped = value.strip()
if not stripped or "://" in stripped or stripped.startswith("/vsi"):
return None
candidate = Path(stripped)
if not candidate.is_absolute():
normalized = stripped.replace("\\", "/")
if normalized.startswith("storage/"):
normalized = normalized.removeprefix("storage/")
if _prefix_match(normalized, PROTECTED_PREFIXES + CLEANUP_PREFIXES) is None:
return None
candidate = storage_root / normalized
try:
resolved = candidate.resolve()
resolved.relative_to(storage_root.resolve())
except (OSError, ValueError):
return None
return resolved
def _add_row_references(target: set[Path], row: Any, storage_root: Path, direct_fields: Iterable[str]) -> None:
for field in direct_fields:
normalized = normalize_storage_reference(getattr(row, field, None), storage_root)
if normalized:
target.add(normalized)
for field in PATH_METADATA_FIELDS:
for value in _iter_strings(getattr(row, field, None)):
normalized = normalize_storage_reference(value, storage_root)
if normalized:
target.add(normalized)
def collect_database_state(db: Any, storage_root: Path) -> dict[str, Any]:
projects = db.query(Project).all()
project_names = {project.id: project.name for project in projects}
datasets = db.query(Dataset).all()
versions = db.query(DatasetVersion).all()
exports = db.query(Export).all()
detections = db.query(Detection).all()
segmentations = db.query(Segmentation).all()
jobs = db.query(Job).all()
analysis_runs = db.query(AnalysisRun).all()
references: set[Path] = set()
for row in datasets:
_add_row_references(references, row, storage_root, ("storage_path",))
for row in versions:
_add_row_references(references, row, storage_root, ("storage_path",))
for row in exports:
_add_row_references(references, row, storage_root, ("storage_path",))
for row in detections:
_add_row_references(references, row, storage_root, ("source_tile_path",))
for row in segmentations:
_add_row_references(references, row, storage_root, ("mask_path", "source_tile_path"))
for row in jobs:
_add_row_references(references, row, storage_root, ())
for row in analysis_runs:
_add_row_references(references, row, storage_root, ())
source_families: dict[str, dict[str, dict[str, Any]]] = {
"national": {},
"regional": {},
"maritime": {},
}
maritime_tokens = ("marine", "maritime", "north_sea", "bathymetry", "rbin", "mdk", "msp")
for dataset in datasets:
source_name = (dataset.source_name or dataset.source or "unknown").strip().lower()
metadata = dataset.source_metadata or {}
zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or []
if isinstance(zones, str):
zones = [zones]
families: set[str] = set()
project_name = project_names.get(dataset.project_id, "")
if project_name == NATIONAL_PROJECT_NAME or "belgium" in zones:
families.add("national")
else:
families.add("regional")
searchable = " ".join([source_name, dataset.name.lower(), *(str(zone).lower() for zone in zones)])
if "belgian_north_sea" in zones or any(token in searchable for token in maritime_tokens):
families.add("maritime")
for family in families:
item = source_families[family].setdefault(
source_name,
{
"source_name": source_name,
"dataset_count": 0,
"ready_count": 0,
"latest_imported_at": None,
"source_versions": set(),
},
)
item["dataset_count"] += 1
item["ready_count"] += int(dataset.status == "ready")
if dataset.source_version:
item["source_versions"].add(dataset.source_version)
if dataset.imported_at and (
item["latest_imported_at"] is None or dataset.imported_at > item["latest_imported_at"]
):
item["latest_imported_at"] = dataset.imported_at
serialized_families: dict[str, list[dict[str, Any]]] = {}
for family, sources in source_families.items():
serialized_families[family] = []
for source in sorted(sources.values(), key=lambda item: item["source_name"]):
serialized_families[family].append(
{
**source,
"latest_imported_at": (
source["latest_imported_at"].isoformat()
if source["latest_imported_at"] is not None
else None
),
"source_versions": sorted(source["source_versions"]),
}
)
failed_cutoff = utc_now() - timedelta(days=7)
return {
"references": references,
"counts": {
"projects": len(projects),
"active_projects": sum(project.status == "active" for project in projects),
"archived_projects": sum(project.status == "archived" for project in projects),
"datasets": len(datasets),
"dataset_versions": len(versions),
"exports": len(exports),
"detections": len(detections),
"segmentations": len(segmentations),
"jobs": len(jobs),
"analysis_runs": len(analysis_runs),
"failed_jobs_older_than_7d": sum(
job.status == "failed" and job.created_at and job.created_at < failed_cutoff for job in jobs
),
"failed_runs_older_than_7d": sum(
run.status == "failed" and run.created_at and run.created_at < failed_cutoff
for run in analysis_runs
),
},
"source_families": serialized_families,
}
def disk_pressure(storage_root: Path) -> dict[str, Any]:
usage = shutil.disk_usage(storage_root)
free_percent = (usage.free / usage.total * 100) if usage.total else 0.0
if usage.free < 10 * 1024**3 or free_percent < 5:
status = "critical"
elif usage.free < 25 * 1024**3 or free_percent < 10:
status = "warning"
else:
status = "ok"
return {
"status": status,
"total_bytes": usage.total,
"used_bytes": usage.used,
"free_bytes": usage.free,
"free_percent": round(free_percent, 2),
"acquisition_allowed": status != "critical",
}
def build_report(
storage_root: Path,
db: Any,
*,
minimum_age_days: int = 7,
max_candidate_records: int = 500,
) -> tuple[dict[str, Any], list[FileRecord]]:
if minimum_age_days < 1:
raise ValueError("minimum_age_days must be at least one")
root = storage_root.expanduser().resolve()
if not root.is_dir():
raise RuntimeError(f"Storage root is not a directory: {root}")
records, skipped_symlinks = inventory_storage(root)
database = collect_database_state(db, root)
references: set[Path] = database.pop("references")
cutoff = utc_now() - timedelta(days=minimum_age_days)
categories: dict[str, dict[str, Any]] = defaultdict(
lambda: {"file_count": 0, "size_bytes": 0, "protected": False, "cleanup_eligible": False}
)
for record in records:
item = categories[record.category]
item["file_count"] += 1
item["size_bytes"] += record.size_bytes
item["protected"] = item["protected"] or record.protected
item["cleanup_eligible"] = item["cleanup_eligible"] or record.cleanup_eligible
existing_paths = {record.path for record in records}
missing_references = sorted(
path.relative_to(root).as_posix()
for path in references
if path not in existing_paths and not path.is_dir()
)
candidates = sorted(
(
record
for record in records
if record.cleanup_eligible
and record.path not in references
and record.modified_at <= cutoff
and record.path.name not in IGNORED_FILENAMES
),
key=lambda item: (item.modified_at, item.relative_path),
)
report = {
"schema_version": 1,
"generated_at": utc_now().isoformat(),
"mode": "read-only",
"storage_root": str(root),
"minimum_age_days": minimum_age_days,
"disk_pressure": disk_pressure(root),
"lifecycle": {
"raw_and_normalized": ["originals", "uploads", "operator-data"],
"derived_and_cache": ["derived", "rasters/derived", "previews", "tiles", "masks"],
"exports": ["exports"],
"ai_models": ["models"],
"immutable_evidence": ["release-evidence", "operator-evidence"],
},
"categories": [
{"category": category, **values}
for category, values in sorted(categories.items())
],
"database": database,
"integrity": {
"referenced_path_count": len(references),
"missing_referenced_path_count": len(missing_references),
"missing_referenced_paths": missing_references[:max_candidate_records],
"skipped_symlinks": skipped_symlinks[:max_candidate_records],
},
"cleanup": {
"candidate_count": len(candidates),
"candidate_bytes": sum(item.size_bytes for item in candidates),
"candidate_records_truncated": len(candidates) > max_candidate_records,
"candidates": [
{
"relative_path": item.relative_path,
"category": item.category,
"size_bytes": item.size_bytes,
"modified_at": item.modified_at.isoformat(),
}
for item in candidates[:max_candidate_records]
],
"protected_prefixes": list(PROTECTED_PREFIXES),
"eligible_prefixes": list(CLEANUP_PREFIXES),
"apply_requires": [
"exact confirmation token",
"recent checksum-verified database and SHA-256 storage backup",
"explicit maximum delete count",
],
},
"limitations": [
"The audit never downloads or refreshes an official source.",
"Failed jobs and analysis runs remain provenance and are reported, not deleted.",
"Unknown storage categories are protected by default.",
"No cleanup is scheduled implicitly.",
],
}
return report, candidates
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--storage-root", type=Path)
parser.add_argument("--minimum-age-days", type=int, default=7)
parser.add_argument("--max-candidate-records", type=int, default=500)
parser.add_argument("--output", type=Path)
parser.add_argument("--fail-on-pressure", choices=("never", "critical", "warning"), default="critical")
parser.add_argument("--fail-on-missing-reference", action="store_true")
return parser.parse_args()
def main() -> int:
args = parse_args()
storage_root = args.storage_root or Path(get_settings().storage_root)
with SessionLocal() as db:
report, _ = build_report(
storage_root,
db,
minimum_age_days=args.minimum_age_days,
max_candidate_records=args.max_candidate_records,
)
serialized = json.dumps(report, indent=2, sort_keys=True)
print(serialized)
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
temporary = args.output.with_suffix(f"{args.output.suffix}.partial")
temporary.write_text(serialized + "\n", encoding="utf-8")
temporary.replace(args.output)
pressure = report["disk_pressure"]["status"]
if args.fail_on_pressure == "critical" and pressure == "critical":
return 1
if args.fail_on_pressure == "warning" and pressure in {"warning", "critical"}:
return 1
if args.fail_on_missing_reference and report["integrity"]["missing_referenced_path_count"]:
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
+1
View File
@@ -18,6 +18,7 @@ def load_backend_script() -> ModuleType:
_impl = load_backend_script()
DEMO_PROJECT_NAME = _impl.DEMO_PROJECT_NAME
DELETE_CONFIRMATION = _impl.DELETE_CONFIRMATION
is_within_storage_root = _impl.is_within_storage_root
export_created_at = _impl.export_created_at
select_cleanup_candidates = _impl.select_cleanup_candidates
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env python3
"""Dry-run-first cleanup for old unreferenced derived/cache artifacts."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from audit_data_operations import build_report
from release_backup_guard import require_confirmation, verify_current_backup
from app.core.config import get_settings
from app.db.session import SessionLocal
CONFIRMATION = "DELETE_STORAGE_ARTIFACTS"
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--storage-root", type=Path)
parser.add_argument("--minimum-age-days", type=int, default=7)
parser.add_argument("--max-delete", type=int, default=25)
parser.add_argument("--apply", action="store_true")
parser.add_argument("--confirm")
parser.add_argument("--backup-dir", type=Path)
parser.add_argument("--backup-max-age-hours", type=float, default=24.0)
return parser.parse_args()
def main() -> int:
args = parse_args()
if args.max_delete < 0:
raise SystemExit("--max-delete must be greater than or equal to zero")
storage_root = (args.storage_root or Path(get_settings().storage_root)).resolve()
with SessionLocal() as db:
report, candidates = build_report(
storage_root,
db,
minimum_age_days=args.minimum_age_days,
max_candidate_records=max(args.max_delete, 500),
)
blocked_reason = None
backup = None
deleted: list[str] = []
if args.apply:
require_confirmation(args.confirm, CONFIRMATION)
if args.backup_dir is None:
raise RuntimeError("--backup-dir is required with --apply")
backup = verify_current_backup(
args.backup_dir,
max_age_hours=args.backup_max_age_hours,
)
if len(candidates) > args.max_delete:
blocked_reason = (
f"candidate_count {len(candidates)} exceeds --max-delete {args.max_delete}; "
"review the dry run and raise the explicit limit"
)
else:
for candidate in candidates:
candidate.path.unlink()
deleted.append(candidate.relative_path)
payload = {
"schema_version": 1,
"mode": "apply" if args.apply else "dry-run",
"storage_root": str(storage_root),
"minimum_age_days": args.minimum_age_days,
"max_delete": args.max_delete,
"candidate_count": len(candidates),
"candidate_bytes": sum(item.size_bytes for item in candidates),
"candidates": [item.relative_path for item in candidates],
"deleted_count": len(deleted),
"deleted": deleted,
"blocked_reason": blocked_reason,
"protected_prefixes": report["cleanup"]["protected_prefixes"],
"backup": (
{
"release_id": backup.release_id,
"created_at": backup.created_at.isoformat(),
"age_hours": round(backup.age_hours, 3),
"git_commit": backup.git_commit,
}
if backup
else None
),
}
print(json.dumps(payload, indent=2, sort_keys=True))
return 1 if blocked_reason else 0
if __name__ == "__main__":
raise SystemExit(main())
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env python3
"""Verification guard shared by destructive GeoIntel operator commands."""
from __future__ import annotations
import hashlib
import json
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
@dataclass(frozen=True)
class VerifiedBackup:
backup_dir: Path
release_id: str
created_at: datetime
age_hours: float
git_commit: str
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _created_at(value: object) -> datetime:
if not isinstance(value, str):
raise RuntimeError("Backup manifest does not contain created_at")
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
def verify_current_backup(
backup_dir: str | Path,
*,
max_age_hours: float = 24.0,
now: datetime | None = None,
) -> VerifiedBackup:
"""Verify checksums and release metadata without mutating the backup."""
if max_age_hours <= 0:
raise ValueError("max_age_hours must be greater than zero")
root = Path(backup_dir).expanduser().resolve()
if not root.is_dir():
raise RuntimeError(f"Backup directory does not exist: {root}")
required = {
"manifest.json",
"CHECKSUMS.sha256",
"database.dump",
"database.list",
"database-metadata.tsv",
"table-counts.tsv",
"storage-manifest.tsv",
}
missing = sorted(name for name in required if not (root / name).is_file())
if missing:
raise RuntimeError(f"Backup is incomplete; missing: {', '.join(missing)}")
checksum_lines = (root / "CHECKSUMS.sha256").read_text(encoding="utf-8").splitlines()
checked: set[str] = set()
for line in checksum_lines:
if not line.strip():
continue
try:
expected, name = line.split(maxsplit=1)
except ValueError as exc:
raise RuntimeError("Backup checksum file has an invalid line") from exc
name = name.lstrip("*")
if "/" in name or "\\" in name or name in {".", ".."}:
raise RuntimeError(f"Backup checksum contains an unsafe path: {name}")
target = root / name
if not target.is_file():
raise RuntimeError(f"Backup checksum target is missing: {name}")
if _sha256(target) != expected.lower():
raise RuntimeError(f"Backup checksum mismatch: {name}")
checked.add(name)
unchecked = sorted((required - {"CHECKSUMS.sha256"}) - checked)
if unchecked:
raise RuntimeError(f"Backup checksum coverage is incomplete: {', '.join(unchecked)}")
manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8"))
if manifest.get("schema_version") != 1 or manifest.get("read_only_source") is not True:
raise RuntimeError("Backup manifest schema or read-only marker is invalid")
if manifest.get("database_password_secure") is not True:
raise RuntimeError("Backup was made from an insecure database configuration")
if manifest.get("inventory_mode") != "sha256" or manifest.get("storage_inventory_requested") is not True:
raise RuntimeError("Destructive maintenance requires a SHA-256 storage inventory backup")
created = _created_at(manifest.get("created_at"))
current = now or datetime.now(timezone.utc)
if current.tzinfo is None:
current = current.replace(tzinfo=timezone.utc)
age_hours = (current.astimezone(timezone.utc) - created).total_seconds() / 3600
if age_hours < -0.1:
raise RuntimeError("Backup timestamp is in the future")
if age_hours > max_age_hours:
raise RuntimeError(
f"Backup is {age_hours:.1f} hours old; maximum allowed age is {max_age_hours:.1f} hours"
)
release_id = manifest.get("release_id")
git_commit = manifest.get("git_commit")
if not isinstance(release_id, str) or not release_id:
raise RuntimeError("Backup release id is missing")
if not isinstance(git_commit, str) or len(git_commit) < 7:
raise RuntimeError("Backup Git commit is missing")
return VerifiedBackup(
backup_dir=root,
release_id=release_id,
created_at=created,
age_hours=age_hours,
git_commit=git_commit,
)
def require_confirmation(actual: str | None, expected: str) -> None:
if actual != expected:
raise RuntimeError(f"Refusing destructive maintenance; pass --confirm {expected}")
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
CONTAINER="${RC10_CONTAINER:-geointel}"
OUTPUT_DIR="${1:-artifacts/rc10-data-operations}"
MINIMUM_AGE_DAYS="${RC10_MINIMUM_AGE_DAYS:-7}"
mkdir -p "$OUTPUT_DIR"
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
echo "Container '$CONTAINER' is not running." >&2
exit 2
fi
read_counts() {
docker exec "$CONTAINER" sh -c '
db="${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"
user="${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"
psql -X -v ON_ERROR_STOP=1 -U "$user" -d "$db" -AtF $'"'"'\t'"'"' \
-c "SELECT table_name,
CASE table_name
WHEN '\''projects'\'' THEN (SELECT count(*) FROM projects)
WHEN '\''datasets'\'' THEN (SELECT count(*) FROM datasets)
WHEN '\''dataset_versions'\'' THEN (SELECT count(*) FROM dataset_versions)
WHEN '\''jobs'\'' THEN (SELECT count(*) FROM jobs)
WHEN '\''analysis_runs'\'' THEN (SELECT count(*) FROM analysis_runs)
WHEN '\''exports'\'' THEN (SELECT count(*) FROM exports)
WHEN '\''detections'\'' THEN (SELECT count(*) FROM detections)
WHEN '\''segmentations'\'' THEN (SELECT count(*) FROM segmentations)
WHEN '\''quality_checks'\'' THEN (SELECT count(*) FROM quality_checks)
END
FROM (VALUES
('\''projects'\''), ('\''datasets'\''), ('\''dataset_versions'\''),
('\''jobs'\''), ('\''analysis_runs'\''), ('\''exports'\''),
('\''detections'\''), ('\''segmentations'\''), ('\''quality_checks'\'')
) AS critical(table_name)
ORDER BY table_name;"
'
}
read_counts > "$OUTPUT_DIR/table-counts-before.tsv"
docker exec "$CONTAINER" python /app/scripts/audit_data_operations.py \
--minimum-age-days "$MINIMUM_AGE_DAYS" \
--fail-on-pressure never \
> "$OUTPUT_DIR/data-operations.json"
docker exec "$CONTAINER" python /app/scripts/cleanup_storage_artifacts.py \
--minimum-age-days "$MINIMUM_AGE_DAYS" \
--max-delete 25 \
> "$OUTPUT_DIR/cleanup-dry-run.json"
read_counts > "$OUTPUT_DIR/table-counts-after.tsv"
python3 - "$OUTPUT_DIR" <<'PY'
import json
import pathlib
import sys
root = pathlib.Path(sys.argv[1])
audit = json.loads((root / "data-operations.json").read_text(encoding="utf-8"))
cleanup = json.loads((root / "cleanup-dry-run.json").read_text(encoding="utf-8"))
before = (root / "table-counts-before.tsv").read_text(encoding="utf-8")
after = (root / "table-counts-after.tsv").read_text(encoding="utf-8")
if before != after:
raise SystemExit("RC10 read-only audit changed one or more critical table counts")
if audit.get("mode") != "read-only":
raise SystemExit("Data operations audit did not report read-only mode")
if cleanup.get("mode") != "dry-run" or cleanup.get("deleted_count") != 0:
raise SystemExit("Storage cleanup audit was not a zero-delete dry run")
if "release-evidence" not in audit.get("cleanup", {}).get("protected_prefixes", []):
raise SystemExit("Release evidence is not protected")
families = audit.get("database", {}).get("source_families", {})
if set(families) != {"national", "regional", "maritime"}:
raise SystemExit("National/regional/maritime source-family report is incomplete")
if audit.get("integrity", {}).get("missing_referenced_path_count", 0):
raise SystemExit("Persisted storage references are missing")
manifest = {
"schema_version": 1,
"status": "passed",
"disk_pressure": audit.get("disk_pressure"),
"storage_category_count": len(audit.get("categories", [])),
"cleanup_candidate_count": cleanup.get("candidate_count", 0),
"cleanup_candidate_bytes": cleanup.get("candidate_bytes", 0),
"critical_table_counts_unchanged": True,
"source_family_counts": {
family: len(items) for family, items in families.items()
},
"missing_referenced_path_count": 0,
}
(root / "manifest.json").write_text(
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
print(
"RC10 data operations audit passed: "
f"pressure={manifest['disk_pressure']['status']}, "
f"candidates={manifest['cleanup_candidate_count']}, "
f"evidence={root / 'manifest.json'}"
)
PY
+4
View File
@@ -119,6 +119,9 @@ ${PYTHON_BIN} -m py_compile scripts/validate_detection_false_negative_review_dec
${PYTHON_BIN} -m py_compile scripts/activate_promoted_yolo_candidate.py
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
${PYTHON_BIN} -m py_compile scripts/archive_technical_projects.py
${PYTHON_BIN} -m py_compile scripts/release_backup_guard.py
${PYTHON_BIN} -m py_compile scripts/audit_data_operations.py
${PYTHON_BIN} -m py_compile scripts/cleanup_storage_artifacts.py
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
${PYTHON_BIN} -m compileall backend/app
(cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning)
@@ -162,4 +165,5 @@ bash -n scripts/verify_demo_cleanup_dry_run.sh
bash -n scripts/capture_workbench_screenshots.sh
bash -n scripts/run_rc8_release_journeys.sh
bash -n scripts/run_rc9_ux_audit.sh
bash -n scripts/run_rc10_data_operations_audit.sh
echo "== Run readiness check passed =="