Initial public release

2026-08-31 21:56:53 +02:00
commit faeb58ef6d
1386 changed files with 263203 additions and 0 deletions
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT"
+
+CONTAINER="${RC10_CONTAINER:-geointel}"
+OUTPUT_DIR="${1:-artifacts/rc10-data-operations}"
+MINIMUM_AGE_DAYS="${RC10_MINIMUM_AGE_DAYS:-7}"
+mkdir -p "$OUTPUT_DIR"
+
+if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
+  echo "Container '$CONTAINER' is not running." >&2
+  exit 2
+fi
+
+read_counts() {
+  docker exec "$CONTAINER" sh -c '
+    db="${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"
+    user="${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"
+    psql -X -v ON_ERROR_STOP=1 -U "$user" -d "$db" -AtF $'"'"'\t'"'"' \
+      -c "SELECT table_name,
+                 CASE table_name
+                   WHEN '\''projects'\'' THEN (SELECT count(*) FROM projects)
+                   WHEN '\''datasets'\'' THEN (SELECT count(*) FROM datasets)
+                   WHEN '\''dataset_versions'\'' THEN (SELECT count(*) FROM dataset_versions)
+                   WHEN '\''jobs'\'' THEN (SELECT count(*) FROM jobs)
+                   WHEN '\''analysis_runs'\'' THEN (SELECT count(*) FROM analysis_runs)
+                   WHEN '\''exports'\'' THEN (SELECT count(*) FROM exports)
+                   WHEN '\''detections'\'' THEN (SELECT count(*) FROM detections)
+                   WHEN '\''segmentations'\'' THEN (SELECT count(*) FROM segmentations)
+                   WHEN '\''quality_checks'\'' THEN (SELECT count(*) FROM quality_checks)
+                 END
+          FROM (VALUES
+            ('\''projects'\''), ('\''datasets'\''), ('\''dataset_versions'\''),
+            ('\''jobs'\''), ('\''analysis_runs'\''), ('\''exports'\''),
+            ('\''detections'\''), ('\''segmentations'\''), ('\''quality_checks'\'')
+          ) AS critical(table_name)
+          ORDER BY table_name;"
+  '
+}
+
+read_counts > "$OUTPUT_DIR/table-counts-before.tsv"
+docker exec "$CONTAINER" python /app/scripts/audit_data_operations.py \
+  --minimum-age-days "$MINIMUM_AGE_DAYS" \
+  --fail-on-pressure never \
+  > "$OUTPUT_DIR/data-operations.json"
+docker exec "$CONTAINER" python /app/scripts/cleanup_storage_artifacts.py \
+  --minimum-age-days "$MINIMUM_AGE_DAYS" \
+  --max-delete 25 \
+  > "$OUTPUT_DIR/cleanup-dry-run.json"
+read_counts > "$OUTPUT_DIR/table-counts-after.tsv"
+
+python3 - "$OUTPUT_DIR" <<'PY'
+import json
+import pathlib
+import sys
+
+root = pathlib.Path(sys.argv[1])
+audit = json.loads((root / "data-operations.json").read_text(encoding="utf-8"))
+cleanup = json.loads((root / "cleanup-dry-run.json").read_text(encoding="utf-8"))
+before = (root / "table-counts-before.tsv").read_text(encoding="utf-8")
+after = (root / "table-counts-after.tsv").read_text(encoding="utf-8")
+
+if before != after:
+    raise SystemExit("RC10 read-only audit changed one or more critical table counts")
+if audit.get("mode") != "read-only":
+    raise SystemExit("Data operations audit did not report read-only mode")
+if cleanup.get("mode") != "dry-run" or cleanup.get("deleted_count") != 0:
+    raise SystemExit("Storage cleanup audit was not a zero-delete dry run")
+if "release-evidence" not in audit.get("cleanup", {}).get("protected_prefixes", []):
+    raise SystemExit("Release evidence is not protected")
+families = audit.get("database", {}).get("source_families", {})
+if set(families) != {"national", "regional", "maritime"}:
+    raise SystemExit("National/regional/maritime source-family report is incomplete")
+if audit.get("integrity", {}).get("missing_referenced_path_count", 0):
+    raise SystemExit("Persisted storage references are missing")
+
+manifest = {
+    "schema_version": 1,
+    "status": "passed",
+    "disk_pressure": audit.get("disk_pressure"),
+    "storage_category_count": len(audit.get("categories", [])),
+    "cleanup_candidate_count": cleanup.get("candidate_count", 0),
+    "cleanup_candidate_bytes": cleanup.get("candidate_bytes", 0),
+    "critical_table_counts_unchanged": True,
+    "source_family_counts": {
+        family: len(items) for family, items in families.items()
+    },
+    "missing_referenced_path_count": 0,
+    "missing_manifest_artifact_count": audit.get("integrity", {}).get(
+        "missing_manifest_artifact_count", 0
+    ),
+}
+(root / "manifest.json").write_text(
+    json.dumps(manifest, indent=2, sort_keys=True) + "\n",
+    encoding="utf-8",
+)
+print(
+    "RC10 data operations audit passed: "
+    f"pressure={manifest['disk_pressure']['status']}, "
+    f"candidates={manifest['cleanup_candidate_count']}, "
+    f"evidence={root / 'manifest.json'}"
+)
+PY