106 lines
4.4 KiB
Bash
106 lines
4.4 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$ROOT"
|
|
|
|
CONTAINER="${RC10_CONTAINER:-geointel}"
|
|
OUTPUT_DIR="${1:-artifacts/rc10-data-operations}"
|
|
MINIMUM_AGE_DAYS="${RC10_MINIMUM_AGE_DAYS:-7}"
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
|
echo "Container '$CONTAINER' is not running." >&2
|
|
exit 2
|
|
fi
|
|
|
|
read_counts() {
|
|
docker exec "$CONTAINER" sh -c '
|
|
db="${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"
|
|
user="${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"
|
|
psql -X -v ON_ERROR_STOP=1 -U "$user" -d "$db" -AtF $'"'"'\t'"'"' \
|
|
-c "SELECT table_name,
|
|
CASE table_name
|
|
WHEN '\''projects'\'' THEN (SELECT count(*) FROM projects)
|
|
WHEN '\''datasets'\'' THEN (SELECT count(*) FROM datasets)
|
|
WHEN '\''dataset_versions'\'' THEN (SELECT count(*) FROM dataset_versions)
|
|
WHEN '\''jobs'\'' THEN (SELECT count(*) FROM jobs)
|
|
WHEN '\''analysis_runs'\'' THEN (SELECT count(*) FROM analysis_runs)
|
|
WHEN '\''exports'\'' THEN (SELECT count(*) FROM exports)
|
|
WHEN '\''detections'\'' THEN (SELECT count(*) FROM detections)
|
|
WHEN '\''segmentations'\'' THEN (SELECT count(*) FROM segmentations)
|
|
WHEN '\''quality_checks'\'' THEN (SELECT count(*) FROM quality_checks)
|
|
END
|
|
FROM (VALUES
|
|
('\''projects'\''), ('\''datasets'\''), ('\''dataset_versions'\''),
|
|
('\''jobs'\''), ('\''analysis_runs'\''), ('\''exports'\''),
|
|
('\''detections'\''), ('\''segmentations'\''), ('\''quality_checks'\'')
|
|
) AS critical(table_name)
|
|
ORDER BY table_name;"
|
|
'
|
|
}
|
|
|
|
read_counts > "$OUTPUT_DIR/table-counts-before.tsv"
|
|
docker exec "$CONTAINER" python /app/scripts/audit_data_operations.py \
|
|
--minimum-age-days "$MINIMUM_AGE_DAYS" \
|
|
--fail-on-pressure never \
|
|
> "$OUTPUT_DIR/data-operations.json"
|
|
docker exec "$CONTAINER" python /app/scripts/cleanup_storage_artifacts.py \
|
|
--minimum-age-days "$MINIMUM_AGE_DAYS" \
|
|
--max-delete 25 \
|
|
> "$OUTPUT_DIR/cleanup-dry-run.json"
|
|
read_counts > "$OUTPUT_DIR/table-counts-after.tsv"
|
|
|
|
python3 - "$OUTPUT_DIR" <<'PY'
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
root = pathlib.Path(sys.argv[1])
|
|
audit = json.loads((root / "data-operations.json").read_text(encoding="utf-8"))
|
|
cleanup = json.loads((root / "cleanup-dry-run.json").read_text(encoding="utf-8"))
|
|
before = (root / "table-counts-before.tsv").read_text(encoding="utf-8")
|
|
after = (root / "table-counts-after.tsv").read_text(encoding="utf-8")
|
|
|
|
if before != after:
|
|
raise SystemExit("RC10 read-only audit changed one or more critical table counts")
|
|
if audit.get("mode") != "read-only":
|
|
raise SystemExit("Data operations audit did not report read-only mode")
|
|
if cleanup.get("mode") != "dry-run" or cleanup.get("deleted_count") != 0:
|
|
raise SystemExit("Storage cleanup audit was not a zero-delete dry run")
|
|
if "release-evidence" not in audit.get("cleanup", {}).get("protected_prefixes", []):
|
|
raise SystemExit("Release evidence is not protected")
|
|
families = audit.get("database", {}).get("source_families", {})
|
|
if set(families) != {"national", "regional", "maritime"}:
|
|
raise SystemExit("National/regional/maritime source-family report is incomplete")
|
|
if audit.get("integrity", {}).get("missing_referenced_path_count", 0):
|
|
raise SystemExit("Persisted storage references are missing")
|
|
|
|
manifest = {
|
|
"schema_version": 1,
|
|
"status": "passed",
|
|
"disk_pressure": audit.get("disk_pressure"),
|
|
"storage_category_count": len(audit.get("categories", [])),
|
|
"cleanup_candidate_count": cleanup.get("candidate_count", 0),
|
|
"cleanup_candidate_bytes": cleanup.get("candidate_bytes", 0),
|
|
"critical_table_counts_unchanged": True,
|
|
"source_family_counts": {
|
|
family: len(items) for family, items in families.items()
|
|
},
|
|
"missing_referenced_path_count": 0,
|
|
"missing_manifest_artifact_count": audit.get("integrity", {}).get(
|
|
"missing_manifest_artifact_count", 0
|
|
),
|
|
}
|
|
(root / "manifest.json").write_text(
|
|
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
print(
|
|
"RC10 data operations audit passed: "
|
|
f"pressure={manifest['disk_pressure']['status']}, "
|
|
f"candidates={manifest['cleanup_candidate_count']}, "
|
|
f"evidence={root / 'manifest.json'}"
|
|
)
|
|
PY
|