fix(release): make deployment backup and rollback immutable
This commit is contained in:
Jens
+19
-9
@@ -2142,11 +2142,17 @@ bash scripts/backup_release_state.sh \
|
||||
```
|
||||
|
||||
The backup is written atomically and contains a PostgreSQL custom-format dump,
|
||||
archive listing, Alembic/PostGIS metadata, critical table counts, optional
|
||||
storage/model inventories and SHA-256 checksums. An empty or known-default
|
||||
database password leaves the release gate failed. For an emergency backup
|
||||
before rotating that password, add `--allow-insecure-password`; the manifest
|
||||
still records the insecure state.
|
||||
archive listing, Alembic/PostGIS metadata, critical table counts and
|
||||
byte-complete SHA-256-verified storage/model snapshots. The first snapshot is a
|
||||
full copy; a later deployment may hard-link only checksum-identical files from
|
||||
another completed, fully verified backup with `--link-dest-backup`. It never
|
||||
hard-links a live source file and never deletes an older backup. An empty or
|
||||
known-default database password leaves the release gate failed. For an
|
||||
emergency backup before rotating that password, add
|
||||
`--allow-insecure-password`; the manifest still records the insecure state.
|
||||
`backup_tool_revision` identifies the candidate source that executed the
|
||||
backup; `running_image_revision` identifies the currently running old image.
|
||||
Rollback is always bound to the retained immutable Docker `image_id`.
|
||||
|
||||
Verify without changing any database:
|
||||
|
||||
@@ -2190,7 +2196,7 @@ docker exec geointel python /app/scripts/audit_data_operations.py \
|
||||
--output /app/storage/release-evidence/rc-current/data-operations.json
|
||||
```
|
||||
|
||||
Preview old unreferenced derived/cache/export candidates without deletion:
|
||||
Preview old unreferenced derived/cache/export candidates without mutation:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
|
||||
@@ -2199,10 +2205,14 @@ docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
|
||||
```
|
||||
|
||||
Apply requires a reviewed candidate count, the exact
|
||||
`DELETE_STORAGE_ARTIFACTS` token and a backup no older than 24 hours with a
|
||||
checksum-verified database dump and SHA-256 storage inventory. The host backup
|
||||
`QUARANTINE_STORAGE_ARTIFACTS` token and a backup no older than 24 hours with a
|
||||
checksum-verified database dump and byte-complete storage snapshot. The host backup
|
||||
root is mounted read-only at `/app/backups`. See
|
||||
`docs/DATA_OPERATIONS_RUNBOOK.md`. No cleanup is scheduled by GeoIntel.
|
||||
`docs/DATA_OPERATIONS_RUNBOOK.md`. Candidates enter protected
|
||||
`operator-evidence/cleanup-quarantine` storage through an interruption-safe
|
||||
hard-link/unlink state machine. `restore_storage_quarantine.py` reverses that
|
||||
move with the exact `RESTORE_QUARANTINED_ARTIFACTS` token and refuses to
|
||||
overwrite an existing original path. No cleanup is scheduled by GeoIntel.
|
||||
|
||||
## RC-8 Belgium/North Sea release journeys
|
||||
|
||||
|
||||
@@ -14,17 +14,26 @@ from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from sqlalchemy import func
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BACKEND_ROOT = ROOT / "backend" if (ROOT / "backend" / "app").is_dir() else ROOT
|
||||
if str(BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_ROOT))
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.session import SessionLocal
|
||||
from sqlalchemy import func
|
||||
|
||||
from app.models import AnalysisRun, Dataset, DatasetVersion, Detection, Export, Job, Project, Segmentation
|
||||
from app.core.config import get_settings # noqa: E402 - imported after backend path bootstrap
|
||||
from app.db.session import SessionLocal # noqa: E402 - imported after backend path bootstrap
|
||||
from app.models import ( # noqa: E402 - imported after backend path bootstrap
|
||||
AnalysisRun,
|
||||
Dataset,
|
||||
DatasetVersion,
|
||||
Detection,
|
||||
Export,
|
||||
Job,
|
||||
Project,
|
||||
Segmentation,
|
||||
)
|
||||
|
||||
|
||||
NATIONAL_PROJECT_NAME = "Belgium and North Sea Workbench"
|
||||
|
||||
+163
-48
@@ -3,12 +3,14 @@ set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
CONTAINER="geointel"
|
||||
OUTPUT_ROOT="backups"
|
||||
OUTPUT_ROOT="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
|
||||
RELEASE_ID="rc-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
STORAGE_PATH=""
|
||||
MODELS_PATH=""
|
||||
INVENTORY_MODE="metadata"
|
||||
ALLOW_INSECURE_PASSWORD="false"
|
||||
LINK_DEST_BACKUP=""
|
||||
ROLLBACK_IMAGE_TAG=""
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
@@ -19,11 +21,16 @@ GeoIntel container. It never deletes or restores application data.
|
||||
|
||||
Options:
|
||||
--container NAME Docker container (default: geointel)
|
||||
--output-root PATH Host backup root (default: backups)
|
||||
--output-root PATH Host backup root (default:
|
||||
/mnt/user/appdata/geointel/backups)
|
||||
--release-id ID Safe backup directory name
|
||||
--storage-path PATH Optional host storage path to inventory
|
||||
--models-path PATH Optional host model path to inventory
|
||||
--inventory-mode metadata|sha256 Hash all inventoried files only with sha256
|
||||
--storage-path PATH Host storage path to snapshot byte-for-byte
|
||||
--models-path PATH Host model path to snapshot byte-for-byte
|
||||
--inventory-mode metadata|sha256 Retained manifest compatibility setting
|
||||
--link-dest-backup PATH Verified older backup used only to hard-link
|
||||
checksum-identical backup-to-backup files
|
||||
--rollback-image-tag TAG Immutable backup-specific tag bound to the
|
||||
running image ID
|
||||
--allow-insecure-password Complete emergency backup despite an
|
||||
empty/default production DB password
|
||||
EOF
|
||||
@@ -37,6 +44,8 @@ while [ "$#" -gt 0 ]; do
|
||||
--storage-path) STORAGE_PATH="$2"; shift 2 ;;
|
||||
--models-path) MODELS_PATH="$2"; shift 2 ;;
|
||||
--inventory-mode) INVENTORY_MODE="$2"; shift 2 ;;
|
||||
--link-dest-backup) LINK_DEST_BACKUP="$2"; shift 2 ;;
|
||||
--rollback-image-tag) ROLLBACK_IMAGE_TAG="$2"; shift 2 ;;
|
||||
--allow-insecure-password) ALLOW_INSECURE_PASSWORD="true"; shift ;;
|
||||
--help|-h) usage; exit 0 ;;
|
||||
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||
@@ -51,12 +60,93 @@ if [ "$INVENTORY_MODE" != "metadata" ] && [ "$INVENTORY_MODE" != "sha256" ]; the
|
||||
echo "--inventory-mode must be metadata or sha256" >&2
|
||||
exit 2
|
||||
fi
|
||||
for required in docker python3 sha256sum git; do
|
||||
for required in docker python3 sha256sum; do
|
||||
if ! command -v "$required" >/dev/null 2>&1; then
|
||||
echo "Missing required command: $required" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
resolve_source_revision() {
|
||||
local controller_sha="" explicit_sha="${GEOINTEL_BUILD_SHA:-}"
|
||||
local gitea_sha="${GITEA_COMMIT_SHA:-}" github_sha="${GITHUB_SHA:-}"
|
||||
local git_head="" git_dirty="false" source=""
|
||||
|
||||
if [ -n "$gitea_sha" ]; then
|
||||
if ! [[ "$gitea_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
|
||||
echo "GITEA_COMMIT_SHA must contain one full 40-character Git commit SHA." >&2
|
||||
return 2
|
||||
fi
|
||||
controller_sha="${gitea_sha,,}"
|
||||
source="GITEA_COMMIT_SHA"
|
||||
fi
|
||||
if [ -n "$github_sha" ]; then
|
||||
if ! [[ "$github_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
|
||||
echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2
|
||||
return 2
|
||||
fi
|
||||
github_sha="${github_sha,,}"
|
||||
if [ -n "$controller_sha" ] && [ "$controller_sha" != "$github_sha" ]; then
|
||||
echo "Controller commit variables disagree." >&2
|
||||
return 2
|
||||
fi
|
||||
controller_sha="$github_sha"
|
||||
source="${source:-GITHUB_SHA}"
|
||||
fi
|
||||
if [ -n "$explicit_sha" ]; then
|
||||
if ! [[ "$explicit_sha" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]]; then
|
||||
echo "GEOINTEL_BUILD_SHA contains an unsafe release revision." >&2
|
||||
return 2
|
||||
fi
|
||||
explicit_sha="${explicit_sha,,}"
|
||||
fi
|
||||
if [ -n "$controller_sha" ]; then
|
||||
if [ -n "$explicit_sha" ] && [ "$explicit_sha" != "$controller_sha" ]; then
|
||||
echo "GEOINTEL_BUILD_SHA differs from the controller revision." >&2
|
||||
return 2
|
||||
fi
|
||||
explicit_sha="$controller_sha"
|
||||
elif [ -n "${GITEA_REPOSITORY:-}${GITHUB_REPOSITORY:-}" ]; then
|
||||
if ! [[ "$explicit_sha" =~ ^[0-9a-f]{40}$ ]]; then
|
||||
echo "Automated backup requires a full controller or GEOINTEL_BUILD_SHA revision." >&2
|
||||
return 2
|
||||
fi
|
||||
source="GEOINTEL_BUILD_SHA"
|
||||
fi
|
||||
|
||||
if command -v git >/dev/null 2>&1 && git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1; then
|
||||
git_head="$(git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)"
|
||||
git_head="${git_head,,}"
|
||||
if ! [[ "$git_head" =~ ^[0-9a-f]{40}$ ]]; then
|
||||
echo "Could not resolve a full Git revision from the source checkout." >&2
|
||||
return 2
|
||||
fi
|
||||
if [ -n "$explicit_sha" ] && [[ "$explicit_sha" =~ ^[0-9a-f]{40}$ ]] && [ "$git_head" != "$explicit_sha" ]; then
|
||||
echo "Source checkout does not match the supplied release revision." >&2
|
||||
return 2
|
||||
fi
|
||||
if [ -n "$(git -C "$ROOT" status --porcelain=v1 2>/dev/null)" ]; then
|
||||
git_dirty="true"
|
||||
fi
|
||||
if [ -z "$explicit_sha" ]; then
|
||||
explicit_sha="$git_head"
|
||||
source="git"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$explicit_sha" ]; then
|
||||
echo "Cannot bind backup to a source revision; provide GEOINTEL_BUILD_SHA or a controller SHA." >&2
|
||||
return 2
|
||||
fi
|
||||
SOURCE_REVISION="$explicit_sha"
|
||||
SOURCE_REVISION_SOURCE="${source:-GEOINTEL_BUILD_SHA}"
|
||||
SOURCE_GIT_DIRTY="$git_dirty"
|
||||
}
|
||||
|
||||
SOURCE_REVISION=""
|
||||
SOURCE_REVISION_SOURCE=""
|
||||
SOURCE_GIT_DIRTY="false"
|
||||
resolve_source_revision
|
||||
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
||||
echo "Container '$CONTAINER' is not running." >&2
|
||||
exit 3
|
||||
@@ -72,6 +162,27 @@ if [ -e "$PARTIAL" ] || [ -e "$FINAL" ]; then
|
||||
fi
|
||||
mkdir -p "$PARTIAL"
|
||||
|
||||
if [ -n "$LINK_DEST_BACKUP" ]; then
|
||||
LINK_DEST_BACKUP="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve(strict=True))' "$LINK_DEST_BACKUP")"
|
||||
python3 - "$OUTPUT_ROOT" "$LINK_DEST_BACKUP" <<'PY'
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
root = pathlib.Path(sys.argv[1])
|
||||
candidate = pathlib.Path(sys.argv[2])
|
||||
try:
|
||||
candidate.relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise SystemExit(f"Link-dest backup must remain below {root}") from exc
|
||||
if candidate == root or candidate.name.startswith("."):
|
||||
raise SystemExit("Link-dest backup must identify one completed immutable backup")
|
||||
PY
|
||||
(
|
||||
cd "$LINK_DEST_BACKUP"
|
||||
sha256sum -c CHECKSUMS.sha256 >/dev/null
|
||||
)
|
||||
fi
|
||||
|
||||
cleanup_partial() {
|
||||
if [ -d "$PARTIAL" ]; then
|
||||
rm -rf -- "$PARTIAL"
|
||||
@@ -113,10 +224,21 @@ test -s "$PARTIAL/database.list"
|
||||
|
||||
IMAGE_ID="$(docker inspect -f '{{.Image}}' "$CONTAINER")"
|
||||
IMAGE_NAME="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER")"
|
||||
GIT_COMMIT="$(git -C "$ROOT" rev-parse HEAD)"
|
||||
GIT_DIRTY="false"
|
||||
if [ -n "$(git -C "$ROOT" status --porcelain=v1)" ]; then
|
||||
GIT_DIRTY="true"
|
||||
RUNNING_IMAGE_REVISION="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$CONTAINER")"
|
||||
RUNNING_IMAGE_REVISION_IS_FULL_SHA="false"
|
||||
if [[ "$RUNNING_IMAGE_REVISION" =~ ^[0-9A-Fa-f]{40}$ ]]; then
|
||||
RUNNING_IMAGE_REVISION="${RUNNING_IMAGE_REVISION,,}"
|
||||
RUNNING_IMAGE_REVISION_IS_FULL_SHA="true"
|
||||
elif ! [[ "$RUNNING_IMAGE_REVISION" =~ ^[A-Za-z0-9._-]{1,128}$ ]]; then
|
||||
echo "Running image has an unsafe or missing OCI revision label." >&2
|
||||
exit 3
|
||||
fi
|
||||
if [ -n "$ROLLBACK_IMAGE_TAG" ]; then
|
||||
TAGGED_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$ROLLBACK_IMAGE_TAG" 2>/dev/null || true)"
|
||||
if [ "$TAGGED_IMAGE_ID" != "$IMAGE_ID" ]; then
|
||||
echo "Backup-specific rollback tag does not resolve to the running image ID." >&2
|
||||
exit 3
|
||||
fi
|
||||
fi
|
||||
|
||||
docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -AtF $'\t' \
|
||||
@@ -132,48 +254,32 @@ for table in projects areas datasets dataset_versions vector_features jobs analy
|
||||
printf '%s\t%s\n' "$table" "$count" >> "$PARTIAL/table-counts.tsv"
|
||||
done
|
||||
|
||||
inventory_path() {
|
||||
snapshot_path() {
|
||||
local source_path="$1"
|
||||
local output_path="$2"
|
||||
local label="$2"
|
||||
local manifest_path="$PARTIAL/${label}-manifest.tsv"
|
||||
local snapshot_path="$PARTIAL/${label}-snapshot"
|
||||
local link_args=()
|
||||
if [ -z "$source_path" ]; then
|
||||
printf 'not_requested\n' > "$output_path"
|
||||
printf 'not_requested\n' > "$manifest_path"
|
||||
return
|
||||
fi
|
||||
python3 - "$source_path" "$output_path" "$INVENTORY_MODE" <<'PY'
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
root = pathlib.Path(sys.argv[1]).expanduser().resolve()
|
||||
output = pathlib.Path(sys.argv[2])
|
||||
mode = sys.argv[3]
|
||||
if not root.is_dir():
|
||||
raise SystemExit(f"Inventory root is not a directory: {root}")
|
||||
|
||||
def digest(path: pathlib.Path) -> str:
|
||||
value = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
value.update(chunk)
|
||||
return value.hexdigest()
|
||||
|
||||
with output.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write("relative_path\tsize_bytes\tmtime_ns\tsha256\n")
|
||||
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
continue
|
||||
stat = path.stat()
|
||||
checksum = digest(path) if mode == "sha256" else ""
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if "\t" in relative or "\n" in relative:
|
||||
raise SystemExit(f"Unsupported inventory path: {relative!r}")
|
||||
handle.write(f"{relative}\t{stat.st_size}\t{stat.st_mtime_ns}\t{checksum}\n")
|
||||
PY
|
||||
if [ -n "$LINK_DEST_BACKUP" ]; then
|
||||
link_args=(
|
||||
--link-dest-snapshot "$LINK_DEST_BACKUP/${label}-snapshot"
|
||||
--link-dest-manifest "$LINK_DEST_BACKUP/${label}-manifest.tsv"
|
||||
)
|
||||
fi
|
||||
python3 "$ROOT/scripts/release_backup_snapshot.py" create \
|
||||
--source "$source_path" \
|
||||
--snapshot "$snapshot_path" \
|
||||
--manifest "$manifest_path" \
|
||||
--label "$label" \
|
||||
"${link_args[@]}"
|
||||
}
|
||||
|
||||
inventory_path "$STORAGE_PATH" "$PARTIAL/storage-manifest.tsv"
|
||||
inventory_path "$MODELS_PATH" "$PARTIAL/models-manifest.tsv"
|
||||
snapshot_path "$STORAGE_PATH" storage
|
||||
snapshot_path "$MODELS_PATH" models
|
||||
|
||||
python3 - "$PARTIAL/manifest.json" <<PY
|
||||
import json
|
||||
@@ -190,16 +296,25 @@ payload = {
|
||||
"database_password_secure": ${PASSWORD_SECURE@Q} == "true",
|
||||
"image_id": ${IMAGE_ID@Q},
|
||||
"image_name": ${IMAGE_NAME@Q},
|
||||
"git_commit": ${GIT_COMMIT@Q},
|
||||
"git_dirty": ${GIT_DIRTY@Q} == "true",
|
||||
"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None,
|
||||
"backup_tool_revision": ${SOURCE_REVISION@Q},
|
||||
"backup_tool_revision_source": ${SOURCE_REVISION_SOURCE@Q},
|
||||
"backup_tool_git_dirty": ${SOURCE_GIT_DIRTY@Q} == "true",
|
||||
"running_image_revision": ${RUNNING_IMAGE_REVISION@Q},
|
||||
"running_image_revision_is_full_sha": ${RUNNING_IMAGE_REVISION_IS_FULL_SHA@Q} == "true",
|
||||
"inventory_mode": ${INVENTORY_MODE@Q},
|
||||
"storage_inventory_requested": bool(${STORAGE_PATH@Q}),
|
||||
"models_inventory_requested": bool(${MODELS_PATH@Q}),
|
||||
"storage_snapshot_requested": bool(${STORAGE_PATH@Q}),
|
||||
"models_snapshot_requested": bool(${MODELS_PATH@Q}),
|
||||
"link_dest_backup": ${LINK_DEST_BACKUP@Q} or None,
|
||||
}
|
||||
path = pathlib.Path(__import__("sys").argv[1])
|
||||
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
PY
|
||||
|
||||
python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$PARTIAL"
|
||||
|
||||
(
|
||||
cd "$PARTIAL"
|
||||
find . -maxdepth 1 -type f ! -name CHECKSUMS.sha256 -printf '%f\n' \
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dry-run-first cleanup for old unreferenced derived/cache artifacts."""
|
||||
"""Dry-run-first quarantine for old unreferenced derived/cache artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from audit_data_operations import build_report
|
||||
from release_backup_guard import require_confirmation, verify_current_backup
|
||||
@@ -14,7 +18,21 @@ from app.core.config import get_settings
|
||||
from app.db.session import SessionLocal
|
||||
|
||||
|
||||
CONFIRMATION = "DELETE_STORAGE_ARTIFACTS"
|
||||
CONFIRMATION = "QUARANTINE_STORAGE_ARTIFACTS"
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_manifest(path: Path, payload: dict[str, object]) -> None:
|
||||
temporary = path.with_suffix(".json.partial")
|
||||
temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
@@ -26,6 +44,11 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--confirm")
|
||||
parser.add_argument("--backup-dir", type=Path)
|
||||
parser.add_argument("--backup-max-age-hours", type=float, default=24.0)
|
||||
parser.add_argument(
|
||||
"--quarantine-root",
|
||||
type=Path,
|
||||
help="Protected destination below the storage root (default: operator-evidence/cleanup-quarantine)",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -44,7 +67,8 @@ def main() -> int:
|
||||
|
||||
blocked_reason = None
|
||||
backup = None
|
||||
deleted: list[str] = []
|
||||
quarantined: list[dict[str, object]] = []
|
||||
quarantine_manifest: Path | None = None
|
||||
if args.apply:
|
||||
require_confirmation(args.confirm, CONFIRMATION)
|
||||
if args.backup_dir is None:
|
||||
@@ -59,9 +83,80 @@ def main() -> int:
|
||||
"review the dry run and raise the explicit limit"
|
||||
)
|
||||
else:
|
||||
quarantine_root = (
|
||||
args.quarantine_root
|
||||
or storage_root / "operator-evidence" / "cleanup-quarantine"
|
||||
).resolve()
|
||||
protected_quarantine_root = (
|
||||
storage_root / "operator-evidence" / "cleanup-quarantine"
|
||||
).resolve()
|
||||
try:
|
||||
quarantine_root.relative_to(protected_quarantine_root)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(
|
||||
"--quarantine-root must remain below "
|
||||
"operator-evidence/cleanup-quarantine in --storage-root"
|
||||
) from exc
|
||||
operation_id = f"cleanup-{datetime.now(timezone.utc):%Y%m%dT%H%M%SZ}-{uuid4().hex[:12]}"
|
||||
operation_root = quarantine_root / operation_id
|
||||
operation_root.mkdir(parents=True, exist_ok=False)
|
||||
quarantine_manifest = operation_root / "manifest.json"
|
||||
entries: list[dict[str, object]] = []
|
||||
for candidate in candidates:
|
||||
if candidate.path.is_symlink():
|
||||
raise RuntimeError(f"Cleanup candidate became a symlink: {candidate.relative_path}")
|
||||
try:
|
||||
candidate.path.resolve().relative_to(storage_root)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(
|
||||
f"Cleanup candidate escaped storage: {candidate.relative_path}"
|
||||
) from exc
|
||||
destination = operation_root / "files" / candidate.relative_path
|
||||
current_size = candidate.path.stat().st_size
|
||||
if current_size != candidate.size_bytes:
|
||||
raise RuntimeError(f"Cleanup candidate changed size: {candidate.relative_path}")
|
||||
entries.append(
|
||||
{
|
||||
"relative_path": candidate.relative_path,
|
||||
"size_bytes": current_size,
|
||||
"sha256": sha256(candidate.path),
|
||||
"status": "planned",
|
||||
"quarantine_relative_path": destination.relative_to(storage_root).as_posix(),
|
||||
}
|
||||
)
|
||||
manifest: dict[str, object] = {
|
||||
"schema_version": 1,
|
||||
"operation_id": operation_id,
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"state": "in_progress",
|
||||
"storage_root": str(storage_root),
|
||||
"backup_release_id": backup.release_id,
|
||||
"entries": entries,
|
||||
}
|
||||
write_manifest(quarantine_manifest, manifest)
|
||||
for candidate, entry in zip(candidates, entries, strict=True):
|
||||
destination = storage_root / str(entry["quarantine_relative_path"])
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
os.link(candidate.path, destination, follow_symlinks=False)
|
||||
except FileExistsError as exc:
|
||||
raise RuntimeError(f"Quarantine destination already exists: {destination}") from exc
|
||||
if (
|
||||
not destination.is_file()
|
||||
or destination.stat().st_size != entry["size_bytes"]
|
||||
or sha256(destination) != entry["sha256"]
|
||||
):
|
||||
destination.unlink(missing_ok=True)
|
||||
raise RuntimeError(f"Quarantine link verification failed: {candidate.relative_path}")
|
||||
entry["status"] = "linked"
|
||||
write_manifest(quarantine_manifest, manifest)
|
||||
candidate.path.unlink()
|
||||
deleted.append(candidate.relative_path)
|
||||
entry["status"] = "quarantined"
|
||||
quarantined.append(dict(entry))
|
||||
write_manifest(quarantine_manifest, manifest)
|
||||
manifest["state"] = "complete"
|
||||
manifest["completed_at"] = datetime.now(timezone.utc).isoformat()
|
||||
write_manifest(quarantine_manifest, manifest)
|
||||
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
@@ -72,8 +167,11 @@ def main() -> int:
|
||||
"candidate_count": len(candidates),
|
||||
"candidate_bytes": sum(item.size_bytes for item in candidates),
|
||||
"candidates": [item.relative_path for item in candidates],
|
||||
"deleted_count": len(deleted),
|
||||
"deleted": deleted,
|
||||
"deleted_count": 0,
|
||||
"deleted": [],
|
||||
"quarantined_count": len(quarantined),
|
||||
"quarantined": quarantined,
|
||||
"quarantine_manifest": str(quarantine_manifest) if quarantine_manifest else None,
|
||||
"blocked_reason": blocked_reason,
|
||||
"protected_prefixes": report["cleanup"]["protected_prefixes"],
|
||||
"backup": (
|
||||
@@ -81,7 +179,7 @@ def main() -> int:
|
||||
"release_id": backup.release_id,
|
||||
"created_at": backup.created_at.isoformat(),
|
||||
"age_hours": round(backup.age_hours, 3),
|
||||
"git_commit": backup.git_commit,
|
||||
"backup_tool_revision": backup.backup_tool_revision,
|
||||
}
|
||||
if backup
|
||||
else None
|
||||
|
||||
@@ -9,6 +9,8 @@ from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from release_backup_snapshot import verify_backup as verify_byte_snapshots
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VerifiedBackup:
|
||||
@@ -16,7 +18,7 @@ class VerifiedBackup:
|
||||
release_id: str
|
||||
created_at: datetime
|
||||
age_hours: float
|
||||
git_commit: str
|
||||
backup_tool_revision: str
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
@@ -62,6 +64,8 @@ def verify_current_backup(
|
||||
missing = sorted(name for name in required if not (root / name).is_file())
|
||||
if missing:
|
||||
raise RuntimeError(f"Backup is incomplete; missing: {', '.join(missing)}")
|
||||
if not (root / "storage-snapshot").is_dir():
|
||||
raise RuntimeError("Backup is incomplete; missing: storage-snapshot")
|
||||
|
||||
checksum_lines = (root / "CHECKSUMS.sha256").read_text(encoding="utf-8").splitlines()
|
||||
checked: set[str] = set()
|
||||
@@ -93,6 +97,9 @@ def verify_current_backup(
|
||||
raise RuntimeError("Backup was made from an insecure database configuration")
|
||||
if manifest.get("inventory_mode") != "sha256" or manifest.get("storage_inventory_requested") is not True:
|
||||
raise RuntimeError("Destructive maintenance requires a SHA-256 storage inventory backup")
|
||||
if manifest.get("storage_snapshot_requested") is not True:
|
||||
raise RuntimeError("Destructive maintenance requires a byte-complete storage snapshot")
|
||||
verify_byte_snapshots(root)
|
||||
|
||||
created = _created_at(manifest.get("created_at"))
|
||||
current = now or datetime.now(timezone.utc)
|
||||
@@ -107,17 +114,17 @@ def verify_current_backup(
|
||||
)
|
||||
|
||||
release_id = manifest.get("release_id")
|
||||
git_commit = manifest.get("git_commit")
|
||||
backup_tool_revision = manifest.get("backup_tool_revision", manifest.get("git_commit"))
|
||||
if not isinstance(release_id, str) or not release_id:
|
||||
raise RuntimeError("Backup release id is missing")
|
||||
if not isinstance(git_commit, str) or len(git_commit) < 7:
|
||||
raise RuntimeError("Backup Git commit is missing")
|
||||
if not isinstance(backup_tool_revision, str) or len(backup_tool_revision) < 7:
|
||||
raise RuntimeError("Backup tool revision is missing")
|
||||
return VerifiedBackup(
|
||||
backup_dir=root,
|
||||
release_id=release_id,
|
||||
created_at=created,
|
||||
age_hours=age_hours,
|
||||
git_commit=git_commit,
|
||||
backup_tool_revision=backup_tool_revision,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create and verify byte-complete, symlink-safe release backup snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
|
||||
MANIFEST_HEADER = "relative_path\tsize_bytes\tmtime_ns\tsha256"
|
||||
SAFE_LABEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
|
||||
FICLONE = 0x40049409
|
||||
FICLONE_FALLBACK_ERRORS = {
|
||||
errno.EXDEV,
|
||||
errno.EOPNOTSUPP,
|
||||
errno.ENOTTY,
|
||||
errno.EINVAL,
|
||||
errno.ENOSYS,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceEntry:
|
||||
path: Path
|
||||
relative_path: str
|
||||
stat_result: os.stat_result
|
||||
is_directory: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ManifestEntry:
|
||||
relative_path: str
|
||||
size_bytes: int
|
||||
mtime_ns: int
|
||||
sha256: str
|
||||
|
||||
|
||||
def _safe_relative(value: str) -> str:
|
||||
if not value or "\t" in value or "\n" in value or "\r" in value:
|
||||
raise RuntimeError(f"Unsupported snapshot path: {value!r}")
|
||||
candidate = PurePosixPath(value)
|
||||
if candidate.is_absolute() or any(part in {"", ".", ".."} for part in candidate.parts):
|
||||
raise RuntimeError(f"Unsafe snapshot path: {value!r}")
|
||||
return candidate.as_posix()
|
||||
|
||||
|
||||
def _collect(root: Path) -> list[SourceEntry]:
|
||||
entries: list[SourceEntry] = []
|
||||
for current, directory_names, file_names in os.walk(root, topdown=True, followlinks=False):
|
||||
directory_names.sort()
|
||||
file_names.sort()
|
||||
current_path = Path(current)
|
||||
for name, is_directory in [
|
||||
*((name, True) for name in directory_names),
|
||||
*((name, False) for name in file_names),
|
||||
]:
|
||||
path = current_path / name
|
||||
details = path.lstat()
|
||||
relative = _safe_relative(path.relative_to(root).as_posix())
|
||||
if stat.S_ISLNK(details.st_mode):
|
||||
raise RuntimeError(f"Release snapshot refuses symlinked content: {relative}")
|
||||
if is_directory and not stat.S_ISDIR(details.st_mode):
|
||||
raise RuntimeError(f"Snapshot directory changed during inventory: {relative}")
|
||||
if not is_directory and not stat.S_ISREG(details.st_mode):
|
||||
raise RuntimeError(f"Release snapshot refuses non-regular content: {relative}")
|
||||
entries.append(SourceEntry(path, relative, details, is_directory))
|
||||
return entries
|
||||
|
||||
|
||||
def _same_file_state(before: os.stat_result, after: os.stat_result) -> bool:
|
||||
return (
|
||||
before.st_dev,
|
||||
before.st_ino,
|
||||
before.st_size,
|
||||
before.st_mtime_ns,
|
||||
) == (
|
||||
after.st_dev,
|
||||
after.st_ino,
|
||||
after.st_size,
|
||||
after.st_mtime_ns,
|
||||
)
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _copy_all(source_descriptor: int, destination_descriptor: int) -> None:
|
||||
while True:
|
||||
value = os.read(source_descriptor, 1024 * 1024)
|
||||
if not value:
|
||||
return
|
||||
view = memoryview(value)
|
||||
while view:
|
||||
written = os.write(destination_descriptor, view)
|
||||
if written <= 0:
|
||||
raise RuntimeError("Snapshot copy stopped before writing all bytes")
|
||||
view = view[written:]
|
||||
|
||||
|
||||
def _clone_or_copy(entry: SourceEntry, destination: Path) -> ManifestEntry:
|
||||
source_flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
||||
destination_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0)
|
||||
source_descriptor = os.open(entry.path, source_flags)
|
||||
destination_descriptor = -1
|
||||
try:
|
||||
opened = os.fstat(source_descriptor)
|
||||
if not stat.S_ISREG(opened.st_mode) or not _same_file_state(entry.stat_result, opened):
|
||||
raise RuntimeError(f"Snapshot file changed before copying: {entry.relative_path}")
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination_descriptor = os.open(destination, destination_flags, stat.S_IMODE(opened.st_mode))
|
||||
cloned = False
|
||||
if os.name == "posix":
|
||||
try:
|
||||
import fcntl
|
||||
|
||||
fcntl.ioctl(destination_descriptor, FICLONE, source_descriptor)
|
||||
cloned = True
|
||||
except OSError as exc:
|
||||
if exc.errno not in FICLONE_FALLBACK_ERRORS:
|
||||
raise
|
||||
if not cloned:
|
||||
os.lseek(source_descriptor, 0, os.SEEK_SET)
|
||||
os.ftruncate(destination_descriptor, 0)
|
||||
_copy_all(source_descriptor, destination_descriptor)
|
||||
os.fsync(destination_descriptor)
|
||||
after = os.fstat(source_descriptor)
|
||||
if not _same_file_state(opened, after):
|
||||
raise RuntimeError(f"Snapshot file changed while copying: {entry.relative_path}")
|
||||
except BaseException:
|
||||
if destination_descriptor >= 0:
|
||||
os.close(destination_descriptor)
|
||||
destination_descriptor = -1
|
||||
destination.unlink(missing_ok=True)
|
||||
raise
|
||||
finally:
|
||||
if destination_descriptor >= 0:
|
||||
os.close(destination_descriptor)
|
||||
os.close(source_descriptor)
|
||||
|
||||
os.chmod(destination, stat.S_IMODE(entry.stat_result.st_mode) & ~0o222, follow_symlinks=False)
|
||||
retained_times = (entry.stat_result.st_atime_ns, entry.stat_result.st_mtime_ns)
|
||||
try:
|
||||
os.utime(destination, ns=retained_times, follow_symlinks=False)
|
||||
except NotImplementedError:
|
||||
# Windows does not expose no-follow utime. The destination was created
|
||||
# exclusively above; recheck it before using the portable call.
|
||||
if destination.is_symlink():
|
||||
destination.unlink(missing_ok=True)
|
||||
raise RuntimeError(f"Snapshot destination became a symlink: {entry.relative_path}")
|
||||
os.utime(destination, ns=retained_times)
|
||||
source_checksum = _sha256(entry.path)
|
||||
snapshot_checksum = _sha256(destination)
|
||||
final_source = entry.path.lstat()
|
||||
if not _same_file_state(entry.stat_result, final_source):
|
||||
raise RuntimeError(f"Snapshot file changed during checksum verification: {entry.relative_path}")
|
||||
if source_checksum != snapshot_checksum:
|
||||
raise RuntimeError(f"Snapshot checksum differs from source: {entry.relative_path}")
|
||||
return ManifestEntry(
|
||||
relative_path=entry.relative_path,
|
||||
size_bytes=entry.stat_result.st_size,
|
||||
mtime_ns=entry.stat_result.st_mtime_ns,
|
||||
sha256=snapshot_checksum,
|
||||
)
|
||||
|
||||
|
||||
def _link_verified_prior(
|
||||
entry: SourceEntry,
|
||||
destination: Path,
|
||||
prior_root: Path,
|
||||
prior_manifest: dict[str, ManifestEntry],
|
||||
) -> ManifestEntry | None:
|
||||
retained = prior_manifest.get(entry.relative_path)
|
||||
if retained is None or retained.size_bytes != entry.stat_result.st_size:
|
||||
return None
|
||||
source_checksum = _sha256(entry.path)
|
||||
final_source = entry.path.lstat()
|
||||
if not _same_file_state(entry.stat_result, final_source):
|
||||
raise RuntimeError(f"Snapshot file changed during prior comparison: {entry.relative_path}")
|
||||
if source_checksum != retained.sha256:
|
||||
return None
|
||||
prior_path = prior_root / entry.relative_path
|
||||
try:
|
||||
prior_details = prior_path.lstat()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if not stat.S_ISREG(prior_details.st_mode) or prior_details.st_size != retained.size_bytes:
|
||||
raise RuntimeError(f"Prior snapshot file is not reusable: {entry.relative_path}")
|
||||
if _sha256(prior_path) != retained.sha256:
|
||||
raise RuntimeError(f"Prior snapshot checksum changed: {entry.relative_path}")
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
os.link(prior_path, destination, follow_symlinks=False)
|
||||
if destination.stat().st_size != retained.size_bytes or _sha256(destination) != retained.sha256:
|
||||
destination.unlink(missing_ok=True)
|
||||
raise RuntimeError(f"Hard-linked snapshot verification failed: {entry.relative_path}")
|
||||
return ManifestEntry(
|
||||
relative_path=entry.relative_path,
|
||||
size_bytes=entry.stat_result.st_size,
|
||||
mtime_ns=entry.stat_result.st_mtime_ns,
|
||||
sha256=source_checksum,
|
||||
)
|
||||
|
||||
|
||||
def read_manifest(path: Path) -> dict[str, ManifestEntry]:
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
if not lines or lines[0] != MANIFEST_HEADER:
|
||||
raise RuntimeError(f"Snapshot inventory has an invalid header: {path}")
|
||||
entries: dict[str, ManifestEntry] = {}
|
||||
for line in lines[1:]:
|
||||
fields = line.split("\t")
|
||||
if len(fields) != 4:
|
||||
raise RuntimeError(f"Snapshot inventory has an invalid row: {line!r}")
|
||||
relative_path, size_text, mtime_text, checksum = fields
|
||||
relative_path = _safe_relative(relative_path)
|
||||
if relative_path in entries:
|
||||
raise RuntimeError(f"Snapshot inventory repeats a path: {relative_path}")
|
||||
try:
|
||||
size_bytes = int(size_text)
|
||||
mtime_ns = int(mtime_text)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"Snapshot inventory has invalid metadata: {relative_path}") from exc
|
||||
if size_bytes < 0 or mtime_ns < 0 or not re.fullmatch(r"[0-9a-f]{64}", checksum):
|
||||
raise RuntimeError(f"Snapshot inventory has invalid retained state: {relative_path}")
|
||||
entries[relative_path] = ManifestEntry(relative_path, size_bytes, mtime_ns, checksum)
|
||||
return entries
|
||||
|
||||
|
||||
def verify_snapshot(snapshot_path: Path, manifest_path: Path) -> None:
|
||||
root = snapshot_path.expanduser().resolve(strict=True)
|
||||
if not root.is_dir():
|
||||
raise RuntimeError(f"Snapshot path is not a directory: {root}")
|
||||
expected = read_manifest(manifest_path)
|
||||
observed_entries = _collect(root)
|
||||
observed_files = {item.relative_path: item for item in observed_entries if not item.is_directory}
|
||||
extra = sorted(set(observed_files) - set(expected))
|
||||
missing = sorted(set(expected) - set(observed_files))
|
||||
if extra:
|
||||
raise RuntimeError(f"Snapshot contains unmanifested files: {', '.join(extra[:10])}")
|
||||
if missing:
|
||||
raise RuntimeError(f"Snapshot omits manifested files: {', '.join(missing[:10])}")
|
||||
for relative, retained in expected.items():
|
||||
current = observed_files[relative]
|
||||
if current.stat_result.st_size != retained.size_bytes:
|
||||
raise RuntimeError(f"Snapshot size differs for: {relative}")
|
||||
if _sha256(current.path) != retained.sha256:
|
||||
raise RuntimeError(f"Snapshot checksum differs for: {relative}")
|
||||
|
||||
|
||||
def create_snapshot(
|
||||
source: Path,
|
||||
snapshot_path: Path,
|
||||
manifest_path: Path,
|
||||
*,
|
||||
label: str,
|
||||
link_dest_snapshot: Path | None = None,
|
||||
link_dest_manifest: Path | None = None,
|
||||
) -> None:
|
||||
if not SAFE_LABEL.fullmatch(label):
|
||||
raise RuntimeError(f"Unsafe snapshot label: {label!r}")
|
||||
root = source.expanduser()
|
||||
if root.is_symlink():
|
||||
raise RuntimeError(f"Release snapshot refuses a symlinked root: {root}")
|
||||
root = root.resolve(strict=True)
|
||||
if not root.is_dir():
|
||||
raise RuntimeError(f"Snapshot source is not a directory: {root}")
|
||||
snapshot_path = snapshot_path.expanduser().resolve()
|
||||
manifest_path = manifest_path.expanduser().resolve()
|
||||
for output in (snapshot_path, manifest_path):
|
||||
try:
|
||||
output.relative_to(root)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise RuntimeError("Release snapshot output must not be inside its source tree")
|
||||
if snapshot_path.exists():
|
||||
raise RuntimeError(f"Snapshot destination already exists: {snapshot_path}")
|
||||
snapshot_path.mkdir(parents=True, exist_ok=False)
|
||||
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
prior_root: Path | None = None
|
||||
prior_manifest: dict[str, ManifestEntry] = {}
|
||||
if (link_dest_snapshot is None) != (link_dest_manifest is None):
|
||||
raise RuntimeError("Prior snapshot and manifest must be supplied together")
|
||||
if link_dest_snapshot is not None and link_dest_manifest is not None:
|
||||
prior_root = link_dest_snapshot.expanduser().resolve(strict=True)
|
||||
prior_manifest = read_manifest(link_dest_manifest.expanduser().resolve(strict=True))
|
||||
|
||||
initial = _collect(root)
|
||||
retained: list[ManifestEntry] = []
|
||||
for entry in initial:
|
||||
destination = snapshot_path / entry.relative_path
|
||||
if entry.is_directory:
|
||||
destination.mkdir(parents=True, exist_ok=False)
|
||||
os.chmod(destination, stat.S_IMODE(entry.stat_result.st_mode), follow_symlinks=False)
|
||||
continue
|
||||
linked = (
|
||||
_link_verified_prior(entry, destination, prior_root, prior_manifest)
|
||||
if prior_root is not None
|
||||
else None
|
||||
)
|
||||
retained.append(linked or _clone_or_copy(entry, destination))
|
||||
final = _collect(root)
|
||||
if [(item.relative_path, item.is_directory) for item in initial] != [
|
||||
(item.relative_path, item.is_directory) for item in final
|
||||
]:
|
||||
raise RuntimeError(f"Snapshot source contents changed while backup was running: {root}")
|
||||
with manifest_path.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(f"{MANIFEST_HEADER}\n")
|
||||
for entry in retained:
|
||||
handle.write(
|
||||
f"{entry.relative_path}\t{entry.size_bytes}\t{entry.mtime_ns}\t{entry.sha256}\n"
|
||||
)
|
||||
verify_snapshot(snapshot_path, manifest_path)
|
||||
|
||||
|
||||
def verify_backup(backup_dir: Path) -> None:
|
||||
root = backup_dir.expanduser().resolve(strict=True)
|
||||
payload = json.loads((root / "manifest.json").read_text(encoding="utf-8"))
|
||||
for label in ("storage", "models"):
|
||||
requested = payload.get(f"{label}_inventory_requested") is True
|
||||
snapshotted = payload.get(f"{label}_snapshot_requested") is True
|
||||
if requested != snapshotted:
|
||||
raise RuntimeError(f"Backup manifest does not bind the {label} inventory to a snapshot")
|
||||
if requested:
|
||||
verify_snapshot(root / f"{label}-snapshot", root / f"{label}-manifest.tsv")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
create = subparsers.add_parser("create")
|
||||
create.add_argument("--source", type=Path, required=True)
|
||||
create.add_argument("--snapshot", type=Path, required=True)
|
||||
create.add_argument("--manifest", type=Path, required=True)
|
||||
create.add_argument("--label", required=True)
|
||||
create.add_argument("--link-dest-snapshot", type=Path)
|
||||
create.add_argument("--link-dest-manifest", type=Path)
|
||||
verify = subparsers.add_parser("verify")
|
||||
verify.add_argument("--snapshot", type=Path, required=True)
|
||||
verify.add_argument("--manifest", type=Path, required=True)
|
||||
verify_backup_parser = subparsers.add_parser("verify-backup")
|
||||
verify_backup_parser.add_argument("--backup-dir", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if args.command == "create":
|
||||
create_snapshot(
|
||||
args.source,
|
||||
args.snapshot,
|
||||
args.manifest,
|
||||
label=args.label,
|
||||
link_dest_snapshot=args.link_dest_snapshot,
|
||||
link_dest_manifest=args.link_dest_manifest,
|
||||
)
|
||||
elif args.command == "verify":
|
||||
verify_snapshot(args.snapshot, args.manifest)
|
||||
else:
|
||||
verify_backup(args.backup_dir)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Restore a traceable GeoIntel cleanup quarantine without overwriting data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
CONFIRMATION = "RESTORE_QUARANTINED_ARTIFACTS"
|
||||
CLEANUP_PREFIXES = ("exports", "previews", "tiles", "masks", "derived", "rasters/derived")
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_manifest(path: Path, payload: dict[str, object]) -> None:
|
||||
temporary = path.with_suffix(".json.partial")
|
||||
temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--storage-root", type=Path, required=True)
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--confirm", required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _within(path: Path, root: Path, *, label: str) -> Path:
|
||||
resolved = path.resolve()
|
||||
try:
|
||||
resolved.relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"{label} escapes the storage root") from exc
|
||||
return resolved
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if args.confirm != CONFIRMATION:
|
||||
raise RuntimeError(f"Refusing restore; pass --confirm {CONFIRMATION}")
|
||||
storage_root = args.storage_root.expanduser().resolve()
|
||||
manifest_path = _within(args.manifest.expanduser(), storage_root, label="Manifest")
|
||||
protected_quarantine_root = storage_root / "operator-evidence" / "cleanup-quarantine"
|
||||
try:
|
||||
manifest_path.relative_to(protected_quarantine_root.resolve())
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("Manifest is outside the protected cleanup quarantine") from exc
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if payload.get("schema_version") != 1:
|
||||
raise RuntimeError("Unsupported quarantine manifest")
|
||||
if payload.get("state") not in {"complete", "in_progress", "restore_in_progress", "restored"}:
|
||||
raise RuntimeError("Quarantine manifest is not in a restorable state")
|
||||
raw_entries = payload.get("entries")
|
||||
if not isinstance(raw_entries, list):
|
||||
raise RuntimeError("Quarantine manifest entries are invalid")
|
||||
|
||||
plans: list[tuple[str, dict[str, object], Path, Path]] = []
|
||||
for raw_entry in raw_entries:
|
||||
if not isinstance(raw_entry, dict):
|
||||
raise RuntimeError("Quarantine manifest entry is invalid")
|
||||
status = raw_entry.get("status")
|
||||
if status not in {"planned", "linked", "quarantined", "restore_linked", "restored"}:
|
||||
raise RuntimeError(f"Quarantine manifest entry has an invalid status: {status!r}")
|
||||
relative_path = raw_entry.get("relative_path")
|
||||
quarantine_relative_path = raw_entry.get("quarantine_relative_path")
|
||||
expected_hash = raw_entry.get("sha256")
|
||||
expected_size = raw_entry.get("size_bytes")
|
||||
if (
|
||||
not isinstance(relative_path, str)
|
||||
or not isinstance(expected_hash, str)
|
||||
or not isinstance(expected_size, int)
|
||||
):
|
||||
raise RuntimeError("Quarantine manifest entry lacks recovery metadata")
|
||||
if not any(
|
||||
relative_path == prefix or relative_path.startswith(f"{prefix}/")
|
||||
for prefix in CLEANUP_PREFIXES
|
||||
):
|
||||
raise RuntimeError(f"Original path is outside the cleanup allowlist: {relative_path}")
|
||||
original = _within(storage_root / relative_path, storage_root, label="Original path")
|
||||
if not isinstance(quarantine_relative_path, str):
|
||||
if status != "planned":
|
||||
raise RuntimeError("Quarantine manifest entry lacks its retained path")
|
||||
quarantine_relative_path = (
|
||||
manifest_path.parent / "files" / relative_path
|
||||
).relative_to(storage_root).as_posix()
|
||||
raw_entry["quarantine_relative_path"] = quarantine_relative_path
|
||||
quarantined = _within(
|
||||
storage_root / quarantine_relative_path,
|
||||
storage_root,
|
||||
label="Quarantine path",
|
||||
)
|
||||
try:
|
||||
quarantined.relative_to(manifest_path.parent.resolve())
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("Quarantine entry escapes its operation directory") from exc
|
||||
original_exists = original.exists()
|
||||
quarantined_exists = quarantined.exists()
|
||||
if original_exists:
|
||||
if not original.is_file() or original.stat().st_size != expected_size or sha256(original) != expected_hash:
|
||||
raise RuntimeError(f"Restore destination already exists with different bytes: {relative_path}")
|
||||
if quarantined_exists:
|
||||
if (
|
||||
not quarantined.is_file()
|
||||
or quarantined.stat().st_size != expected_size
|
||||
or sha256(quarantined) != expected_hash
|
||||
):
|
||||
raise RuntimeError(f"Quarantined artifact checksum mismatch: {quarantine_relative_path}")
|
||||
if original_exists and quarantined_exists:
|
||||
if not os.path.samefile(original, quarantined):
|
||||
raise RuntimeError(f"Restore destination already exists: {relative_path}")
|
||||
plans.append(("remove_duplicate_link", raw_entry, quarantined, original))
|
||||
elif original_exists:
|
||||
plans.append(("mark_restored", raw_entry, quarantined, original))
|
||||
elif quarantined_exists:
|
||||
plans.append(("restore", raw_entry, quarantined, original))
|
||||
else:
|
||||
raise RuntimeError(f"Both original and quarantined artifacts are missing: {relative_path}")
|
||||
|
||||
payload["state"] = "restore_in_progress"
|
||||
write_manifest(manifest_path, payload)
|
||||
for action, entry, quarantined, original in plans:
|
||||
if action == "restore":
|
||||
original.parent.mkdir(parents=True, exist_ok=True)
|
||||
_within(original, storage_root, label="Original path")
|
||||
try:
|
||||
os.link(quarantined, original, follow_symlinks=False)
|
||||
except FileExistsError as exc:
|
||||
raise RuntimeError(f"Restore destination was created concurrently: {original}") from exc
|
||||
if not os.path.samefile(quarantined, original):
|
||||
original.unlink(missing_ok=True)
|
||||
raise RuntimeError(f"Restore link verification failed: {original}")
|
||||
entry["status"] = "restore_linked"
|
||||
write_manifest(manifest_path, payload)
|
||||
quarantined.unlink()
|
||||
elif action == "remove_duplicate_link":
|
||||
quarantined.unlink()
|
||||
entry["status"] = "restored"
|
||||
entry["restored_at"] = datetime.now(timezone.utc).isoformat()
|
||||
write_manifest(manifest_path, payload)
|
||||
payload["state"] = "restored"
|
||||
payload["restored_at"] = datetime.now(timezone.utc).isoformat()
|
||||
write_manifest(manifest_path, payload)
|
||||
print(json.dumps({"state": "restored", "restored_count": len(plans)}, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -43,6 +43,8 @@ echo "== GeoIntel run readiness check =="
|
||||
"$PYTHON_BIN" -m py_compile scripts/build_release_package.py
|
||||
"$PYTHON_BIN" -m py_compile scripts/verify_python_lock.py
|
||||
"$PYTHON_BIN" scripts/verify_python_lock.py
|
||||
"$PYTHON_BIN" -m py_compile scripts/verify_repository_layout.py
|
||||
"$PYTHON_BIN" scripts/verify_repository_layout.py
|
||||
"$PYTHON_BIN" -m py_compile scripts/verify_security_exceptions.py
|
||||
"$PYTHON_BIN" scripts/verify_security_exceptions.py
|
||||
bash -n scripts/backup_release_state.sh
|
||||
@@ -55,6 +57,7 @@ bash -n scripts/scan_container_image.sh
|
||||
bash -n scripts/audit_python_dependencies.sh
|
||||
echo "Using Python: ${PYTHON_BIN}"
|
||||
bash scripts/check_repo_structure.sh
|
||||
"$PYTHON_BIN" -m ruff check backend scripts tests
|
||||
${PYTHON_BIN} scripts/smoke_docs.py
|
||||
${PYTHON_BIN} scripts/validate_fixtures.py
|
||||
${PYTHON_BIN} scripts/smoke_contracts.py
|
||||
@@ -129,8 +132,10 @@ ${PYTHON_BIN} -m py_compile scripts/migrate_runtime_model_provenance.py
|
||||
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
||||
${PYTHON_BIN} -m py_compile scripts/archive_technical_projects.py
|
||||
${PYTHON_BIN} -m py_compile scripts/release_backup_guard.py
|
||||
${PYTHON_BIN} -m py_compile scripts/release_backup_snapshot.py
|
||||
${PYTHON_BIN} -m py_compile scripts/audit_data_operations.py
|
||||
${PYTHON_BIN} -m py_compile scripts/cleanup_storage_artifacts.py
|
||||
${PYTHON_BIN} -m py_compile scripts/restore_storage_quarantine.py
|
||||
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
||||
${PYTHON_BIN} -m compileall backend/app
|
||||
(cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning)
|
||||
@@ -149,6 +154,7 @@ bash -n deploy/unraid/gosu-setpriv
|
||||
bash -n deploy/unraid/run-dockerman-container.sh
|
||||
bash -n deploy/unraid/deploy-release.sh
|
||||
bash -n deploy/unraid/rollback-dockerman-container.sh
|
||||
bash -n deploy/unraid/restore-predeploy-database.sh
|
||||
bash -n scripts/verify_browser_runtime.sh
|
||||
bash -n scripts/verify_demo_export_workflow.sh
|
||||
bash -n scripts/verify_demo_raster_workflow.sh
|
||||
|
||||
@@ -21,8 +21,17 @@ esac
|
||||
docker image inspect "$TARGET_IMAGE" >/dev/null
|
||||
mkdir -p "$ROOT/$(dirname "$OUTPUT")" "$CACHE_DIR"
|
||||
"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py"
|
||||
"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" \
|
||||
--print-container-ids > "$IGNORE_FILE"
|
||||
mapfile -t ignored_container_ids < <(
|
||||
"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" \
|
||||
--print-container-ids | tr -d '\r'
|
||||
)
|
||||
ignore_args=()
|
||||
trivy_ignore_args=()
|
||||
if [ "${#ignored_container_ids[@]}" -gt 0 ]; then
|
||||
printf '%s\n' "${ignored_container_ids[@]}" > "$IGNORE_FILE"
|
||||
ignore_args=(-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro")
|
||||
trivy_ignore_args=(--ignorefile "$CONTAINER_IGNORE_FILE")
|
||||
fi
|
||||
|
||||
# Keep the complete report, including vulnerabilities without an available fix.
|
||||
docker run --rm \
|
||||
@@ -45,14 +54,14 @@ docker run --rm \
|
||||
docker run --rm \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v "$CACHE_DIR:/root/.cache/trivy" \
|
||||
-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro" \
|
||||
"${ignore_args[@]}" \
|
||||
"$TRIVY_IMAGE" \
|
||||
image \
|
||||
--scanners vuln \
|
||||
--timeout 20m \
|
||||
--skip-version-check \
|
||||
--ignore-unfixed \
|
||||
--ignorefile "$CONTAINER_IGNORE_FILE" \
|
||||
"${trivy_ignore_args[@]}" \
|
||||
--skip-files /usr/local/bin/gosu \
|
||||
--severity HIGH,CRITICAL \
|
||||
--exit-code 1 \
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
CONTAINER="geointel"
|
||||
BACKUP_DIR=""
|
||||
|
||||
@@ -59,15 +60,22 @@ required = {
|
||||
"database_name",
|
||||
"database_user",
|
||||
"image_id",
|
||||
"git_commit",
|
||||
}
|
||||
missing = sorted(required - payload.keys())
|
||||
if missing:
|
||||
raise SystemExit(f"Backup manifest misses: {', '.join(missing)}")
|
||||
if payload["schema_version"] != 1 or payload["read_only_source"] is not True:
|
||||
raise SystemExit("Unsupported or unsafe backup manifest")
|
||||
tool_revision = payload.get("backup_tool_revision", payload.get("git_commit"))
|
||||
if not isinstance(tool_revision, str) or len(tool_revision) < 7:
|
||||
raise SystemExit("Backup manifest lacks its backup-tool revision")
|
||||
running_revision = payload.get("running_image_revision")
|
||||
if running_revision is not None and not isinstance(running_revision, str):
|
||||
raise SystemExit("Backup manifest has an invalid running-image revision")
|
||||
PY
|
||||
|
||||
python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$BACKUP_DIR"
|
||||
|
||||
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
||||
echo "Container '$CONTAINER' is required to run pg_restore --list." >&2
|
||||
exit 3
|
||||
|
||||
@@ -17,29 +17,39 @@ EXCEPTIONS_PATH = ROOT / "security" / "pip-audit-exceptions.json"
|
||||
def load_and_validate() -> tuple[dict[str, object], list[str]]:
|
||||
payload = json.loads(EXCEPTIONS_PATH.read_text(encoding="utf-8"))
|
||||
errors: list[str] = []
|
||||
try:
|
||||
review_by = dt.date.fromisoformat(str(payload["review_by"]))
|
||||
except (KeyError, ValueError):
|
||||
errors.append("review_by must be an ISO date")
|
||||
review_by = dt.date.min
|
||||
if review_by < dt.date.today():
|
||||
errors.append(f"dependency exception review expired on {review_by.isoformat()}")
|
||||
if payload.get("package") != "starlette":
|
||||
errors.append("only the documented Starlette compatibility exception is allowed")
|
||||
controls = payload.get("compensating_controls")
|
||||
if not isinstance(controls, list) or len(controls) < 3:
|
||||
errors.append("at least three compensating controls are required")
|
||||
if set(payload) != {"schema_version", "advisories"}:
|
||||
errors.append("exception policy must contain only schema_version and advisories")
|
||||
if payload.get("schema_version") != 1:
|
||||
errors.append("schema_version must be 1")
|
||||
advisories = payload.get("advisories")
|
||||
if not isinstance(advisories, list) or not advisories:
|
||||
errors.append("at least one advisory exception is required")
|
||||
if not isinstance(advisories, list):
|
||||
errors.append("advisories must be a list")
|
||||
else:
|
||||
ids = [str(item.get("id", "")) for item in advisories if isinstance(item, dict)]
|
||||
if len(ids) != len(set(ids)) or any(not item.startswith("PYSEC-") for item in ids):
|
||||
errors.append("advisory IDs must be unique PYSEC identifiers")
|
||||
for item in advisories:
|
||||
if not isinstance(item, dict) or len(str(item.get("reason", ""))) < 30:
|
||||
if not isinstance(item, dict):
|
||||
errors.append("every advisory must be an object")
|
||||
continue
|
||||
required = {"id", "package", "review_by", "reason"}
|
||||
allowed = required | {"aliases"}
|
||||
if not required.issubset(item) or not set(item).issubset(allowed):
|
||||
errors.append("every advisory must match the documented exception schema")
|
||||
if not str(item.get("package", "")).strip():
|
||||
errors.append("every advisory requires a package")
|
||||
if len(str(item.get("reason", ""))) < 30:
|
||||
errors.append("every advisory requires a specific reason")
|
||||
break
|
||||
try:
|
||||
review_by = dt.date.fromisoformat(str(item["review_by"]))
|
||||
except (KeyError, ValueError):
|
||||
errors.append("every advisory review_by must be an ISO date")
|
||||
else:
|
||||
if review_by < dt.date.today():
|
||||
errors.append(
|
||||
f"dependency exception {item.get('id', '')} expired on "
|
||||
f"{review_by.isoformat()}"
|
||||
)
|
||||
aliases = [
|
||||
str(alias)
|
||||
for item in advisories
|
||||
@@ -72,8 +82,7 @@ def main() -> int:
|
||||
print(alias)
|
||||
else:
|
||||
print(
|
||||
"Dependency exceptions valid through "
|
||||
f"{payload['review_by']} with documented compensating controls."
|
||||
f"Dependency exception policy valid; {len(payload['advisories'])} active exception(s)."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user