Harden demo export cleanup safety
GeoIntel CI / docs-smoke (push) Has been cancelled
GeoIntel CI / contract-smoke (push) Has been cancelled

This commit is contained in:
Codex
2026-06-17 23:47:49 +02:00
parent 212be9ae48
commit 949614e336
9 changed files with 140 additions and 9 deletions
+3 -1
View File
@@ -187,7 +187,9 @@ bash scripts/live_migration_smoke.sh
- No new migrations, product lines, live providers or AI dependencies are introduced by this export pass.
- Old offline demo export artifacts can be inspected with `python scripts/cleanup_demo_artifacts.py`
and removed only with an explicit `--apply`. The script keeps the newest exports
per demo project and refuses to delete files outside `STORAGE_ROOT`.
per demo project, refuses to delete files outside `STORAGE_ROOT`, and blocks
apply runs above `--max-delete` until the cap is raised after a dry-run review.
Use repeated `--export-type` values to target only specific artifact kinds.
In Docker, use `docker compose exec -T backend python scripts/cleanup_demo_artifacts.py`.
## Run locally
+46 -2
View File
@@ -40,6 +40,13 @@ def select_cleanup_candidates(exports: list[Any], keep_latest: int) -> tuple[lis
return ordered[:keep_latest], ordered[keep_latest:]
def filter_exports_by_type(exports: list[Any], export_types: list[str] | None) -> list[Any]:
if not export_types:
return exports
allowed = set(export_types)
return [export for export in exports if getattr(export, "export_type", None) in allowed]
def export_path(export: Any) -> Path:
return Path(str(getattr(export, "storage_path")))
@@ -58,15 +65,27 @@ def prune_empty_parents(start_path: Path, storage_root: Path) -> list[str]:
return pruned
def cleanup_demo_exports(project_name: str, keep_latest: int, apply: bool) -> dict[str, Any]:
def cleanup_demo_exports(
project_name: str,
keep_latest: int,
apply: bool,
max_delete: int,
export_types: list[str] | None = None,
) -> dict[str, Any]:
if max_delete < 0:
raise ValueError("max_delete must be greater than or equal to zero")
settings = get_settings()
storage_root = Path(settings.storage_root).resolve()
summary: dict[str, Any] = {
"dry_run": not apply,
"project_name": project_name,
"keep_latest": keep_latest,
"max_delete": max_delete,
"export_types": export_types or [],
"storage_root": str(storage_root),
"projects": [],
"matched_export_count": 0,
"type_filtered_export_count": 0,
"selected_export_count": 0,
"deleted_export_count": 0,
"candidate_files": [],
@@ -92,12 +111,21 @@ def cleanup_demo_exports(project_name: str, keep_latest: int, apply: bool) -> di
.order_by(Export.created_at.desc())
.all()
)
kept, candidates = select_cleanup_candidates(exports, keep_latest)
filtered_exports = filter_exports_by_type(exports, export_types)
kept, candidates = select_cleanup_candidates(filtered_exports, keep_latest)
summary["projects"].append(str(project.id))
summary["matched_export_count"] += len(exports)
summary["type_filtered_export_count"] += len(filtered_exports)
summary["selected_export_count"] += len(candidates)
summary["kept_export_ids"].extend(str(export.id) for export in kept)
if apply and len(candidates) > max_delete:
summary["blocked_reason"] = (
f"selected_export_count {len(candidates)} exceeds --max-delete {max_delete}; "
"raise --max-delete after reviewing a dry run"
)
continue
for export in candidates:
path = export_path(export)
if not is_within_storage_root(path, storage_root):
@@ -140,6 +168,18 @@ def build_parser() -> argparse.ArgumentParser:
default=3,
help="Number of newest export records/files to keep per matching project.",
)
parser.add_argument(
"--max-delete",
type=int,
default=25,
help="Maximum export rows/files allowed to be deleted per matching project when --apply is set.",
)
parser.add_argument(
"--export-type",
action="append",
default=None,
help="Restrict cleanup to an export_type. Repeat for multiple types.",
)
parser.add_argument("--apply", action="store_true", help="Delete selected export rows and files.")
return parser
@@ -149,11 +189,15 @@ def main() -> int:
args = parser.parse_args()
if args.keep_latest < 0:
parser.error("--keep-latest must be greater than or equal to zero")
if args.max_delete < 0:
parser.error("--max-delete must be greater than or equal to zero")
summary = cleanup_demo_exports(
project_name=args.project_name,
keep_latest=args.keep_latest,
apply=args.apply,
max_delete=args.max_delete,
export_types=args.export_type,
)
print(json.dumps(summary, indent=2, sort_keys=True))
return 0
@@ -22,6 +22,7 @@ class ExportRow:
id: str
created_at: datetime | None
storage_path: str
export_type: str = "project_metadata_json"
def test_cleanup_candidate_selection_keeps_newest_exports() -> None:
@@ -51,6 +52,20 @@ def test_cleanup_candidate_selection_rejects_negative_keep_latest() -> None:
raise AssertionError("negative keep_latest should fail")
def test_cleanup_can_filter_candidates_by_export_type() -> None:
cleanup = load_cleanup_module()
base = datetime(2026, 1, 1, 12, 0, 0)
exports = [
ExportRow("metadata", base + timedelta(days=2), "/tmp/metadata.json", "project_metadata_json"),
ExportRow("report", base + timedelta(days=1), "/tmp/report.html", "project_report_html"),
ExportRow("dataset", base, "/tmp/dataset.geojson", "dataset_geojson"),
]
filtered = cleanup.filter_exports_by_type(exports, ["project_report_html"])
assert [export.id for export in filtered] == ["report"]
def test_cleanup_path_safety_requires_storage_root_containment(tmp_path: Path) -> None:
cleanup = load_cleanup_module()
storage_root = tmp_path / "storage"
@@ -72,12 +87,40 @@ def test_cleanup_script_defaults_to_explicit_demo_project() -> None:
assert args.project_name == cleanup.DEMO_PROJECT_NAME
assert args.keep_latest == 3
assert args.max_delete == 25
assert args.export_type is None
assert args.apply is False
def test_cleanup_script_accepts_max_delete_and_repeated_export_type() -> None:
cleanup = load_cleanup_module()
parser = cleanup.build_parser()
args = parser.parse_args(
[
"--keep-latest",
"10",
"--max-delete",
"100",
"--export-type",
"project_report_html",
"--export-type",
"project_metadata_json",
"--apply",
]
)
assert args.keep_latest == 10
assert args.max_delete == 100
assert args.export_type == ["project_report_html", "project_metadata_json"]
assert args.apply is True
def test_cleanup_script_reports_dry_run_candidates_separately() -> None:
script = Path(__file__).resolve().parents[2] / "backend" / "scripts" / "cleanup_demo_artifacts.py"
content = script.read_text(encoding="utf-8")
assert '"candidate_files": []' in content
assert 'summary["candidate_files"].append(str(path))' in content
assert '"max_delete": max_delete' in content
assert "blocked_reason" in content