Make demo cleanup runnable in backend container
GeoIntel CI / docs-smoke (push) Has been cancelled
GeoIntel CI / contract-smoke (push) Has been cancelled

This commit is contained in:
Codex
2026-06-17 01:48:08 +02:00
parent baf830fb5b
commit 6bf0610147
9 changed files with 194 additions and 149 deletions
+1
View File
@@ -10,6 +10,7 @@
## Sprint 24 demo/export artifact cleanup tooling (2026-06-17) ## Sprint 24 demo/export artifact cleanup tooling (2026-06-17)
- Added `scripts/cleanup_demo_artifacts.py`, a dry-run-first maintenance script for old offline demo export artifacts. - Added `scripts/cleanup_demo_artifacts.py`, a dry-run-first maintenance script for old offline demo export artifacts.
- Added the same cleanup entrypoint under `backend/scripts/` so it can run inside the backend Docker container.
- The cleanup keeps the newest exports per matching demo project, deletes only explicit `exports` rows/files when `--apply` is set and refuses file deletion outside `STORAGE_ROOT`. - The cleanup keeps the newest exports per matching demo project, deletes only explicit `exports` rows/files when `--apply` is set and refuses file deletion outside `STORAGE_ROOT`.
- Added tests for cleanup candidate selection, storage-root path safety and readiness gate coverage. - Added tests for cleanup candidate selection, storage-root path safety and readiness gate coverage.
- Added the cleanup script to the main readiness gate via Python compile validation. - Added the cleanup script to the main readiness gate via Python compile validation.
+1
View File
@@ -188,6 +188,7 @@ bash scripts/live_migration_smoke.sh
- Old offline demo export artifacts can be inspected with `python scripts/cleanup_demo_artifacts.py` - Old offline demo export artifacts can be inspected with `python scripts/cleanup_demo_artifacts.py`
and removed only with an explicit `--apply`. The script keeps the newest exports and removed only with an explicit `--apply`. The script keeps the newest exports
per demo project and refuses to delete files outside `STORAGE_ROOT`. per demo project and refuses to delete files outside `STORAGE_ROOT`.
In Docker, use `docker compose exec -T backend python scripts/cleanup_demo_artifacts.py`.
## Run locally ## Run locally
+160
View File
@@ -0,0 +1,160 @@
from __future__ import annotations
import argparse
import json
import sys
from datetime import datetime
from pathlib import Path
from typing import Any
BACKEND_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(BACKEND_ROOT))
from app.core.config import get_settings
from app.db.session import SessionLocal
from app.models import Export, Project
DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
def is_within_storage_root(path: Path, storage_root: Path) -> bool:
try:
path.resolve().relative_to(storage_root.resolve())
except ValueError:
return False
return True
def export_created_at(export: Any) -> datetime:
created_at = getattr(export, "created_at", None)
if isinstance(created_at, datetime):
return created_at
return datetime.min
def select_cleanup_candidates(exports: list[Any], keep_latest: int) -> tuple[list[Any], list[Any]]:
if keep_latest < 0:
raise ValueError("keep_latest must be greater than or equal to zero")
ordered = sorted(exports, key=export_created_at, reverse=True)
return ordered[:keep_latest], ordered[keep_latest:]
def export_path(export: Any) -> Path:
return Path(str(getattr(export, "storage_path")))
def prune_empty_parents(start_path: Path, storage_root: Path) -> list[str]:
pruned: list[str] = []
parent = start_path.resolve().parent
stop_at = storage_root.resolve()
while parent != stop_at and is_within_storage_root(parent, stop_at):
try:
parent.rmdir()
except OSError:
break
pruned.append(str(parent))
parent = parent.parent
return pruned
def cleanup_demo_exports(project_name: str, keep_latest: int, apply: bool) -> dict[str, Any]:
settings = get_settings()
storage_root = Path(settings.storage_root).resolve()
summary: dict[str, Any] = {
"dry_run": not apply,
"project_name": project_name,
"storage_root": str(storage_root),
"projects": [],
"matched_export_count": 0,
"selected_export_count": 0,
"deleted_export_count": 0,
"deleted_files": [],
"missing_files": [],
"skipped_outside_storage": [],
"pruned_dirs": [],
"kept_export_ids": [],
}
with SessionLocal() as db:
projects = (
db.query(Project)
.filter(Project.name == project_name)
.filter(Project.status != "deleted")
.order_by(Project.created_at.desc())
.all()
)
for project in projects:
exports = (
db.query(Export)
.filter(Export.project_id == project.id)
.order_by(Export.created_at.desc())
.all()
)
kept, candidates = select_cleanup_candidates(exports, keep_latest)
summary["projects"].append(str(project.id))
summary["matched_export_count"] += len(exports)
summary["selected_export_count"] += len(candidates)
summary["kept_export_ids"].extend(str(export.id) for export in kept)
for export in candidates:
path = export_path(export)
if not is_within_storage_root(path, storage_root):
summary["skipped_outside_storage"].append(
{"export_id": str(export.id), "storage_path": str(path)}
)
continue
if path.exists():
if apply:
path.unlink()
summary["pruned_dirs"].extend(prune_empty_parents(path, storage_root))
summary["deleted_files"].append(str(path))
else:
summary["missing_files"].append({"export_id": str(export.id), "storage_path": str(path)})
if apply:
db.delete(export)
summary["deleted_export_count"] += 1
if apply:
db.commit()
return summary
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description=(
"Clean old offline demo export artifacts. The script is dry-run by default "
"and only targets the explicit GeoIntel demo project unless overridden."
)
)
parser.add_argument("--project-name", default=DEMO_PROJECT_NAME, help="Exact project name to clean.")
parser.add_argument(
"--keep-latest",
type=int,
default=3,
help="Number of newest export records/files to keep per matching project.",
)
parser.add_argument("--apply", action="store_true", help="Delete selected export rows and files.")
return parser
def main() -> int:
parser = build_parser()
args = parser.parse_args()
if args.keep_latest < 0:
parser.error("--keep-latest must be greater than or equal to zero")
summary = cleanup_demo_exports(
project_name=args.project_name,
keep_latest=args.keep_latest,
apply=args.apply,
)
print(json.dumps(summary, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+1
View File
@@ -27,6 +27,7 @@ def test_readiness_gate_compiles_demo_cleanup_script() -> None:
content = script.read_text(encoding="utf-8") content = script.read_text(encoding="utf-8")
assert "-m py_compile scripts/cleanup_demo_artifacts.py" in content assert "-m py_compile scripts/cleanup_demo_artifacts.py" in content
assert "-m py_compile backend/scripts/cleanup_demo_artifacts.py" in content
def test_demo_export_workflow_script_verifies_export_endpoints() -> None: def test_demo_export_workflow_script_verifies_export_endpoints() -> None:
+2 -1
View File
@@ -2,6 +2,7 @@
Changed: Changed:
- Added `scripts/cleanup_demo_artifacts.py` for dry-run-first cleanup of old offline demo export artifacts. - Added `scripts/cleanup_demo_artifacts.py` for dry-run-first cleanup of old offline demo export artifacts.
- Added `backend/scripts/cleanup_demo_artifacts.py` so the same cleanup can run inside the backend Docker container.
- Cleanup is constrained to an exact demo project name by default, keeps the newest exports per project and refuses file deletion outside `STORAGE_ROOT`. - Cleanup is constrained to an exact demo project name by default, keeps the newest exports per project and refuses file deletion outside `STORAGE_ROOT`.
- Added regression tests for cleanup selection, path safety and readiness gate coverage. - Added regression tests for cleanup selection, path safety and readiness gate coverage.
- Added Python compile validation for the cleanup script to `scripts/run_readiness_check.sh`. - Added Python compile validation for the cleanup script to `scripts/run_readiness_check.sh`.
@@ -10,7 +11,7 @@ Changed:
Validation: Validation:
- `python -m py_compile scripts/cleanup_demo_artifacts.py` passed. - `python -m py_compile scripts/cleanup_demo_artifacts.py` passed.
- `python -m pytest backend/tests/test_sprint24_cleanup_demo_artifacts.py backend/tests/test_readiness_gate.py` passed: 10 tests. - `python -m pytest backend/tests/test_sprint24_cleanup_demo_artifacts.py backend/tests/test_readiness_gate.py` passed: 10 tests.
- `bash scripts/run_readiness_check.sh` passed: 151 backend tests, frontend typecheck/build, Alembic head check and script syntax checks. - `bash scripts/run_readiness_check.sh` passed twice after adding the backend container entrypoint: 151 backend tests, frontend typecheck/build, Alembic head check and script syntax checks.
- `python -m compileall backend/app` passed. - `python -m compileall backend/app` passed.
- `cd backend && python -m alembic upgrade head --sql` passed. - `cd backend && python -m alembic upgrade head --sql` passed.
- `bash -n scripts/live_migration_smoke.sh` and `bash -n scripts/verify_demo_export_workflow.sh` passed. - `bash -n scripts/live_migration_smoke.sh` and `bash -n scripts/verify_demo_export_workflow.sh` passed.
+1
View File
@@ -104,6 +104,7 @@ Offline demo export artifacts can be inspected and cleaned with:
```bash ```bash
python scripts/cleanup_demo_artifacts.py python scripts/cleanup_demo_artifacts.py
python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply
docker compose exec -T backend python scripts/cleanup_demo_artifacts.py
``` ```
The script is dry-run by default, targets only the explicit The script is dry-run by default, targets only the explicit
+7
View File
@@ -30,6 +30,13 @@ python scripts/cleanup_demo_artifacts.py
python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply
``` ```
Against the Docker runtime, run the backend-container entrypoint:
```bash
docker compose exec -T backend python scripts/cleanup_demo_artifacts.py
docker compose exec -T backend python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply
```
The cleanup script is dry-run by default. It only targets the explicit The cleanup script is dry-run by default. It only targets the explicit
`GeoIntel Demo - Building QA` project unless `--project-name` is provided, keeps `GeoIntel Demo - Building QA` project unless `--project-name` is provided, keeps
the newest exports per matching project, deletes only `exports` rows/files when the newest exports per matching project, deletes only `exports` rows/files when
+20 -148
View File
@@ -1,159 +1,31 @@
from __future__ import annotations from __future__ import annotations
import argparse import importlib.util
import json
import sys
from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Any from types import ModuleType
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "backend"))
from app.core.config import get_settings
from app.db.session import SessionLocal
from app.models import Export, Project
DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA" def load_backend_script() -> ModuleType:
script = Path(__file__).resolve().parents[1] / "backend" / "scripts" / "cleanup_demo_artifacts.py"
spec = importlib.util.spec_from_file_location("backend_cleanup_demo_artifacts", script)
if spec is None or spec.loader is None:
raise RuntimeError(f"Could not load cleanup implementation from {script}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def is_within_storage_root(path: Path, storage_root: Path) -> bool: _impl = load_backend_script()
try:
path.resolve().relative_to(storage_root.resolve())
except ValueError:
return False
return True
DEMO_PROJECT_NAME = _impl.DEMO_PROJECT_NAME
def export_created_at(export: Any) -> datetime: is_within_storage_root = _impl.is_within_storage_root
created_at = getattr(export, "created_at", None) export_created_at = _impl.export_created_at
if isinstance(created_at, datetime): select_cleanup_candidates = _impl.select_cleanup_candidates
return created_at export_path = _impl.export_path
return datetime.min prune_empty_parents = _impl.prune_empty_parents
cleanup_demo_exports = _impl.cleanup_demo_exports
build_parser = _impl.build_parser
def select_cleanup_candidates(exports: list[Any], keep_latest: int) -> tuple[list[Any], list[Any]]: main = _impl.main
if keep_latest < 0:
raise ValueError("keep_latest must be greater than or equal to zero")
ordered = sorted(exports, key=export_created_at, reverse=True)
return ordered[:keep_latest], ordered[keep_latest:]
def export_path(export: Any) -> Path:
return Path(str(getattr(export, "storage_path")))
def prune_empty_parents(start_path: Path, storage_root: Path) -> list[str]:
pruned: list[str] = []
parent = start_path.resolve().parent
stop_at = storage_root.resolve()
while parent != stop_at and is_within_storage_root(parent, stop_at):
try:
parent.rmdir()
except OSError:
break
pruned.append(str(parent))
parent = parent.parent
return pruned
def cleanup_demo_exports(project_name: str, keep_latest: int, apply: bool) -> dict[str, Any]:
settings = get_settings()
storage_root = Path(settings.storage_root).resolve()
summary: dict[str, Any] = {
"dry_run": not apply,
"project_name": project_name,
"storage_root": str(storage_root),
"projects": [],
"matched_export_count": 0,
"selected_export_count": 0,
"deleted_export_count": 0,
"deleted_files": [],
"missing_files": [],
"skipped_outside_storage": [],
"pruned_dirs": [],
"kept_export_ids": [],
}
with SessionLocal() as db:
projects = (
db.query(Project)
.filter(Project.name == project_name)
.filter(Project.status != "deleted")
.order_by(Project.created_at.desc())
.all()
)
for project in projects:
exports = (
db.query(Export)
.filter(Export.project_id == project.id)
.order_by(Export.created_at.desc())
.all()
)
kept, candidates = select_cleanup_candidates(exports, keep_latest)
summary["projects"].append(str(project.id))
summary["matched_export_count"] += len(exports)
summary["selected_export_count"] += len(candidates)
summary["kept_export_ids"].extend(str(export.id) for export in kept)
for export in candidates:
path = export_path(export)
if not is_within_storage_root(path, storage_root):
summary["skipped_outside_storage"].append(
{"export_id": str(export.id), "storage_path": str(path)}
)
continue
if path.exists():
if apply:
path.unlink()
summary["pruned_dirs"].extend(prune_empty_parents(path, storage_root))
summary["deleted_files"].append(str(path))
else:
summary["missing_files"].append({"export_id": str(export.id), "storage_path": str(path)})
if apply:
db.delete(export)
summary["deleted_export_count"] += 1
if apply:
db.commit()
return summary
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description=(
"Clean old offline demo export artifacts. The script is dry-run by default "
"and only targets the explicit GeoIntel demo project unless overridden."
)
)
parser.add_argument("--project-name", default=DEMO_PROJECT_NAME, help="Exact project name to clean.")
parser.add_argument(
"--keep-latest",
type=int,
default=3,
help="Number of newest export records/files to keep per matching project.",
)
parser.add_argument("--apply", action="store_true", help="Delete selected export rows and files.")
return parser
def main() -> int:
parser = build_parser()
args = parser.parse_args()
if args.keep_latest < 0:
parser.error("--keep-latest must be greater than or equal to zero")
summary = cleanup_demo_exports(
project_name=args.project_name,
keep_latest=args.keep_latest,
apply=args.apply,
)
print(json.dumps(summary, indent=2, sort_keys=True))
return 0
if __name__ == "__main__": if __name__ == "__main__":
+1
View File
@@ -38,6 +38,7 @@ ${PYTHON_BIN} scripts/validate_m14_launch_assets.py
${PYTHON_BIN} -m py_compile scripts/gis_import_smoke.py ${PYTHON_BIN} -m py_compile scripts/gis_import_smoke.py
${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py ${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
${PYTHON_BIN} -m compileall backend/app ${PYTHON_BIN} -m compileall backend/app
(cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning) (cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning)
(cd backend && ${PYTHON_BIN} -m alembic heads) (cd backend && ${PYTHON_BIN} -m alembic heads)