diff --git a/CHANGELOG.md b/CHANGELOG.md index af0cb53e..99896fd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ## Sprint 24 demo/export artifact cleanup tooling (2026-06-17) - Added `scripts/cleanup_demo_artifacts.py`, a dry-run-first maintenance script for old offline demo export artifacts. +- Added the same cleanup entrypoint under `backend/scripts/` so it can run inside the backend Docker container. - The cleanup keeps the newest exports per matching demo project, deletes only explicit `exports` rows/files when `--apply` is set and refuses file deletion outside `STORAGE_ROOT`. - Added tests for cleanup candidate selection, storage-root path safety and readiness gate coverage. - Added the cleanup script to the main readiness gate via Python compile validation. diff --git a/backend/README.md b/backend/README.md index 7ce1378a..cd49dfba 100644 --- a/backend/README.md +++ b/backend/README.md @@ -188,6 +188,7 @@ bash scripts/live_migration_smoke.sh - Old offline demo export artifacts can be inspected with `python scripts/cleanup_demo_artifacts.py` and removed only with an explicit `--apply`. The script keeps the newest exports per demo project and refuses to delete files outside `STORAGE_ROOT`. + In Docker, use `docker compose exec -T backend python scripts/cleanup_demo_artifacts.py`. ## Run locally diff --git a/backend/scripts/cleanup_demo_artifacts.py b/backend/scripts/cleanup_demo_artifacts.py new file mode 100644 index 00000000..9cd88111 --- /dev/null +++ b/backend/scripts/cleanup_demo_artifacts.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime +from pathlib import Path +from typing import Any + +BACKEND_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(BACKEND_ROOT)) + +from app.core.config import get_settings +from app.db.session import SessionLocal +from app.models import Export, Project + + +DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA" + + +def is_within_storage_root(path: Path, storage_root: Path) -> bool: + try: + path.resolve().relative_to(storage_root.resolve()) + except ValueError: + return False + return True + + +def export_created_at(export: Any) -> datetime: + created_at = getattr(export, "created_at", None) + if isinstance(created_at, datetime): + return created_at + return datetime.min + + +def select_cleanup_candidates(exports: list[Any], keep_latest: int) -> tuple[list[Any], list[Any]]: + if keep_latest < 0: + raise ValueError("keep_latest must be greater than or equal to zero") + ordered = sorted(exports, key=export_created_at, reverse=True) + return ordered[:keep_latest], ordered[keep_latest:] + + +def export_path(export: Any) -> Path: + return Path(str(getattr(export, "storage_path"))) + + +def prune_empty_parents(start_path: Path, storage_root: Path) -> list[str]: + pruned: list[str] = [] + parent = start_path.resolve().parent + stop_at = storage_root.resolve() + while parent != stop_at and is_within_storage_root(parent, stop_at): + try: + parent.rmdir() + except OSError: + break + pruned.append(str(parent)) + parent = parent.parent + return pruned + + +def cleanup_demo_exports(project_name: str, keep_latest: int, apply: bool) -> dict[str, Any]: + settings = get_settings() + storage_root = Path(settings.storage_root).resolve() + summary: dict[str, Any] = { + "dry_run": not apply, + "project_name": project_name, + "storage_root": str(storage_root), + "projects": [], + "matched_export_count": 0, + "selected_export_count": 0, + "deleted_export_count": 0, + "deleted_files": [], + "missing_files": [], + "skipped_outside_storage": [], + "pruned_dirs": [], + "kept_export_ids": [], + } + + with SessionLocal() as db: + projects = ( + db.query(Project) + .filter(Project.name == project_name) + .filter(Project.status != "deleted") + .order_by(Project.created_at.desc()) + .all() + ) + for project in projects: + exports = ( + db.query(Export) + .filter(Export.project_id == project.id) + .order_by(Export.created_at.desc()) + .all() + ) + kept, candidates = select_cleanup_candidates(exports, keep_latest) + summary["projects"].append(str(project.id)) + summary["matched_export_count"] += len(exports) + summary["selected_export_count"] += len(candidates) + summary["kept_export_ids"].extend(str(export.id) for export in kept) + + for export in candidates: + path = export_path(export) + if not is_within_storage_root(path, storage_root): + summary["skipped_outside_storage"].append( + {"export_id": str(export.id), "storage_path": str(path)} + ) + continue + + if path.exists(): + if apply: + path.unlink() + summary["pruned_dirs"].extend(prune_empty_parents(path, storage_root)) + summary["deleted_files"].append(str(path)) + else: + summary["missing_files"].append({"export_id": str(export.id), "storage_path": str(path)}) + + if apply: + db.delete(export) + summary["deleted_export_count"] += 1 + + if apply: + db.commit() + + return summary + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Clean old offline demo export artifacts. The script is dry-run by default " + "and only targets the explicit GeoIntel demo project unless overridden." + ) + ) + parser.add_argument("--project-name", default=DEMO_PROJECT_NAME, help="Exact project name to clean.") + parser.add_argument( + "--keep-latest", + type=int, + default=3, + help="Number of newest export records/files to keep per matching project.", + ) + parser.add_argument("--apply", action="store_true", help="Delete selected export rows and files.") + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + if args.keep_latest < 0: + parser.error("--keep-latest must be greater than or equal to zero") + + summary = cleanup_demo_exports( + project_name=args.project_name, + keep_latest=args.keep_latest, + apply=args.apply, + ) + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tests/test_readiness_gate.py b/backend/tests/test_readiness_gate.py index 3b77682d..e72c329a 100644 --- a/backend/tests/test_readiness_gate.py +++ b/backend/tests/test_readiness_gate.py @@ -27,6 +27,7 @@ def test_readiness_gate_compiles_demo_cleanup_script() -> None: content = script.read_text(encoding="utf-8") assert "-m py_compile scripts/cleanup_demo_artifacts.py" in content + assert "-m py_compile backend/scripts/cleanup_demo_artifacts.py" in content def test_demo_export_workflow_script_verifies_export_endpoints() -> None: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index a725716d..63362bff 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -2,6 +2,7 @@ Changed: - Added `scripts/cleanup_demo_artifacts.py` for dry-run-first cleanup of old offline demo export artifacts. +- Added `backend/scripts/cleanup_demo_artifacts.py` so the same cleanup can run inside the backend Docker container. - Cleanup is constrained to an exact demo project name by default, keeps the newest exports per project and refuses file deletion outside `STORAGE_ROOT`. - Added regression tests for cleanup selection, path safety and readiness gate coverage. - Added Python compile validation for the cleanup script to `scripts/run_readiness_check.sh`. @@ -10,7 +11,7 @@ Changed: Validation: - `python -m py_compile scripts/cleanup_demo_artifacts.py` passed. - `python -m pytest backend/tests/test_sprint24_cleanup_demo_artifacts.py backend/tests/test_readiness_gate.py` passed: 10 tests. -- `bash scripts/run_readiness_check.sh` passed: 151 backend tests, frontend typecheck/build, Alembic head check and script syntax checks. +- `bash scripts/run_readiness_check.sh` passed twice after adding the backend container entrypoint: 151 backend tests, frontend typecheck/build, Alembic head check and script syntax checks. - `python -m compileall backend/app` passed. - `cd backend && python -m alembic upgrade head --sql` passed. - `bash -n scripts/live_migration_smoke.sh` and `bash -n scripts/verify_demo_export_workflow.sh` passed. diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index 64a3c4ea..6fff8a42 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -104,6 +104,7 @@ Offline demo export artifacts can be inspected and cleaned with: ```bash python scripts/cleanup_demo_artifacts.py python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply +docker compose exec -T backend python scripts/cleanup_demo_artifacts.py ``` The script is dry-run by default, targets only the explicit diff --git a/scripts/README.md b/scripts/README.md index 9e01ce93..7a444dba 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -30,6 +30,13 @@ python scripts/cleanup_demo_artifacts.py python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply ``` +Against the Docker runtime, run the backend-container entrypoint: + +```bash +docker compose exec -T backend python scripts/cleanup_demo_artifacts.py +docker compose exec -T backend python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply +``` + The cleanup script is dry-run by default. It only targets the explicit `GeoIntel Demo - Building QA` project unless `--project-name` is provided, keeps the newest exports per matching project, deletes only `exports` rows/files when diff --git a/scripts/cleanup_demo_artifacts.py b/scripts/cleanup_demo_artifacts.py index 2bc8f9f0..b7a2504b 100644 --- a/scripts/cleanup_demo_artifacts.py +++ b/scripts/cleanup_demo_artifacts.py @@ -1,159 +1,31 @@ from __future__ import annotations -import argparse -import json -import sys -from datetime import datetime +import importlib.util from pathlib import Path -from typing import Any - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "backend")) - -from app.core.config import get_settings -from app.db.session import SessionLocal -from app.models import Export, Project +from types import ModuleType -DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA" +def load_backend_script() -> ModuleType: + script = Path(__file__).resolve().parents[1] / "backend" / "scripts" / "cleanup_demo_artifacts.py" + spec = importlib.util.spec_from_file_location("backend_cleanup_demo_artifacts", script) + if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load cleanup implementation from {script}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module -def is_within_storage_root(path: Path, storage_root: Path) -> bool: - try: - path.resolve().relative_to(storage_root.resolve()) - except ValueError: - return False - return True +_impl = load_backend_script() - -def export_created_at(export: Any) -> datetime: - created_at = getattr(export, "created_at", None) - if isinstance(created_at, datetime): - return created_at - return datetime.min - - -def select_cleanup_candidates(exports: list[Any], keep_latest: int) -> tuple[list[Any], list[Any]]: - if keep_latest < 0: - raise ValueError("keep_latest must be greater than or equal to zero") - ordered = sorted(exports, key=export_created_at, reverse=True) - return ordered[:keep_latest], ordered[keep_latest:] - - -def export_path(export: Any) -> Path: - return Path(str(getattr(export, "storage_path"))) - - -def prune_empty_parents(start_path: Path, storage_root: Path) -> list[str]: - pruned: list[str] = [] - parent = start_path.resolve().parent - stop_at = storage_root.resolve() - while parent != stop_at and is_within_storage_root(parent, stop_at): - try: - parent.rmdir() - except OSError: - break - pruned.append(str(parent)) - parent = parent.parent - return pruned - - -def cleanup_demo_exports(project_name: str, keep_latest: int, apply: bool) -> dict[str, Any]: - settings = get_settings() - storage_root = Path(settings.storage_root).resolve() - summary: dict[str, Any] = { - "dry_run": not apply, - "project_name": project_name, - "storage_root": str(storage_root), - "projects": [], - "matched_export_count": 0, - "selected_export_count": 0, - "deleted_export_count": 0, - "deleted_files": [], - "missing_files": [], - "skipped_outside_storage": [], - "pruned_dirs": [], - "kept_export_ids": [], - } - - with SessionLocal() as db: - projects = ( - db.query(Project) - .filter(Project.name == project_name) - .filter(Project.status != "deleted") - .order_by(Project.created_at.desc()) - .all() - ) - for project in projects: - exports = ( - db.query(Export) - .filter(Export.project_id == project.id) - .order_by(Export.created_at.desc()) - .all() - ) - kept, candidates = select_cleanup_candidates(exports, keep_latest) - summary["projects"].append(str(project.id)) - summary["matched_export_count"] += len(exports) - summary["selected_export_count"] += len(candidates) - summary["kept_export_ids"].extend(str(export.id) for export in kept) - - for export in candidates: - path = export_path(export) - if not is_within_storage_root(path, storage_root): - summary["skipped_outside_storage"].append( - {"export_id": str(export.id), "storage_path": str(path)} - ) - continue - - if path.exists(): - if apply: - path.unlink() - summary["pruned_dirs"].extend(prune_empty_parents(path, storage_root)) - summary["deleted_files"].append(str(path)) - else: - summary["missing_files"].append({"export_id": str(export.id), "storage_path": str(path)}) - - if apply: - db.delete(export) - summary["deleted_export_count"] += 1 - - if apply: - db.commit() - - return summary - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description=( - "Clean old offline demo export artifacts. The script is dry-run by default " - "and only targets the explicit GeoIntel demo project unless overridden." - ) - ) - parser.add_argument("--project-name", default=DEMO_PROJECT_NAME, help="Exact project name to clean.") - parser.add_argument( - "--keep-latest", - type=int, - default=3, - help="Number of newest export records/files to keep per matching project.", - ) - parser.add_argument("--apply", action="store_true", help="Delete selected export rows and files.") - return parser - - -def main() -> int: - parser = build_parser() - args = parser.parse_args() - if args.keep_latest < 0: - parser.error("--keep-latest must be greater than or equal to zero") - - summary = cleanup_demo_exports( - project_name=args.project_name, - keep_latest=args.keep_latest, - apply=args.apply, - ) - print(json.dumps(summary, indent=2, sort_keys=True)) - return 0 +DEMO_PROJECT_NAME = _impl.DEMO_PROJECT_NAME +is_within_storage_root = _impl.is_within_storage_root +export_created_at = _impl.export_created_at +select_cleanup_candidates = _impl.select_cleanup_candidates +export_path = _impl.export_path +prune_empty_parents = _impl.prune_empty_parents +cleanup_demo_exports = _impl.cleanup_demo_exports +build_parser = _impl.build_parser +main = _impl.main if __name__ == "__main__": diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 47611d61..ae343390 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -38,6 +38,7 @@ ${PYTHON_BIN} scripts/validate_m14_launch_assets.py ${PYTHON_BIN} -m py_compile scripts/gis_import_smoke.py ${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py +${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m compileall backend/app (cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning) (cd backend && ${PYTHON_BIN} -m alembic heads)