Make demo cleanup runnable in backend container
This commit is contained in:
@@ -10,6 +10,7 @@
|
|||||||
## Sprint 24 demo/export artifact cleanup tooling (2026-06-17)
|
## Sprint 24 demo/export artifact cleanup tooling (2026-06-17)
|
||||||
|
|
||||||
- Added `scripts/cleanup_demo_artifacts.py`, a dry-run-first maintenance script for old offline demo export artifacts.
|
- Added `scripts/cleanup_demo_artifacts.py`, a dry-run-first maintenance script for old offline demo export artifacts.
|
||||||
|
- Added the same cleanup entrypoint under `backend/scripts/` so it can run inside the backend Docker container.
|
||||||
- The cleanup keeps the newest exports per matching demo project, deletes only explicit `exports` rows/files when `--apply` is set and refuses file deletion outside `STORAGE_ROOT`.
|
- The cleanup keeps the newest exports per matching demo project, deletes only explicit `exports` rows/files when `--apply` is set and refuses file deletion outside `STORAGE_ROOT`.
|
||||||
- Added tests for cleanup candidate selection, storage-root path safety and readiness gate coverage.
|
- Added tests for cleanup candidate selection, storage-root path safety and readiness gate coverage.
|
||||||
- Added the cleanup script to the main readiness gate via Python compile validation.
|
- Added the cleanup script to the main readiness gate via Python compile validation.
|
||||||
|
|||||||
@@ -188,6 +188,7 @@ bash scripts/live_migration_smoke.sh
|
|||||||
- Old offline demo export artifacts can be inspected with `python scripts/cleanup_demo_artifacts.py`
|
- Old offline demo export artifacts can be inspected with `python scripts/cleanup_demo_artifacts.py`
|
||||||
and removed only with an explicit `--apply`. The script keeps the newest exports
|
and removed only with an explicit `--apply`. The script keeps the newest exports
|
||||||
per demo project and refuses to delete files outside `STORAGE_ROOT`.
|
per demo project and refuses to delete files outside `STORAGE_ROOT`.
|
||||||
|
In Docker, use `docker compose exec -T backend python scripts/cleanup_demo_artifacts.py`.
|
||||||
|
|
||||||
## Run locally
|
## Run locally
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,160 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
BACKEND_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(BACKEND_ROOT))
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
|
from app.db.session import SessionLocal
|
||||||
|
from app.models import Export, Project
|
||||||
|
|
||||||
|
|
||||||
|
DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
|
||||||
|
|
||||||
|
|
||||||
|
def is_within_storage_root(path: Path, storage_root: Path) -> bool:
|
||||||
|
try:
|
||||||
|
path.resolve().relative_to(storage_root.resolve())
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def export_created_at(export: Any) -> datetime:
|
||||||
|
created_at = getattr(export, "created_at", None)
|
||||||
|
if isinstance(created_at, datetime):
|
||||||
|
return created_at
|
||||||
|
return datetime.min
|
||||||
|
|
||||||
|
|
||||||
|
def select_cleanup_candidates(exports: list[Any], keep_latest: int) -> tuple[list[Any], list[Any]]:
|
||||||
|
if keep_latest < 0:
|
||||||
|
raise ValueError("keep_latest must be greater than or equal to zero")
|
||||||
|
ordered = sorted(exports, key=export_created_at, reverse=True)
|
||||||
|
return ordered[:keep_latest], ordered[keep_latest:]
|
||||||
|
|
||||||
|
|
||||||
|
def export_path(export: Any) -> Path:
|
||||||
|
return Path(str(getattr(export, "storage_path")))
|
||||||
|
|
||||||
|
|
||||||
|
def prune_empty_parents(start_path: Path, storage_root: Path) -> list[str]:
|
||||||
|
pruned: list[str] = []
|
||||||
|
parent = start_path.resolve().parent
|
||||||
|
stop_at = storage_root.resolve()
|
||||||
|
while parent != stop_at and is_within_storage_root(parent, stop_at):
|
||||||
|
try:
|
||||||
|
parent.rmdir()
|
||||||
|
except OSError:
|
||||||
|
break
|
||||||
|
pruned.append(str(parent))
|
||||||
|
parent = parent.parent
|
||||||
|
return pruned
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_demo_exports(project_name: str, keep_latest: int, apply: bool) -> dict[str, Any]:
|
||||||
|
settings = get_settings()
|
||||||
|
storage_root = Path(settings.storage_root).resolve()
|
||||||
|
summary: dict[str, Any] = {
|
||||||
|
"dry_run": not apply,
|
||||||
|
"project_name": project_name,
|
||||||
|
"storage_root": str(storage_root),
|
||||||
|
"projects": [],
|
||||||
|
"matched_export_count": 0,
|
||||||
|
"selected_export_count": 0,
|
||||||
|
"deleted_export_count": 0,
|
||||||
|
"deleted_files": [],
|
||||||
|
"missing_files": [],
|
||||||
|
"skipped_outside_storage": [],
|
||||||
|
"pruned_dirs": [],
|
||||||
|
"kept_export_ids": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
with SessionLocal() as db:
|
||||||
|
projects = (
|
||||||
|
db.query(Project)
|
||||||
|
.filter(Project.name == project_name)
|
||||||
|
.filter(Project.status != "deleted")
|
||||||
|
.order_by(Project.created_at.desc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
for project in projects:
|
||||||
|
exports = (
|
||||||
|
db.query(Export)
|
||||||
|
.filter(Export.project_id == project.id)
|
||||||
|
.order_by(Export.created_at.desc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
kept, candidates = select_cleanup_candidates(exports, keep_latest)
|
||||||
|
summary["projects"].append(str(project.id))
|
||||||
|
summary["matched_export_count"] += len(exports)
|
||||||
|
summary["selected_export_count"] += len(candidates)
|
||||||
|
summary["kept_export_ids"].extend(str(export.id) for export in kept)
|
||||||
|
|
||||||
|
for export in candidates:
|
||||||
|
path = export_path(export)
|
||||||
|
if not is_within_storage_root(path, storage_root):
|
||||||
|
summary["skipped_outside_storage"].append(
|
||||||
|
{"export_id": str(export.id), "storage_path": str(path)}
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if path.exists():
|
||||||
|
if apply:
|
||||||
|
path.unlink()
|
||||||
|
summary["pruned_dirs"].extend(prune_empty_parents(path, storage_root))
|
||||||
|
summary["deleted_files"].append(str(path))
|
||||||
|
else:
|
||||||
|
summary["missing_files"].append({"export_id": str(export.id), "storage_path": str(path)})
|
||||||
|
|
||||||
|
if apply:
|
||||||
|
db.delete(export)
|
||||||
|
summary["deleted_export_count"] += 1
|
||||||
|
|
||||||
|
if apply:
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description=(
|
||||||
|
"Clean old offline demo export artifacts. The script is dry-run by default "
|
||||||
|
"and only targets the explicit GeoIntel demo project unless overridden."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
parser.add_argument("--project-name", default=DEMO_PROJECT_NAME, help="Exact project name to clean.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--keep-latest",
|
||||||
|
type=int,
|
||||||
|
default=3,
|
||||||
|
help="Number of newest export records/files to keep per matching project.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--apply", action="store_true", help="Delete selected export rows and files.")
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = build_parser()
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.keep_latest < 0:
|
||||||
|
parser.error("--keep-latest must be greater than or equal to zero")
|
||||||
|
|
||||||
|
summary = cleanup_demo_exports(
|
||||||
|
project_name=args.project_name,
|
||||||
|
keep_latest=args.keep_latest,
|
||||||
|
apply=args.apply,
|
||||||
|
)
|
||||||
|
print(json.dumps(summary, indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -27,6 +27,7 @@ def test_readiness_gate_compiles_demo_cleanup_script() -> None:
|
|||||||
content = script.read_text(encoding="utf-8")
|
content = script.read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert "-m py_compile scripts/cleanup_demo_artifacts.py" in content
|
assert "-m py_compile scripts/cleanup_demo_artifacts.py" in content
|
||||||
|
assert "-m py_compile backend/scripts/cleanup_demo_artifacts.py" in content
|
||||||
|
|
||||||
|
|
||||||
def test_demo_export_workflow_script_verifies_export_endpoints() -> None:
|
def test_demo_export_workflow_script_verifies_export_endpoints() -> None:
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
Changed:
|
Changed:
|
||||||
- Added `scripts/cleanup_demo_artifacts.py` for dry-run-first cleanup of old offline demo export artifacts.
|
- Added `scripts/cleanup_demo_artifacts.py` for dry-run-first cleanup of old offline demo export artifacts.
|
||||||
|
- Added `backend/scripts/cleanup_demo_artifacts.py` so the same cleanup can run inside the backend Docker container.
|
||||||
- Cleanup is constrained to an exact demo project name by default, keeps the newest exports per project and refuses file deletion outside `STORAGE_ROOT`.
|
- Cleanup is constrained to an exact demo project name by default, keeps the newest exports per project and refuses file deletion outside `STORAGE_ROOT`.
|
||||||
- Added regression tests for cleanup selection, path safety and readiness gate coverage.
|
- Added regression tests for cleanup selection, path safety and readiness gate coverage.
|
||||||
- Added Python compile validation for the cleanup script to `scripts/run_readiness_check.sh`.
|
- Added Python compile validation for the cleanup script to `scripts/run_readiness_check.sh`.
|
||||||
@@ -10,7 +11,7 @@ Changed:
|
|||||||
Validation:
|
Validation:
|
||||||
- `python -m py_compile scripts/cleanup_demo_artifacts.py` passed.
|
- `python -m py_compile scripts/cleanup_demo_artifacts.py` passed.
|
||||||
- `python -m pytest backend/tests/test_sprint24_cleanup_demo_artifacts.py backend/tests/test_readiness_gate.py` passed: 10 tests.
|
- `python -m pytest backend/tests/test_sprint24_cleanup_demo_artifacts.py backend/tests/test_readiness_gate.py` passed: 10 tests.
|
||||||
- `bash scripts/run_readiness_check.sh` passed: 151 backend tests, frontend typecheck/build, Alembic head check and script syntax checks.
|
- `bash scripts/run_readiness_check.sh` passed twice after adding the backend container entrypoint: 151 backend tests, frontend typecheck/build, Alembic head check and script syntax checks.
|
||||||
- `python -m compileall backend/app` passed.
|
- `python -m compileall backend/app` passed.
|
||||||
- `cd backend && python -m alembic upgrade head --sql` passed.
|
- `cd backend && python -m alembic upgrade head --sql` passed.
|
||||||
- `bash -n scripts/live_migration_smoke.sh` and `bash -n scripts/verify_demo_export_workflow.sh` passed.
|
- `bash -n scripts/live_migration_smoke.sh` and `bash -n scripts/verify_demo_export_workflow.sh` passed.
|
||||||
|
|||||||
@@ -104,6 +104,7 @@ Offline demo export artifacts can be inspected and cleaned with:
|
|||||||
```bash
|
```bash
|
||||||
python scripts/cleanup_demo_artifacts.py
|
python scripts/cleanup_demo_artifacts.py
|
||||||
python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply
|
python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply
|
||||||
|
docker compose exec -T backend python scripts/cleanup_demo_artifacts.py
|
||||||
```
|
```
|
||||||
|
|
||||||
The script is dry-run by default, targets only the explicit
|
The script is dry-run by default, targets only the explicit
|
||||||
|
|||||||
@@ -30,6 +30,13 @@ python scripts/cleanup_demo_artifacts.py
|
|||||||
python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply
|
python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Against the Docker runtime, run the backend-container entrypoint:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose exec -T backend python scripts/cleanup_demo_artifacts.py
|
||||||
|
docker compose exec -T backend python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply
|
||||||
|
```
|
||||||
|
|
||||||
The cleanup script is dry-run by default. It only targets the explicit
|
The cleanup script is dry-run by default. It only targets the explicit
|
||||||
`GeoIntel Demo - Building QA` project unless `--project-name` is provided, keeps
|
`GeoIntel Demo - Building QA` project unless `--project-name` is provided, keeps
|
||||||
the newest exports per matching project, deletes only `exports` rows/files when
|
the newest exports per matching project, deletes only `exports` rows/files when
|
||||||
|
|||||||
@@ -1,159 +1,31 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import importlib.util
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
from datetime import datetime
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from types import ModuleType
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
sys.path.insert(0, str(ROOT / "backend"))
|
|
||||||
|
|
||||||
from app.core.config import get_settings
|
|
||||||
from app.db.session import SessionLocal
|
|
||||||
from app.models import Export, Project
|
|
||||||
|
|
||||||
|
|
||||||
DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
|
def load_backend_script() -> ModuleType:
|
||||||
|
script = Path(__file__).resolve().parents[1] / "backend" / "scripts" / "cleanup_demo_artifacts.py"
|
||||||
|
spec = importlib.util.spec_from_file_location("backend_cleanup_demo_artifacts", script)
|
||||||
|
if spec is None or spec.loader is None:
|
||||||
|
raise RuntimeError(f"Could not load cleanup implementation from {script}")
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
def is_within_storage_root(path: Path, storage_root: Path) -> bool:
|
_impl = load_backend_script()
|
||||||
try:
|
|
||||||
path.resolve().relative_to(storage_root.resolve())
|
|
||||||
except ValueError:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
DEMO_PROJECT_NAME = _impl.DEMO_PROJECT_NAME
|
||||||
def export_created_at(export: Any) -> datetime:
|
is_within_storage_root = _impl.is_within_storage_root
|
||||||
created_at = getattr(export, "created_at", None)
|
export_created_at = _impl.export_created_at
|
||||||
if isinstance(created_at, datetime):
|
select_cleanup_candidates = _impl.select_cleanup_candidates
|
||||||
return created_at
|
export_path = _impl.export_path
|
||||||
return datetime.min
|
prune_empty_parents = _impl.prune_empty_parents
|
||||||
|
cleanup_demo_exports = _impl.cleanup_demo_exports
|
||||||
|
build_parser = _impl.build_parser
|
||||||
def select_cleanup_candidates(exports: list[Any], keep_latest: int) -> tuple[list[Any], list[Any]]:
|
main = _impl.main
|
||||||
if keep_latest < 0:
|
|
||||||
raise ValueError("keep_latest must be greater than or equal to zero")
|
|
||||||
ordered = sorted(exports, key=export_created_at, reverse=True)
|
|
||||||
return ordered[:keep_latest], ordered[keep_latest:]
|
|
||||||
|
|
||||||
|
|
||||||
def export_path(export: Any) -> Path:
|
|
||||||
return Path(str(getattr(export, "storage_path")))
|
|
||||||
|
|
||||||
|
|
||||||
def prune_empty_parents(start_path: Path, storage_root: Path) -> list[str]:
|
|
||||||
pruned: list[str] = []
|
|
||||||
parent = start_path.resolve().parent
|
|
||||||
stop_at = storage_root.resolve()
|
|
||||||
while parent != stop_at and is_within_storage_root(parent, stop_at):
|
|
||||||
try:
|
|
||||||
parent.rmdir()
|
|
||||||
except OSError:
|
|
||||||
break
|
|
||||||
pruned.append(str(parent))
|
|
||||||
parent = parent.parent
|
|
||||||
return pruned
|
|
||||||
|
|
||||||
|
|
||||||
def cleanup_demo_exports(project_name: str, keep_latest: int, apply: bool) -> dict[str, Any]:
|
|
||||||
settings = get_settings()
|
|
||||||
storage_root = Path(settings.storage_root).resolve()
|
|
||||||
summary: dict[str, Any] = {
|
|
||||||
"dry_run": not apply,
|
|
||||||
"project_name": project_name,
|
|
||||||
"storage_root": str(storage_root),
|
|
||||||
"projects": [],
|
|
||||||
"matched_export_count": 0,
|
|
||||||
"selected_export_count": 0,
|
|
||||||
"deleted_export_count": 0,
|
|
||||||
"deleted_files": [],
|
|
||||||
"missing_files": [],
|
|
||||||
"skipped_outside_storage": [],
|
|
||||||
"pruned_dirs": [],
|
|
||||||
"kept_export_ids": [],
|
|
||||||
}
|
|
||||||
|
|
||||||
with SessionLocal() as db:
|
|
||||||
projects = (
|
|
||||||
db.query(Project)
|
|
||||||
.filter(Project.name == project_name)
|
|
||||||
.filter(Project.status != "deleted")
|
|
||||||
.order_by(Project.created_at.desc())
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
for project in projects:
|
|
||||||
exports = (
|
|
||||||
db.query(Export)
|
|
||||||
.filter(Export.project_id == project.id)
|
|
||||||
.order_by(Export.created_at.desc())
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
kept, candidates = select_cleanup_candidates(exports, keep_latest)
|
|
||||||
summary["projects"].append(str(project.id))
|
|
||||||
summary["matched_export_count"] += len(exports)
|
|
||||||
summary["selected_export_count"] += len(candidates)
|
|
||||||
summary["kept_export_ids"].extend(str(export.id) for export in kept)
|
|
||||||
|
|
||||||
for export in candidates:
|
|
||||||
path = export_path(export)
|
|
||||||
if not is_within_storage_root(path, storage_root):
|
|
||||||
summary["skipped_outside_storage"].append(
|
|
||||||
{"export_id": str(export.id), "storage_path": str(path)}
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
if path.exists():
|
|
||||||
if apply:
|
|
||||||
path.unlink()
|
|
||||||
summary["pruned_dirs"].extend(prune_empty_parents(path, storage_root))
|
|
||||||
summary["deleted_files"].append(str(path))
|
|
||||||
else:
|
|
||||||
summary["missing_files"].append({"export_id": str(export.id), "storage_path": str(path)})
|
|
||||||
|
|
||||||
if apply:
|
|
||||||
db.delete(export)
|
|
||||||
summary["deleted_export_count"] += 1
|
|
||||||
|
|
||||||
if apply:
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
return summary
|
|
||||||
|
|
||||||
|
|
||||||
def build_parser() -> argparse.ArgumentParser:
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description=(
|
|
||||||
"Clean old offline demo export artifacts. The script is dry-run by default "
|
|
||||||
"and only targets the explicit GeoIntel demo project unless overridden."
|
|
||||||
)
|
|
||||||
)
|
|
||||||
parser.add_argument("--project-name", default=DEMO_PROJECT_NAME, help="Exact project name to clean.")
|
|
||||||
parser.add_argument(
|
|
||||||
"--keep-latest",
|
|
||||||
type=int,
|
|
||||||
default=3,
|
|
||||||
help="Number of newest export records/files to keep per matching project.",
|
|
||||||
)
|
|
||||||
parser.add_argument("--apply", action="store_true", help="Delete selected export rows and files.")
|
|
||||||
return parser
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = build_parser()
|
|
||||||
args = parser.parse_args()
|
|
||||||
if args.keep_latest < 0:
|
|
||||||
parser.error("--keep-latest must be greater than or equal to zero")
|
|
||||||
|
|
||||||
summary = cleanup_demo_exports(
|
|
||||||
project_name=args.project_name,
|
|
||||||
keep_latest=args.keep_latest,
|
|
||||||
apply=args.apply,
|
|
||||||
)
|
|
||||||
print(json.dumps(summary, indent=2, sort_keys=True))
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ ${PYTHON_BIN} scripts/validate_m14_launch_assets.py
|
|||||||
${PYTHON_BIN} -m py_compile scripts/gis_import_smoke.py
|
${PYTHON_BIN} -m py_compile scripts/gis_import_smoke.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py
|
${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
||||||
|
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
||||||
${PYTHON_BIN} -m compileall backend/app
|
${PYTHON_BIN} -m compileall backend/app
|
||||||
(cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning)
|
(cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning)
|
||||||
(cd backend && ${PYTHON_BIN} -m alembic heads)
|
(cd backend && ${PYTHON_BIN} -m alembic heads)
|
||||||
|
|||||||
Reference in New Issue
Block a user