From baf830fb5b5562b31cd3cfe7ee6e814558b9c5c8 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 17 Jun 2026 01:42:00 +0200 Subject: [PATCH] Add demo export cleanup tooling --- CHANGELOG.md | 8 + backend/README.md | 3 + backend/tests/test_readiness_gate.py | 7 + .../test_sprint24_cleanup_demo_artifacts.py | 75 ++++++++ docs/CODEX_EXECUTION_LOG.md | 22 ++- docs/STORAGE_ARCHITECTURE.md | 14 ++ docs/TODO.md | 1 + scripts/README.md | 13 ++ scripts/cleanup_demo_artifacts.py | 160 ++++++++++++++++++ scripts/run_readiness_check.sh | 1 + 10 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_sprint24_cleanup_demo_artifacts.py create mode 100644 scripts/cleanup_demo_artifacts.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9903d059..af0cb53e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ # Changelog +## Sprint 24 demo/export artifact cleanup tooling (2026-06-17) + +- Added `scripts/cleanup_demo_artifacts.py`, a dry-run-first maintenance script for old offline demo export artifacts. +- The cleanup keeps the newest exports per matching demo project, deletes only explicit `exports` rows/files when `--apply` is set and refuses file deletion outside `STORAGE_ROOT`. +- Added tests for cleanup candidate selection, storage-root path safety and readiness gate coverage. +- Added the cleanup script to the main readiness gate via Python compile validation. +- No API contracts, migrations, product features, provider fetching, AI inference or source dataset cleanup behavior were changed. + ## Sprint 23 V1 report handoff summary (2026-06-17) - Added V1 readiness summary data to project metadata exports. diff --git a/backend/README.md b/backend/README.md index 9fe412c4..7ce1378a 100644 --- a/backend/README.md +++ b/backend/README.md @@ -185,6 +185,9 @@ bash scripts/live_migration_smoke.sh - `GET /api/v1/exports/{export_id}/content` - Exported detection and segmentation GeoJSON is generated from persisted first-class geometry rows. - No new migrations, product lines, live providers or AI dependencies are introduced by this export pass. +- Old offline demo export artifacts can be inspected with `python scripts/cleanup_demo_artifacts.py` + and removed only with an explicit `--apply`. The script keeps the newest exports + per demo project and refuses to delete files outside `STORAGE_ROOT`. ## Run locally diff --git a/backend/tests/test_readiness_gate.py b/backend/tests/test_readiness_gate.py index 43cafb7f..3b77682d 100644 --- a/backend/tests/test_readiness_gate.py +++ b/backend/tests/test_readiness_gate.py @@ -22,6 +22,13 @@ def test_readiness_gate_checks_demo_export_workflow_script_syntax() -> None: assert "bash -n scripts/verify_demo_export_workflow.sh" in content +def test_readiness_gate_compiles_demo_cleanup_script() -> None: + script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh" + content = script.read_text(encoding="utf-8") + + assert "-m py_compile scripts/cleanup_demo_artifacts.py" in content + + def test_demo_export_workflow_script_verifies_export_endpoints() -> None: script = Path(__file__).resolve().parents[2] / "scripts" / "verify_demo_export_workflow.sh" content = script.read_text(encoding="utf-8") diff --git a/backend/tests/test_sprint24_cleanup_demo_artifacts.py b/backend/tests/test_sprint24_cleanup_demo_artifacts.py new file mode 100644 index 00000000..6abd060c --- /dev/null +++ b/backend/tests/test_sprint24_cleanup_demo_artifacts.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import importlib.util +from dataclasses import dataclass +from datetime import datetime, timedelta +from pathlib import Path +from types import ModuleType + + +def load_cleanup_module() -> ModuleType: + script = Path(__file__).resolve().parents[2] / "scripts" / "cleanup_demo_artifacts.py" + spec = importlib.util.spec_from_file_location("cleanup_demo_artifacts", script) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@dataclass +class ExportRow: + id: str + created_at: datetime | None + storage_path: str + + +def test_cleanup_candidate_selection_keeps_newest_exports() -> None: + cleanup = load_cleanup_module() + base = datetime(2026, 1, 1, 12, 0, 0) + exports = [ + ExportRow("old", base, "/tmp/old.json"), + ExportRow("new", base + timedelta(days=2), "/tmp/new.json"), + ExportRow("middle", base + timedelta(days=1), "/tmp/middle.json"), + ExportRow("unknown", None, "/tmp/unknown.json"), + ] + + kept, candidates = cleanup.select_cleanup_candidates(exports, keep_latest=2) + + assert [export.id for export in kept] == ["new", "middle"] + assert [export.id for export in candidates] == ["old", "unknown"] + + +def test_cleanup_candidate_selection_rejects_negative_keep_latest() -> None: + cleanup = load_cleanup_module() + + try: + cleanup.select_cleanup_candidates([], keep_latest=-1) + except ValueError as exc: + assert "keep_latest" in str(exc) + else: + raise AssertionError("negative keep_latest should fail") + + +def test_cleanup_path_safety_requires_storage_root_containment(tmp_path: Path) -> None: + cleanup = load_cleanup_module() + storage_root = tmp_path / "storage" + safe_export = storage_root / "exports" / "project" / "report.html" + unsafe_export = tmp_path / "outside" / "report.html" + + safe_export.parent.mkdir(parents=True) + unsafe_export.parent.mkdir(parents=True) + + assert cleanup.is_within_storage_root(safe_export, storage_root) is True + assert cleanup.is_within_storage_root(unsafe_export, storage_root) is False + + +def test_cleanup_script_defaults_to_explicit_demo_project() -> None: + cleanup = load_cleanup_module() + parser = cleanup.build_parser() + + args = parser.parse_args([]) + + assert args.project_name == cleanup.DEMO_PROJECT_NAME + assert args.keep_latest == 3 + assert args.apply is False diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index aa3caa81..a725716d 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,23 @@ +## Sprint 24 demo/export artifact cleanup tooling (2026-06-17) + +Changed: +- Added `scripts/cleanup_demo_artifacts.py` for dry-run-first cleanup of old offline demo export artifacts. +- Cleanup is constrained to an exact demo project name by default, keeps the newest exports per project and refuses file deletion outside `STORAGE_ROOT`. +- Added regression tests for cleanup selection, path safety and readiness gate coverage. +- Added Python compile validation for the cleanup script to `scripts/run_readiness_check.sh`. +- Documented cleanup usage in `scripts/README.md`, `docs/STORAGE_ARCHITECTURE.md`, `backend/README.md`, `docs/TODO.md` and `CHANGELOG.md`. + +Validation: +- `python -m py_compile scripts/cleanup_demo_artifacts.py` passed. +- `python -m pytest backend/tests/test_sprint24_cleanup_demo_artifacts.py backend/tests/test_readiness_gate.py` passed: 10 tests. +- `bash scripts/run_readiness_check.sh` passed: 151 backend tests, frontend typecheck/build, Alembic head check and script syntax checks. +- `python -m compileall backend/app` passed. +- `cd backend && python -m alembic upgrade head --sql` passed. +- `bash -n scripts/live_migration_smoke.sh` and `bash -n scripts/verify_demo_export_workflow.sh` passed. +- Local `docker compose config` could not run because the Windows Docker CLI is not installed in this Codex environment. + +Notes: +- No API contracts, migrations, product features, provider fetching, AI inference or source dataset cleanup behavior changed. ## Sprint 23 V1 report handoff summary (2026-06-17) Changed: @@ -385,7 +405,7 @@ Date: 2026-06-11 ### Next recommended pass - Run full raster end-to-end tests with real GeoTIFF fixtures and validate output dataset metadata persistence. -## Pass 16 — Sprint 4 raster operations foundation hardening +## Pass 16 � Sprint 4 raster operations foundation hardening Date: 2026-06-11 ### Completed diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index 77193478..64a3c4ea 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -99,6 +99,20 @@ Mask files are provenance/debug artifacts. QA, map display and GeoJSON output mu Do not delete originals automatically. Derived outputs may be cleaned through explicit cache management. +Offline demo export artifacts can be inspected and cleaned with: + +```bash +python scripts/cleanup_demo_artifacts.py +python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply +``` + +The script is dry-run by default, targets only the explicit +`GeoIntel Demo - Building QA` project unless an exact `--project-name` is +provided, keeps the newest export artifacts per matching project and refuses to +delete files outside `STORAGE_ROOT`. It cleans `exports` records/files only; it +does not remove original uploads, vector features, QA/QC rows, projects, areas, +tiles, rasters or masks. + ## Model storage Model artifacts live under: diff --git a/docs/TODO.md b/docs/TODO.md index 552fbbe9..ecda7374 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -33,6 +33,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] V1 readiness handoff summary in project metadata/report exports. - [x] Browser-facing demo/export workflow smoke script with connected V1 state checks. - [x] Compact V1 workbench status strip for project, AOI, datasets, map, QA/QC and exports. +- [x] Dry-run-first demo export artifact cleanup tooling. - [x] Live Docker/PostGIS validation on Tower/Unraid. - [ ] Real YOLO compatibility smoke with optional AI extras and local model file. - [ ] Further frontend state/module decomposition beyond Sprint 10 extraction. diff --git a/scripts/README.md b/scripts/README.md index a01cccaa..9e01ce93 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -23,6 +23,19 @@ datasets, vector FeatureCollection content, vector feature summary, persisted QA/QC metrics, creates metadata/report/vector GeoJSON exports, lists exports and downloads the JSON/GeoJSON/HTML artifacts through the frontend proxy. +Clean old offline demo export artifacts without touching uploaded source data: + +```bash +python scripts/cleanup_demo_artifacts.py +python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply +``` + +The cleanup script is dry-run by default. It only targets the explicit +`GeoIntel Demo - Building QA` project unless `--project-name` is provided, keeps +the newest exports per matching project, deletes only `exports` rows/files when +`--apply` is set, and refuses to remove files outside the configured +`STORAGE_ROOT`. + ## Tower deployment Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime: diff --git a/scripts/cleanup_demo_artifacts.py b/scripts/cleanup_demo_artifacts.py new file mode 100644 index 00000000..2bc8f9f0 --- /dev/null +++ b/scripts/cleanup_demo_artifacts.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "backend")) + +from app.core.config import get_settings +from app.db.session import SessionLocal +from app.models import Export, Project + + +DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA" + + +def is_within_storage_root(path: Path, storage_root: Path) -> bool: + try: + path.resolve().relative_to(storage_root.resolve()) + except ValueError: + return False + return True + + +def export_created_at(export: Any) -> datetime: + created_at = getattr(export, "created_at", None) + if isinstance(created_at, datetime): + return created_at + return datetime.min + + +def select_cleanup_candidates(exports: list[Any], keep_latest: int) -> tuple[list[Any], list[Any]]: + if keep_latest < 0: + raise ValueError("keep_latest must be greater than or equal to zero") + ordered = sorted(exports, key=export_created_at, reverse=True) + return ordered[:keep_latest], ordered[keep_latest:] + + +def export_path(export: Any) -> Path: + return Path(str(getattr(export, "storage_path"))) + + +def prune_empty_parents(start_path: Path, storage_root: Path) -> list[str]: + pruned: list[str] = [] + parent = start_path.resolve().parent + stop_at = storage_root.resolve() + while parent != stop_at and is_within_storage_root(parent, stop_at): + try: + parent.rmdir() + except OSError: + break + pruned.append(str(parent)) + parent = parent.parent + return pruned + + +def cleanup_demo_exports(project_name: str, keep_latest: int, apply: bool) -> dict[str, Any]: + settings = get_settings() + storage_root = Path(settings.storage_root).resolve() + summary: dict[str, Any] = { + "dry_run": not apply, + "project_name": project_name, + "storage_root": str(storage_root), + "projects": [], + "matched_export_count": 0, + "selected_export_count": 0, + "deleted_export_count": 0, + "deleted_files": [], + "missing_files": [], + "skipped_outside_storage": [], + "pruned_dirs": [], + "kept_export_ids": [], + } + + with SessionLocal() as db: + projects = ( + db.query(Project) + .filter(Project.name == project_name) + .filter(Project.status != "deleted") + .order_by(Project.created_at.desc()) + .all() + ) + for project in projects: + exports = ( + db.query(Export) + .filter(Export.project_id == project.id) + .order_by(Export.created_at.desc()) + .all() + ) + kept, candidates = select_cleanup_candidates(exports, keep_latest) + summary["projects"].append(str(project.id)) + summary["matched_export_count"] += len(exports) + summary["selected_export_count"] += len(candidates) + summary["kept_export_ids"].extend(str(export.id) for export in kept) + + for export in candidates: + path = export_path(export) + if not is_within_storage_root(path, storage_root): + summary["skipped_outside_storage"].append( + {"export_id": str(export.id), "storage_path": str(path)} + ) + continue + + if path.exists(): + if apply: + path.unlink() + summary["pruned_dirs"].extend(prune_empty_parents(path, storage_root)) + summary["deleted_files"].append(str(path)) + else: + summary["missing_files"].append({"export_id": str(export.id), "storage_path": str(path)}) + + if apply: + db.delete(export) + summary["deleted_export_count"] += 1 + + if apply: + db.commit() + + return summary + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Clean old offline demo export artifacts. The script is dry-run by default " + "and only targets the explicit GeoIntel demo project unless overridden." + ) + ) + parser.add_argument("--project-name", default=DEMO_PROJECT_NAME, help="Exact project name to clean.") + parser.add_argument( + "--keep-latest", + type=int, + default=3, + help="Number of newest export records/files to keep per matching project.", + ) + parser.add_argument("--apply", action="store_true", help="Delete selected export rows and files.") + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + if args.keep_latest < 0: + parser.error("--keep-latest must be greater than or equal to zero") + + summary = cleanup_demo_exports( + project_name=args.project_name, + keep_latest=args.keep_latest, + apply=args.apply, + ) + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 00097f4c..47611d61 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -37,6 +37,7 @@ ${PYTHON_BIN} scripts/validate_m13_codex_assets.py ${PYTHON_BIN} scripts/validate_m14_launch_assets.py ${PYTHON_BIN} -m py_compile scripts/gis_import_smoke.py ${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py +${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m compileall backend/app (cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning) (cd backend && ${PYTHON_BIN} -m alembic heads)