diff --git a/CHANGELOG.md b/CHANGELOG.md index a6946647..163c05b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -635,3 +635,11 @@ Added: - Slightly compacted export action buttons so filters are visible earlier on standard desktop viewports. - Added regression coverage for filtering controls, filtered list limiting and the no-match state. - No API contracts, migrations, product capabilities, live provider fetching or AI/model dependency changes were introduced. + +## Sprint 57 safer demo export cleanup (2026-06-17) + +- Hardened the existing dry-run-first demo export cleanup command with a `--max-delete` safety cap. +- Added repeated `--export-type` filters so operators can clean only selected artifact kinds. +- Cleanup apply runs now report a `blocked_reason` instead of deleting when selected candidates exceed the cap. +- Updated root and backend cleanup entrypoints, docs and regression coverage. +- No API contracts, migrations, product capabilities, live provider fetching or AI/model dependency changes were introduced. diff --git a/backend/README.md b/backend/README.md index 9fa501a5..5a3dbbec 100644 --- a/backend/README.md +++ b/backend/README.md @@ -187,7 +187,9 @@ bash scripts/live_migration_smoke.sh - No new migrations, product lines, live providers or AI dependencies are introduced by this export pass. - Old offline demo export artifacts can be inspected with `python scripts/cleanup_demo_artifacts.py` and removed only with an explicit `--apply`. The script keeps the newest exports - per demo project and refuses to delete files outside `STORAGE_ROOT`. + per demo project, refuses to delete files outside `STORAGE_ROOT`, and blocks + apply runs above `--max-delete` until the cap is raised after a dry-run review. + Use repeated `--export-type` values to target only specific artifact kinds. In Docker, use `docker compose exec -T backend python scripts/cleanup_demo_artifacts.py`. ## Run locally diff --git a/backend/scripts/cleanup_demo_artifacts.py b/backend/scripts/cleanup_demo_artifacts.py index c5cb0019..fe2859a6 100644 --- a/backend/scripts/cleanup_demo_artifacts.py +++ b/backend/scripts/cleanup_demo_artifacts.py @@ -40,6 +40,13 @@ def select_cleanup_candidates(exports: list[Any], keep_latest: int) -> tuple[lis return ordered[:keep_latest], ordered[keep_latest:] +def filter_exports_by_type(exports: list[Any], export_types: list[str] | None) -> list[Any]: + if not export_types: + return exports + allowed = set(export_types) + return [export for export in exports if getattr(export, "export_type", None) in allowed] + + def export_path(export: Any) -> Path: return Path(str(getattr(export, "storage_path"))) @@ -58,15 +65,27 @@ def prune_empty_parents(start_path: Path, storage_root: Path) -> list[str]: return pruned -def cleanup_demo_exports(project_name: str, keep_latest: int, apply: bool) -> dict[str, Any]: +def cleanup_demo_exports( + project_name: str, + keep_latest: int, + apply: bool, + max_delete: int, + export_types: list[str] | None = None, +) -> dict[str, Any]: + if max_delete < 0: + raise ValueError("max_delete must be greater than or equal to zero") settings = get_settings() storage_root = Path(settings.storage_root).resolve() summary: dict[str, Any] = { "dry_run": not apply, "project_name": project_name, + "keep_latest": keep_latest, + "max_delete": max_delete, + "export_types": export_types or [], "storage_root": str(storage_root), "projects": [], "matched_export_count": 0, + "type_filtered_export_count": 0, "selected_export_count": 0, "deleted_export_count": 0, "candidate_files": [], @@ -92,12 +111,21 @@ def cleanup_demo_exports(project_name: str, keep_latest: int, apply: bool) -> di .order_by(Export.created_at.desc()) .all() ) - kept, candidates = select_cleanup_candidates(exports, keep_latest) + filtered_exports = filter_exports_by_type(exports, export_types) + kept, candidates = select_cleanup_candidates(filtered_exports, keep_latest) summary["projects"].append(str(project.id)) summary["matched_export_count"] += len(exports) + summary["type_filtered_export_count"] += len(filtered_exports) summary["selected_export_count"] += len(candidates) summary["kept_export_ids"].extend(str(export.id) for export in kept) + if apply and len(candidates) > max_delete: + summary["blocked_reason"] = ( + f"selected_export_count {len(candidates)} exceeds --max-delete {max_delete}; " + "raise --max-delete after reviewing a dry run" + ) + continue + for export in candidates: path = export_path(export) if not is_within_storage_root(path, storage_root): @@ -140,6 +168,18 @@ def build_parser() -> argparse.ArgumentParser: default=3, help="Number of newest export records/files to keep per matching project.", ) + parser.add_argument( + "--max-delete", + type=int, + default=25, + help="Maximum export rows/files allowed to be deleted per matching project when --apply is set.", + ) + parser.add_argument( + "--export-type", + action="append", + default=None, + help="Restrict cleanup to an export_type. Repeat for multiple types.", + ) parser.add_argument("--apply", action="store_true", help="Delete selected export rows and files.") return parser @@ -149,11 +189,15 @@ def main() -> int: args = parser.parse_args() if args.keep_latest < 0: parser.error("--keep-latest must be greater than or equal to zero") + if args.max_delete < 0: + parser.error("--max-delete must be greater than or equal to zero") summary = cleanup_demo_exports( project_name=args.project_name, keep_latest=args.keep_latest, apply=args.apply, + max_delete=args.max_delete, + export_types=args.export_type, ) print(json.dumps(summary, indent=2, sort_keys=True)) return 0 diff --git a/backend/tests/test_sprint24_cleanup_demo_artifacts.py b/backend/tests/test_sprint24_cleanup_demo_artifacts.py index 9db100f4..0d9f98a8 100644 --- a/backend/tests/test_sprint24_cleanup_demo_artifacts.py +++ b/backend/tests/test_sprint24_cleanup_demo_artifacts.py @@ -22,6 +22,7 @@ class ExportRow: id: str created_at: datetime | None storage_path: str + export_type: str = "project_metadata_json" def test_cleanup_candidate_selection_keeps_newest_exports() -> None: @@ -51,6 +52,20 @@ def test_cleanup_candidate_selection_rejects_negative_keep_latest() -> None: raise AssertionError("negative keep_latest should fail") +def test_cleanup_can_filter_candidates_by_export_type() -> None: + cleanup = load_cleanup_module() + base = datetime(2026, 1, 1, 12, 0, 0) + exports = [ + ExportRow("metadata", base + timedelta(days=2), "/tmp/metadata.json", "project_metadata_json"), + ExportRow("report", base + timedelta(days=1), "/tmp/report.html", "project_report_html"), + ExportRow("dataset", base, "/tmp/dataset.geojson", "dataset_geojson"), + ] + + filtered = cleanup.filter_exports_by_type(exports, ["project_report_html"]) + + assert [export.id for export in filtered] == ["report"] + + def test_cleanup_path_safety_requires_storage_root_containment(tmp_path: Path) -> None: cleanup = load_cleanup_module() storage_root = tmp_path / "storage" @@ -72,12 +87,40 @@ def test_cleanup_script_defaults_to_explicit_demo_project() -> None: assert args.project_name == cleanup.DEMO_PROJECT_NAME assert args.keep_latest == 3 + assert args.max_delete == 25 + assert args.export_type is None assert args.apply is False +def test_cleanup_script_accepts_max_delete_and_repeated_export_type() -> None: + cleanup = load_cleanup_module() + parser = cleanup.build_parser() + + args = parser.parse_args( + [ + "--keep-latest", + "10", + "--max-delete", + "100", + "--export-type", + "project_report_html", + "--export-type", + "project_metadata_json", + "--apply", + ] + ) + + assert args.keep_latest == 10 + assert args.max_delete == 100 + assert args.export_type == ["project_report_html", "project_metadata_json"] + assert args.apply is True + + def test_cleanup_script_reports_dry_run_candidates_separately() -> None: script = Path(__file__).resolve().parents[2] / "backend" / "scripts" / "cleanup_demo_artifacts.py" content = script.read_text(encoding="utf-8") assert '"candidate_files": []' in content assert 'summary["candidate_files"].append(str(path))' in content + assert '"max_delete": max_delete' in content + assert "blocked_reason" in content diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 3027c0b4..68a7521c 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -2267,3 +2267,27 @@ Limitations: Next recommended pass: - Consider a safe export retention/cleanup command if the artifact table keeps growing beyond demo needs. + +## Sprint 57 safer demo export cleanup (2026-06-17) + +Changed: +- Hardened the existing dry-run-first demo export cleanup command instead of creating a parallel cleanup path. +- Added `--max-delete` with a default cap of 25 so large `--apply` runs are blocked until explicitly reviewed and raised. +- Added repeatable `--export-type` filters for targeted cleanup, e.g. reports only. +- Extended the cleanup summary with `keep_latest`, `max_delete`, `export_types`, `type_filtered_export_count` and `blocked_reason`. +- Updated the root wrapper to expose the new filter helper. +- Updated `scripts/README.md`, `docs/STORAGE_ARCHITECTURE.md`, `backend/README.md`, `docs/TODO.md` and `CHANGELOG.md`. +- Added regression coverage for export-type filtering, parser defaults and max-delete options. + +Tested: +- `python -m py_compile scripts/cleanup_demo_artifacts.py backend/scripts/cleanup_demo_artifacts.py` +- `cd backend && python -m pytest tests/test_sprint24_cleanup_demo_artifacts.py tests/test_readiness_gate.py -q` (`16 passed`) + +Open: +- Run full readiness, deploy Tower and optionally perform a live cleanup dry run inside the all-in-one container before using `--apply`. + +Limitations: +- Cleanup still targets demo export records/files only. It does not delete source uploads, vector features, projects, AOIs, QA/QC records, rasters, tiles, masks or production data. + +Next recommended pass: +- Add a small live maintenance smoke that runs cleanup in dry-run mode through the deployed all-in-one container. diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index 6fff8a42..d0f8a0eb 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -103,7 +103,8 @@ Offline demo export artifacts can be inspected and cleaned with: ```bash python scripts/cleanup_demo_artifacts.py -python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply +python scripts/cleanup_demo_artifacts.py --keep-latest 10 --export-type project_report_html +python scripts/cleanup_demo_artifacts.py --keep-latest 10 --max-delete 100 --apply docker compose exec -T backend python scripts/cleanup_demo_artifacts.py ``` @@ -112,7 +113,9 @@ The script is dry-run by default, targets only the explicit provided, keeps the newest export artifacts per matching project and refuses to delete files outside `STORAGE_ROOT`. It cleans `exports` records/files only; it does not remove original uploads, vector features, QA/QC rows, projects, areas, -tiles, rasters or masks. +tiles, rasters or masks. `--max-delete` defaults to 25 and blocks oversized +apply runs until the operator increases the cap after reviewing dry-run output. +Repeat `--export-type` to restrict cleanup to selected artifact kinds. ## Model storage diff --git a/docs/TODO.md b/docs/TODO.md index c42720d5..083b8fdd 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -326,4 +326,5 @@ This file now starts with the current implementation status. Older preparation/b - [x] Improve populated Data/Exports readability after a demo workflow run. - [x] Improve live visual shell width, scroll behavior and Map workspace layout at 1280px. - [x] Add export history filtering controls for long-running demo environments. -- [ ] Add a safe export retention/cleanup command for demo environments. +- [x] Add a safe export retention/cleanup command for demo environments. +- [ ] Add a live dry-run maintenance smoke for demo export cleanup. diff --git a/scripts/README.md b/scripts/README.md index 42f29b79..7b3aac2e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -94,21 +94,26 @@ Clean old offline demo export artifacts without touching uploaded source data: ```bash python scripts/cleanup_demo_artifacts.py -python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply +python scripts/cleanup_demo_artifacts.py --keep-latest 10 --export-type project_report_html +python scripts/cleanup_demo_artifacts.py --keep-latest 10 --max-delete 100 --apply ``` Against the Docker runtime, run the backend-container entrypoint: ```bash docker compose exec -T backend python scripts/cleanup_demo_artifacts.py -docker compose exec -T backend python scripts/cleanup_demo_artifacts.py --keep-latest 3 --apply +docker compose exec -T backend python scripts/cleanup_demo_artifacts.py --keep-latest 10 --export-type project_report_html +docker compose exec -T backend python scripts/cleanup_demo_artifacts.py --keep-latest 10 --max-delete 100 --apply ``` The cleanup script is dry-run by default. It only targets the explicit `GeoIntel Demo - Building QA` project unless `--project-name` is provided, keeps the newest exports per matching project, deletes only `exports` rows/files when `--apply` is set, and refuses to remove files outside the configured -`STORAGE_ROOT`. +`STORAGE_ROOT`. `--max-delete` defaults to 25 and blocks large cleanup runs until +the operator raises it after reviewing dry-run output. Repeat `--export-type` to +limit cleanup to specific artifact kinds such as `project_report_html` or +`project_metadata_json`. ## Tower deployment diff --git a/scripts/cleanup_demo_artifacts.py b/scripts/cleanup_demo_artifacts.py index b7a2504b..8532e16c 100644 --- a/scripts/cleanup_demo_artifacts.py +++ b/scripts/cleanup_demo_artifacts.py @@ -21,6 +21,7 @@ DEMO_PROJECT_NAME = _impl.DEMO_PROJECT_NAME is_within_storage_root = _impl.is_within_storage_root export_created_at = _impl.export_created_at select_cleanup_candidates = _impl.select_cleanup_candidates +filter_exports_by_type = _impl.filter_exports_by_type export_path = _impl.export_path prune_empty_parents = _impl.prune_empty_parents cleanup_demo_exports = _impl.cleanup_demo_exports