From 6a72c14c572afa6e23ebd7442472d287e0f92d94 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 18 Jul 2026 07:32:57 +0200 Subject: [PATCH] Separate RC10 manifest provenance warnings --- backend/tests/test_rc10_data_operations.py | 3 +++ docs/DATA_OPERATIONS_RUNBOOK.md | 4 ++++ scripts/audit_data_operations.py | 22 +++++++++++++++++----- scripts/run_rc10_data_operations_audit.sh | 3 +++ 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/backend/tests/test_rc10_data_operations.py b/backend/tests/test_rc10_data_operations.py index c39dab3b..352ae41b 100644 --- a/backend/tests/test_rc10_data_operations.py +++ b/backend/tests/test_rc10_data_operations.py @@ -144,6 +144,8 @@ def test_storage_audit_only_selects_old_unreferenced_allowlisted_files( assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"] assert report["cleanup"]["candidate_count"] == 1 assert "release-evidence" in report["cleanup"]["protected_prefixes"] + assert report["integrity"]["missing_referenced_path_count"] == 0 + assert report["integrity"]["missing_manifest_artifact_count"] == 0 def test_referenced_tile_manifest_protects_its_tiles(tmp_path: Path) -> None: @@ -342,3 +344,4 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N assert "table-counts-before.tsv" in live_audit assert "table-counts-after.tsv" in live_audit assert "deleted_count" in live_audit + assert "missing_manifest_artifact_count" in live_audit diff --git a/docs/DATA_OPERATIONS_RUNBOOK.md b/docs/DATA_OPERATIONS_RUNBOOK.md index 777a20f6..b56ecdec 100644 --- a/docs/DATA_OPERATIONS_RUNBOOK.md +++ b/docs/DATA_OPERATIONS_RUNBOOK.md @@ -137,6 +137,10 @@ The older demo-export cleanup has the same gate and uses confirmation token - `disk_pressure=critical`: stop new source acquisition and resolve capacity. - `missing_referenced_path_count>0`: persisted provenance points to absent files; investigate before cleanup or release. +- `missing_manifest_artifact_count>0`: a retained operator manifest names an + absent non-authoritative intermediate. The direct database artifacts remain + the release gate; retain this warning as provenance debt and do not delete + the protected manifest or its other evidence. - checksum or backup-age failure: create and verify a new backup. - candidate count above `--max-delete`: keep dry-run mode and review; never increase the limit blindly. diff --git a/scripts/audit_data_operations.py b/scripts/audit_data_operations.py index cc25fdcd..ee0f3369 100644 --- a/scripts/audit_data_operations.py +++ b/scripts/audit_data_operations.py @@ -426,7 +426,9 @@ def build_report( raise RuntimeError(f"Storage root is not a directory: {root}") records, skipped_symlinks = inventory_storage(root) database = collect_database_state(db, root) - references: set[Path] = expand_manifest_references(database.pop("references"), root) + direct_references: set[Path] = database.pop("references") + references = expand_manifest_references(direct_references, root) + manifest_references = references - direct_references cutoff = utc_now() - timedelta(days=minimum_age_days) categories: dict[str, dict[str, Any]] = defaultdict( @@ -440,9 +442,14 @@ def build_report( item["cleanup_eligible"] = item["cleanup_eligible"] or record.cleanup_eligible existing_paths = {record.path for record in records} - missing_references = sorted( + missing_direct_references = sorted( path.relative_to(root).as_posix() - for path in references + for path in direct_references + if path not in existing_paths and not path.is_dir() + ) + missing_manifest_references = sorted( + path.relative_to(root).as_posix() + for path in manifest_references if path not in existing_paths and not path.is_dir() ) referenced_directories = tuple(path for path in references if path.is_dir()) @@ -479,8 +486,12 @@ def build_report( "database": database, "integrity": { "referenced_path_count": len(references), - "missing_referenced_path_count": len(missing_references), - "missing_referenced_paths": missing_references[:max_candidate_records], + "direct_database_reference_count": len(direct_references), + "missing_referenced_path_count": len(missing_direct_references), + "missing_referenced_paths": missing_direct_references[:max_candidate_records], + "manifest_artifact_reference_count": len(manifest_references), + "missing_manifest_artifact_count": len(missing_manifest_references), + "missing_manifest_artifacts": missing_manifest_references[:max_candidate_records], "skipped_symlinks": skipped_symlinks[:max_candidate_records], }, "cleanup": { @@ -507,6 +518,7 @@ def build_report( "limitations": [ "The audit never downloads or refreshes an official source.", "Failed jobs and analysis runs remain provenance and are reported, not deleted.", + "Missing non-authoritative manifest intermediates are reported separately from direct database artifacts.", "Unknown storage categories are protected by default.", "No cleanup is scheduled implicitly.", ], diff --git a/scripts/run_rc10_data_operations_audit.sh b/scripts/run_rc10_data_operations_audit.sh index eb7cf371..e6299f6f 100644 --- a/scripts/run_rc10_data_operations_audit.sh +++ b/scripts/run_rc10_data_operations_audit.sh @@ -88,6 +88,9 @@ manifest = { family: len(items) for family, items in families.items() }, "missing_referenced_path_count": 0, + "missing_manifest_artifact_count": audit.get("integrity", {}).get( + "missing_manifest_artifact_count", 0 + ), } (root / "manifest.json").write_text( json.dumps(manifest, indent=2, sort_keys=True) + "\n",