Separate RC10 manifest provenance warnings
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-18 07:32:57 +02:00
parent dbdc594f94
commit 6a72c14c57
4 changed files with 27 additions and 5 deletions
@@ -144,6 +144,8 @@ def test_storage_audit_only_selects_old_unreferenced_allowlisted_files(
assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"] assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"]
assert report["cleanup"]["candidate_count"] == 1 assert report["cleanup"]["candidate_count"] == 1
assert "release-evidence" in report["cleanup"]["protected_prefixes"] assert "release-evidence" in report["cleanup"]["protected_prefixes"]
assert report["integrity"]["missing_referenced_path_count"] == 0
assert report["integrity"]["missing_manifest_artifact_count"] == 0
def test_referenced_tile_manifest_protects_its_tiles(tmp_path: Path) -> None: def test_referenced_tile_manifest_protects_its_tiles(tmp_path: Path) -> None:
@@ -342,3 +344,4 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
assert "table-counts-before.tsv" in live_audit assert "table-counts-before.tsv" in live_audit
assert "table-counts-after.tsv" in live_audit assert "table-counts-after.tsv" in live_audit
assert "deleted_count" in live_audit assert "deleted_count" in live_audit
assert "missing_manifest_artifact_count" in live_audit
+4
View File
@@ -137,6 +137,10 @@ The older demo-export cleanup has the same gate and uses confirmation token
- `disk_pressure=critical`: stop new source acquisition and resolve capacity. - `disk_pressure=critical`: stop new source acquisition and resolve capacity.
- `missing_referenced_path_count>0`: persisted provenance points to absent - `missing_referenced_path_count>0`: persisted provenance points to absent
files; investigate before cleanup or release. files; investigate before cleanup or release.
- `missing_manifest_artifact_count>0`: a retained operator manifest names an
absent non-authoritative intermediate. The direct database artifacts remain
the release gate; retain this warning as provenance debt and do not delete
the protected manifest or its other evidence.
- checksum or backup-age failure: create and verify a new backup. - checksum or backup-age failure: create and verify a new backup.
- candidate count above `--max-delete`: keep dry-run mode and review; never - candidate count above `--max-delete`: keep dry-run mode and review; never
increase the limit blindly. increase the limit blindly.
+17 -5
View File
@@ -426,7 +426,9 @@ def build_report(
raise RuntimeError(f"Storage root is not a directory: {root}") raise RuntimeError(f"Storage root is not a directory: {root}")
records, skipped_symlinks = inventory_storage(root) records, skipped_symlinks = inventory_storage(root)
database = collect_database_state(db, root) database = collect_database_state(db, root)
references: set[Path] = expand_manifest_references(database.pop("references"), root) direct_references: set[Path] = database.pop("references")
references = expand_manifest_references(direct_references, root)
manifest_references = references - direct_references
cutoff = utc_now() - timedelta(days=minimum_age_days) cutoff = utc_now() - timedelta(days=minimum_age_days)
categories: dict[str, dict[str, Any]] = defaultdict( categories: dict[str, dict[str, Any]] = defaultdict(
@@ -440,9 +442,14 @@ def build_report(
item["cleanup_eligible"] = item["cleanup_eligible"] or record.cleanup_eligible item["cleanup_eligible"] = item["cleanup_eligible"] or record.cleanup_eligible
existing_paths = {record.path for record in records} existing_paths = {record.path for record in records}
missing_references = sorted( missing_direct_references = sorted(
path.relative_to(root).as_posix() path.relative_to(root).as_posix()
for path in references for path in direct_references
if path not in existing_paths and not path.is_dir()
)
missing_manifest_references = sorted(
path.relative_to(root).as_posix()
for path in manifest_references
if path not in existing_paths and not path.is_dir() if path not in existing_paths and not path.is_dir()
) )
referenced_directories = tuple(path for path in references if path.is_dir()) referenced_directories = tuple(path for path in references if path.is_dir())
@@ -479,8 +486,12 @@ def build_report(
"database": database, "database": database,
"integrity": { "integrity": {
"referenced_path_count": len(references), "referenced_path_count": len(references),
"missing_referenced_path_count": len(missing_references), "direct_database_reference_count": len(direct_references),
"missing_referenced_paths": missing_references[:max_candidate_records], "missing_referenced_path_count": len(missing_direct_references),
"missing_referenced_paths": missing_direct_references[:max_candidate_records],
"manifest_artifact_reference_count": len(manifest_references),
"missing_manifest_artifact_count": len(missing_manifest_references),
"missing_manifest_artifacts": missing_manifest_references[:max_candidate_records],
"skipped_symlinks": skipped_symlinks[:max_candidate_records], "skipped_symlinks": skipped_symlinks[:max_candidate_records],
}, },
"cleanup": { "cleanup": {
@@ -507,6 +518,7 @@ def build_report(
"limitations": [ "limitations": [
"The audit never downloads or refreshes an official source.", "The audit never downloads or refreshes an official source.",
"Failed jobs and analysis runs remain provenance and are reported, not deleted.", "Failed jobs and analysis runs remain provenance and are reported, not deleted.",
"Missing non-authoritative manifest intermediates are reported separately from direct database artifacts.",
"Unknown storage categories are protected by default.", "Unknown storage categories are protected by default.",
"No cleanup is scheduled implicitly.", "No cleanup is scheduled implicitly.",
], ],
@@ -88,6 +88,9 @@ manifest = {
family: len(items) for family, items in families.items() family: len(items) for family, items in families.items()
}, },
"missing_referenced_path_count": 0, "missing_referenced_path_count": 0,
"missing_manifest_artifact_count": audit.get("integrity", {}).get(
"missing_manifest_artifact_count", 0
),
} }
(root / "manifest.json").write_text( (root / "manifest.json").write_text(
json.dumps(manifest, indent=2, sort_keys=True) + "\n", json.dumps(manifest, indent=2, sort_keys=True) + "\n",