Separate RC10 manifest provenance warnings
This commit is contained in:
@@ -144,6 +144,8 @@ def test_storage_audit_only_selects_old_unreferenced_allowlisted_files(
|
||||
assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"]
|
||||
assert report["cleanup"]["candidate_count"] == 1
|
||||
assert "release-evidence" in report["cleanup"]["protected_prefixes"]
|
||||
assert report["integrity"]["missing_referenced_path_count"] == 0
|
||||
assert report["integrity"]["missing_manifest_artifact_count"] == 0
|
||||
|
||||
|
||||
def test_referenced_tile_manifest_protects_its_tiles(tmp_path: Path) -> None:
|
||||
@@ -342,3 +344,4 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
|
||||
assert "table-counts-before.tsv" in live_audit
|
||||
assert "table-counts-after.tsv" in live_audit
|
||||
assert "deleted_count" in live_audit
|
||||
assert "missing_manifest_artifact_count" in live_audit
|
||||
|
||||
@@ -137,6 +137,10 @@ The older demo-export cleanup has the same gate and uses confirmation token
|
||||
- `disk_pressure=critical`: stop new source acquisition and resolve capacity.
|
||||
- `missing_referenced_path_count>0`: persisted provenance points to absent
|
||||
files; investigate before cleanup or release.
|
||||
- `missing_manifest_artifact_count>0`: a retained operator manifest names an
|
||||
absent non-authoritative intermediate. The direct database artifacts remain
|
||||
the release gate; retain this warning as provenance debt and do not delete
|
||||
the protected manifest or its other evidence.
|
||||
- checksum or backup-age failure: create and verify a new backup.
|
||||
- candidate count above `--max-delete`: keep dry-run mode and review; never
|
||||
increase the limit blindly.
|
||||
|
||||
@@ -426,7 +426,9 @@ def build_report(
|
||||
raise RuntimeError(f"Storage root is not a directory: {root}")
|
||||
records, skipped_symlinks = inventory_storage(root)
|
||||
database = collect_database_state(db, root)
|
||||
references: set[Path] = expand_manifest_references(database.pop("references"), root)
|
||||
direct_references: set[Path] = database.pop("references")
|
||||
references = expand_manifest_references(direct_references, root)
|
||||
manifest_references = references - direct_references
|
||||
cutoff = utc_now() - timedelta(days=minimum_age_days)
|
||||
|
||||
categories: dict[str, dict[str, Any]] = defaultdict(
|
||||
@@ -440,9 +442,14 @@ def build_report(
|
||||
item["cleanup_eligible"] = item["cleanup_eligible"] or record.cleanup_eligible
|
||||
|
||||
existing_paths = {record.path for record in records}
|
||||
missing_references = sorted(
|
||||
missing_direct_references = sorted(
|
||||
path.relative_to(root).as_posix()
|
||||
for path in references
|
||||
for path in direct_references
|
||||
if path not in existing_paths and not path.is_dir()
|
||||
)
|
||||
missing_manifest_references = sorted(
|
||||
path.relative_to(root).as_posix()
|
||||
for path in manifest_references
|
||||
if path not in existing_paths and not path.is_dir()
|
||||
)
|
||||
referenced_directories = tuple(path for path in references if path.is_dir())
|
||||
@@ -479,8 +486,12 @@ def build_report(
|
||||
"database": database,
|
||||
"integrity": {
|
||||
"referenced_path_count": len(references),
|
||||
"missing_referenced_path_count": len(missing_references),
|
||||
"missing_referenced_paths": missing_references[:max_candidate_records],
|
||||
"direct_database_reference_count": len(direct_references),
|
||||
"missing_referenced_path_count": len(missing_direct_references),
|
||||
"missing_referenced_paths": missing_direct_references[:max_candidate_records],
|
||||
"manifest_artifact_reference_count": len(manifest_references),
|
||||
"missing_manifest_artifact_count": len(missing_manifest_references),
|
||||
"missing_manifest_artifacts": missing_manifest_references[:max_candidate_records],
|
||||
"skipped_symlinks": skipped_symlinks[:max_candidate_records],
|
||||
},
|
||||
"cleanup": {
|
||||
@@ -507,6 +518,7 @@ def build_report(
|
||||
"limitations": [
|
||||
"The audit never downloads or refreshes an official source.",
|
||||
"Failed jobs and analysis runs remain provenance and are reported, not deleted.",
|
||||
"Missing non-authoritative manifest intermediates are reported separately from direct database artifacts.",
|
||||
"Unknown storage categories are protected by default.",
|
||||
"No cleanup is scheduled implicitly.",
|
||||
],
|
||||
|
||||
@@ -88,6 +88,9 @@ manifest = {
|
||||
family: len(items) for family, items in families.items()
|
||||
},
|
||||
"missing_referenced_path_count": 0,
|
||||
"missing_manifest_artifact_count": audit.get("integrity", {}).get(
|
||||
"missing_manifest_artifact_count", 0
|
||||
),
|
||||
}
|
||||
(root / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
||||
|
||||
Reference in New Issue
Block a user