Separate RC10 manifest provenance warnings
This commit is contained in:
@@ -144,6 +144,8 @@ def test_storage_audit_only_selects_old_unreferenced_allowlisted_files(
|
|||||||
assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"]
|
assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"]
|
||||||
assert report["cleanup"]["candidate_count"] == 1
|
assert report["cleanup"]["candidate_count"] == 1
|
||||||
assert "release-evidence" in report["cleanup"]["protected_prefixes"]
|
assert "release-evidence" in report["cleanup"]["protected_prefixes"]
|
||||||
|
assert report["integrity"]["missing_referenced_path_count"] == 0
|
||||||
|
assert report["integrity"]["missing_manifest_artifact_count"] == 0
|
||||||
|
|
||||||
|
|
||||||
def test_referenced_tile_manifest_protects_its_tiles(tmp_path: Path) -> None:
|
def test_referenced_tile_manifest_protects_its_tiles(tmp_path: Path) -> None:
|
||||||
@@ -342,3 +344,4 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
|
|||||||
assert "table-counts-before.tsv" in live_audit
|
assert "table-counts-before.tsv" in live_audit
|
||||||
assert "table-counts-after.tsv" in live_audit
|
assert "table-counts-after.tsv" in live_audit
|
||||||
assert "deleted_count" in live_audit
|
assert "deleted_count" in live_audit
|
||||||
|
assert "missing_manifest_artifact_count" in live_audit
|
||||||
|
|||||||
@@ -137,6 +137,10 @@ The older demo-export cleanup has the same gate and uses confirmation token
|
|||||||
- `disk_pressure=critical`: stop new source acquisition and resolve capacity.
|
- `disk_pressure=critical`: stop new source acquisition and resolve capacity.
|
||||||
- `missing_referenced_path_count>0`: persisted provenance points to absent
|
- `missing_referenced_path_count>0`: persisted provenance points to absent
|
||||||
files; investigate before cleanup or release.
|
files; investigate before cleanup or release.
|
||||||
|
- `missing_manifest_artifact_count>0`: a retained operator manifest names an
|
||||||
|
absent non-authoritative intermediate. The direct database artifacts remain
|
||||||
|
the release gate; retain this warning as provenance debt and do not delete
|
||||||
|
the protected manifest or its other evidence.
|
||||||
- checksum or backup-age failure: create and verify a new backup.
|
- checksum or backup-age failure: create and verify a new backup.
|
||||||
- candidate count above `--max-delete`: keep dry-run mode and review; never
|
- candidate count above `--max-delete`: keep dry-run mode and review; never
|
||||||
increase the limit blindly.
|
increase the limit blindly.
|
||||||
|
|||||||
@@ -426,7 +426,9 @@ def build_report(
|
|||||||
raise RuntimeError(f"Storage root is not a directory: {root}")
|
raise RuntimeError(f"Storage root is not a directory: {root}")
|
||||||
records, skipped_symlinks = inventory_storage(root)
|
records, skipped_symlinks = inventory_storage(root)
|
||||||
database = collect_database_state(db, root)
|
database = collect_database_state(db, root)
|
||||||
references: set[Path] = expand_manifest_references(database.pop("references"), root)
|
direct_references: set[Path] = database.pop("references")
|
||||||
|
references = expand_manifest_references(direct_references, root)
|
||||||
|
manifest_references = references - direct_references
|
||||||
cutoff = utc_now() - timedelta(days=minimum_age_days)
|
cutoff = utc_now() - timedelta(days=minimum_age_days)
|
||||||
|
|
||||||
categories: dict[str, dict[str, Any]] = defaultdict(
|
categories: dict[str, dict[str, Any]] = defaultdict(
|
||||||
@@ -440,9 +442,14 @@ def build_report(
|
|||||||
item["cleanup_eligible"] = item["cleanup_eligible"] or record.cleanup_eligible
|
item["cleanup_eligible"] = item["cleanup_eligible"] or record.cleanup_eligible
|
||||||
|
|
||||||
existing_paths = {record.path for record in records}
|
existing_paths = {record.path for record in records}
|
||||||
missing_references = sorted(
|
missing_direct_references = sorted(
|
||||||
path.relative_to(root).as_posix()
|
path.relative_to(root).as_posix()
|
||||||
for path in references
|
for path in direct_references
|
||||||
|
if path not in existing_paths and not path.is_dir()
|
||||||
|
)
|
||||||
|
missing_manifest_references = sorted(
|
||||||
|
path.relative_to(root).as_posix()
|
||||||
|
for path in manifest_references
|
||||||
if path not in existing_paths and not path.is_dir()
|
if path not in existing_paths and not path.is_dir()
|
||||||
)
|
)
|
||||||
referenced_directories = tuple(path for path in references if path.is_dir())
|
referenced_directories = tuple(path for path in references if path.is_dir())
|
||||||
@@ -479,8 +486,12 @@ def build_report(
|
|||||||
"database": database,
|
"database": database,
|
||||||
"integrity": {
|
"integrity": {
|
||||||
"referenced_path_count": len(references),
|
"referenced_path_count": len(references),
|
||||||
"missing_referenced_path_count": len(missing_references),
|
"direct_database_reference_count": len(direct_references),
|
||||||
"missing_referenced_paths": missing_references[:max_candidate_records],
|
"missing_referenced_path_count": len(missing_direct_references),
|
||||||
|
"missing_referenced_paths": missing_direct_references[:max_candidate_records],
|
||||||
|
"manifest_artifact_reference_count": len(manifest_references),
|
||||||
|
"missing_manifest_artifact_count": len(missing_manifest_references),
|
||||||
|
"missing_manifest_artifacts": missing_manifest_references[:max_candidate_records],
|
||||||
"skipped_symlinks": skipped_symlinks[:max_candidate_records],
|
"skipped_symlinks": skipped_symlinks[:max_candidate_records],
|
||||||
},
|
},
|
||||||
"cleanup": {
|
"cleanup": {
|
||||||
@@ -507,6 +518,7 @@ def build_report(
|
|||||||
"limitations": [
|
"limitations": [
|
||||||
"The audit never downloads or refreshes an official source.",
|
"The audit never downloads or refreshes an official source.",
|
||||||
"Failed jobs and analysis runs remain provenance and are reported, not deleted.",
|
"Failed jobs and analysis runs remain provenance and are reported, not deleted.",
|
||||||
|
"Missing non-authoritative manifest intermediates are reported separately from direct database artifacts.",
|
||||||
"Unknown storage categories are protected by default.",
|
"Unknown storage categories are protected by default.",
|
||||||
"No cleanup is scheduled implicitly.",
|
"No cleanup is scheduled implicitly.",
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -88,6 +88,9 @@ manifest = {
|
|||||||
family: len(items) for family, items in families.items()
|
family: len(items) for family, items in families.items()
|
||||||
},
|
},
|
||||||
"missing_referenced_path_count": 0,
|
"missing_referenced_path_count": 0,
|
||||||
|
"missing_manifest_artifact_count": audit.get("integrity", {}).get(
|
||||||
|
"missing_manifest_artifact_count", 0
|
||||||
|
),
|
||||||
}
|
}
|
||||||
(root / "manifest.json").write_text(
|
(root / "manifest.json").write_text(
|
||||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
||||||
|
|||||||
Reference in New Issue
Block a user