Protect manifest artifacts in RC10 audit

backend/tests/test_rc10_data_operations.py

@@ -146,6 +146,43 @@ def test_storage_audit_only_selects_old_unreferenced_allowlisted_files(
     assert "release-evidence" in report["cleanup"]["protected_prefixes"]
 
 
+def test_referenced_tile_manifest_protects_its_tiles(tmp_path: Path) -> None:
+    audit = load_script("audit_data_operations.py")
+    storage = tmp_path / "storage"
+    manifest = storage / "tiles" / "dataset" / "set" / "manifest.json"
+    tile = manifest.parent / "tile_0000.tif"
+    tile.parent.mkdir(parents=True)
+    tile.write_bytes(b"tile")
+    manifest.write_text(json.dumps({"tiles": [{"path": "tile_0000.tif"}]}), encoding="utf-8")
+
+    expanded = audit.expand_manifest_references({manifest.resolve()}, storage)
+
+    assert manifest.resolve() in expanded
+    assert tile.resolve() in expanded
+
+
+def test_disk_pressure_uses_absolute_headroom_for_large_arrays(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    audit = load_script("audit_data_operations.py")
+    monkeypatch.setattr(
+        audit.shutil,
+        "disk_usage",
+        lambda _path: SimpleNamespace(
+            total=56 * 1024**4,
+            used=(56 * 1024**4) - (700 * 1024**3),
+            free=700 * 1024**3,
+        ),
+    )
+
+    pressure = audit.disk_pressure(tmp_path)
+
+    assert pressure["free_percent"] < 2
+    assert pressure["status"] == "ok"
+    assert pressure["acquisition_allowed"] is True
+
+
 def test_source_family_report_covers_national_regional_and_maritime(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,

docs/DATA_OPERATIONS_RUNBOOK.md

@@ -48,8 +48,10 @@ The command:
 
 Use `--fail-on-pressure warning` for a stricter acquisition preflight and
 `--fail-on-missing-reference` for release gates. `critical` disk pressure
-blocks acquisition by default. The thresholds are 10 GiB or 5% free for
+blocks acquisition by default. Critical means less than 10 GiB free. Warning
-critical and 25 GiB or 10% free for warning.
+means less than 50 GiB free, or less than 2% and less than 250 GiB free. The
+combined percentage/absolute rule avoids falsely blocking a large Unraid
+array that still has hundreds of GiB available.
 
 The existing project-level freshness policy remains authoritative:
 

scripts/audit_data_operations.py

@@ -165,6 +165,33 @@ def normalize_storage_reference(value: str | None, storage_root: Path) -> Path |
     return resolved
 
 
+def expand_manifest_references(references: set[Path], storage_root: Path) -> set[Path]:
+    """Protect files named by referenced, bounded JSON manifests."""
+    root = storage_root.resolve()
+    expanded = set(references)
+    for manifest in list(references):
+        if manifest.suffix.lower() != ".json" or not manifest.is_file():
+            continue
+        try:
+            if manifest.stat().st_size > 16 * 1024 * 1024:
+                continue
+            payload = json.loads(manifest.read_text(encoding="utf-8"))
+        except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+            continue
+        for value in _iter_strings(payload):
+            normalized = normalize_storage_reference(value, root)
+            if normalized is None and "://" not in value:
+                try:
+                    relative = (manifest.parent / value).resolve()
+                    relative.relative_to(root)
+                except (OSError, ValueError):
+                    continue
+                normalized = relative
+            if normalized is not None:
+                expanded.add(normalized)
+    return expanded
+
+
 def _add_row_references(target: set[Path], row: Any, storage_root: Path, direct_fields: Iterable[str]) -> None:
     for field in direct_fields:
         normalized = normalize_storage_reference(getattr(row, field, None), storage_root)
@@ -327,9 +354,11 @@ def collect_database_state(db: Any, storage_root: Path) -> dict[str, Any]:
 def disk_pressure(storage_root: Path) -> dict[str, Any]:
     usage = shutil.disk_usage(storage_root)
     free_percent = (usage.free / usage.total * 100) if usage.total else 0.0
-    if usage.free < 10 * 1024**3 or free_percent < 5:
+    # Absolute headroom governs very large Unraid arrays: a low percentage of
+    # tens of terabytes can still leave hundreds of GiB safely available.
+    if usage.free < 10 * 1024**3:
         status = "critical"
-    elif usage.free < 25 * 1024**3 or free_percent < 10:
+    elif usage.free < 50 * 1024**3 or (free_percent < 2 and usage.free < 250 * 1024**3):
         status = "warning"
     else:
         status = "ok"
@@ -357,7 +386,7 @@ def build_report(
         raise RuntimeError(f"Storage root is not a directory: {root}")
     records, skipped_symlinks = inventory_storage(root)
     database = collect_database_state(db, root)
-    references: set[Path] = database.pop("references")
+    references: set[Path] = expand_manifest_references(database.pop("references"), root)
     cutoff = utc_now() - timedelta(days=minimum_age_days)
 
     categories: dict[str, dict[str, Any]] = defaultdict(
@@ -376,12 +405,14 @@ def build_report(
         for path in references
         if path not in existing_paths and not path.is_dir()
     )
+    referenced_directories = tuple(path for path in references if path.is_dir())
     candidates = sorted(
         (
             record
             for record in records
             if record.cleanup_eligible
             and record.path not in references
+            and not any(record.path.is_relative_to(reference) for reference in referenced_directories)
             and record.modified_at <= cutoff
             and record.path.name not in IGNORED_FILENAMES
         ),