diff --git a/backend/tests/test_rc10_data_operations.py b/backend/tests/test_rc10_data_operations.py index dc49f508..809891d3 100644 --- a/backend/tests/test_rc10_data_operations.py +++ b/backend/tests/test_rc10_data_operations.py @@ -146,6 +146,43 @@ def test_storage_audit_only_selects_old_unreferenced_allowlisted_files( assert "release-evidence" in report["cleanup"]["protected_prefixes"] +def test_referenced_tile_manifest_protects_its_tiles(tmp_path: Path) -> None: + audit = load_script("audit_data_operations.py") + storage = tmp_path / "storage" + manifest = storage / "tiles" / "dataset" / "set" / "manifest.json" + tile = manifest.parent / "tile_0000.tif" + tile.parent.mkdir(parents=True) + tile.write_bytes(b"tile") + manifest.write_text(json.dumps({"tiles": [{"path": "tile_0000.tif"}]}), encoding="utf-8") + + expanded = audit.expand_manifest_references({manifest.resolve()}, storage) + + assert manifest.resolve() in expanded + assert tile.resolve() in expanded + + +def test_disk_pressure_uses_absolute_headroom_for_large_arrays( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + audit = load_script("audit_data_operations.py") + monkeypatch.setattr( + audit.shutil, + "disk_usage", + lambda _path: SimpleNamespace( + total=56 * 1024**4, + used=(56 * 1024**4) - (700 * 1024**3), + free=700 * 1024**3, + ), + ) + + pressure = audit.disk_pressure(tmp_path) + + assert pressure["free_percent"] < 2 + assert pressure["status"] == "ok" + assert pressure["acquisition_allowed"] is True + + def test_source_family_report_covers_national_regional_and_maritime( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/docs/DATA_OPERATIONS_RUNBOOK.md b/docs/DATA_OPERATIONS_RUNBOOK.md index 0156e721..777a20f6 100644 --- a/docs/DATA_OPERATIONS_RUNBOOK.md +++ b/docs/DATA_OPERATIONS_RUNBOOK.md @@ -48,8 +48,10 @@ The command: Use `--fail-on-pressure warning` for a stricter acquisition preflight and `--fail-on-missing-reference` for release gates. `critical` disk pressure -blocks acquisition by default. The thresholds are 10 GiB or 5% free for -critical and 25 GiB or 10% free for warning. +blocks acquisition by default. Critical means less than 10 GiB free. Warning +means less than 50 GiB free, or less than 2% and less than 250 GiB free. The +combined percentage/absolute rule avoids falsely blocking a large Unraid +array that still has hundreds of GiB available. The existing project-level freshness policy remains authoritative: diff --git a/scripts/audit_data_operations.py b/scripts/audit_data_operations.py index a4aa55b2..92de8cd5 100644 --- a/scripts/audit_data_operations.py +++ b/scripts/audit_data_operations.py @@ -165,6 +165,33 @@ def normalize_storage_reference(value: str | None, storage_root: Path) -> Path | return resolved +def expand_manifest_references(references: set[Path], storage_root: Path) -> set[Path]: + """Protect files named by referenced, bounded JSON manifests.""" + root = storage_root.resolve() + expanded = set(references) + for manifest in list(references): + if manifest.suffix.lower() != ".json" or not manifest.is_file(): + continue + try: + if manifest.stat().st_size > 16 * 1024 * 1024: + continue + payload = json.loads(manifest.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + continue + for value in _iter_strings(payload): + normalized = normalize_storage_reference(value, root) + if normalized is None and "://" not in value: + try: + relative = (manifest.parent / value).resolve() + relative.relative_to(root) + except (OSError, ValueError): + continue + normalized = relative + if normalized is not None: + expanded.add(normalized) + return expanded + + def _add_row_references(target: set[Path], row: Any, storage_root: Path, direct_fields: Iterable[str]) -> None: for field in direct_fields: normalized = normalize_storage_reference(getattr(row, field, None), storage_root) @@ -327,9 +354,11 @@ def collect_database_state(db: Any, storage_root: Path) -> dict[str, Any]: def disk_pressure(storage_root: Path) -> dict[str, Any]: usage = shutil.disk_usage(storage_root) free_percent = (usage.free / usage.total * 100) if usage.total else 0.0 - if usage.free < 10 * 1024**3 or free_percent < 5: + # Absolute headroom governs very large Unraid arrays: a low percentage of + # tens of terabytes can still leave hundreds of GiB safely available. + if usage.free < 10 * 1024**3: status = "critical" - elif usage.free < 25 * 1024**3 or free_percent < 10: + elif usage.free < 50 * 1024**3 or (free_percent < 2 and usage.free < 250 * 1024**3): status = "warning" else: status = "ok" @@ -357,7 +386,7 @@ def build_report( raise RuntimeError(f"Storage root is not a directory: {root}") records, skipped_symlinks = inventory_storage(root) database = collect_database_state(db, root) - references: set[Path] = database.pop("references") + references: set[Path] = expand_manifest_references(database.pop("references"), root) cutoff = utc_now() - timedelta(days=minimum_age_days) categories: dict[str, dict[str, Any]] = defaultdict( @@ -376,12 +405,14 @@ def build_report( for path in references if path not in existing_paths and not path.is_dir() ) + referenced_directories = tuple(path for path in references if path.is_dir()) candidates = sorted( ( record for record in records if record.cleanup_eligible and record.path not in references + and not any(record.path.is_relative_to(reference) for reference in referenced_directories) and record.modified_at <= cutoff and record.path.name not in IGNORED_FILENAMES ),