Parse only explicit manifest paths
This commit is contained in:
@@ -173,6 +173,30 @@ def test_ordinary_json_export_is_not_treated_as_an_artifact_manifest(tmp_path: P
|
|||||||
assert expanded == {export.resolve()}
|
assert expanded == {export.resolve()}
|
||||||
|
|
||||||
|
|
||||||
|
def test_manifest_ids_dates_and_labels_are_not_treated_as_paths(tmp_path: Path) -> None:
|
||||||
|
audit = load_script("audit_data_operations.py")
|
||||||
|
storage = tmp_path / "storage"
|
||||||
|
manifest = storage / "operator-data" / "scope" / "scope.manifest.json"
|
||||||
|
actual = manifest.parent / "scope.geojson"
|
||||||
|
manifest.parent.mkdir(parents=True)
|
||||||
|
actual.write_text("{}", encoding="utf-8")
|
||||||
|
manifest.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"municipality_ids": ["13025", "11001"],
|
||||||
|
"generated_at": "2026-07-18T00:00:00Z",
|
||||||
|
"label": "Belgium",
|
||||||
|
"output_path": "scope.geojson",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
expanded = audit.expand_manifest_references({manifest.resolve()}, storage)
|
||||||
|
|
||||||
|
assert expanded == {manifest.resolve(), actual.resolve()}
|
||||||
|
|
||||||
|
|
||||||
def test_disk_pressure_uses_absolute_headroom_for_large_arrays(
|
def test_disk_pressure_uses_absolute_headroom_for_large_arrays(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
|||||||
@@ -143,6 +143,23 @@ def _iter_strings(value: Any) -> Iterable[str]:
|
|||||||
yield from _iter_strings(nested)
|
yield from _iter_strings(nested)
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_manifest_path_values(value: Any, parent_key: str = "") -> Iterable[str]:
|
||||||
|
path_tokens = ("path", "file", "artifact", "manifest", "output")
|
||||||
|
if isinstance(value, dict):
|
||||||
|
for key, nested in value.items():
|
||||||
|
normalized_key = str(key).strip().lower()
|
||||||
|
if isinstance(nested, str) and any(token in normalized_key for token in path_tokens):
|
||||||
|
yield nested
|
||||||
|
else:
|
||||||
|
yield from _iter_manifest_path_values(nested, normalized_key)
|
||||||
|
elif isinstance(value, (list, tuple)):
|
||||||
|
for nested in value:
|
||||||
|
if isinstance(nested, str) and any(token in parent_key for token in path_tokens):
|
||||||
|
yield nested
|
||||||
|
else:
|
||||||
|
yield from _iter_manifest_path_values(nested, parent_key)
|
||||||
|
|
||||||
|
|
||||||
def normalize_storage_reference(value: str | None, storage_root: Path) -> Path | None:
|
def normalize_storage_reference(value: str | None, storage_root: Path) -> Path | None:
|
||||||
if not value:
|
if not value:
|
||||||
return None
|
return None
|
||||||
@@ -182,7 +199,7 @@ def expand_manifest_references(references: set[Path], storage_root: Path) -> set
|
|||||||
payload = json.loads(manifest.read_text(encoding="utf-8"))
|
payload = json.loads(manifest.read_text(encoding="utf-8"))
|
||||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
||||||
continue
|
continue
|
||||||
for value in _iter_strings(payload):
|
for value in _iter_manifest_path_values(payload):
|
||||||
normalized = normalize_storage_reference(value, root)
|
normalized = normalize_storage_reference(value, root)
|
||||||
if normalized is None and "://" not in value:
|
if normalized is None and "://" not in value:
|
||||||
try:
|
try:
|
||||||
|
|||||||
Reference in New Issue
Block a user