consume only artifacts the runtime produced

tile_manifest_path arrives in the detection and segmentation request and was
read straight off disk, and a manifest entry may name an absolute tile path.
That makes an API field an unbounded reference to the host filesystem, and it
contradicts the rule the persistence model rests on: only a governed,
runtime-produced artifact may be consumed, and a file outside the storage root
is not one.

Both the manifest and every tile it names now resolve under STORAGE_ROOT.
Resolution happens before the comparison, so ".." cannot climb out and a
sibling that merely shares a name prefix does not pass.
GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS opts out for provisioning workflows that
stage tiles before ingest.

The check honours the Settings the caller is operating under rather than the
process-wide ones, because every analysis path already threads its own.

The affected tests write manifests into tmp_path, so they now declare tmp_path
as the storage root — which is what a deployment does, and makes the fixtures
more honest than they were.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Jens
2026-08-22 16:16:25 +02:00
co-authored by Claude Opus 5
parent 16dedeb670
commit e2f586c029
14 changed files with 243 additions and 21 deletions
+5
View File
@@ -186,3 +186,8 @@ GEOINTEL_AOI_WORKER_POLL_SECONDS=2
# /detection/run-async, so tiled GPU inference never blocks an HTTP request.
GEOINTEL_ANALYSIS_WORKER_ENABLED=false
GEOINTEL_ANALYSIS_WORKER_POLL_SECONDS=2
# Analysis consumes only artifacts under STORAGE_ROOT: a tile manifest path
# arrives in the request and a manifest entry may name an absolute tile
# path, so without this an API field is an unbounded filesystem reference.
# Provisioning workflows that stage tiles elsewhere before ingest can opt out.
GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS=false
+5
View File
@@ -49,6 +49,11 @@ class Settings(BaseSettings):
validation_alias="DATABASE_URL",
)
storage_root: str = Field(default="./storage", validation_alias="STORAGE_ROOT")
# Analysis consumes only artifacts under storage_root. Provisioning
# workflows that stage tiles elsewhere before ingest can opt out.
allow_external_artifact_paths: bool = Field(
default=False, validation_alias="GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS"
)
max_upload_mb: int = Field(default=500, validation_alias="MAX_UPLOAD_MB")
orthophoto_enabled: bool = Field(default=True, validation_alias="ORTHOPHOTO_ENABLED")
orthophoto_wms_url: str = Field(
+14 -6
View File
@@ -29,6 +29,7 @@ from app.services.model_asset_catalog_service import ModelAssetCatalogService
from app.services.model_registry_service import ModelRegistryService
from app.services.model_validation_scope_service import ModelValidationScopeService
from app.services.qa_service import QaService
from app.services.storage_service import StorageService
from app.services.quality_service import QualityService
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
from app.services.temporal_compatibility_service import TemporalCompatibilityService
@@ -450,7 +451,7 @@ class DetectionService:
coverage = None
if manifest_path:
manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles)
manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles, resolved_settings)
coverage = DetectionQaService.build_tile_coverage(
manifest,
manifest_path=manifest_path,
@@ -942,7 +943,7 @@ class DetectionService:
settings: Settings,
yolo_adapter_class: Type[YoloDetectionAdapter],
) -> tuple[list[Detection], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings)
model_path = Path(settings.yolo_model_path or "").expanduser()
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
@@ -965,7 +966,7 @@ class DetectionService:
raster_bounds = DetectionService._bounds_to_epsg4326(manifest.get("bounds"), manifest_crs)
tiles = list(manifest["tiles"])
tile_paths = [
DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser()) for tile in tiles
DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser(), settings) for tile in tiles
]
# Batched so the GPU is not idle between tiles; each tile keeps its own
# transform for georeferencing, so results stay per tile and in order.
@@ -1248,14 +1249,18 @@ class DetectionService:
return 1.5 * max((right - left) / width, (top - bottom) / height)
@staticmethod
def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int) -> dict[str, Any]:
def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int, settings: Settings | None = None) -> dict[str, Any]:
if not tile_manifest_path:
raise AppError(
code="DETECTION_TILE_MANIFEST_REQUIRED",
message="Configured YOLO inference requires an existing raster tile manifest path",
status_code=400,
)
manifest_path = Path(tile_manifest_path).expanduser()
# The path arrives in the request, so it must name a governed artifact
# rather than an arbitrary file on the host.
manifest_path = StorageService.assert_within_storage_root(
tile_manifest_path, label="tile manifest", settings=settings
)
if not manifest_path.exists() or not manifest_path.is_file():
raise AppError(
code="DETECTION_TILE_MANIFEST_NOT_FOUND",
@@ -1280,13 +1285,16 @@ class DetectionService:
return manifest
@staticmethod
def _resolve_tile_path(tile: dict[str, Any], manifest_path: Path) -> Path:
def _resolve_tile_path(tile: dict[str, Any], manifest_path: Path, settings: Settings | None = None) -> Path:
raw_path = tile.get("path")
if not isinstance(raw_path, str) or not raw_path:
raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile manifest entries require a path", status_code=422)
tile_path = Path(raw_path).expanduser()
if not tile_path.is_absolute():
tile_path = manifest_path.parent / tile_path
# A manifest entry may name an absolute path; it is still only allowed
# to point at a tile the runtime itself produced.
tile_path = StorageService.assert_within_storage_root(tile_path, label="raster tile", settings=settings)
if not tile_path.exists() or not tile_path.is_file():
raise AppError(
code="DETECTION_TILE_NOT_FOUND",
+6 -3
View File
@@ -380,7 +380,10 @@ class SegmentationService:
manifest_path = DetectionQaService.tile_manifest_path(run_parameters)
coverage = None
if manifest_path:
manifest = DetectionService._load_tile_manifest(manifest_path, get_settings().yolo_max_tiles)
settings_for_qa = get_settings()
manifest = DetectionService._load_tile_manifest(
manifest_path, settings_for_qa.yolo_max_tiles, settings_for_qa
)
coverage = DetectionQaService.build_tile_coverage(
manifest,
manifest_path=manifest_path,
@@ -700,7 +703,7 @@ class SegmentationService:
yolo_seg_adapter_class: type[YoloSegmentationAdapter],
sam_adapter_class: type[SamSegmentationAdapter],
) -> tuple[list[Segmentation], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings)
if model_name == settings.sam_model_id:
model_path = Path(settings.sam_model_path or "").expanduser()
allowed_frameworks = ("ultralytics/sam", "sam", "ultralytics", "pytorch")
@@ -728,7 +731,7 @@ class SegmentationService:
manifest_crs = DetectionService._require_manifest_crs(manifest)
candidates: list[dict[str, Any]] = []
for tile in manifest["tiles"]:
tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser())
tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser(), settings)
for raw in adapter.predict_tile(model, tile_path, confidence_threshold):
model_class_name = str(raw.get("class_name") or "").strip()
class_name = DetectionService._canonical_class_name(model_class_name)
+48
View File
@@ -6,6 +6,7 @@ from pathlib import Path
from typing import Any
from app.core.config import get_settings
from app.core.errors import AppError
class StorageService:
@@ -13,6 +14,53 @@ class StorageService:
def _base_dir() -> Path:
return Path(get_settings().storage_root).resolve()
@staticmethod
def assert_within_storage_root(
path: str | Path,
*,
label: str = "artifact",
settings: Any = None,
) -> Path:
"""Resolve a path and refuse anything outside the configured storage root.
``tile_manifest_path`` arrives in the analysis request and a manifest
entry may name an absolute tile path, so without this an API field is an
unbounded reference to the host filesystem. It is also the persistence
rule the product already states: only a governed, runtime-produced
artifact may be consumed, and a file outside the root is not one.
Resolution happens before the comparison, so ``..`` cannot climb out and
a sibling that merely shares a name prefix does not pass.
"""
from app.core.config import get_settings
settings = settings or get_settings()
raw = str(path or "").strip()
if not raw:
raise AppError(
code="STORAGE_PATH_OUTSIDE_ROOT",
message=f"A {label} path is required.",
status_code=400,
)
resolved = Path(raw).expanduser().resolve()
if getattr(settings, "allow_external_artifact_paths", False):
return resolved
root = Path(settings.storage_root).resolve()
if resolved != root and root not in resolved.parents:
raise AppError(
code="STORAGE_PATH_OUTSIDE_ROOT",
message=(
f"The {label} path lies outside the configured storage root and is therefore not a "
"governed artifact."
),
details={"storage_root": str(root)},
status_code=400,
)
return resolved
@staticmethod
def normalize_dataset_type(dataset_type: str) -> str:
normalized = dataset_type.strip().lower()
@@ -149,8 +149,11 @@ class YoloPreflightService:
return result
try:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, resolved_settings.yolo_max_tiles)
tile_paths = [DetectionService._resolve_tile_path(tile, Path(tile_manifest_path).expanduser()) for tile in manifest["tiles"]]
manifest = DetectionService._load_tile_manifest(tile_manifest_path, resolved_settings.yolo_max_tiles, resolved_settings)
tile_paths = [
DetectionService._resolve_tile_path(tile, Path(tile_manifest_path).expanduser(), resolved_settings)
for tile in manifest["tiles"]
]
except AppError as exc:
result["status"] = "manifest_invalid"
result["message"] = exc.message
+4 -1
View File
@@ -287,7 +287,10 @@ def test_model_assets_api_returns_canonical_envelope(monkeypatch, tmp_path: Path
assert payload["data"]["items"][0]["will_download_models"] is False
def test_detection_run_persists_selected_model_asset_parameters(tmp_path: Path) -> None:
def test_detection_run_persists_selected_model_asset_parameters(tmp_path, monkeypatch: Path) -> None:
# A manifest written into tmp_path is only a governed artifact if
# tmp_path is the storage root.
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
model_file = tmp_path / "building-detector.pt"
model_file.write_bytes(b"local model")
db, project_id, dataset_id = _project_and_raster_dataset()
@@ -140,6 +140,9 @@ def _project_and_dataset(dataset_type: str = "raster"):
def _settings(tmp_path: Path, **overrides) -> Settings:
values = {
# The runtime only consumes artifacts under the storage root, so a
# test that writes tiles into tmp_path must say that is the root.
"storage_root": str(tmp_path),
"yolo_seg_enabled": True,
"yolo_seg_model_path": str(tmp_path / "seg.pt"),
"sam_enabled": True,
@@ -114,7 +114,10 @@ def _session(tmp_path: Path, *, with_manifest: bool):
return db, analysis_run_id, reference_dataset_id
def test_segmentation_qa_scores_only_inside_persisted_tile_coverage(tmp_path: Path) -> None:
def test_segmentation_qa_scores_only_inside_persisted_tile_coverage(tmp_path: Path, monkeypatch) -> None:
# A manifest written into tmp_path is only a governed artifact if
# tmp_path is the storage root.
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
db, analysis_run_id, reference_dataset_id = _session(tmp_path, with_manifest=True)
result = SegmentationService.compare_segmentations_with_reference(
@@ -151,7 +154,10 @@ def test_segmentation_qa_without_manifest_reports_unbounded_coverage(tmp_path: P
assert result["coverage"]["mode"] == "unbounded_no_manifest"
def test_segmentation_qa_rejects_reference_entirely_outside_coverage(tmp_path: Path) -> None:
def test_segmentation_qa_rejects_reference_entirely_outside_coverage(tmp_path: Path, monkeypatch) -> None:
# A manifest written into tmp_path is only a governed artifact if
# tmp_path is the storage root.
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
db, analysis_run_id, reference_dataset_id = _session(tmp_path, with_manifest=True)
db.query_rows[VectorFeature] = [
_reference(reference_dataset_id, box(8.0, 8.0, 8.1, 8.1)),
+36 -5
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
from hashlib import sha256
import json
import os
import subprocess
import sys
from pathlib import Path
@@ -138,7 +139,12 @@ def test_yolo_preflight_validates_model_and_manifest_without_importing_yolo(tmp_
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"weights")
manifest_path = _manifest(tmp_path, tile_count=2)
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
settings = Settings(
storage_root=str(tmp_path),
yolo_enabled=True,
yolo_model_path=str(model_path),
yolo_max_tiles=4,
)
_write_model_sidecar(model_path, settings)
result = YoloPreflightService.run(
@@ -162,7 +168,12 @@ def test_yolo_preflight_marks_assumed_dependencies_in_runtime_details(tmp_path:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"weights")
manifest_path = _manifest(tmp_path, tile_count=1)
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
settings = Settings(
storage_root=str(tmp_path),
yolo_enabled=True,
yolo_model_path=str(model_path),
yolo_max_tiles=4,
)
_write_model_sidecar(model_path, settings)
result = YoloPreflightService.run(
@@ -182,7 +193,12 @@ def test_yolo_preflight_can_explicitly_smoke_load_local_model(tmp_path: Path) ->
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"weights")
manifest_path = _manifest(tmp_path)
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
settings = Settings(
storage_root=str(tmp_path),
yolo_enabled=True,
yolo_model_path=str(model_path),
yolo_max_tiles=4,
)
_write_model_sidecar(model_path, settings)
result = YoloPreflightService.run(
@@ -202,7 +218,12 @@ def test_yolo_preflight_can_explicitly_smoke_load_local_model(tmp_path: Path) ->
def test_yolo_preflight_reports_explicit_model_load_failure(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"weights")
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
settings = Settings(
storage_root=str(tmp_path),
yolo_enabled=True,
yolo_model_path=str(model_path),
yolo_max_tiles=4,
)
_write_model_sidecar(model_path, settings)
result = YoloPreflightService.run(
@@ -238,6 +259,9 @@ def test_yolo_preflight_script_outputs_json(tmp_path: Path) -> None:
check=True,
capture_output=True,
text=True,
# The script reads process settings; the manifest it is asked to
# validate lives here, so this is the storage root for that run.
env={**os.environ, "STORAGE_ROOT": str(tmp_path)},
)
payload = json.loads(result.stdout)
@@ -250,7 +274,13 @@ def test_yolo_preflight_script_uses_environment_configuration(tmp_path: Path, mo
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"weights")
manifest_path = _manifest(tmp_path)
_write_model_sidecar(model_path, Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4))
_write_model_sidecar(model_path, Settings(
storage_root=str(tmp_path),
yolo_enabled=True,
yolo_model_path=str(model_path),
yolo_max_tiles=4,
))
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
monkeypatch.setenv("YOLO_ENABLED", "true")
monkeypatch.setenv("YOLO_MODEL_PATH", str(model_path))
monkeypatch.setenv("YOLO_MAX_TILES", "4")
@@ -314,6 +344,7 @@ def test_yolo_preflight_script_rejects_assumed_dependencies_for_model_load(tmp_p
def test_yolo_preflight_api_returns_canonical_envelope(monkeypatch, tmp_path: Path) -> None:
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
monkeypatch.setenv("YOLO_ENABLED", "false")
monkeypatch.setenv("YOLO_MODEL_PATH", str(tmp_path / "missing.pt"))
monkeypatch.setenv("YOLO_CONFIG_DIR", str(tmp_path / "ultralytics"))
@@ -208,6 +208,9 @@ def _project_and_dataset(dataset_type: str = "raster"):
def _settings(tmp_path: Path, **overrides) -> Settings:
model_path = tmp_path / "model.pt"
values = {
# The runtime only consumes artifacts under the storage root, so a
# test that writes tiles into tmp_path must say that is the root.
"storage_root": str(tmp_path),
"yolo_enabled": True,
"yolo_model_path": str(model_path),
"yolo_max_tiles": 4,
@@ -460,7 +460,10 @@ def _coverage_manifest(tmp_path, dataset_id, bounds=(-1.0, -1.0, 3.0, 3.0)):
return manifest_path
def test_detection_qa_excludes_references_outside_persisted_tile_coverage(tmp_path) -> None:
def test_detection_qa_excludes_references_outside_persisted_tile_coverage(tmp_path, monkeypatch) -> None:
# QA reads process settings; a manifest written into tmp_path is only a
# governed artifact if tmp_path is the storage root.
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
project_id = uuid4()
dataset_id = uuid4()
reference_dataset_id = uuid4()
@@ -525,7 +528,10 @@ def test_detection_qa_excludes_references_outside_persisted_tile_coverage(tmp_pa
assert quality_check.findings_json["coverage"] == result["coverage"]
def test_detection_qa_reports_box_to_footprint_diagnostic_without_changing_strict_metrics(tmp_path) -> None:
def test_detection_qa_reports_box_to_footprint_diagnostic_without_changing_strict_metrics(tmp_path, monkeypatch) -> None:
# QA reads process settings; a manifest written into tmp_path is only a
# governed artifact if tmp_path is the storage root.
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
project_id = uuid4()
dataset_id = uuid4()
reference_dataset_id = uuid4()
+90
View File
@@ -0,0 +1,90 @@
"""Analysis may only read artifacts the runtime itself produced.
``tile_manifest_path`` arrives in the detection and segmentation request and was
read straight off disk, and a manifest entry may name an absolute tile path. A
manifest outside the storage root is by definition not a governed artifact, so
consuming one contradicts the rule the whole persistence model rests on — and
it turns an API field into an unbounded reference to the host filesystem.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from app.core.errors import AppError
from app.services.storage_service import StorageService
def test_a_path_inside_the_root_is_returned_resolved(tmp_path: Path, monkeypatch) -> None:
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
target = tmp_path / "tiles" / "manifest.json"
target.parent.mkdir(parents=True)
target.write_text("{}", encoding="utf-8")
resolved = StorageService.assert_within_storage_root(str(target), label="tile manifest")
assert resolved == target.resolve()
def test_a_path_outside_the_root_is_refused(tmp_path: Path, monkeypatch) -> None:
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path / "storage"))
(tmp_path / "storage").mkdir()
outside = tmp_path / "elsewhere.json"
outside.write_text("{}", encoding="utf-8")
with pytest.raises(AppError) as exc_info:
StorageService.assert_within_storage_root(str(outside), label="tile manifest")
assert exc_info.value.code == "STORAGE_PATH_OUTSIDE_ROOT"
assert "tile manifest" in exc_info.value.message
def test_a_traversal_sequence_cannot_climb_out(tmp_path: Path, monkeypatch) -> None:
root = tmp_path / "storage"
root.mkdir()
monkeypatch.setenv("STORAGE_ROOT", str(root))
secret = tmp_path / "secret.json"
secret.write_text("{}", encoding="utf-8")
with pytest.raises(AppError) as exc_info:
StorageService.assert_within_storage_root(str(root / ".." / "secret.json"), label="tile")
assert exc_info.value.code == "STORAGE_PATH_OUTSIDE_ROOT"
def test_a_sibling_directory_sharing_a_name_prefix_is_refused(tmp_path: Path, monkeypatch) -> None:
"""``/data/storage-old`` is not inside ``/data/storage``."""
root = tmp_path / "storage"
root.mkdir()
sibling = tmp_path / "storage-old"
sibling.mkdir()
monkeypatch.setenv("STORAGE_ROOT", str(root))
target = sibling / "manifest.json"
target.write_text("{}", encoding="utf-8")
with pytest.raises(AppError):
StorageService.assert_within_storage_root(str(target), label="tile manifest")
def test_an_empty_path_is_refused(tmp_path: Path, monkeypatch) -> None:
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
with pytest.raises(AppError):
StorageService.assert_within_storage_root("", label="tile manifest")
def test_the_check_can_be_disabled_for_an_operator_provisioning_workflow(
tmp_path: Path, monkeypatch
) -> None:
"""Provisioning scripts stage tiles outside the root before ingest."""
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path / "storage"))
monkeypatch.setenv("GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS", "true")
(tmp_path / "storage").mkdir()
outside = tmp_path / "elsewhere.json"
outside.write_text("{}", encoding="utf-8")
assert StorageService.assert_within_storage_root(str(outside), label="tile") == outside.resolve()
+8
View File
@@ -1715,6 +1715,14 @@ reproducibility. Clients must not submit arbitrary model paths.
GeoIntel does not download model weights automatically. Configured YOLO runs read existing tile files from the manifest, convert YOLO pixel-space boxes to EPSG:4326 detection polygons and persist detections as first-class records.
The manifest path, and every tile path inside it, must resolve under
`STORAGE_ROOT`. The path arrives in the request and a manifest entry may name an
absolute tile path, so without that check the field is an unbounded reference to
the host filesystem — and a file outside the root is by definition not the
governed, runtime-produced artifact the persistence model requires. Rejection is
`STORAGE_PATH_OUTSIDE_ROOT`; `GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS` opts out
for provisioning workflows that stage tiles before ingest.
The manifest must carry explicit CRS metadata (`crs`, `source_crs` or
`dataset_crs`). A manifest without it fails with
`DETECTION_TILE_MANIFEST_INVALID` rather than being georeferenced against an