Initial public release
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s

This commit is contained in:
Jens
2026-08-31 21:56:53 +02:00
commit faeb58ef6d
1386 changed files with 263203 additions and 0 deletions
View File
+19
View File
@@ -0,0 +1,19 @@
"""Canonical backend-test import boundary.
Pytest is intentionally runnable from ``backend/`` because that is the CI
entrypoint. Some contract tests exercise repository-level deterministic
scripts; put the canonical repository root ahead of the legacy
``backend/scripts`` helper directory so those imports resolve to the code that
is actually shipped by the root Docker build.
"""
from __future__ import annotations
import sys
from pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
repository_root_text = str(REPOSITORY_ROOT)
if repository_root_text not in sys.path:
sys.path.insert(0, repository_root_text)
+181
View File
@@ -0,0 +1,181 @@
"""Helpers for asserting frontend wiring from the backend test suite.
Most of this suite checks frontend behaviour by reading TSX files and
asserting that literal substrings occur in them. That reds the suite on every
rename and every copy change while proving nothing about behaviour: renaming a
button label is not a regression, and ``source.count("useEffect(") == 1`` is a
formatting rule, not a contract.
These helpers keep the useful half of that idea — that a documented product
contract must remain wired somewhere in the frontend — and drop the brittle
half. Assert on identifiers, API paths and prop names, which only change when
the wiring genuinely changes. Do not assert on user-visible copy; put that in
a frontend component test where the rendered output can be checked properly.
"""
from __future__ import annotations
import re
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
FRONTEND_SRC = ROOT / "frontend" / "src"
# A feature is one behaviour spread over several modules: a container, its
# hooks, its domain layer, its pure helpers. A contract belongs to the feature,
# not to whichever file currently holds it, so moving code between siblings
# must not red the suite. Missing entries are skipped, so a group survives a
# module being split further, renamed or merged back.
#
# Use these for *positive* contracts ("this is wired"). A negative contract
# ("this component performs no transport") is a statement about one file and
# must keep reading that file, or widening it would quietly weaken the check.
FEATURE_SOURCES: dict[str, tuple[str, ...]] = {
"map_workspace": (
"components/map/MapWorkspace.tsx",
"components/map/mapWorkspaceProps.ts",
"components/map/mapWorkspaceThemes.ts",
"components/map/mapWorkspaceUtils.ts",
"components/map/useMapWorkspaceViewModel.ts",
"components/map/MapExplorerView.tsx",
"components/map/MapAdvancedWorkbench.tsx",
"components/map/MunicipalitySearch.tsx",
"hooks/useMapImageOverlays.ts",
"hooks/useMapRectangleSelection.ts",
"hooks/useFullGisWorkflow.ts",
"hooks/useMapThemeSelectionInsights.ts",
"hooks/useMapSelectionExtract.ts",
"hooks/useMapWorkspaceState.ts",
"hooks/useMapSelectionDataset.ts",
"hooks/useMapSelectionQa.ts",
"hooks/useMapThemeSelectionInsights.ts",
"hooks/useTemporalComparison.ts",
"hooks/useCoverageResolver.ts",
"hooks/useOfficialMapProducts.ts",
),
# The presentational half of the map workspace. Transport belongs to the
# hooks, so "this performs no transport" is a contract about these modules
# and would fail — correctly — against the whole feature.
"map_workspace_presentation": (
"components/map/MapWorkspace.tsx",
"components/map/MapExplorerView.tsx",
"components/map/MapAdvancedWorkbench.tsx",
),
"detection": (
"components/detection/DetectionLab.tsx",
"components/detection/DetectionModelManagement.tsx",
"components/detection/detectionProfiles.ts",
"components/models/ModelSelector.tsx",
"components/models/modelOptions.ts",
"hooks/useDetectionWorkflow.ts",
),
"segmentation": (
"components/segmentation/SegmentationLab.tsx",
"hooks/useSegmentationWorkflow.ts",
),
"quality": (
"components/quality/QualityResultsPanel.tsx",
"components/quality/DetectionReviewPanel.tsx",
"hooks/useQualityWorkflow.ts",
),
"datasets": (
"components/datasets/DatasetPanel.tsx",
"components/datasets/DatasetDetailPanel.tsx",
"components/datasets/RasterControls.tsx",
"components/datasets/VectorControls.tsx",
"components/datasets/SourceCatalogPanel.tsx",
"hooks/useDatasetWorkflow.ts",
"services/api/datasets.ts",
),
"exports": (
"components/exports/ExportCenter.tsx",
"hooks/useExportWorkflow.ts",
),
"shell": (
"App.tsx",
"WorkbenchApp.tsx",
"components/shell/WorkbenchNavigation.tsx",
"components/shell/SecondaryDisplay.tsx",
"components/inspector/WorkbenchInspector.tsx",
"components/overview/OverviewWorkspace.tsx",
"hooks/useProjectWorkspace.ts",
"hooks/useWorkbenchBootstrap.ts",
),
}
MAP_WORKSPACE_SOURCES = FEATURE_SOURCES["map_workspace"]
def read_frontend(relative_path: str) -> str:
"""Read one frontend source file relative to ``frontend/src``."""
return (FRONTEND_SRC / relative_path).read_text(encoding="utf-8")
def read_frontend_area(*relative_paths: str) -> str:
"""Read several related sources as one body of code.
Files that do not exist are skipped, so this survives a module being split
further or merged back.
"""
parts: list[str] = []
for relative_path in relative_paths:
path = FRONTEND_SRC / relative_path
if path.is_file():
parts.append(path.read_text(encoding="utf-8"))
return chr(10).join(parts)
def read_feature(name: str) -> str:
"""Every module of one feature, whichever files it is currently split into."""
try:
sources = FEATURE_SOURCES[name]
except KeyError: # pragma: no cover - a typo should fail loudly
raise AssertionError(
f"Unknown frontend feature {name!r}; known: {sorted(FEATURE_SOURCES)}"
) from None
return read_frontend_area(*sources)
def read_map_workspace() -> str:
"""The whole map workspace feature, whichever modules it is split into."""
return read_feature("map_workspace")
def assert_wired(source: str, *identifiers: str, context: str = "frontend source") -> None:
"""Every identifier must appear in ``source``.
Use for symbols, hook names, prop names and API paths — things a refactor
renames deliberately — never for sentences shown to a user.
"""
missing = [identifier for identifier in identifiers if identifier not in source]
assert not missing, f"{context} no longer wires: {missing}"
def assert_calls(source: str, function_name: str, *, first_argument: str) -> None:
"""Assert ``function_name`` is called with ``first_argument`` as argument 1.
Tolerates whatever the remaining arguments have been refactored into, which
is the part that keeps changing while the wiring stays the same.
"""
pattern = rf"{re.escape(function_name)}\(\s*{re.escape(first_argument)}\s*[,)]"
assert re.search(pattern, source), f"{function_name}({first_argument}, …) is no longer called"
def assert_mentions(source: str, *phrases: str, context: str = "frontend source") -> None:
"""Case-insensitive check that a concept is still surfaced to the operator.
A deliberately weak assertion: it survives rewording but still fails if a
whole explanation is deleted. Prefer ``assert_wired`` where an identifier
exists to check instead.
"""
folded = source.casefold()
missing = [phrase for phrase in phrases if phrase.casefold() not in folded]
assert not missing, f"{context} no longer mentions: {missing}"
@@ -0,0 +1,34 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "run_accuracy_phase2_foundation_audit.py"
SPEC = importlib.util.spec_from_file_location("accuracy_phase2_foundation_audit", SCRIPT)
assert SPEC is not None and SPEC.loader is not None
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def test_phase2_foundation_audit_enumerates_exact_source_and_contract_policies() -> None:
payload = MODULE.collect()
assert payload["phase"] == "P2"
assert payload["migration_revision"] == "202608010001"
assert payload["source_registry"]["definition_count"] >= 40
assert payload["source_registry"]["required_building_policy"] == {
"grb_primary_building_validation": "primary",
"buildings_register_classification": "authoritative",
"sentinel_2_classification": "contextual",
"dhmv_classification": "authoritative",
"osm_ground_truth_allowed": False,
}
assert {(item["key"], item["version"]) for item in payload["data_contracts"]} == {
("geointel.vector.geojson", "1.0.0"),
("geointel.raster.geotiff", "1.0.0"),
("geointel.label.yolo", "1.0.0"),
("geointel.label.yolo", "1.1.0"),
("geointel.model.pytorch", "1.0.0"),
}
@@ -0,0 +1,772 @@
from __future__ import annotations
from datetime import timedelta
from importlib.util import module_from_spec, spec_from_file_location
from pathlib import Path
from uuid import uuid4
import pytest
from sqlalchemy import CheckConstraint
from app.core.errors import AppError
from app.models import (
Dataset,
DatasetLineageEdge,
DatasetQuarantine,
DatasetVersion,
SourceRegistry,
SourceSnapshot,
)
from app.services.coverage_registry_service import SOURCE_DEFINITIONS
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.dataset_service import DatasetService
from app.services.source_registry_service import (
SERVER_OWNED_SOURCE_DEFINITIONS,
SourceRegistryService,
)
class _Query:
def __init__(self, session: "InMemorySession", model: type) -> None:
self.session = session
self.model = model
self.predicates = []
def filter(self, *predicates):
self.predicates.extend(predicates)
return self
def one_or_none(self):
matches = self._matches()
if len(matches) > 1:
raise AssertionError(
f"Expected one {self.model.__name__}, found {len(matches)}"
)
return matches[0] if matches else None
def all(self):
return self._matches()
def _matches(self):
matches = list(self.session.objects.get(self.model, []))
for predicate in self.predicates:
field_name = predicate.left.key
expected = predicate.right.value
operator_name = getattr(predicate.operator, "__name__", "")
if operator_name == "in_op":
matches = [
item for item in matches if getattr(item, field_name) in expected
]
else:
matches = [
item for item in matches if getattr(item, field_name) == expected
]
return matches
class InMemorySession:
def __init__(self, *objects: object) -> None:
self.objects: dict[type, list[object]] = {}
self.added: list[object] = []
self.flushes = 0
for item in objects:
self._store(item)
def query(self, model: type) -> _Query:
return _Query(self, model)
def add(self, item: object) -> None:
if getattr(item, "id", None) is None:
setattr(item, "id", uuid4())
self._store(item)
self.added.append(item)
def flush(self) -> None:
self.flushes += 1
def _store(self, item: object) -> None:
self.objects.setdefault(type(item), []).append(item)
def _registry(source_key: str) -> SourceRegistry:
definition = SERVER_OWNED_SOURCE_DEFINITIONS[source_key]
return SourceRegistry(id=uuid4(), **definition.as_model_values())
def _dataset() -> Dataset:
return Dataset(
id=uuid4(),
project_id=uuid4(),
name="candidate.tif",
dataset_type="raster",
source="governed",
status="ready",
validation_status="not_validated",
provenance_status="incomplete",
lineage_status="incomplete",
quarantine_status="not_quarantined",
)
def test_server_owned_definitions_encode_building_authority_and_non_ground_truth_sources() -> (
None
):
grb = SERVER_OWNED_SOURCE_DEFINITIONS["grb"]
buildings_register = SERVER_OWNED_SOURCE_DEFINITIONS[
"digitaal_vlaanderen_buildings_addresses_register"
]
sentinel = SERVER_OWNED_SOURCE_DEFINITIONS["sentinel_2"]
dhmv = SERVER_OWNED_SOURCE_DEFINITIONS["digitaal_vlaanderen_dhmv"]
osm = SERVER_OWNED_SOURCE_DEFINITIONS["osm"]
assert grb.classification == "authoritative"
assert grb.usage_policy["ground_truth_allowed"] is True
assert grb.usage_policy["validation_authority"]["building_validation"] == "primary"
assert buildings_register.classification == "authoritative"
assert buildings_register.usage_policy["ground_truth_allowed"] is False
assert (
buildings_register.usage_policy["validation_authority"]["building_validation"]
== "corroborative"
)
assert (
buildings_register.usage_policy["validation_authority"][
"building_register_validation"
]
== "primary"
)
assert sentinel.classification == "contextual"
assert dhmv.classification == "authoritative"
assert dhmv.usage_policy["ground_truth_allowed"] is False
assert (
dhmv.usage_policy["validation_authority"]["building_validation"]
== "corroborative"
)
assert (
dhmv.usage_policy["validation_authority"]["elevation_validation"] == "primary"
)
assert osm.classification == "contextual"
assert osm.usage_policy["ground_truth_allowed"] is False
assert osm.usage_policy["automatic_ground_truth"] is False
assert osm.usage_policy["training_allowed"] is False
assert {
"ngi_adminvector",
"rbins_marine_reporting_units",
"rbins_msp_2026",
"grb",
"digitaal_vlaanderen",
"vrbg",
"digitaal_vlaanderen_buildings_addresses_register",
"digitaal_vlaanderen_orthophoto",
"spw_orthophoto",
"urbis_orthophoto",
"digitaal_vlaanderen_dhmv",
"spw_terrain",
"spw_walous_land_cover",
"spw_geoportail",
"spw_picc",
"urbis",
"vmm_flood_hazard",
"vmm_vha_bathymetry_profiles",
"dov_soil_map",
"statbel",
"waterinfo",
"agentschap_landbouw_zeevisserij_agricultural_parcels",
"sentinel_2",
"osm",
"manual",
"fixture",
"map_selection",
"derived",
"training_label",
"model",
"experimental",
"mdk_bcp_bathymetry",
}.issubset(SERVER_OWNED_SOURCE_DEFINITIONS)
for umbrella_key in ("digitaal_vlaanderen", "spw_geoportail"):
definition = SERVER_OWNED_SOURCE_DEFINITIONS[umbrella_key]
assert definition.classification == "authoritative"
assert definition.usage_policy["ground_truth_allowed"] is False
assert definition.usage_policy["automatic_ground_truth"] is False
assert (
SERVER_OWNED_SOURCE_DEFINITIONS["mdk_bcp_bathymetry"].ingest_status
== "not_configured"
)
def test_coverage_and_direct_adapter_source_keys_are_registry_backed() -> None:
coverage_source_keys = {
definition.contract.source_name for definition in SOURCE_DEFINITIONS
}
coverage_materialization_keys = {
source_key
for definition in SOURCE_DEFINITIONS
for source_key in definition.materialized_source_names
}
direct_adapter_source_keys = {
"digitaal_vlaanderen",
"spw_geoportail",
"mdk_bcp_bathymetry",
}
assert (
coverage_source_keys
| coverage_materialization_keys
| direct_adapter_source_keys
<= set(SERVER_OWNED_SOURCE_DEFINITIONS)
)
def test_new_adapter_source_seed_rows_match_server_owned_registry_semantics() -> None:
migration_path = (
Path(__file__).resolve().parents[1]
/ "alembic"
/ "versions"
/ "202608010001_source_registry_provenance.py"
)
spec = spec_from_file_location("phase2_source_registry_migration", migration_path)
assert spec is not None and spec.loader is not None
migration = module_from_spec(spec)
spec.loader.exec_module(migration)
seed_rows = {row["source_key"]: row for row in migration._seed_rows()}
for source_key in ("digitaal_vlaanderen", "spw_geoportail", "mdk_bcp_bathymetry"):
expected = SERVER_OWNED_SOURCE_DEFINITIONS[source_key].as_model_values()
observed = seed_rows[source_key]
for field in (
"source_key",
"display_name",
"classification",
"authority_name",
"authority_scope_json",
"provider_adapter_key",
"source_url",
"default_crs",
"default_units",
"geographic_coverage_json",
"usage_policy_json",
"freshness_status",
"ingest_status",
"known_limitations_json",
):
assert observed[field] == expected[field]
def test_ensure_source_is_idempotent_and_rejects_caller_owned_unknown_sources() -> None:
grb = _registry("grb")
session = InMemorySession(grb)
assert SourceRegistryService.ensure_server_owned_source(session, "GRB") is grb
assert session.added == []
with pytest.raises(AppError) as exc_info:
SourceRegistryService.ensure_server_owned_source(session, "caller_claimed_grb")
assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND"
def test_snapshot_is_checksum_bound_and_idempotent() -> None:
grb = _registry("grb")
session = InMemorySession(grb)
checksum = "a" * 64
snapshot = SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key="2026-08-01-gbg",
checksum_sha256=checksum,
source_version="2026-08-01",
crs="EPSG:31370",
units="metres",
)
assert snapshot.source_registry_id == grb.id
assert snapshot.checksum_sha256 == checksum
assert snapshot.ingest_status == "ingested"
assert (
SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key="2026-08-01-gbg",
checksum_sha256=checksum,
)
is snapshot
)
with pytest.raises(AppError) as exc_info:
SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key="2026-08-01-gbg",
checksum_sha256="b" * 64,
)
assert exc_info.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
with pytest.raises(AppError) as version_conflict:
SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key="2026-08-01-gbg",
checksum_sha256=checksum,
source_version="2026-08-02",
)
assert version_conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
with pytest.raises(AppError) as invalid_checksum:
SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key="bad-checksum",
checksum_sha256="not-a-checksum",
)
assert invalid_checksum.value.code == "SOURCE_SNAPSHOT_CHECKSUM_INVALID"
def test_governed_import_reuses_an_identical_snapshot_without_rewriting_fetched_at() -> None:
"""A second project may bind the same immutable source snapshot safely."""
grb = _registry("grb")
session = InMemorySession(grb)
checksum = "a" * 64
observed_at = None
metadata = {
"dataset_type": "vector",
"bounds_json": {"min_x": 4.0, "min_y": 50.0, "max_x": 4.1, "max_y": 50.1},
}
source_metadata = {"source_url": "https://example.test/grb", "units": "metres"}
# Dataset ingest keys are project-scoped, while a source snapshot is
# globally keyed by immutable source evidence. This represents the same
# source file arriving through two independently resumable imports.
project_one, project_two = uuid4(), uuid4()
assert (
DatasetService._ingest_key(
project_id=project_one,
source_key="grb",
checksum_sha256=checksum,
dataset_type="vector",
dataset_role="source",
area_id=None,
reference_layer_name=None,
source_version="2026-08-01",
)
!= DatasetService._ingest_key(
project_id=project_two,
source_key="grb",
checksum_sha256=checksum,
dataset_type="vector",
dataset_role="source",
area_id=None,
reference_layer_name=None,
source_version="2026-08-01",
)
)
first_source, first_snapshot = DatasetService._record_snapshot(
session,
source_key="grb",
checksum_sha256=checksum,
source_version="2026-08-01",
observed_at=observed_at,
valid_from=None,
valid_to=None,
source_crs="EPSG:31370",
source_metadata=source_metadata,
metadata=metadata,
)
original_fetched_at = first_snapshot.fetched_at
replay_source, replay_snapshot = DatasetService._record_snapshot(
session,
source_key="grb",
checksum_sha256=checksum,
source_version="2026-08-01",
observed_at=observed_at,
valid_from=None,
valid_to=None,
source_crs="EPSG:31370",
source_metadata=source_metadata,
metadata=metadata,
)
assert replay_source is first_source
assert replay_snapshot is first_snapshot
assert replay_snapshot.fetched_at == original_fetched_at
assert session.objects[SourceSnapshot] == [first_snapshot]
# Outside the governed replay path, a contradictory acquisition timestamp
# remains immutable evidence and is still rejected.
with pytest.raises(AppError) as fetched_at_conflict:
SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key=first_snapshot.snapshot_key,
checksum_sha256=checksum,
fetched_at=original_fetched_at + timedelta(seconds=1),
)
assert fetched_at_conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
# Replay mode is narrow: a changed immutable evidence field still fails.
with pytest.raises(AppError) as conflict:
SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key=first_snapshot.snapshot_key,
checksum_sha256=checksum,
crs="EPSG:4326",
reuse_existing_snapshot=True,
)
assert conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
def test_snapshot_schema_requires_a_canonical_sha256() -> None:
constraints = {
constraint.name: str(constraint.sqltext)
for constraint in SourceSnapshot.__table__.constraints
if isinstance(constraint, CheckConstraint)
}
assert SourceSnapshot.__table__.c.checksum_sha256.nullable is False
assert "ck_source_snapshots_checksum_sha256" in constraints
assert (
"lower(checksum_sha256)" in constraints["ck_source_snapshots_checksum_sha256"]
)
def test_complete_provenance_binding_is_required_before_authoritative_validation() -> (
None
):
grb = _registry("grb")
snapshot = SourceSnapshot(
id=uuid4(),
source_registry_id=grb.id,
snapshot_key="governed-grb",
checksum_sha256="c" * 64,
ingest_status="ingested",
)
dataset = _dataset()
SourceRegistryService.bind_dataset_provenance(
dataset,
source=grb,
snapshot=snapshot,
data_contract_key="vector.grb.buildings",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="complete",
)
assert SourceRegistryService.is_dataset_eligible_for_authoritative_validation(
dataset,
source=grb,
snapshot=snapshot,
task="building_validation",
)
osm = _registry("osm")
dataset.source_registry_id = osm.id
snapshot.source_registry_id = osm.id
assert not SourceRegistryService.is_dataset_eligible_for_authoritative_validation(
dataset,
source=osm,
snapshot=snapshot,
task="building_validation",
)
def test_lineage_and_quarantine_are_fail_closed_and_observable() -> None:
session = InMemorySession()
parent_id = uuid4()
child_id = uuid4()
edge = SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=parent_id,
child_dataset_id=child_id,
relation_type="derived_from",
transformation_name="vector_clip",
input_checksum_sha256="d" * 64,
output_checksum_sha256="e" * 64,
)
assert isinstance(edge, DatasetLineageEdge)
assert (
SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=parent_id,
child_dataset_id=child_id,
relation_type="derived_from",
transformation_name="vector_clip",
input_checksum_sha256="d" * 64,
output_checksum_sha256="e" * 64,
)
is edge
)
with pytest.raises(AppError) as self_reference:
SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=parent_id,
child_dataset_id=parent_id,
relation_type="derived_from",
transformation_name="vector_clip",
)
assert self_reference.value.code == "DATASET_LINEAGE_SELF_REFERENCE"
grandchild_id = uuid4()
SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=child_id,
child_dataset_id=grandchild_id,
relation_type="derived_from",
transformation_name="vector_buffer",
)
with pytest.raises(AppError) as cycle:
SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=grandchild_id,
child_dataset_id=parent_id,
relation_type="derived_from",
transformation_name="vector_union",
)
assert cycle.value.code == "DATASET_LINEAGE_CYCLE_DETECTED"
dataset = _dataset()
record = SourceRegistryService.quarantine_dataset(
session,
dataset=dataset,
stage="vector_ingest",
reason_code="CRS_UNVERIFIED",
details={"observed_crs": None},
)
assert isinstance(record, DatasetQuarantine)
assert dataset.status == "quarantined"
assert dataset.quarantine_status == "quarantined"
assert dataset.validation_status == "failed"
version_parent = _dataset()
version = DatasetVersion(id=uuid4(), dataset_id=version_parent.id, version=1)
version_session = InMemorySession(version_parent, version)
version_record = SourceRegistryService.quarantine_dataset(
version_session,
dataset_version=version,
stage="dataset_version_validation",
reason_code="CHECKSUM_MISMATCH",
)
assert version_record.dataset_id == version_parent.id
assert version_record.dataset_version_id == version.id
assert version_parent.status == "quarantined"
assert version_parent.quarantine_status == "quarantined"
assert version_parent.validation_status == "failed"
assert version_parent.provenance_status == "incomplete"
assert version_parent.lineage_status == "incomplete"
assert version.validation_status == "failed"
assert version.provenance_status == "incomplete"
snapshot = SourceSnapshot(
id=uuid4(),
source_registry_id=uuid4(),
snapshot_key="quarantined-source",
checksum_sha256="f" * 64,
ingest_status="ingested",
)
SourceRegistryService.quarantine_dataset(
session,
source_snapshot=snapshot,
stage="source_snapshot_validation",
reason_code="CHECKSUM_MISMATCH",
)
assert snapshot.ingest_status == "quarantined"
def test_quarantine_propagates_transitively_to_descendant_dataset_and_version_consumption_gates() -> (
None
):
"""A -> B -> C must fail closed when the governing A artifact is rejected."""
source = _registry("grb")
snapshot = SourceSnapshot(
id=uuid4(),
source_registry_id=source.id,
snapshot_key="transitive-quarantine-source",
checksum_sha256="a" * 64,
freshness_status="current",
ingest_status="ingested",
)
def governed_dataset(name: str) -> Dataset:
dataset = _dataset()
dataset.name = name
dataset.source = "grb"
dataset.source_name = "grb"
dataset.dataset_role = "source"
dataset.checksum_sha256 = snapshot.checksum_sha256
dataset.source_registry_id = source.id
dataset.source_snapshot_id = snapshot.id
dataset.data_contract_key = "geointel.raster.geotiff"
dataset.data_contract_version = "1.0.0"
dataset.validation_status = "passed"
dataset.provenance_status = "complete"
dataset.lineage_status = "complete"
dataset.quarantine_status = "not_quarantined"
dataset.status = "ready"
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
parent = governed_dataset("parent.tif")
child = governed_dataset("child.tif")
grandchild = governed_dataset("grandchild.tif")
parent_version = DatasetVersion(
id=uuid4(),
dataset_id=parent.id,
version=1,
validation_status="passed",
provenance_status="complete",
lineage_status="complete",
)
child_version = DatasetVersion(
id=uuid4(),
dataset_id=child.id,
version=1,
validation_status="passed",
provenance_status="complete",
lineage_status="complete",
)
grandchild_version = DatasetVersion(
id=uuid4(),
dataset_id=grandchild.id,
version=1,
validation_status="passed",
provenance_status="complete",
lineage_status="complete",
)
session = InMemorySession(
parent,
child,
grandchild,
parent_version,
child_version,
grandchild_version,
)
SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=parent.id,
child_dataset_id=child.id,
relation_type="derived_from",
transformation_name="clip",
)
SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=child.id,
child_dataset_id=grandchild.id,
relation_type="derived_from",
transformation_name="buffer",
)
assert (
DatasetConsumptionGate.evaluate(child, purpose="production_inference").eligible
is True
)
assert (
DatasetConsumptionGate.evaluate(
grandchild, purpose="production_inference"
).eligible
is True
)
SourceRegistryService.quarantine_dataset(
session,
dataset=parent,
stage="contract_validation",
reason_code="CHECKSUM_MISMATCH",
)
for dataset in (parent, child, grandchild):
decision = DatasetConsumptionGate.evaluate(
dataset, purpose="production_inference"
)
assert dataset.status == "quarantined"
assert dataset.quarantine_status == "quarantined"
assert dataset.validation_status == "failed"
assert dataset.provenance_status == "incomplete"
assert dataset.lineage_status == "incomplete"
assert decision.eligible is False
assert "dataset_quarantined" in decision.reasons
for dataset_version in (parent_version, child_version, grandchild_version):
assert dataset_version.validation_status == "failed"
assert dataset_version.provenance_status == "incomplete"
assert dataset_version.lineage_status == "incomplete"
def test_ingest_keys_are_scoped_and_migration_keeps_unknown_legacy_unbound() -> None:
project_id = uuid4()
dataset = _dataset()
dataset.project_id = project_id
dataset.ingest_key = "grb:2026-08-01:gbg:area-sha"
version = DatasetVersion(
id=uuid4(),
dataset_id=dataset.id,
ingest_key=dataset.ingest_key,
validation_status="not_validated",
provenance_status="incomplete",
lineage_status="incomplete",
)
session = InMemorySession(dataset, version)
assert (
SourceRegistryService.find_dataset_by_ingest_key(
session, project_id, dataset.ingest_key
)
is dataset
)
assert (
SourceRegistryService.find_dataset_version_by_ingest_key(
session, dataset.id, dataset.ingest_key
)
is version
)
with pytest.raises(AppError) as invalid_key:
SourceRegistryService.find_dataset_by_ingest_key(session, project_id, " ")
assert invalid_key.value.code == "INGEST_KEY_INVALID"
migration = (
Path(__file__).resolve().parents[1]
/ "alembic"
/ "versions"
/ "202608010001_source_registry_provenance.py"
).read_text(encoding="utf-8")
assert "uuid_generate_v5" not in migration
assert "__unregistered_legacy_source__" in migration
assert "uq_datasets_project_ingest_key" in migration
assert "uq_dataset_versions_dataset_ingest_key" in migration
def test_migration_contains_database_guards_for_snapshot_pairing_contract_lineage_and_quarantine() -> (
None
):
migration = (
Path(__file__).resolve().parents[1]
/ "alembic"
/ "versions"
/ "202608010001_source_registry_provenance.py"
).read_text(encoding="utf-8")
assert "trg_datasets_snapshot_registry_guard" in migration
assert "trg_dataset_versions_snapshot_registry_guard" in migration
assert "trg_source_registry_write_guard" in migration
assert "trg_source_snapshots_evidence_immutable" in migration
assert "geointel_phase2_contract_report_guard" in migration
assert "trg_datasets_contract_report_guard" in migration
assert "trg_dataset_versions_contract_report_guard" in migration
assert "matching complete validation report" in migration
assert "geointel_phase2_lineage_cycle_guard" in migration
assert "trg_dataset_lineage_edges_cycle_guard" in migration
assert "geointel_phase2_lineage_edge_immutable_guard" in migration
assert "trg_dataset_lineage_edges_immutable" in migration
assert "WITH RECURSIVE descendants" in migration
assert "geointel_phase2_quarantine_lineage_descendants" in migration
assert "geointel_phase2_quarantine_state_guard" in migration
assert "trg_dataset_quarantines_state_guard" in migration
assert "accepted dataset artifact and provenance evidence is immutable" in migration
@@ -0,0 +1,131 @@
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "run_accuracy_phase3_full_data_scan.py"
def run_scan(
repo: Path,
output: Path,
*,
resume: bool = False,
unreachable_scopes: tuple[str, ...] = (),
) -> dict:
command = [
sys.executable,
str(SCRIPT),
"--repo-root",
str(repo),
"--output-dir",
str(output),
"--roots",
"data",
"--batch-size",
"2",
]
if resume:
command.append("--resume")
for scope in unreachable_scopes:
command.extend(("--unreachable-scope", scope))
completed = subprocess.run(command, check=True, capture_output=True, text=True)
return json.loads(completed.stdout)
def test_phase3_scan_reconciles_and_resumes_deterministically(tmp_path: Path) -> None:
data = tmp_path / "data"
data.mkdir()
(data / "valid.geojson").write_text(
json.dumps(
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {"source": "GRB"},
"geometry": {"type": "Point", "coordinates": [4.4, 50.8]},
}
],
}
),
encoding="utf-8",
)
invalid = data / "invalid.geojson"
invalid.write_text(
'{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0,0],[1,1],[1,0],[0,1],[0,0]]]}}]}',
encoding="utf-8",
)
duplicate_payload = '{"schema_version":1,"value":"same"}'
(data / "one.json").write_text(duplicate_payload, encoding="utf-8")
(data / "two.json").write_text(duplicate_payload, encoding="utf-8")
(data / "broken.tif").write_bytes(b"not a geotiff")
output = tmp_path / "evidence"
unreachable_scopes = (
"external://tower-corpora=Tower corpora are not mounted in this fixture",
"external://mounted-model-volumes=Model volumes are not mounted in this fixture",
"external://production-postgis-or-api=Production database is not configured in this fixture",
)
first = run_scan(tmp_path, output, unreachable_scopes=unreachable_scopes)
second = run_scan(
tmp_path,
output,
resume=True,
unreachable_scopes=unreachable_scopes,
)
manifest = json.loads((output / "full-scan-manifest.json").read_text(encoding="utf-8"))
quarantine = json.loads((output / "quarantine-manifest.json").read_text(encoding="utf-8"))
assert first["reconciliation"] == {"examined": 5, "skipped": 0, "unreachable": 3, "inventory_total": 8, "reconciles": True}
assert second["content_hash"] == first["content_hash"]
assert manifest["determinism"]["content_hash"] == first["content_hash"]
assert any(item["path"] == "data/broken.tif" for item in quarantine["items"])
assert any(item["path"] == "data/invalid.geojson" for item in quarantine["items"])
assert len(manifest["duplicates"]["exact_duplicate_groups"]) == 1
def test_phase3_scan_does_not_invent_unreachable_production_boundaries(
tmp_path: Path,
) -> None:
data = tmp_path / "data"
data.mkdir()
(data / "present.json").write_text("{}", encoding="utf-8")
output = tmp_path / "evidence"
result = run_scan(tmp_path, output)
assert result["reconciliation"] == {
"examined": 1,
"skipped": 0,
"unreachable": 0,
"inventory_total": 1,
"reconciles": True,
}
def test_phase3_scan_records_a_missing_requested_root(tmp_path: Path) -> None:
output = tmp_path / "evidence"
completed = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--repo-root",
str(tmp_path),
"--output-dir",
str(output),
"--roots",
"missing-data",
],
check=True,
capture_output=True,
text=True,
)
result = json.loads(completed.stdout)
manifest = json.loads((output / "full-scan-manifest.json").read_text(encoding="utf-8"))
assert result["reconciliation"]["unreachable"] == 1
assert manifest["items"][0]["path"] == "root://missing-data"
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,718 @@
from __future__ import annotations
import copy
import hashlib
import json
import math
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
from accuracy_phase4_evaluator import ( # noqa: E402
TASKS,
EXPECTED_PROTECTED_POLICY,
canonical_hash,
count_metrics,
detection_ap,
evaluate_cases,
evaluate_object_detection,
evaluate_footprint_segmentation,
evaluate_raster_classification,
evaluate_terrain,
evaluate_validation,
evaluate_vector_comparison,
subgroup_report,
task_inventory,
)
METADATA = {
"region": "flanders",
"municipality": "Mol",
"urbanity": "urban",
"object_size": "medium",
"source": "synthetic-source",
"sensor": "synthetic-sensor",
"resolution_m": 0.25,
"context": "dense_urban",
"season": "summer",
"date": "2026-01-01",
"vegetation": "partial",
"occlusion": "none",
"difficulty": "normal",
}
def lineage(sample_id: str) -> dict:
return {
"reference": {
"source_id": f"synthetic:{sample_id}:reference",
"source_version": "1",
"derivation": "hand_authored_contract_fixture",
},
"prediction": {
"source_id": f"synthetic:{sample_id}:prediction",
"source_version": "1",
"derivation": "hand_authored_fixed_output",
},
}
def detection_case(sample_id: str = "det-1") -> dict:
return {
"sample_id": sample_id,
"task": "object_detection",
"split": "test",
"metadata": copy.deepcopy(METADATA),
"config": {"confidence_threshold": 0.5, "match_iou": 0.5},
"lineage": lineage(sample_id),
"classes": ["building", "tank"],
"references": [{"id": "r-building", "class": "building", "bbox": [0, 0, 4, 4]}],
"predictions": [
{
"id": "p-building",
"class": "building",
"bbox": [0, 0, 4, 4],
"confidence": 0.8,
},
{
"id": "p-filtered",
"class": "building",
"bbox": [10, 10, 12, 12],
"confidence": 0.2,
},
],
}
def raster_case(sample_id: str = "raster-1") -> dict:
reference_side = {
"crs": "EPSG:31370",
"transform": [1, 0, 100000, 0, -1, 200000],
"shape": [2, 2],
"nodata": -9999,
"mask": [[True, True], [True, True]],
}
return {
"sample_id": sample_id,
"task": "raster_classification",
"split": "test",
"metadata": copy.deepcopy(METADATA),
"config": {},
"lineage": lineage(sample_id),
"classes": [0, 1],
"references": [[0, 1], [1, 0]],
"predictions": [[0, 1], [1, 0]],
"raster_context": {
"reference": reference_side,
"prediction": copy.deepcopy(reference_side),
},
}
def polygon_case(task: str = "vector_comparison") -> dict:
sample_id = f"{task}-1"
config = {"match_iou": 0.5}
if task == "footprint_segmentation":
config["boundary_tolerance_m"] = 1.0
polygon = [
[100000, 200000],
[100010, 200000],
[100010, 200010],
[100000, 200010],
[100000, 200000],
]
return {
"sample_id": sample_id,
"task": task,
"split": "test",
"metadata": copy.deepcopy(METADATA),
"config": config,
"lineage": lineage(sample_id),
"classes": ["building"],
"spatial_context": {
"crs": "EPSG:31370",
"coordinate_units": "m",
"metric": True,
},
"references": [{"id": "reference", "class": "building", "polygon": polygon}],
"predictions": [{"id": "prediction", "class": "building", "polygon": polygon}],
}
def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> None:
case = detection_case()
portfolio = {
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "synthetic-hardening-test",
"portfolio_lineage": {
"origin": "repository_fixture",
"source_path": "synthetic.json",
"version": "1",
},
"split_roles": ["test"],
"selection_policy": "Fixed before evaluation; no selection.",
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
"cases": [case],
}
path = tmp_path / "portfolio.json"
path.write_text(
json.dumps(portfolio, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
report = evaluate_cases(path, {case["sample_id"]})
raw = report["results"][0]["raw"]
assert raw["references"] == case["references"]
assert raw["predictions_pre_filter"] == case["predictions"]
assert raw["predictions_post_filter"] == case["predictions"][:1]
assert raw["config"] == case["config"]
assert raw["split"] == "test"
assert raw["input_lineage"] == case["lineage"]
assert raw["portfolio_lineage"]["declared"] == portfolio["portfolio_lineage"]
assert raw["hashes"]["case_input_canonical_json_sha256"] == canonical_hash(case)
assert raw["hashes"]["references_canonical_json_sha256"] == canonical_hash(
case["references"]
)
assert (
report["portfolio_file_sha256"] == hashlib.sha256(path.read_bytes()).hexdigest()
)
assert report["portfolio_canonical_json_sha256"] == canonical_hash(portfolio)
assert report["results_canonical_json_sha256"] == canonical_hash(report["results"])
high_threshold = next(
row
for row in report["results"][0]["metrics"]["coverage_risk"]
if row["threshold"] == 0.9
)
assert high_threshold["retained_prediction_coverage"] == 0.0
assert high_threshold["reference_coverage"] == 0.0
assert high_threshold["false_negative_count"] == 1
assert high_threshold["risk"] == 1.0
challenge_exposed = copy.deepcopy(portfolio)
challenge_exposed["challenge_labels"] = []
path.write_text(json.dumps(challenge_exposed, ensure_ascii=False), encoding="utf-8")
with pytest.raises(ValueError, match="Challenge cases and labels"):
evaluate_cases(path, {case["sample_id"]})
def test_portfolio_schema_policy_metadata_and_lineage_are_strict(
tmp_path: Path,
) -> None:
case = detection_case("strict-contract")
portfolio = {
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "synthetic-strict-contract",
"portfolio_lineage": {
"origin": "repository_fixture",
"source_path": "synthetic.json",
"version": "1",
},
"split_roles": ["test"],
"selection_policy": "Fixed before evaluation; no selection.",
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
"cases": [case],
}
path = tmp_path / "strict.json"
def evaluate(value: dict) -> dict:
path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
return evaluate_cases(path, {case["sample_id"]})
assert evaluate(portfolio)["case_count"] == 1
for invalid_version in (1, True, "2"):
invalid = copy.deepcopy(portfolio)
invalid["schema_version"] = invalid_version
with pytest.raises(
ValueError, match="schema_version must be exactly integer 2"
):
evaluate(invalid)
invalid_policy = copy.deepcopy(portfolio)
invalid_policy["protected_policy"]["test_feedback_allowed"] = True
with pytest.raises(ValueError, match="protected_policy must exactly equal"):
evaluate(invalid_policy)
invalid_metadata = copy.deepcopy(portfolio)
invalid_metadata["cases"][0]["metadata"]["source"] = "unknown"
with pytest.raises(ValueError, match="metadata.source must be a meaningful"):
evaluate(invalid_metadata)
invalid_resolution = copy.deepcopy(portfolio)
invalid_resolution["cases"][0]["metadata"]["resolution_m"] = 0
with pytest.raises(ValueError, match="metadata.resolution_m must be positive"):
evaluate(invalid_resolution)
invalid_lineage = copy.deepcopy(portfolio)
del invalid_lineage["cases"][0]["lineage"]["prediction"]["derivation"]
with pytest.raises(ValueError, match="lineage.prediction missing"):
evaluate(invalid_lineage)
def test_portfolio_kind_separates_synthetic_and_governed_product_claims(
tmp_path: Path,
) -> None:
fixture_path = (
ROOT / "fixtures" / "accuracy" / "p4" / "protected-baseline-cases.json"
)
synthetic = json.loads(fixture_path.read_text(encoding="utf-8"))
allowed = {item["sample_id"] for item in synthetic["cases"]}
path = tmp_path / "portfolio.json"
missing_kind = copy.deepcopy(synthetic)
del missing_kind["portfolio_kind"]
path.write_text(json.dumps(missing_kind), encoding="utf-8")
with pytest.raises(ValueError, match="portfolio_kind"):
evaluate_cases(path, allowed)
confused = copy.deepcopy(synthetic)
confused["claim_boundary"] = "Governed product baseline accuracy evidence."
path.write_text(json.dumps(confused), encoding="utf-8")
with pytest.raises(ValueError, match="synthetic_contract"):
evaluate_cases(path, allowed)
governed = json.loads(
json.dumps(synthetic)
.replace("Synthetic", "Governed")
.replace("synthetic", "governed")
.replace("repository_fixture", "governed_product_evaluation")
)
governed["portfolio_kind"] = "governed_product_baseline"
governed["portfolio_id"] = "governed-product-baseline-test"
governed["claim_boundary"] = (
"Governed product baseline metrics recomputed from protected raw cases; "
"inference provenance is validated separately."
)
path.write_text(json.dumps(governed), encoding="utf-8")
report = evaluate_cases(path, allowed)
assert report["portfolio_kind"] == "governed_product_baseline"
assert set(report["evaluated_task_families"]) == TASKS
governed["cases"] = governed["cases"][:-1]
path.write_text(json.dumps(governed), encoding="utf-8")
def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}]
predictions = [
{
"id": "z-true",
"class": "building",
"bbox": [0, 0, 4, 4],
"confidence": 0.8,
},
{
"id": "a-false",
"class": "building",
"bbox": [10, 10, 12, 12],
"confidence": 0.8,
},
]
forward = detection_ap(predictions, references, 0.5)
reverse = detection_ap(list(reversed(predictions)), references, 0.5)
assert forward == reverse == pytest.approx(0.5)
wrong_class = copy.deepcopy(predictions)
wrong_class[1] = {
"id": "a-tank",
"class": "tank",
"bbox": [0, 0, 4, 4],
"confidence": 0.95,
}
assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5)
def test_detection_ap_and_calibration_are_pooled_globally_and_per_subgroup(
tmp_path: Path,
) -> None:
first = detection_case("a-case")
first["predictions"] = [
{
"id": "p-true",
"class": "building",
"bbox": [0, 0, 4, 4],
"confidence": 0.9,
}
]
second = detection_case("b-case")
second["predictions"] = [
{
"id": "p-false",
"class": "building",
"bbox": [10, 10, 12, 12],
"confidence": 0.9,
},
{
"id": "p-true",
"class": "building",
"bbox": [0, 0, 4, 4],
"confidence": 0.8,
},
]
portfolio = {
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "synthetic-pooled-detection",
"portfolio_lineage": {
"origin": "repository_fixture",
"source_path": "pooled.json",
"version": "1",
},
"split_roles": ["test"],
"selection_policy": "Fixed before evaluation; no selection.",
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
"cases": [first, second],
}
path = tmp_path / "pooled.json"
path.write_text(json.dumps(portfolio, ensure_ascii=False), encoding="utf-8")
report = evaluate_cases(path, {"a-case", "b-case"})
case_ap = [item["metrics"]["ap50"] for item in report["results"]]
pooled = report["portfolio_metrics"]["object_detection"]["micro"]
expected_pooled = detection_ap(
[
{**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
for item in first["predictions"]
]
+ [
{**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
for item in second["predictions"]
],
[
{**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
for item in first["references"]
]
+ [
{**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
for item in second["references"]
],
0.5,
)
assert pooled["ap50"] == expected_pooled
assert pooled["ap50"] != pytest.approx(sum(case_ap) / len(case_ap))
assert sum(item["count"] for item in pooled["calibration"]["bins"]) == 3
subgroup = report["subgroups"]["dimensions"]["region"]["strata"]["flanders"]
subgroup_calibration = subgroup["task_metrics"]["object_detection"]["micro"][
"calibration"
]
assert sum(item["count"] for item in subgroup_calibration["bins"]) == 3
def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None:
invalid_case = detection_case("invalid-class")
invalid_case["predictions"][0]["class"] = "road"
with pytest.raises(ValueError, match="outside the declared ontology"):
evaluate_object_detection(invalid_case)
valid = raster_case()
valid["predictions"][0][1] = -9999
valid["raster_context"]["prediction"]["mask"][0][1] = False
result = evaluate_raster_classification(valid)
assert result["metrics"]["prediction_coverage"] == pytest.approx(0.75)
assert result["metrics"]["per_class"]["1"]["false_negative"] == 1
jagged = raster_case("jagged")
jagged["predictions"][1].pop()
with pytest.raises(ValueError, match="exactly rectangular"):
evaluate_raster_classification(jagged)
missing_metadata = raster_case("missing-metadata")
del missing_metadata["raster_context"]["prediction"]["crs"]
with pytest.raises(ValueError, match="missing"):
evaluate_raster_classification(missing_metadata)
shifted = raster_case("shifted")
shifted["raster_context"]["prediction"]["transform"][2] += 1
with pytest.raises(ValueError, match="affine alignment differs"):
evaluate_raster_classification(shifted)
invalid_class = raster_case("invalid-class")
invalid_class["predictions"][0][0] = 3
with pytest.raises(ValueError, match="prediction class outside ontology"):
evaluate_raster_classification(invalid_class)
invalid_nodata = raster_case("invalid-nodata")
invalid_nodata["predictions"][0][0] = -9999
with pytest.raises(ValueError, match="marks nodata as valid"):
evaluate_raster_classification(invalid_nodata)
singular = raster_case("singular")
for side in ("reference", "prediction"):
singular["raster_context"][side]["transform"] = [1, 2, 0, 2, 4, 0]
with pytest.raises(ValueError, match="affine transform is singular"):
evaluate_raster_classification(singular)
def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None:
assert evaluate_vector_comparison(polygon_case())["metrics"]["f1"] == 1.0
assert (
evaluate_footprint_segmentation(polygon_case("footprint_segmentation"))[
"metrics"
]["mean_iou"]
== 1.0
)
outer = [
[100000, 200000],
[100020, 200000],
[100020, 200020],
[100000, 200020],
[100000, 200000],
]
hole = [
[100005, 200005],
[100010, 200005],
[100010, 200010],
[100005, 200010],
[100005, 200005],
]
polygon_geometry = {"type": "Polygon", "coordinates": [outer, hole]}
geojson_polygon = polygon_case()
for side in ("references", "predictions"):
del geojson_polygon[side][0]["polygon"]
geojson_polygon[side][0]["geometry"] = copy.deepcopy(polygon_geometry)
polygon_result = evaluate_vector_comparison(geojson_polygon)
assert polygon_result["metrics"]["mean_iou"] == 1.0
assert polygon_result["raw"]["references"][0]["geometry"] == polygon_geometry
second = [
[100030, 200000],
[100040, 200000],
[100040, 200010],
[100030, 200010],
[100030, 200000],
]
multipolygon_geometry = {
"type": "MultiPolygon",
"coordinates": [[outer, hole], [second]],
}
geojson_multi = polygon_case()
for side in ("references", "predictions"):
del geojson_multi[side][0]["polygon"]
geojson_multi[side][0]["geometry"] = copy.deepcopy(multipolygon_geometry)
assert evaluate_vector_comparison(geojson_multi)["metrics"]["mean_iou"] == 1.0
geographic = polygon_case()
geographic["spatial_context"]["crs"] = "EPSG:4326"
with pytest.raises(ValueError, match="projected CRS"):
evaluate_vector_comparison(geographic)
mercator = polygon_case()
mercator["spatial_context"]["crs"] = "EPSG:3857"
with pytest.raises(ValueError, match="Mercator is unsuitable"):
evaluate_vector_comparison(mercator)
wrong_geography = polygon_case()
wrong_geography["spatial_context"]["crs"] = "EPSG:32660"
with pytest.raises(ValueError, match="does not overlap"):
evaluate_vector_comparison(wrong_geography)
wrong_units = polygon_case()
wrong_units["spatial_context"]["coordinate_units"] = "degree"
with pytest.raises(ValueError, match="must be 'm'"):
evaluate_vector_comparison(wrong_units)
bowtie = polygon_case()
bowtie["predictions"][0]["polygon"] = [
[100000, 200000],
[100010, 200010],
[100010, 200000],
[100000, 200010],
[100000, 200000],
]
with pytest.raises(ValueError, match="positive-area and valid"):
evaluate_vector_comparison(bowtie)
def test_failure_gallery_covers_geometry_raster_calibration_and_contexts() -> None:
segmentation = polygon_case("footprint_segmentation")
segmentation["predictions"][0]["polygon"] = [
[100000, 200000],
[100012, 200000],
[100012, 200010],
[100000, 200010],
[100000, 200000],
]
segmentation_result = evaluate_footprint_segmentation(segmentation)
segmentation_codes = {
item["error_code"] for item in segmentation_result["failures"]
}
assert {"M-BOUNDARY", "M-AREA-BIAS"} <= segmentation_codes
raster = raster_case("raster-taxonomy")
raster["metadata"]["tile_edge"] = True
raster["predictions"][0][0] = 1
raster_result = evaluate_raster_classification(raster)
raster_failure = next(
item
for item in raster_result["failures"]
if item["kind"] == "raster_misclassification"
)
assert raster_failure["error_code"] == "M-CLASS"
assert "tile_edge" in raster_failure["contexts"]
detection = detection_case("context-taxonomy")
detection["references"] = []
detection["predictions"] = [
{
"id": "high-confidence-fp",
"class": "building",
"bbox": [10, 10, 12, 12],
"confidence": 0.95,
}
]
detection["config"]["fixed_diagnostic_risk_thresholds"] = [0.5, 0.9]
detection["metadata"]["tile_edge"] = True
detection["metadata"]["ood"] = True
detection_result = evaluate_object_detection(detection)
false_positive = next(
item
for item in detection_result["failures"]
if item["kind"] == "false_positive"
)
assert {"tile_edge", "high_confidence", "out_of_distribution"} <= set(
false_positive["contexts"]
)
assert {"M-MISCALIBRATED", "M-OOD"} <= set(false_positive["secondary_error_codes"])
assert any(
item["error_code"] == "M-MISCALIBRATED" for item in detection_result["failures"]
)
def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> (
None
):
terrain = {
"sample_id": "terrain",
"task": "terrain_interpretation",
"split": "test",
"metadata": copy.deepcopy(METADATA),
"config": {},
"lineage": lineage("terrain"),
"units": "m_TAW",
"references": [1.0, 2.0],
"predictions": [1.1, None],
}
assert evaluate_terrain(terrain)["metrics"]["coverage"] == 0.5
for field, value in (("references", math.nan), ("predictions", math.inf)):
invalid = copy.deepcopy(terrain)
invalid[field][0] = value
with pytest.raises(ValueError, match="finite number"):
evaluate_terrain(invalid)
validation = {
"sample_id": "validation",
"task": "geospatial_data_validation",
"split": "test",
"metadata": copy.deepcopy(METADATA),
"config": {},
"lineage": lineage("validation"),
"expected_anomalies": [{"code": "D-MAJOR", "severity": "major"}],
"observed_anomalies": [],
}
assert (
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
== 0
)
validation["expected_anomalies"].append(
{"code": "D-CRITICAL", "severity": "critical"}
)
assert (
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
== 1
)
validation["expected_anomalies"] = [{"code": "D-SEVERITY", "severity": "critical"}]
validation["observed_anomalies"] = [{"code": "D-SEVERITY", "severity": "minor"}]
severity_result = evaluate_validation(validation)
assert severity_result["metrics"]["true_positive"] == 0
assert severity_result["metrics"]["false_positive"] == 1
assert severity_result["metrics"]["false_negative"] == 1
assert severity_result["metrics"]["severity_mismatch_count"] == 1
assert severity_result["metrics"]["blocker_or_critical_miss_count"] == 1
validation["expected_anomalies"] = ["D-NO-SEVERITY"]
with pytest.raises(ValueError, match="include code and severity"):
evaluate_validation(validation)
def _subgroup_result(region: str, index: int, tp: int, fp: int, fn: int) -> dict:
metadata = copy.deepcopy(METADATA)
metadata["region"] = region
sample_id = f"{region}-{index}"
reference = {"id": "r", "class": "building", "bbox": [0, 0, 1, 1]}
prediction = {
"id": "p",
"class": "building",
"bbox": [0, 0, 1, 1],
"confidence": 0.8,
}
return {
"sample_id": sample_id,
"task": "object_detection",
"metadata": metadata,
"metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4},
"raw": {
"sample_id": sample_id,
"classes": ["building"],
"references": [reference],
"predictions_pre_filter": [prediction],
"predictions_post_filter": [prediction],
"matches": [
{
"prediction_id": "p",
"reference_id": "r",
"overlap": 1.0,
"confidence": 0.8,
"class": "building",
}
],
},
"failures": [],
}
def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
results = [
*[_subgroup_result("strong", index, 10, 0, 0) for index in range(5)],
*[_subgroup_result("weak", index, 1, 4, 4) for index in range(5)],
]
report = subgroup_report(results)
region = report["dimensions"]["region"]
weak = region["strata"]["weak"]["task_metrics"]["object_detection"]
assert weak["status"] == "evaluable"
assert weak["case_support"] == 5
assert weak["micro"]["precision_ci95_wilson"]["status"] == "computed"
assert weak["macro"]["f1_case_support"] == 5
assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak"
insufficient = subgroup_report([_subgroup_result("thin", 0, 1, 0, 0)])
thin = insufficient["dimensions"]["region"]["strata"]["thin"]
assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support"
assert thin["release_gate_status"] == "not_evaluable"
assert insufficient["overall_status"] == "not_evaluable"
def test_capability_inventory_is_comprehensive_and_honest() -> None:
inventory = task_inventory()
assert {item["task"] for item in inventory} == TASKS
assert len(inventory) >= 15
assert all(item["implementation_paths"] for item in inventory)
assert all(item["suitable_metrics"] for item in inventory)
assistant = next(
item
for item in inventory
if item["capability_id"] == "geo_assistant_orchestration"
)
assert assistant["evaluation_status"].startswith("no_independent_accuracy_score")
@@ -0,0 +1,538 @@
from __future__ import annotations
import copy
import hashlib
import json
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
from generate_accuracy_phase4_splits import ( # noqa: E402
LeakageError,
assert_training_inputs_safe,
build_manifests,
generate,
)
SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json"
def load_source() -> dict:
return json.loads(SOURCE.read_text(encoding="utf-8"))
def build_fixture_manifests(source: dict) -> tuple[dict, dict, dict]:
return build_manifests(source, trusted_fixture_mode=True)
def assert_fixture_training_inputs_safe(
input_paths: list[Path], input_records: list[dict], protected: dict
) -> None:
assert_training_inputs_safe(
input_paths, input_records, protected, trusted_fixture_mode=True
)
def test_normative_roles_hashes_and_source_order_are_enforced() -> None:
source = load_source()
development, protected, leakage = build_fixture_manifests(source)
reversed_source = copy.deepcopy(source)
reversed_source["samples"].reverse()
reversed_development, reversed_protected, reversed_leakage = (
build_fixture_manifests(reversed_source)
)
assert leakage["status"] == "pass"
assert leakage["finding_count"] == 0
assert leakage["split_counts"] == {
"background-test": 2,
"calibration": 2,
"challenge": 4,
"test": 7,
"train": 3,
"val": 3,
}
assert leakage["crs_validation"] == {
"status": "pass",
"crs": "EPSG:31370",
"distance_units": "m",
}
assert development["training_access_allowed_by_split"] == {
"train": True,
"val": False,
"calibration": False,
}
assert protected["labels_available_by_split"]["challenge"] == "sealed_external"
assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
assert reversed_leakage == leakage
@pytest.mark.parametrize(
("field", "expected_code"),
[
("group_id", "S-SPATIAL-GROUP"),
("source_family", "S-SOURCE-FAMILY"),
("temporal_family", "S-TEMPORAL-FAMILY"),
("raw_image_sha256", "S-RAW-IMAGE-DUPLICATE"),
("processed_image_sha256", "S-PROCESSED-IMAGE-DUPLICATE"),
("label_sha256", "S-LABEL-DUPLICATE"),
("label_geometry_hash", "S-LABEL-GEOMETRY-DUPLICATE"),
("parent_raster_id", "S-PARENT-RASTER"),
("acquisition_id", "S-ACQUISITION"),
],
)
def test_cross_split_lineage_and_content_collisions_fail(
field: str, expected_code: str
) -> None:
source = load_source()
source["samples"][8][field] = source["samples"][0][field]
_development, _protected, leakage = build_fixture_manifests(source)
assert leakage["status"] == "fail"
assert expected_code in {item["code"] for item in leakage["findings"]}
@pytest.mark.parametrize(
("field", "expected_code"),
[
("perceptual_image_hash", "S-PERCEPTUAL-IMAGE-NEAR-DUPLICATE"),
("label_geometry_fingerprint", "S-LABEL-GEOMETRY-NEAR-DUPLICATE"),
],
)
def test_near_duplicate_fingerprints_fail(field: str, expected_code: str) -> None:
source = load_source()
source["samples"][8][field] = source["samples"][0][field]
_development, _protected, leakage = build_fixture_manifests(source)
assert leakage["status"] == "fail"
assert expected_code in {item["code"] for item in leakage["findings"]}
def test_object_native_feature_and_spatial_collisions_fail() -> None:
source = load_source()
source["samples"][8]["object_ids"] = source["samples"][0]["object_ids"]
source["samples"][9]["native_feature_ids"] = source["samples"][1][
"native_feature_ids"
]
source["samples"][10]["bbox"] = source["samples"][2]["bbox"]
_development, _protected, leakage = build_fixture_manifests(source)
codes = {item["code"] for item in leakage["findings"]}
assert {"S-OBJECT-INSTANCE", "S-NATIVE-FEATURE", "S-SPATIAL-OVERLAP"} <= codes
def test_non_metric_crs_and_missing_normative_role_fail_closed() -> None:
geographic = load_source()
geographic["crs"] = "EPSG:4326"
with pytest.raises(LeakageError, match="projected in metres"):
build_fixture_manifests(geographic)
missing = load_source()
missing["samples"] = [
item for item in missing["samples"] if item["split"] != "calibration"
]
with pytest.raises(LeakageError, match="Required splits are absent"):
build_fixture_manifests(missing)
def test_training_firewall_only_allows_train_and_binds_protected_lineage() -> None:
development, protected, leakage = build_fixture_manifests(load_source())
assert leakage["status"] == "pass"
train = [item for item in development["samples"] if item["split"] == "train"]
validation = next(item for item in development["samples"] if item["split"] == "val")
protected_item = protected["samples"][0]
assert_fixture_training_inputs_safe([], train, protected)
with pytest.raises(LeakageError, match="non_train_role"):
assert_fixture_training_inputs_safe([], [validation], protected)
with pytest.raises(LeakageError, match="protected_identity"):
disguised = copy.deepcopy(train[0])
disguised["source_family"] = protected_item["source_family"]
assert_fixture_training_inputs_safe([], [disguised], protected)
with pytest.raises(LeakageError, match="protected_path"):
assert_fixture_training_inputs_safe(
[Path("vault/protected/test.json")], [], protected
)
def test_failed_generation_writes_status_but_no_consumable_manifests(
tmp_path: Path,
) -> None:
source = load_source()
source["samples"][8]["group_id"] = source["samples"][0]["group_id"]
source_path = tmp_path / "source.json"
source_path.write_text(json.dumps(source), encoding="utf-8")
output = tmp_path / "out"
with pytest.raises(LeakageError, match="Leakage gate failed"):
generate(source_path, output, trusted_fixture_mode=True)
status = json.loads((output / "generation-status.json").read_text(encoding="utf-8"))
assert status["status"] == "fail"
assert status["consumable_manifests_valid"] is False
for name in (
"development-split-manifest.json",
"protected-split-manifest.json",
):
tombstone = json.loads((output / name).read_text(encoding="utf-8"))
assert tombstone["status"] == "invalidated"
assert tombstone["consumable"] is False
def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together() -> (
None
):
source = load_source()
source["assignment_mode"] = "deterministic_grouped"
source["split_assignment"] = {
"seed": "fixed-phase4-test-seed",
"roles": [
"train",
"val",
"calibration",
"test",
"background-test",
"challenge",
],
"weights": {
"train": 6,
"val": 2,
"calibration": 1,
"test": 2,
"background-test": 1,
"challenge": 1,
},
"stratify_by": ["task"],
}
for item in source["samples"]:
item.pop("split")
source["samples"][1]["group_id"] = source["samples"][0]["group_id"]
development, protected, leakage = build_fixture_manifests(source)
reversed_source = copy.deepcopy(source)
reversed_source["samples"].reverse()
reversed_development, reversed_protected, reversed_leakage = (
build_fixture_manifests(reversed_source)
)
assigned = {
item["sample_id"]: item["split"]
for item in development["samples"] + protected["samples"]
}
assert assigned["det-train-a"] == assigned["seg-train-a"]
assert set(leakage["split_counts"]) == {
"train",
"val",
"calibration",
"test",
"background-test",
"challenge",
}
assert leakage["status"] == "pass"
assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
assert reversed_leakage == leakage
def test_source_cannot_weaken_mandatory_roles_or_policy_floors() -> None:
source = load_source()
source["required_splits"] = ["train", "val", "test"]
with pytest.raises(LeakageError, match="mandatory role order"):
build_fixture_manifests(source)
grouped = load_source()
grouped["assignment_mode"] = "deterministic_grouped"
grouped["split_assignment"] = {"roles": ["train", "val"]}
for item in grouped["samples"]:
item.pop("split")
with pytest.raises(LeakageError, match="mandatory role order"):
build_fixture_manifests(grouped)
for field, value in (
("independence_buffer_m", 1999),
("perceptual_hamming_threshold", 3),
("label_geometry_hamming_threshold", 1),
):
weakened = load_source()
weakened[field] = value
with pytest.raises(LeakageError, match="code-owned minimum"):
build_fixture_manifests(weakened)
def test_task_coverage_gap_and_even_justified_exemption_fail_honestly() -> None:
source = load_source()
source["samples"] = [
item
for item in source["samples"]
if item["sample_id"] != "validation-test-national"
]
_development, _protected, leakage = build_fixture_manifests(source)
assert leakage["status"] == "fail"
assert "S-PROTECTED-TASK-COVERAGE-MISSING" in {
finding["code"] for finding in leakage["findings"]
}
source["protected_task_exemptions"] = {
"geospatial_data_validation": (
"No evaluator-visible reference exists; challenge data remains sealed."
)
}
_development, _protected, leakage = build_fixture_manifests(source)
assert leakage["status"] == "fail"
assert "S-PROTECTED-TASK-COVERAGE-EXEMPTED" in {
finding["code"] for finding in leakage["findings"]
}
def test_identifiers_and_acquisition_dates_are_canonical_leakage_keys() -> None:
source = load_source()
source["samples"][8]["group_id"] = " G01 "
_development, _protected, leakage = build_fixture_manifests(source)
assert "S-SPATIAL-GROUP" in {item["code"] for item in leakage["findings"]}
temporal = load_source()
temporal["samples"][8]["acquisition_date"] = temporal["samples"][0][
"acquisition_date"
]
_development, _protected, leakage = build_fixture_manifests(temporal)
assert "S-ACQUISITION-DATE" in {item["code"] for item in leakage["findings"]}
ambiguous = load_source()
ambiguous["samples"][8]["sample_id"] = " DET-TRAIN-A "
with pytest.raises(LeakageError, match="ambiguous canonical sample_id"):
build_fixture_manifests(ambiguous)
def test_challenge_is_sealed_in_standard_manifest() -> None:
_development, protected, leakage = build_fixture_manifests(load_source())
assert leakage["status"] == "pass"
challenge = [item for item in protected["samples"] if item["split"] == "challenge"]
forbidden = {
"label_sha256",
"label_geometry_hash",
"label_geometry_fingerprint",
"object_ids",
"native_feature_ids",
"record_sha256",
"label_path",
"label_geometry_path",
}
assert challenge
assert all(item["sealed"] is True for item in challenge)
assert all(not (forbidden & set(item)) for item in challenge)
def test_firewall_rejects_empty_tampered_wrong_and_renamed_manifests(
tmp_path: Path,
) -> None:
development, protected, leakage = build_fixture_manifests(load_source())
assert leakage["status"] == "pass"
train = [item for item in development["samples"] if item["split"] == "train"]
with pytest.raises(LeakageError, match="empty manifest"):
assert_fixture_training_inputs_safe([], train, {})
with pytest.raises(LeakageError, match="missing fields"):
assert_fixture_training_inputs_safe([], train, development)
tampered = copy.deepcopy(protected)
tampered["samples"].pop()
with pytest.raises(LeakageError, match="checksum mismatch"):
assert_fixture_training_inputs_safe([], train, tampered)
renamed = tmp_path / "ordinary-training-input.json"
renamed.write_text(json.dumps(protected), encoding="utf-8")
with pytest.raises(LeakageError, match="protected_manifest_content"):
assert_fixture_training_inputs_safe([renamed], train, protected)
def _canonical_json_sha256(value: object) -> str:
return hashlib.sha256(
json.dumps(
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
).encode("utf-8")
).hexdigest()
def make_governed_source(tmp_path: Path) -> dict:
source = load_source()
source["dataset_version"] = "governed-production-v1"
source["claim_boundary"] = "Governed production split source."
p3_items: list[dict] = []
provenance_records: list[dict] = []
asset_fields = {
"raw_image": ("raw_image_path", "raw_image_sha256"),
"processed_image": ("processed_image_path", "processed_image_sha256"),
"label": ("label_path", "label_sha256"),
"label_geometry": ("label_geometry_path", "label_geometry_hash"),
}
for sample in source["samples"]:
assets: dict[str, dict] = {}
p3_ids: dict[str, str] = {}
for role, (path_field, hash_field) in asset_fields.items():
relative = Path("assets") / sample["sample_id"] / f"{role}.bin"
path = tmp_path / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(f"{sample['sample_id']}:{role}:governed".encode())
digest = hashlib.sha256(path.read_bytes()).hexdigest()
p3_id = hashlib.sha256(
f"{sample['sample_id']}:{role}".encode()
).hexdigest()[:20]
relative_posix = relative.as_posix()
sample[path_field] = relative_posix
sample[hash_field] = digest
p3_ids[role] = p3_id
assets[role] = {
"path": relative_posix,
"sha256": digest,
"size_bytes": path.stat().st_size,
"p3_item_id": p3_id,
}
p3_items.append(
{
"item_id": p3_id,
"path": relative_posix,
"sha256": digest,
"size_bytes": path.stat().st_size,
"status": "examined",
"read_status": "readable",
"recommended_action": "accept",
"empty_content": False,
"schema_conformity": "conformant",
"anomalies": [],
}
)
provenance_id = (
"prov-" + hashlib.sha256(sample["sample_id"].encode()).hexdigest()[:24]
)
sample["governance_binding"] = {
"source_provenance_record_id": provenance_id,
"p3_item_ids": p3_ids,
}
provenance_records.append(
{
"record_id": provenance_id,
"sample_id": sample["sample_id"],
"status": "accepted",
"lineage_status": "complete",
"training_allowed": True,
"perceptual_image_hash": sample["perceptual_image_hash"],
"label_geometry_fingerprint": sample["label_geometry_fingerprint"],
"assets": assets,
}
)
p3 = {
"schema_version": 1,
"scan_id": "p3-test-governed",
"scanner_version": "3.0.3",
"completed_at": "2026-08-02T12:00:00+02:00",
"items": p3_items,
"reconciliation": {
"examined": len(p3_items),
"skipped": 0,
"unreachable": 0,
"inventory_total": len(p3_items),
"reconciles": True,
},
}
provenance = {
"schema_version": 1,
"manifest_type": "geointel_phase4_source_provenance",
"status": "pass",
"records": provenance_records,
"records_canonical_json_sha256": _canonical_json_sha256(provenance_records),
}
p3_path = tmp_path / "p3.json"
provenance_path = tmp_path / "provenance.json"
p3_path.write_text(json.dumps(p3), encoding="utf-8")
provenance_path.write_text(json.dumps(provenance), encoding="utf-8")
source["governance_evidence"] = {
"p3_scan_manifest": {
"path": p3_path.name,
"sha256": hashlib.sha256(p3_path.read_bytes()).hexdigest(),
},
"source_provenance_manifest": {
"path": provenance_path.name,
"sha256": hashlib.sha256(provenance_path.read_bytes()).hexdigest(),
},
}
return source
def test_fixture_mode_is_explicit_and_source_metadata_cannot_enable_it() -> None:
with pytest.raises(LeakageError, match="explicit trusted_fixture_mode"):
build_manifests(load_source())
source = load_source()
source["trusted_fixture_mode"] = True
with pytest.raises(LeakageError, match="cannot be enabled by source metadata"):
build_manifests(source, trusted_fixture_mode=True)
def test_empty_or_arbitrary_governance_json_is_rejected(tmp_path: Path) -> None:
source = load_source()
source["dataset_version"] = "governed-production-v1"
source["claim_boundary"] = "Governed production split source."
p3 = tmp_path / "p3.json"
provenance = tmp_path / "provenance.json"
p3.write_text("{}", encoding="utf-8")
provenance.write_text("{}", encoding="utf-8")
source["governance_evidence"] = {
"p3_scan_manifest": {
"path": p3.name,
"sha256": hashlib.sha256(p3.read_bytes()).hexdigest(),
},
"source_provenance_manifest": {
"path": provenance.name,
"sha256": hashlib.sha256(provenance.read_bytes()).hexdigest(),
},
}
with pytest.raises(LeakageError, match="non-empty JSON object"):
build_manifests(source, source_root=tmp_path)
def test_governed_records_require_exact_provenance_paths_and_live_bytes(
tmp_path: Path,
) -> None:
source = make_governed_source(tmp_path)
development, protected, leakage = build_manifests(source, source_root=tmp_path)
assert leakage["status"] == "pass"
assert protected["source_trust"]["production_accuracy_use_allowed"] is True
train = [item for item in development["samples"] if item["split"] == "train"]
assert_training_inputs_safe([], train, protected)
no_paths = copy.deepcopy(train[0])
no_paths.pop("content_path_bindings")
with pytest.raises(LeakageError, match="missing_accessible_content_paths"):
assert_training_inputs_safe([], [no_paths], protected)
relabeled = copy.deepcopy(
next(item for item in protected["samples"] if item["split"] == "test")
)
relabeled["split"] = "train"
relabeled.pop("content_path_bindings")
for field in ("sample_id", "group_id", "source_family", "temporal_family"):
relabeled[field] = f"spoofed-{field}"
with pytest.raises(
LeakageError, match="unavailable_provenance_record|protected_identity"
):
assert_training_inputs_safe([], [relabeled], protected)
broken_binding = copy.deepcopy(source)
broken_binding["samples"][0]["governance_binding"]["p3_item_ids"]["raw_image"] = (
"0" * 20
)
with pytest.raises(LeakageError, match="provenance binding mismatch|P3 record"):
build_manifests(broken_binding, source_root=tmp_path)
raw_path = Path(train[0]["content_path_bindings"]["raw_image"]["resolved_path"])
raw_path.write_bytes(b"mutated after manifest creation")
with pytest.raises(LeakageError, match="record_path_hash_binding_mismatch"):
assert_training_inputs_safe([], train, protected)
@@ -0,0 +1,10 @@
from pathlib import Path
def test_alembic_logging_formatter_uses_runtime_interpolation_tokens() -> None:
config = Path(__file__).resolve().parents[1] / "alembic.ini"
content = config.read_text(encoding="utf-8")
assert "format = %(levelname)-5.5s [%(name)s] %(message)s" in content
assert "%%(levelname)" not in content
assert "%%(message)" not in content
@@ -0,0 +1,135 @@
"""A queued run must be claimed once, even if two workers look at it.
The worker selected queued jobs and then set them to running in a second
statement. Two workers — an API restart overlapping the previous process, or a
second replica — could both select the same row and both start tiled GPU
inference on it, producing duplicate analysis runs and doubling the GPU load.
The AOI worker beside it already claims with ``FOR UPDATE SKIP LOCKED``. This
uses a conditional update, which is the same guarantee expressed in one
statement: exactly one caller sees a row count of 1.
"""
from __future__ import annotations
from uuid import uuid4
from app.models import Job
from app.services.analysis_job_worker import AnalysisJobWorker
class _Update:
"""Mimics a conditional UPDATE: the first caller wins, the rest see zero."""
def __init__(self, store: dict, job_id):
self.store = store
self.job_id = job_id
def update(self, values, **_kwargs) -> int:
if self.store.get(self.job_id) != "queued":
return 0
self.store[self.job_id] = "running"
return 1
class _Query:
def __init__(self, session, model):
self.session = session
self.model = model
self.job_id = None
def filter(self, *criteria):
for criterion in criteria:
right = getattr(criterion, "right", None)
value = getattr(right, "value", None)
if isinstance(value, type(uuid4())):
self.job_id = value
return self
def update(self, values, **kwargs) -> int:
return _Update(self.session.statuses, self.job_id).update(values, **kwargs)
def order_by(self, *_args):
return self
def limit(self, _count):
return self
def all(self):
return list(self.session.rows)
class _Session:
def __init__(self, rows: list[Job]):
self.rows = rows
self.statuses = {row.id: row.status for row in rows}
self.committed = 0
def query(self, model):
return _Query(self, model)
def get(self, _model, item_id):
return next((row for row in self.rows if row.id == item_id), None)
def add(self, _item):
return None
def commit(self):
self.committed += 1
def rollback(self):
return None
def close(self):
return None
def _job() -> Job:
return Job(
id=uuid4(),
job_type="detection.run",
status="queued",
project_id=uuid4(),
parameters_json={},
)
def test_the_first_claim_wins() -> None:
job = _job()
session = _Session([job])
assert AnalysisJobWorker.claim(session, job) is True
assert job.status == "running"
def test_a_second_claim_on_the_same_job_is_refused() -> None:
job = _job()
session = _Session([job])
assert AnalysisJobWorker.claim(session, job) is True
assert AnalysisJobWorker.claim(session, job) is False
def test_a_job_that_is_no_longer_queued_cannot_be_claimed() -> None:
job = _job()
session = _Session([job])
session.statuses[job.id] = "success"
assert AnalysisJobWorker.claim(session, job) is False
def test_an_unclaimable_job_is_skipped_rather_than_run(monkeypatch) -> None:
job = _job()
session = _Session([job])
session.statuses[job.id] = "running"
dispatched: list[Job] = []
monkeypatch.setattr(
AnalysisJobWorker,
"_dispatch",
staticmethod(lambda _db, item: dispatched.append(item)),
)
processed = AnalysisJobWorker.run_once(db=session)
assert processed == 0
assert dispatched == []
+186
View File
@@ -0,0 +1,186 @@
"""Tiled GPU inference must not run inside an HTTP request.
A configured YOLO run walks up to ``YOLO_MAX_TILES`` tiles through the GPU.
Doing that in the request handler holds a worker thread for minutes, gives the
operator no progress, and times the client out before the result exists. The
run is queued as a Job instead and executed by a background worker, which is
the same pattern the AOI operations already use.
"""
from __future__ import annotations
from uuid import uuid4
import pytest
from app.core.errors import AppError
from app.models import Job
from app.services.analysis_job_worker import AnalysisJobWorker
from app.services.detection_service import DetectionService
class FakeQuery:
def __init__(self, rows):
self.rows = list(rows)
def filter(self, *criteria):
return self
def update(self, values, **_kwargs) -> int:
"""Stand in for the conditional claim: succeeds while still queued."""
claimed = 0
for row in self.rows:
if getattr(row, "status", None) == "queued":
row.status = "running"
claimed += 1
return claimed
def order_by(self, *_args):
return self
def limit(self, count):
self.rows = self.rows[:count]
return self
def all(self):
return list(self.rows)
class FakeSession:
def __init__(self, objects=None, query_rows=None):
self.objects = dict(objects or {})
self.query_rows = query_rows or {}
self.added = []
def get(self, model, item_id):
return self.objects.get((model, item_id))
def query(self, model):
return FakeQuery(self.query_rows.get(model, []))
def add(self, item):
self.added.append(item)
if getattr(item, "id", None) is not None:
self.objects[(item.__class__, item.id)] = item
def commit(self):
return None
def rollback(self):
return None
def refresh(self, _item):
return None
def close(self):
return None
def _queued_job(**parameters) -> Job:
payload = {
"project_id": str(uuid4()),
"dataset_id": str(uuid4()),
"model_id": "yolo-configured",
"confidence_threshold": 0.4,
"class_filter": ["building"],
"tile_manifest_path": "/tiles/manifest.json",
"parameters_json": {},
}
payload.update(parameters)
return Job(
id=uuid4(),
job_type="detection.run",
status="queued",
project_id=uuid4(),
parameters_json=payload,
)
def test_queued_detection_job_is_dispatched_to_the_detection_service(monkeypatch) -> None:
job = _queued_job()
db = FakeSession(objects={(Job, job.id): job}, query_rows={Job: [job]})
calls: list[dict] = []
def fake_run(**kwargs):
calls.append(kwargs)
return type(
"Result",
(),
{
"status": "success",
"detection_count": 3,
"analysis_run_id": uuid4(),
"job_id": kwargs["existing_job"].id,
"model_dump": lambda self, **_: {"status": "success", "detection_count": 3},
},
)()
monkeypatch.setattr(DetectionService, "run_detection", staticmethod(fake_run))
processed = AnalysisJobWorker.run_once(db=db)
assert processed == 1
assert calls[0]["model_id"] == "yolo-configured"
assert calls[0]["confidence_threshold"] == 0.4
assert calls[0]["tile_manifest_path"] == "/tiles/manifest.json"
assert calls[0]["existing_job"] is job
assert job.status == "success"
def test_a_failing_run_marks_the_job_failed_instead_of_leaving_it_running(monkeypatch) -> None:
job = _queued_job()
db = FakeSession(objects={(Job, job.id): job}, query_rows={Job: [job]})
def exploding(**_kwargs):
raise AppError(code="DETECTION_TILE_NOT_FOUND", message="missing tile", status_code=422)
monkeypatch.setattr(DetectionService, "run_detection", staticmethod(exploding))
processed = AnalysisJobWorker.run_once(db=db)
assert processed == 1
assert job.status == "failed"
assert job.error_message == "missing tile"
assert job.result_json["error_code"] == "DETECTION_TILE_NOT_FOUND"
def test_an_unexpected_error_still_closes_the_job(monkeypatch) -> None:
job = _queued_job()
db = FakeSession(objects={(Job, job.id): job}, query_rows={Job: [job]})
def exploding(**_kwargs):
raise RuntimeError("CUDA out of memory")
monkeypatch.setattr(DetectionService, "run_detection", staticmethod(exploding))
AnalysisJobWorker.run_once(db=db)
assert job.status == "failed"
assert job.result_json["error_code"] == "ANALYSIS_JOB_INTERNAL_ERROR"
def test_job_types_the_worker_does_not_own_are_left_alone() -> None:
job = _queued_job()
job.job_type = "raster.clip"
db = FakeSession(objects={(Job, job.id): job}, query_rows={Job: [job]})
assert AnalysisJobWorker.run_once(db=db) == 0
assert job.status == "queued"
def test_enqueue_validates_before_accepting_the_job() -> None:
"""A bad request is rejected up front, not minutes later in the worker."""
db = FakeSession()
with pytest.raises(AppError) as exc_info:
DetectionService.enqueue_detection(
db=db,
project_id=uuid4(),
dataset_id=uuid4(),
model_id="yolo-configured",
confidence_threshold=0.4,
)
assert exc_info.value.code == "PROJECT_NOT_FOUND"
+159
View File
@@ -0,0 +1,159 @@
from __future__ import annotations
from uuid import uuid4
from geoalchemy2.shape import from_shape, to_shape
from pyproj import Transformer
import pytest
from shapely.geometry import Polygon, mapping
from shapely.ops import transform
from app.core.errors import AppError
from app.models import Area, Project
from app.schemas.area import AreaCreate, AreaUpdate
from app.services.area_service import AreaService
from app.utils.geometry import area_m2, normalize_area_to_epsg4326
class FakeSession:
def __init__(self, objects=None) -> None:
self.objects = objects or {}
self.added = []
self.commits = 0
self.refreshes = []
def get(self, model, item_id):
return self.objects.get((model, item_id))
def add(self, item) -> None:
self.added.append(item)
def commit(self) -> None:
self.commits += 1
def refresh(self, item) -> None:
self.refreshes.append(item)
def _wgs84_polygon(offset: float = 0.0) -> Polygon:
return Polygon(
[
(5.00 + offset, 51.00),
(5.01 + offset, 51.00),
(5.01 + offset, 51.01),
(5.00 + offset, 51.01),
(5.00 + offset, 51.00),
]
)
def _to_lambert(geometry: Polygon) -> Polygon:
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
return transform(transformer.transform, geometry)
def test_create_area_transforms_declared_lambert_geometry_before_storage() -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
source = _wgs84_polygon()
area = AreaService.create_area(
db,
project_id,
AreaCreate(name="Lambert AOI", geometry=mapping(_to_lambert(source)), crs="EPSG:31370"),
)
stored = to_shape(area.geometry)
assert stored.bounds == pytest.approx(source.bounds, abs=1e-7)
assert area.original_crs == "EPSG:31370"
assert area.area_m2 == pytest.approx(area_m2(normalize_area_to_epsg4326(mapping(source), "EPSG:4326")[0]))
assert area.area_m2 and area.area_m2 > 0
assert to_shape(area.bbox).bounds == pytest.approx(source.bounds, abs=1e-7)
def test_patch_area_replaces_geometry_and_recomputes_all_spatial_fields() -> None:
area_id = uuid4()
project_id = uuid4()
original = _wgs84_polygon()
normalized, _ = normalize_area_to_epsg4326(mapping(original), "EPSG:4326")
area = Area(
id=area_id,
project_id=project_id,
name="Original",
geometry=from_shape(normalized, srid=4326),
bbox=from_shape(normalized.envelope, srid=4326),
original_crs="EPSG:4326",
area_m2=area_m2(normalized),
)
db = FakeSession({(Area, area_id): area})
replacement = _wgs84_polygon(offset=0.05)
updated = AreaService.update_area(
db,
area_id,
AreaUpdate(
name="Replacement",
geometry=mapping(_to_lambert(replacement)),
crs="EPSG:31370",
),
)
assert updated.name == "Replacement"
assert updated.original_crs == "EPSG:31370"
assert to_shape(updated.geometry).bounds == pytest.approx(replacement.bounds, abs=1e-7)
assert to_shape(updated.bbox).bounds == pytest.approx(replacement.bounds, abs=1e-7)
assert updated.area_m2 and updated.area_m2 > 0
assert db.commits == 1
@pytest.mark.parametrize(
("geometry", "crs", "message_fragment"),
[
(mapping(_wgs84_polygon()), "EPSG:not-real", "unknown or invalid"),
(mapping(_wgs84_polygon()), "EPSG:4979", "exactly two spatial axes"),
(
{
"type": "Polygon",
"coordinates": [[[5.0, 51.0], [float("nan"), 51.0], [5.1, 51.1], [5.0, 51.0]]],
},
"EPSG:4326",
"finite",
),
(mapping(Polygon([(10.0, 51.0), (10.1, 51.0), (10.1, 51.1), (10.0, 51.0)])), "EPSG:4326", "workbench domain"),
(
{
"type": "Polygon",
"coordinates": [[[5.0, 51.0], [5.1, 51.1], [5.1, 51.0], [5.0, 51.1], [5.0, 51.0]]],
},
"EPSG:4326",
"invalid",
),
({"type": "Point", "coordinates": [5.0, 51.0]}, "EPSG:4326", "Polygon or MultiPolygon"),
],
)
def test_create_area_rejects_invalid_crs_nonfinite_and_out_of_domain_geometry(
geometry: dict,
crs: str,
message_fragment: str,
) -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
with pytest.raises(AppError) as exc_info:
AreaService.create_area(db, project_id, AreaCreate(name="Invalid", geometry=geometry, crs=crs))
assert exc_info.value.code == "INVALID_GEOMETRY"
assert message_fragment in exc_info.value.message
assert db.commits == 0
def test_patch_area_rejects_crs_without_replacement_geometry() -> None:
area_id = uuid4()
area = Area(id=area_id, project_id=uuid4(), name="AOI", original_crs="EPSG:4326")
db = FakeSession({(Area, area_id): area})
with pytest.raises(AppError) as exc_info:
AreaService.update_area(db, area_id, AreaUpdate(crs="EPSG:31370"))
assert exc_info.value.code == "INVALID_AREA_CRS_UPDATE"
assert db.commits == 0
@@ -0,0 +1,88 @@
"""Estimate disclosure must come from the data, not from patching prose.
``ensure_estimate_disclosure`` rewrites the model's sentences with regular
expressions to insert the word "schatting". That only fires when the generated
text happens to contain one of the phrasings it knows, so whether a number is
labelled an estimate depends on how the language model worded it. The
disclosure is derived from the metric metadata instead, so the honesty of the
answer no longer depends on string matching.
"""
from __future__ import annotations
from datetime import datetime, timezone
from uuid import uuid4
from app.schemas.assistant import AssistantContextMetric
from app.services.geo_assistant_service import GeoAssistantService
def _metric(theme: str, label: str, *, is_estimate: bool) -> AssistantContextMetric:
return AssistantContextMetric(
theme=theme,
label=label,
value=36783.0,
unit="inwoners",
source="Statbel",
dataset_id=uuid4(),
observed_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
is_estimate=is_estimate,
)
def test_every_estimated_metric_produces_a_disclosure() -> None:
metrics = [
_metric("population", "Inwoners", is_estimate=True),
_metric("buildings", "Gebouwen", is_estimate=False),
]
disclosures = GeoAssistantService.estimate_disclosures(metrics)
assert len(disclosures) == 1
assert disclosures[0].theme == "population"
assert disclosures[0].label == "Inwoners"
assert disclosures[0].source == "Statbel"
assert disclosures[0].dataset_id == metrics[0].dataset_id
assert "schatting" in disclosures[0].reason.casefold()
def test_disclosure_does_not_depend_on_the_generated_wording() -> None:
"""The regex path only fires on phrasings it recognises; this does not."""
metrics = [_metric("population", "Inwoners", is_estimate=True)]
patched = GeoAssistantService.ensure_estimate_disclosure(
"Er wonen daar 36.783 mensen.", metrics
)
disclosures = GeoAssistantService.estimate_disclosures(metrics)
# The prose was left untouched because no known phrase matched...
assert "Datakwaliteit" not in patched
# ...but the structured disclosure is present regardless.
assert len(disclosures) == 1
def test_no_estimates_means_no_disclosures() -> None:
metrics = [_metric("buildings", "Gebouwen", is_estimate=False)]
assert GeoAssistantService.estimate_disclosures(metrics) == []
def test_disclosures_are_deduplicated_per_theme_and_dataset() -> None:
shared = _metric("population", "Inwoners", is_estimate=True)
duplicate = AssistantContextMetric(**{**shared.model_dump(), "label": "Inwoners (2024)"})
disclosures = GeoAssistantService.estimate_disclosures([shared, duplicate])
assert len(disclosures) == 1
def test_disclosures_are_ordered_deterministically() -> None:
metrics = [
_metric("space_occupation", "Ruimtebeslag", is_estimate=True),
_metric("population", "Inwoners", is_estimate=True),
]
themes = [item.theme for item in GeoAssistantService.estimate_disclosures(metrics)]
assert themes == ["population", "space_occupation"]
+506
View File
@@ -0,0 +1,506 @@
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
from uuid import UUID, uuid4
from fastapi.testclient import TestClient
from app.core.config import get_settings
from app.core.public_demo import PUBLIC_DEMO_PROJECT_ID
from app.db.session import get_db
from app.main import create_app
from app.schemas.demo import DemoWorkflowResponse
from app.services.auth_service import AuthService
from app.services.change_detection_service import ChangeDetectionService
from app.services.dataset_service import DatasetService
from app.services.detection_service import DetectionService
from app.services.demo_workflow_service import DemoWorkflowService
from app.services.raster_operations_service import RasterOperationsService
from app.services.segmentation_service import SegmentationService
from app.services.job_service import JobService
def auth_client(
monkeypatch,
*,
guest_access: bool = False,
require_https: bool = False,
base_url: str = "http://testserver",
) -> TestClient:
password_hash = AuthService.hash_password(
"correct horse battery staple",
salt=b"geointel-test-salt",
iterations=100_000,
)
monkeypatch.setenv("GEOINTEL_AUTH_ENABLED", "true")
monkeypatch.setenv("GEOINTEL_AUTH_REQUIRE_HTTPS", "true" if require_https else "false")
monkeypatch.setenv("GEOINTEL_AUTH_USERNAME", "operator")
monkeypatch.setenv("GEOINTEL_AUTH_PASSWORD_HASH", password_hash)
monkeypatch.setenv("GEOINTEL_AUTH_SESSION_SECRET", "test-session-secret-that-is-long-enough")
monkeypatch.setenv("GEOINTEL_GUEST_ACCESS_ENABLED", "true" if guest_access else "false")
monkeypatch.setenv("GEOINTEL_GUEST_DISPLAY_NAME", "Gast")
monkeypatch.setenv("GEOINTEL_GUEST_SESSION_TTL_SECONDS", "7200")
return TestClient(create_app(), base_url=base_url)
def test_guest_access_defaults_off_when_operator_authentication_is_enabled(monkeypatch) -> None:
password_hash = AuthService.hash_password(
"correct horse battery staple",
salt=b"geointel-test-salt",
iterations=100_000,
)
monkeypatch.setenv("GEOINTEL_AUTH_ENABLED", "true")
monkeypatch.setenv("GEOINTEL_AUTH_USERNAME", "operator")
monkeypatch.setenv("GEOINTEL_AUTH_PASSWORD_HASH", password_hash)
monkeypatch.setenv("GEOINTEL_AUTH_SESSION_SECRET", "test-session-secret-that-is-long-enough")
monkeypatch.delenv("GEOINTEL_GUEST_ACCESS_ENABLED", raising=False)
client = TestClient(create_app())
session = client.get("/api/v1/auth/session")
assert session.status_code == 200
assert session.json()["data"]["authentication_required"] is True
assert session.json()["data"]["guest_access_enabled"] is False
def test_guest_default_is_inactive_but_valid_when_operator_authentication_is_disabled(monkeypatch) -> None:
monkeypatch.setenv("GEOINTEL_AUTH_ENABLED", "false")
monkeypatch.delenv("GEOINTEL_AUTH_USERNAME", raising=False)
monkeypatch.delenv("GEOINTEL_AUTH_PASSWORD_HASH", raising=False)
monkeypatch.delenv("GEOINTEL_AUTH_SESSION_SECRET", raising=False)
monkeypatch.delenv("GEOINTEL_GUEST_ACCESS_ENABLED", raising=False)
client = TestClient(create_app())
session = client.get("/api/v1/auth/session")
assert session.status_code == 200
assert session.json()["data"]["authentication_required"] is False
assert session.json()["data"]["authenticated"] is True
assert session.json()["data"]["guest_access_enabled"] is False
def test_auth_session_and_health_are_public_but_api_is_protected(monkeypatch) -> None:
client = auth_client(monkeypatch)
session = client.get("/api/v1/auth/session")
protected = client.get("/api/v1/protected-probe")
health = client.get("/health/live")
assert session.status_code == 200
assert session.json()["data"] == {
"authentication_required": True,
"authenticated": False,
"username": None,
"expires_at": None,
"role": None,
"guest_access_enabled": False,
"authentik_enabled": False,
"guest_project_id": None,
}
assert protected.status_code == 401
assert protected.json()["error"] == "AUTHENTICATION_REQUIRED"
assert health.status_code == 200
def test_login_uses_http_only_session_cookie_and_logout_revokes_browser_access(monkeypatch) -> None:
client = auth_client(monkeypatch, guest_access=True)
invalid = client.post(
"/api/v1/auth/login",
json={"username": "operator", "password": "wrong"},
)
login = client.post(
"/api/v1/auth/login",
json={"username": "operator", "password": "correct horse battery staple"},
)
authenticated = client.get("/api/v1/auth/session")
protected_after_login = client.get("/api/v1/protected-probe")
logout = client.post("/api/v1/auth/logout")
protected_after_logout = client.get("/api/v1/protected-probe")
assert invalid.status_code == 401
assert invalid.json()["error"] == "INVALID_CREDENTIALS"
assert login.status_code == 200
assert login.json()["data"] == {
"authentication_required": True,
"authenticated": True,
"username": "operator",
"expires_at": login.json()["data"]["expires_at"],
"role": "operator",
"guest_access_enabled": True,
"authentik_enabled": False,
"guest_project_id": None,
}
cookie = login.headers["set-cookie"].lower()
assert "httponly" in cookie
assert "samesite=strict" in cookie
assert authenticated.json()["data"]["authenticated"] is True
assert authenticated.json()["data"]["role"] == "operator"
assert protected_after_login.status_code == 404
assert logout.status_code == 200
assert logout.json()["data"]["guest_access_enabled"] is True
assert protected_after_logout.status_code == 401
def test_operator_login_can_require_https(monkeypatch) -> None:
insecure_client = auth_client(monkeypatch, require_https=True)
rejected = insecure_client.post(
"/api/v1/auth/login",
json={"username": "operator", "password": "correct horse battery staple"},
)
spoofed = insecure_client.post(
"/api/v1/auth/login",
headers={"x-forwarded-proto": "https", "x-real-ip": "203.0.113.9"},
json={"username": "operator", "password": "correct horse battery staple"},
)
secure_client = auth_client(
monkeypatch,
require_https=True,
base_url="https://testserver",
)
accepted = secure_client.post(
"/api/v1/auth/login",
json={"username": "operator", "password": "correct horse battery staple"},
)
assert rejected.status_code == 426
assert rejected.json()["error"] == "AUTH_HTTPS_REQUIRED"
assert spoofed.status_code == 426
assert accepted.status_code == 200
assert "secure" in accepted.headers["set-cookie"].lower()
def test_guest_login_exposes_models_but_rejects_management_and_cross_project_requests(monkeypatch) -> None:
project_id = PUBLIC_DEMO_PROJECT_ID
demo = DemoWorkflowResponse(
project_id=project_id,
area_id=UUID("00000000-0000-0000-0000-000000000124"),
reference_dataset_id=UUID("00000000-0000-0000-0000-000000000125"),
candidate_dataset_id=UUID("00000000-0000-0000-0000-000000000126"),
raster_dataset_id=UUID("00000000-0000-0000-0000-000000000127"),
quality_check_id=UUID("00000000-0000-0000-0000-000000000128"),
metric_count=6,
status="ok",
message="Demo ready",
created=False,
)
monkeypatch.setattr(DemoWorkflowService, "seed", staticmethod(lambda _db: demo))
client = auth_client(monkeypatch, guest_access=True)
def fake_db():
yield object()
client.app.dependency_overrides[get_db] = fake_db
guest_login = client.post("/api/v1/auth/guest")
guest_session = client.get("/api/v1/auth/session")
mutation = client.post("/api/v1/projects", json={"name": "Not allowed"})
other_project = client.get("/api/v1/projects/00000000-0000-0000-0000-000000000999")
detection_models = client.get("/api/v1/detection/models")
segmentation_models = client.get("/api/v1/segmentation/models")
global_source_registry = client.get("/api/v1/source-registry/grb")
cross_project_runs = client.get(
"/api/v1/detection/runs?project_id=00000000-0000-0000-0000-000000000999"
)
cross_project_coverage = client.post(
"/api/v1/external/coverage/resolve",
json={
"project_id": "00000000-0000-0000-0000-000000000999",
"bbox": {"minx": 4.9, "miny": 51.0, "maxx": 5.0, "maxy": 51.1},
"themes": [],
},
)
bounded_acquisition = client.post(
f"/api/v1/projects/{project_id}/datasets/orthophoto/acquire",
json={},
)
cross_project_acquisition = client.post(
"/api/v1/projects/00000000-0000-0000-0000-000000000999/datasets/orthophoto/acquire",
json={},
)
bounded_derived_selection = client.post(
f"/api/v1/projects/{project_id}/datasets/{demo.candidate_dataset_id}/vector/select/derive",
json={},
)
assert guest_login.status_code == 200
assert guest_login.json()["data"]["role"] == "guest"
assert guest_login.json()["data"]["username"] == "Gast"
assert guest_login.json()["data"]["guest_project_id"] == str(project_id)
assert "httponly" in guest_login.headers["set-cookie"].lower()
assert guest_session.json()["data"]["role"] == "guest"
assert mutation.status_code == 403
assert mutation.json()["error"] == "GUEST_READ_ONLY"
assert other_project.status_code == 403
assert other_project.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
assert detection_models.status_code == 200
assert detection_models.json()["data"]["models"]
assert segmentation_models.status_code == 200
assert global_source_registry.status_code == 403
assert global_source_registry.json()["error"] == "GUEST_ROUTE_NOT_AVAILABLE"
assert cross_project_runs.status_code == 403
assert cross_project_runs.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
assert cross_project_coverage.status_code == 403
assert cross_project_coverage.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
assert bounded_acquisition.status_code == 422
assert bounded_acquisition.json()["error"] != "GUEST_READ_ONLY"
assert cross_project_acquisition.status_code == 403
assert cross_project_acquisition.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
assert bounded_derived_selection.status_code == 422
assert bounded_derived_selection.json()["error"] != "GUEST_READ_ONLY"
def test_guest_change_detection_binds_both_datasets_to_signed_demo_project(monkeypatch) -> None:
project_id = PUBLIC_DEMO_PROJECT_ID
other_project_id = UUID("00000000-0000-0000-0000-000000000999")
source_dataset_id = UUID("00000000-0000-0000-0000-000000000125")
target_dataset_id = UUID("00000000-0000-0000-0000-000000000126")
cross_project_dataset_id = UUID("00000000-0000-0000-0000-000000000998")
demo = DemoWorkflowResponse(
project_id=project_id,
area_id=UUID("00000000-0000-0000-0000-000000000124"),
reference_dataset_id=source_dataset_id,
candidate_dataset_id=target_dataset_id,
raster_dataset_id=UUID("00000000-0000-0000-0000-000000000127"),
quality_check_id=UUID("00000000-0000-0000-0000-000000000128"),
metric_count=6,
status="ok",
message="Demo ready",
created=False,
)
monkeypatch.setattr(DemoWorkflowService, "seed", staticmethod(lambda _db: demo))
class FakeDb:
def get(self, _model, dataset_id):
bound_project_id = other_project_id if dataset_id == cross_project_dataset_id else project_id
return SimpleNamespace(id=dataset_id, project_id=bound_project_id, dataset_type="vector")
validated_datasets: list[tuple[UUID, UUID, str]] = []
def validate_dataset(_db, dataset_id, requested_project_id, label):
validated_datasets.append((dataset_id, requested_project_id, label))
return SimpleNamespace(id=dataset_id, project_id=requested_project_id, dataset_type="vector")
monkeypatch.setattr(
ChangeDetectionService,
"_get_project_vector_dataset",
staticmethod(validate_dataset),
)
monkeypatch.setattr(
JobService,
"run_sync_job",
staticmethod(
lambda **kwargs: SimpleNamespace(
id=uuid4(),
job_type=kwargs["job_type"],
status="success",
project_id=kwargs["project_id"],
dataset_id=source_dataset_id,
input_dataset_id=source_dataset_id,
output_dataset_id=None,
parameters_json=kwargs["parameters"],
result_json={},
error_message=None,
created_at=None,
started_at=None,
finished_at=None,
)
),
)
client = auth_client(monkeypatch, guest_access=True)
def fake_db():
yield FakeDb()
client.app.dependency_overrides[get_db] = fake_db
assert client.post("/api/v1/auth/guest").status_code == 200
accepted = client.post(
"/api/v1/analysis/change-detection",
json={
"source_dataset_id": str(source_dataset_id),
"target_dataset_id": str(target_dataset_id),
},
)
rejected = client.post(
"/api/v1/analysis/change-detection",
json={
"source_dataset_id": str(cross_project_dataset_id),
"target_dataset_id": str(target_dataset_id),
},
)
assert accepted.status_code == 200
assert accepted.json()["data"]["project_id"] == str(project_id)
assert validated_datasets == [
(source_dataset_id, project_id, "Source"),
(target_dataset_id, project_id, "Target"),
]
assert rejected.status_code == 403
assert rejected.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
def test_guest_can_prepare_tiles_and_queue_project_scoped_detection(monkeypatch) -> None:
project_id = PUBLIC_DEMO_PROJECT_ID
raster_dataset_id = UUID("00000000-0000-0000-0000-000000000127")
manifest_path = "/app/storage/tiles/demo/manifest.json"
demo = DemoWorkflowResponse(
project_id=project_id,
area_id=UUID("00000000-0000-0000-0000-000000000124"),
reference_dataset_id=UUID("00000000-0000-0000-0000-000000000125"),
candidate_dataset_id=UUID("00000000-0000-0000-0000-000000000126"),
raster_dataset_id=raster_dataset_id,
quality_check_id=UUID("00000000-0000-0000-0000-000000000128"),
metric_count=6,
status="ok",
message="Demo ready",
created=False,
)
monkeypatch.setattr(DemoWorkflowService, "seed", staticmethod(lambda _db: demo))
monkeypatch.setattr(
DatasetService,
"get_dataset",
staticmethod(lambda _db, _dataset_id: SimpleNamespace(project_id=project_id)),
)
def job(*, job_type: str, result_json: dict | None = None):
return SimpleNamespace(
id=uuid4(),
job_type=job_type,
status="success" if result_json else "queued",
project_id=project_id,
dataset_id=raster_dataset_id,
input_dataset_id=raster_dataset_id,
output_dataset_id=None,
parameters_json={},
result_json=result_json,
error_message=None,
created_at=None,
started_at=None,
finished_at=None,
)
tile_parameters: dict = {}
def tile(_db, _dataset_id, **kwargs):
tile_parameters.update(kwargs)
return {"manifest_path": manifest_path}
monkeypatch.setattr(RasterOperationsService, "tile", staticmethod(tile))
monkeypatch.setattr(
"app.api.routes.datasets._run_job_sync",
lambda **kwargs: job(job_type="raster.tile", result_json=kwargs["operation"]()),
)
queued_parameters: dict = {}
def enqueue_detection(**kwargs):
queued_parameters.update(kwargs)
return job(job_type="detection.run")
monkeypatch.setattr(DetectionService, "enqueue_detection", staticmethod(enqueue_detection))
queued_segmentation_parameters: dict = {}
def enqueue_segmentation(**kwargs):
queued_segmentation_parameters.update(kwargs)
return job(job_type="segmentation.run")
monkeypatch.setattr(SegmentationService, "enqueue_segmentation", staticmethod(enqueue_segmentation))
client = auth_client(monkeypatch, guest_access=True)
def fake_db():
yield object()
client.app.dependency_overrides[get_db] = fake_db
assert client.post("/api/v1/auth/guest").status_code == 200
tile_response = client.post(
f"/api/v1/projects/{project_id}/datasets/{raster_dataset_id}/raster/tile",
json={"tile_size": 512, "overlap": 64},
)
detection_response = client.post(
f"/api/v1/detection/run-async?project_id={project_id}",
json={
"project_id": str(project_id),
"dataset_id": str(raster_dataset_id),
"model_id": "yolo-configured",
"model_asset_id": "active-model",
"confidence_threshold": 0.15,
"tile_manifest_path": manifest_path,
"parameters_json": {},
},
)
segmentation_response = client.post(
f"/api/v1/segmentation/run-async?project_id={project_id}",
json={
"project_id": str(project_id),
"dataset_id": str(raster_dataset_id),
"model_id": "sam-configured",
"confidence_threshold": 0.5,
"tile_manifest_path": manifest_path,
"parameters_json": {},
},
)
assert tile_response.status_code == 201
assert tile_response.json()["data"]["result_json"]["manifest_path"] == manifest_path
assert detection_response.status_code == 200
assert detection_response.json()["data"]["status"] == "queued"
assert segmentation_response.status_code == 200
assert segmentation_response.json()["data"]["status"] == "queued"
assert queued_parameters["project_id"] == project_id
assert queued_parameters["dataset_id"] == raster_dataset_id
assert queued_parameters["tile_manifest_path"] == manifest_path
assert queued_segmentation_parameters["project_id"] == project_id
assert queued_segmentation_parameters["dataset_id"] == raster_dataset_id
assert queued_segmentation_parameters["tile_manifest_path"] == manifest_path
assert tile_parameters["max_tiles"] == get_settings().yolo_max_tiles
def test_guest_session_tokens_fail_closed_without_a_project_scope(monkeypatch) -> None:
auth_client(monkeypatch, guest_access=True)
settings = get_settings()
try:
AuthService.create_session_token("Gast", settings, role="guest")
except ValueError as error:
assert "demo project" in str(error)
else: # pragma: no cover - defensive assertion
raise AssertionError("An unscoped guest token should not be created")
def test_password_hash_and_session_signatures_fail_closed(monkeypatch) -> None:
client = auth_client(monkeypatch)
login = client.post(
"/api/v1/auth/login",
json={"username": "operator", "password": "correct horse battery staple"},
)
token = login.cookies.get("geointel_session")
assert token
client.cookies.set("geointel_session", f"{token}tampered")
session = client.get("/api/v1/auth/session")
assert session.status_code == 200
assert session.json()["data"]["authenticated"] is False
def test_unraid_runtime_carries_only_hashed_operator_credentials() -> None:
root = Path(__file__).resolve().parents[2]
runner = (root / "deploy/unraid/run-dockerman-container.sh").read_text(encoding="utf-8")
example = (root / "deploy/unraid/geointel.env.example").read_text(encoding="utf-8")
browser_smoke = (root / "scripts/verify_browser_runtime.sh").read_text(encoding="utf-8")
assert '-e GEOINTEL_AUTH_PASSWORD_HASH="$GEOINTEL_AUTH_PASSWORD_HASH"' in runner
assert '-e GEOINTEL_AUTHENTIK_CLIENT_SECRET="$GEOINTEL_AUTHENTIK_CLIENT_SECRET"' in runner
assert "GEOINTEL_AUTH_PASSWORD_HASH=" in example
assert "GEOINTEL_AUTHENTIK_CLIENT_SECRET=" in example
assert "GEOINTEL_AUTH_PASSWORD=" not in runner
assert "GEOINTEL_AUTH_ENABLED=true" in example
assert "GEOINTEL_AUTH_REQUIRE_HTTPS=true" in example
assert "GEOINTEL_GUEST_ACCESS_ENABLED=false" in example
assert 'GEOINTEL_AUTH_ENABLED="${GEOINTEL_AUTH_ENABLED:-true}"' in runner
assert 'GEOINTEL_AUTH_REQUIRE_HTTPS="${GEOINTEL_AUTH_REQUIRE_HTTPS:-true}"' in runner
assert 'GEOINTEL_GUEST_ACCESS_ENABLED="${GEOINTEL_GUEST_ACCESS_ENABLED:-false}"' in runner
assert "/api/v1/auth/session" in browser_smoke
@@ -0,0 +1,183 @@
from __future__ import annotations
import time
from urllib.parse import parse_qs, urlsplit
import jwt
import pytest
from cryptography.hazmat.primitives.asymmetric import rsa
from pydantic import ValidationError
from app.core.config import Settings
from app.services.authentik_oidc_service import (
MAX_OIDC_JSON_BYTES,
AuthentikOidcService,
)
ISSUER = "https://auth.example.test/application/o/geointel"
def configured_settings(**overrides: object) -> Settings:
values: dict[str, object] = {
"auth_enabled": True,
"auth_username": "ITWorx",
"auth_password_hash": "pbkdf2_sha256$1$salt$digest",
"auth_session_secret": "s" * 48,
"authentik_issuer": ISSUER,
"authentik_client_id": "geointel-client",
"authentik_client_secret": "client-secret",
"authentik_allowed_email": "operator@example.test",
"public_base_url": "https://geointel.example.test",
}
values.update(overrides)
return Settings(_env_file=None, **values)
def discovery_document() -> dict[str, str]:
return {
"issuer": ISSUER,
"authorization_endpoint": f"{ISSUER}/authorize",
"token_endpoint": f"{ISSUER}/token",
"jwks_uri": f"{ISSUER}/jwks",
}
def test_authentik_configuration_is_all_or_nothing_and_https_only() -> None:
with pytest.raises(ValidationError, match="configured together"):
configured_settings(authentik_client_secret=None)
with pytest.raises(ValidationError, match="absolute HTTPS URL"):
configured_settings(authentik_issuer="http://auth.example.test/issuer")
with pytest.raises(ValidationError, match="must not contain a path"):
configured_settings(public_base_url="https://geointel.example.test/app")
def test_start_uses_same_origin_discovery_and_pkce(monkeypatch: pytest.MonkeyPatch) -> None:
service = AuthentikOidcService(configured_settings())
monkeypatch.setattr(service, "_fetch_json", lambda *_args, **_kwargs: discovery_document())
location, flow_cookie = service.start()
parsed = urlsplit(location)
query = parse_qs(parsed.query)
flow = service.serializer.loads(flow_cookie, max_age=600)
assert f"{parsed.scheme}://{parsed.netloc}{parsed.path}" == f"{ISSUER}/authorize"
assert query["redirect_uri"] == [
"https://geointel.example.test/api/v1/auth/authentik/callback"
]
assert query["code_challenge_method"] == ["S256"]
assert query["state"] == [flow["state"]]
assert query["nonce"] == [flow["nonce"]]
assert query["code_challenge"][0]
def test_discovery_rejects_cross_origin_endpoints(monkeypatch: pytest.MonkeyPatch) -> None:
service = AuthentikOidcService(configured_settings())
document = discovery_document()
document["jwks_uri"] = "https://attacker.example.test/jwks"
monkeypatch.setattr(service, "_fetch_json", lambda *_args, **_kwargs: document)
with pytest.raises(ValueError, match="outside the configured issuer origin"):
service._discovery()
def test_finish_verifies_signature_nonce_and_exact_allowed_email(
monkeypatch: pytest.MonkeyPatch,
) -> None:
service = AuthentikOidcService(configured_settings())
state, nonce, verifier = "state-value", "nonce-value", "verifier-value"
flow_cookie = service.serializer.dumps(
{"state": state, "nonce": nonce, "verifier": verifier}
)
private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
public_jwk = jwt.algorithms.RSAAlgorithm.to_jwk(
private_key.public_key(), as_dict=True
)
public_jwk["kid"] = "operator-key"
now = int(time.time())
token = jwt.encode(
{
"iss": ISSUER,
"aud": "geointel-client",
"sub": "authentik-user-id",
"iat": now,
"exp": now + 300,
"nonce": nonce,
"email": "Operator@Example.Test",
"email_verified": True,
},
private_key,
algorithm="RS256",
headers={"kid": "operator-key"},
)
token_holder = {"value": token}
def fetch(url: str, data: dict[str, str] | None = None) -> dict:
if url.endswith("openid-configuration"):
return discovery_document()
if url.endswith("/token"):
assert data is not None
assert data["code_verifier"] == verifier
return {"id_token": token_holder["value"]}
if url.endswith("/jwks"):
return {"keys": [public_jwk]}
raise AssertionError(url)
monkeypatch.setattr(service, "_fetch_json", fetch)
claims = service.finish(code="authorization-code", state=state, flow_cookie=flow_cookie)
assert claims["sub"] == "authentik-user-id"
token_holder["value"] = jwt.encode(
{
"iss": ISSUER,
"aud": "geointel-client",
"sub": "different-user",
"iat": now,
"exp": now + 300,
"nonce": nonce,
"email": "other@example.test",
"email_verified": True,
},
private_key,
algorithm="RS256",
headers={"kid": "operator-key"},
)
with pytest.raises(ValueError, match="not authorized"):
service.finish(code="authorization-code", state=state, flow_cookie=flow_cookie)
with pytest.raises(ValueError, match="state mismatch"):
service.finish(
code="authorization-code",
state="different-state",
flow_cookie=flow_cookie,
)
def test_fetch_json_rejects_declared_oversize_response(
monkeypatch: pytest.MonkeyPatch,
) -> None:
service = AuthentikOidcService(configured_settings())
class OversizeResponse:
headers = {"Content-Length": str(MAX_OIDC_JSON_BYTES + 1)}
def __enter__(self):
return self
def __exit__(self, *_args: object) -> None:
return None
def read(self, _size: int) -> bytes:
raise AssertionError("oversized responses must not be read")
class Opener:
def open(self, *_args: object, **_kwargs: object) -> OversizeResponse:
return OversizeResponse()
monkeypatch.setattr(
"app.services.authentik_oidc_service.build_opener",
lambda *_args: Opener(),
)
with pytest.raises(ValueError, match="size limit"):
service._fetch_json(f"{ISSUER}/oversized")
@@ -0,0 +1,128 @@
"""A provider that ignores ``resultOffset`` must not produce duplicated data.
ArcGIS layers without ``supportsPagination`` accept ``resultOffset`` and ignore
it, answering every page with the first one. The profile reader advanced its
offset by the page length and stopped when it reached the announced count, so
for a count that is a multiple of the page size it collected N copies of page
one, matched the expected total exactly, and stored that as an official
dataset. The watercourse-name reader had no bound at all: it looped for as long
as the provider kept setting ``exceededTransferLimit``.
The sibling reader for official vector products already refuses a repeated page
and deduplicates on feature identity. These tests hold this reader to the same
rule.
"""
from __future__ import annotations
import json
from typing import Any
from urllib.parse import parse_qs, urlparse
import pytest
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.services.bathymetry_profile_acquisition_service import (
BathymetryProfileAcquisitionService,
)
BBOX = (4.30, 51.20, 4.32, 51.22)
def _settings(**overrides: Any) -> Settings:
base = get_settings()
return base.model_copy(update={"bathymetry_profiles_page_size": 2, **overrides})
class _Response:
def __init__(self, payload: dict[str, Any]) -> None:
self._body = json.dumps(payload).encode("utf-8")
def read(self, _limit: int | None = None) -> bytes:
return self._body
def __enter__(self) -> "_Response":
return self
def __exit__(self, *_exc: object) -> bool:
return False
def _profile(object_id: int) -> dict[str, Any]:
return {
"attributes": {"OBJECTID": object_id, "vhag": 7, "opg_diepte": 1.5},
"geometry": {"x": 4.31, "y": 51.21},
}
class StuckProvider:
"""Answers every page with the same records, as an unpaged layer does."""
def __init__(self, *, count: int, page: list[dict[str, Any]]) -> None:
self.count = count
self.page = page
self.requests: list[str] = []
def __call__(self, request: Any, **_kwargs: Any) -> _Response:
url = request.full_url if hasattr(request, "full_url") else str(request)
self.requests.append(url)
query = parse_qs(urlparse(url).query)
if query.get("returnCountOnly") == ["true"]:
return _Response({"count": self.count})
return _Response({"features": list(self.page), "exceededTransferLimit": True})
def test_a_provider_that_ignores_the_offset_is_refused_not_duplicated() -> None:
"""Four announced records, two per page, the same two every time.
Advancing by page length reaches the announced total after two pages, so the
completeness check passed while every record was stored twice.
"""
provider = StuckProvider(count=4, page=[_profile(1), _profile(2)])
with pytest.raises(AppError) as exc_info:
BathymetryProfileAcquisitionService._fetch_profiles(BBOX, _settings(), provider)
assert exc_info.value.code == "BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION"
assert exc_info.value.status_code == 502
def test_honest_pagination_still_returns_every_record() -> None:
"""The guard must not reject a provider that pages correctly."""
pages = [[_profile(1), _profile(2)], [_profile(3), _profile(4)]]
def opener(request: Any, **_kwargs: Any) -> _Response:
url = request.full_url
query = parse_qs(urlparse(url).query)
if query.get("returnCountOnly") == ["true"]:
return _Response({"count": 4})
offset = int(query.get("resultOffset", ["0"])[0])
index = offset // 2
page = pages[index] if index < len(pages) else []
return _Response({"features": page})
features, provenance = BathymetryProfileAcquisitionService._fetch_profiles(
BBOX, _settings(), opener
)
assert [item["attributes"]["OBJECTID"] for item in features] == [1, 2, 3, 4]
assert provenance["candidate_count"] == 4
def test_watercourse_names_stop_instead_of_looping_forever() -> None:
"""``exceededTransferLimit`` forever is not a reason to request forever."""
provider = StuckProvider(
count=0,
page=[{"attributes": {"wlasvl.vhag": 7, "VHAG_TABEL.naam": "Schelde"}}],
)
with pytest.raises(AppError) as exc_info:
BathymetryProfileAcquisitionService._fetch_watercourse_names({7}, _settings(), provider)
assert exc_info.value.code == "BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION"
# Bounded, and bounded early: it must not have hammered the provider first.
assert len(provider.requests) <= 3
@@ -0,0 +1,40 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
SCRIPT = Path(__file__).parents[2] / "scripts" / "evaluate_belgium_building_candidate.py"
SPEC = importlib.util.spec_from_file_location("candidate_evaluation", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def test_iou_and_one_to_one_matching() -> None:
reference = [(0.0, 0.0, 10.0, 10.0)]
predictions = [((0.0, 0.0, 10.0, 10.0), 0.9), ((0.0, 0.0, 10.0, 10.0), 0.8)]
assert MODULE.iou(reference[0], reference[0]) == 1.0
assert MODULE.match_boxes(predictions, reference, confidence=0.25, match_iou=0.5) == (1, 1, 0)
def test_empty_reference_counts_false_positives() -> None:
predictions = [((0.0, 0.0, 10.0, 10.0), 0.4)]
assert MODULE.match_boxes(predictions, [], confidence=0.25, match_iou=0.5) == (0, 1, 0)
assert MODULE.match_boxes(predictions, [], confidence=0.5, match_iou=0.5) == (0, 0, 0)
def test_box_scaling_preserves_center() -> None:
assert MODULE.scale_box((10.0, 20.0, 30.0, 40.0), 1.5) == (5.0, 15.0, 35.0, 45.0)
def test_containment_suppression_removes_nested_lower_score_box() -> None:
predictions = [
((0.0, 0.0, 20.0, 20.0), 0.9),
((5.0, 5.0, 15.0, 15.0), 0.8),
((25.0, 0.0, 35.0, 10.0), 0.7),
]
assert MODULE.suppress_contained_predictions(predictions, 0.8) == [
predictions[0],
predictions[2],
]
@@ -0,0 +1,37 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
import pytest
SCRIPT = Path(__file__).parents[2] / "scripts" / "assess_belgium_building_training_iteration.py"
SPEC = importlib.util.spec_from_file_location("iteration_assessment", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def test_calibration_selection_prefers_worst_region_then_aggregate() -> None:
report = {
"sweeps": [
{"threshold": 0.1, "pure_empty_false_positives": 0, "aggregate": {"f1": 0.8}, "regions": {"a": {"f1": 0.2}}},
{"threshold": 0.2, "pure_empty_false_positives": 0, "aggregate": {"f1": 0.6}, "regions": {"a": {"f1": 0.5}}},
]
}
assert MODULE.select_calibration_threshold(report)["threshold"] == 0.2
def test_threshold_lookup_is_exact() -> None:
report = {"sweeps": [{"threshold": 0.25, "aggregate": {}}]}
assert MODULE.find_threshold(report, 0.25)["threshold"] == 0.25
def test_release_assessment_rejects_changed_inference_configuration() -> None:
calibration = {field: None for field in MODULE.INFERENCE_CONFIG_FIELDS}
calibration.update({"model": "/models/candidate.pt", "nms_iou": 0.3, "containment_nms": 0.95})
test = dict(calibration)
test["nms_iou"] = 0.4
with pytest.raises(ValueError, match="nms_iou"):
MODULE.assert_same_inference_config(calibration, test, "test")
+510
View File
@@ -0,0 +1,510 @@
from __future__ import annotations
import importlib.util
import hashlib
import json
import subprocess
import sys
from pathlib import Path
SCRIPT = Path(__file__).parents[2] / "scripts" / "run_belgium_building_training_loop.py"
SPEC = importlib.util.spec_from_file_location("training_loop", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
from training_release_manifest import create_training_release_manifest # noqa: E402
def write_fixture_manifest(path: Path) -> None:
policy = "geointel-training-source-eligibility/v1"
def eligible(sample_slug: str) -> dict[str, object]:
return {
"policy_version": policy,
"eligible": True,
"fixture_mode": True,
"raster": {
"eligible": True,
"reasons": [],
"evidence": {
"dataset_id": f"raster:{sample_slug}",
"checksum_sha256": "a" * 64,
"source_registry_id": "fixture-raster",
"source_snapshot_id": "fixture-raster-snapshot",
},
},
"reference": {
"eligible": True,
"reasons": [],
"evidence": {
"dataset_id": f"reference:{sample_slug}",
"checksum_sha256": "b" * 64,
"source_registry_id": "fixture-reference",
"source_snapshot_id": "fixture-reference-snapshot",
},
},
}
path.write_text(
json.dumps(
{
"training_eligibility": {
"policy_version": policy,
"status": "eligible",
"fixture_mode": True,
},
"samples": [
{
"sample_slug": sample_slug,
"split": split,
"raster_dataset_id": f"raster:{sample_slug}",
"reference_dataset_id": f"reference:{sample_slug}",
"training_eligibility": eligible(sample_slug),
}
for sample_slug, split in (("fixture-train", "train"), ("fixture-val", "val"))
],
}
),
encoding="utf-8",
)
(path.parent / "corpus-freeze.json").write_text(
json.dumps(
{
"schema_version": 2,
"manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
"immutable": True,
"training_eligibility_policy": policy,
"fixture_mode": True,
}
),
encoding="utf-8",
)
def write_fixture_training_release(tmp_path: Path, manifest: Path) -> Path:
dataset_dir = tmp_path / "fixture-dataset"
for split, sample_slug in (("train", "fixture-train"), ("val", "fixture-val")):
image = dataset_dir / "images" / split / f"{sample_slug}.png"
label = dataset_dir / "labels" / split / f"{sample_slug}.txt"
image.parent.mkdir(parents=True, exist_ok=True)
label.parent.mkdir(parents=True, exist_ok=True)
image.write_bytes(split.encode("utf-8"))
label.write_text("0 0.5 0.5 0.2 0.2\n", encoding="utf-8")
yaml_path = dataset_dir / "dataset.yaml"
yaml_path.write_text(
f"path: {dataset_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
encoding="utf-8",
)
create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=manifest,
fixture_mode=True,
)
return yaml_path
def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_path: Path) -> None:
command = MODULE.training_command(
"yolo",
model=tmp_path / "base.pt",
data=tmp_path / "dataset.yaml",
project=tmp_path / "runs",
name="iteration-001",
epochs=160,
seed=42,
batch=2,
workers=4,
)
assert command[:2] == ["yolo", "train"]
assert "device=0" in command
assert "deterministic=True" in command
assert "seed=42" in command
assert "epochs=160" in command
assert "patience=18" in command
assert "max_det=1000" in command
assert "imgsz=640" in command
assert "optimizer=auto" in command
assert "mosaic=1.0" in command
def test_training_command_supports_conservative_aerial_finetuning(tmp_path: Path) -> None:
command = MODULE.training_command(
"yolo",
model=tmp_path / "base.pt",
data=tmp_path / "dataset.yaml",
project=tmp_path / "runs",
name="aerial",
epochs=50,
seed=42,
batch=2,
workers=0,
optimizer="AdamW",
lr0=0.0001,
mosaic=0.0,
scale=0.2,
translate=0.05,
)
assert "optimizer=AdamW" in command
assert "lr0=0.0001" in command
assert "mosaic=0.0" in command
assert "scale=0.2" in command
assert "translate=0.05" in command
assert "degrees=0.0" in command
assert "flipud=0.0" in command
assert "fliplr=0.5" in command
assert "warmup_epochs=1.0" in command
assert "warmup_bias_lr=0.01" in command
assert "hsv_h=0.01" in command
assert "hsv_s=0.2" in command
assert "hsv_v=0.15" in command
assert f"data={tmp_path / 'dataset.yaml'}" in command
def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_path: Path) -> None:
command = MODULE.failure_sampling_command(
scripts_dir=tmp_path / "scripts",
train_summary=tmp_path / "train-summary.json",
corpus_manifest=tmp_path / "manifest.json",
assessment=tmp_path / "assessment.json",
output_dir=tmp_path / "iteration-001" / "failure-driven-training",
review_audit=tmp_path / "review-audit.json",
)
assert command[1].endswith("build_failure_driven_yolo_sampling.py")
assert command[command.index("--summary") + 1].endswith("train-summary.json")
assert command[command.index("--assessment") + 1].endswith("assessment.json")
assert command[command.index("--output-dir") + 1].endswith("failure-driven-training")
def test_protected_assessment_uses_frozen_threshold_and_configured_gates(
tmp_path: Path,
) -> None:
command = MODULE.protected_assessment_command(
scripts_dir=tmp_path / "scripts",
calibration=tmp_path / "calibration.json",
test=tmp_path / "test.json",
background=tmp_path / "background.json",
output=tmp_path / "assessment.json",
selected_threshold=0.275,
min_aggregate_f1=0.71,
min_region_f1=0.62,
min_region_precision=0.73,
min_region_recall=0.58,
max_pure_empty_fp=1,
)
assert command[command.index("--selected-threshold") + 1] == "0.275"
assert command[command.index("--min-aggregate-f1") + 1] == "0.71"
assert command[command.index("--min-region-f1") + 1] == "0.62"
assert command[command.index("--min-region-precision") + 1] == "0.73"
assert command[command.index("--min-region-recall") + 1] == "0.58"
assert command[command.index("--max-pure-empty-fp") + 1] == "1"
def test_partial_iteration_resume_uses_exact_checkpoint_and_cuda(tmp_path: Path) -> None:
checkpoint = tmp_path / "runs" / "iteration-002" / "weights" / "last.pt"
assert MODULE.resumable_training_command("yolo", checkpoint) == [
"yolo", "train", f"resume={checkpoint}", "device=0"
]
def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -> None:
audit = tmp_path / "audit.json"
audit.write_text(json.dumps({
"status": "needs_human_review", "failures": [],
"manifest_immutable": True, "spatial_leakage_status": "ok",
}))
quality = tmp_path / "quality.json"
quality.write_text(json.dumps({
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}))
manifest = tmp_path / "manifest.json"
write_fixture_manifest(manifest)
train_yaml = write_fixture_training_release(tmp_path, manifest)
result = subprocess.run(
[
sys.executable, str(SCRIPT),
"--initial-model", str(tmp_path / "candidate.pt"),
"--train-yaml", str(train_yaml),
"--train-summary", str(tmp_path / "train-summary.json"),
"--dataset-audit", str(audit),
"--train-quality-audit", str(quality),
"--calibration-summary", str(tmp_path / "cal.json"),
"--test-summary", str(tmp_path / "test.json"),
"--background-summary", str(tmp_path / "background.json"),
"--corpus-manifest", str(manifest),
"--output-dir", str(tmp_path / "output"),
"--evaluate-initial-model", "--fixture-mode", "--dry-run",
], capture_output=True, text=True, check=False,
)
assert result.returncode == 0
assert json.loads(result.stdout) == {
"training_command": None, "evaluate_existing": True, "resume_partial": False
}
def test_existing_checkpoint_iteration_directory_can_be_created_without_yolo(tmp_path: Path) -> None:
iteration_dir = tmp_path / "closed-loop" / "iteration-001"
iteration_dir.mkdir(parents=True, exist_ok=True)
assert iteration_dir.is_dir()
def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
audit = tmp_path / "audit.json"
audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4}))
quality = tmp_path / "quality.json"
quality.write_text(json.dumps({
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}))
manifest = tmp_path / "manifest.json"
write_fixture_manifest(manifest)
train_yaml = write_fixture_training_release(tmp_path, manifest)
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--initial-model",
str(tmp_path / "base.pt"),
"--train-yaml",
str(train_yaml),
"--train-summary",
str(tmp_path / "train-summary.json"),
"--dataset-audit",
str(audit),
"--train-quality-audit",
str(quality),
"--calibration-summary",
str(tmp_path / "cal.json"),
"--test-summary",
str(tmp_path / "test.json"),
"--background-summary",
str(tmp_path / "background.json"),
"--corpus-manifest",
str(manifest),
"--output-dir",
str(tmp_path / "output"),
"--fixture-mode",
],
capture_output=True,
text=True,
check=False,
)
assert result.returncode != 0
assert "Dataset audit is not eligible for training" in result.stderr
def test_loop_rejects_manifest_without_source_eligibility_before_cuda_training(tmp_path: Path) -> None:
audit = tmp_path / "audit.json"
audit.write_text(json.dumps({
"status": "needs_human_review", "failures": [],
"manifest_immutable": True, "spatial_leakage_status": "ok",
}))
quality = tmp_path / "quality.json"
quality.write_text(json.dumps({
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}))
manifest = tmp_path / "manifest.json"
manifest.write_text(json.dumps({"samples": [{"sample_slug": "unproven"}]}), encoding="utf-8")
result = subprocess.run(
[
sys.executable, str(SCRIPT),
"--initial-model", str(tmp_path / "candidate.pt"),
"--train-yaml", str(tmp_path / "dataset.yaml"),
"--train-summary", str(tmp_path / "train-summary.json"),
"--dataset-audit", str(audit),
"--train-quality-audit", str(quality),
"--calibration-summary", str(tmp_path / "cal.json"),
"--test-summary", str(tmp_path / "test.json"),
"--background-summary", str(tmp_path / "background.json"),
"--corpus-manifest", str(manifest),
"--output-dir", str(tmp_path / "output"),
"--evaluate-initial-model", "--dry-run",
],
capture_output=True,
text=True,
check=False,
)
assert result.returncode != 0
assert "manifest_training_eligibility_missing" in result.stderr
def test_pending_human_review_blocks_operational_training() -> None:
audit = {
"status": "needs_human_review",
"failures": [],
"manifest_immutable": True,
"spatial_leakage_status": "ok",
"low_variance_positive_tile_count": 0,
"review_complete": False,
}
quality = {
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}
failures = MODULE.dataset_audit_failures(audit, quality)
assert "unsupported audit status: needs_human_review" in failures
assert "review_complete_not_true" in failures
assert "accepted_human_review_evidence_missing" in failures
def test_fixture_mode_can_relax_review_only_after_fixture_manifest_gate() -> None:
audit = {
"status": "needs_human_review",
"failures": [],
"manifest_immutable": True,
"spatial_leakage_status": "ok",
"review_complete": False,
}
quality = {
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}
assert MODULE.dataset_audit_failures(audit, quality, fixture_mode=True) == []
def test_operational_dataset_audit_must_be_the_one_bound_into_the_release(tmp_path: Path) -> None:
bound = tmp_path / "bound-audit.json"
other = tmp_path / "other-audit.json"
bound.write_text("{}", encoding="utf-8")
other.write_text("{}", encoding="utf-8")
release = {"human_review": {"audit_path": str(bound.resolve())}}
MODULE.assert_dataset_audit_bound_to_release(
release=release,
dataset_audit=bound,
fixture_mode=False,
)
try:
MODULE.assert_dataset_audit_bound_to_release(
release=release,
dataset_audit=other,
fixture_mode=False,
)
except MODULE.TrainingReleaseError as exc:
assert "does not match" in str(exc)
else:
raise AssertionError("unbound dataset audit was accepted")
def test_protected_assessment_feedback_is_terminal_and_cannot_seed_another_yaml() -> None:
assert MODULE.protected_feedback_roles(
{"status": "continue_training_loop", "test": {"aggregate": {}}, "background": None}
) == ["test"]
assert MODULE.protected_feedback_roles(
{"status": "continue_training_loop", "test": None, "background": {"aggregate": {}}}
) == ["background"]
def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None:
audit = {
"status": "needs_human_review",
"failures": ["wallonia/test below minimum"],
"manifest_immutable": False,
"spatial_leakage_status": "failed",
"low_variance_positive_tile_count": 2,
}
quality = {
"status": "failed", "low_variance_positive_tile_count": 2,
"label_stats": {"invalid_label_count": 1, "missing_label_file_count": 1},
}
failures = MODULE.dataset_audit_failures(audit, quality)
assert "wallonia/test below minimum" in failures
assert "corpus manifest is not immutable" in failures
assert "spatial leakage audit is not ok" in failures
assert "dataset contains blank/low-variance positive tiles" in failures
assert "train tile quality audit is not ok" in failures
assert "train tile quality audit contains invalid labels" in failures
assert "train tile quality audit contains missing label files" in failures
def test_missing_tile_quality_evidence_fails_closed() -> None:
audit = {
"status": "needs_human_review", "failures": [],
"manifest_immutable": True, "spatial_leakage_status": "ok",
}
failures = MODULE.dataset_audit_failures(audit, {})
assert "train tile quality audit is not ok" in failures
assert "train tile quality audit contains invalid labels" in failures
assert "train tile quality audit contains missing label files" in failures
assert "dataset contains blank/low-variance positive tiles" in failures
def test_calibration_failure_blocks_protected_evaluation() -> None:
chosen = {
"threshold": 0.1,
"aggregate": {"f1": 0.54},
"regions": {
"flanders": {"f1": 0.44, "precision": 0.49, "recall": 0.39},
"wallonia": {"f1": 0.6, "precision": 0.6, "recall": 0.6},
},
"pure_empty_false_positives": 0,
}
failures = MODULE.calibration_failures(
chosen,
min_aggregate_f1=0.55,
min_region_f1=0.45,
min_region_precision=0.5,
min_region_recall=0.4,
max_pure_empty_fp=0,
)
assert failures == [
"calibration_aggregate_f1_below_gate",
"calibration_flanders_f1_below_gate",
"calibration_flanders_precision_below_gate",
"calibration_flanders_recall_below_gate",
]
def test_threshold_selection_uses_worst_region_then_aggregate() -> None:
report = {
"sweeps": [
{
"threshold": 0.1,
"aggregate": {"f1": 0.8},
"regions": {"a": {"f1": 0.4}, "b": {"f1": 0.7}},
"pure_empty_false_positives": 0,
},
{
"threshold": 0.2,
"aggregate": {"f1": 0.6},
"regions": {"a": {"f1": 0.5}, "b": {"f1": 0.5}},
"pure_empty_false_positives": 0,
},
]
}
assert MODULE.select_calibration_threshold(report)["threshold"] == 0.2
def test_rejected_candidate_score_prioritizes_weakest_release_gate() -> None:
gates = {
"min_aggregate_f1": 0.55,
"min_region_f1": 0.45,
"min_region_precision": 0.5,
"min_region_recall": 0.4,
}
incumbent = {
"gates": gates,
"calibration": {
"aggregate": {"f1": 0.58},
"regions": {
"flanders": {"f1": 0.34, "precision": 0.38, "recall": 0.31},
"wallonia": {"f1": 0.60, "precision": 0.50, "recall": 0.75},
},
},
}
regressed = {
"gates": gates,
"calibration": {
"aggregate": {"f1": 0.60},
"regions": {
"flanders": {"f1": 0.31, "precision": 0.45, "recall": 0.24},
"wallonia": {"f1": 0.62, "precision": 0.52, "recall": 0.77},
},
},
}
assert MODULE.rejected_candidate_score(incumbent) > MODULE.rejected_candidate_score(regressed)
@@ -0,0 +1,50 @@
from __future__ import annotations
import importlib.util
import sys
from collections import Counter
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SPEC = importlib.util.spec_from_file_location(
"building_portfolio", ROOT / "scripts" / "provision_belgium_building_training_portfolio.py"
)
assert SPEC and SPEC.loader
module = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = module
SPEC.loader.exec_module(module)
def test_portfolio_covers_every_region_split_and_context_family() -> None:
assert len({aoi.slug for aoi in module.AOIS}) == len(module.AOIS)
counts = Counter((aoi.region, aoi.split) for aoi in module.AOIS)
for region in module.REGION_CONTRACT:
assert counts[(region, "train")] >= 15
assert counts[(region, "val")] >= 2
assert counts[(region, "calibration")] >= 3
assert counts[(region, "test")] >= 3
assert counts[(region, "background-test")] >= 2
def test_portfolio_bbox_is_metric_sized() -> None:
bbox = module.bbox_for_center(4.35, 50.85, 256.0)
to_metric = module.Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
bounds = to_metric.transform_bounds(bbox["min_x"], bbox["min_y"], bbox["max_x"], bbox["max_y"])
assert 255 <= bounds[2] - bounds[0] <= 258
assert 255 <= bounds[3] - bounds[1] <= 258
def test_failure_driven_expansion_is_train_only_and_context_complete() -> None:
additions = [aoi for aoi in module.AOIS if aoi.slug.endswith("-v31") or "-v31-bg" in aoi.slug]
assert len(additions) == 18
assert {aoi.split for aoi in additions} == {"train"}
assert {aoi.region for aoi in additions} == {"flanders", "wallonia"}
contexts = {(aoi.region, aoi.context) for aoi in additions}
assert {
("flanders", "industrial"),
("flanders", "ribbon-development"),
("flanders", "coastal-urban"),
("wallonia", "dense-urban"),
("wallonia", "rural-town"),
("wallonia", "regional-architecture"),
} <= contexts
@@ -0,0 +1,261 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import numpy as np
import rasterio
from rasterio.transform import from_origin
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "normalize_belgium_building_labels.py"
SPEC = importlib.util.spec_from_file_location("normalize_buildings", SCRIPT)
assert SPEC and SPEC.loader
module = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(module)
ASSEMBLER_SPEC = importlib.util.spec_from_file_location(
"assemble_building_corpus", ROOT / "scripts" / "assemble_belgium_building_corpus.py"
)
assert ASSEMBLER_SPEC and ASSEMBLER_SPEC.loader
assembler = importlib.util.module_from_spec(ASSEMBLER_SPEC)
ASSEMBLER_SPEC.loader.exec_module(assembler)
def test_normalizer_retains_native_identity_and_records_rejections(tmp_path: Path) -> None:
raster_path = tmp_path / "image.tif"
with rasterio.open(
raster_path,
"w",
driver="GTiff",
width=100,
height=100,
count=3,
dtype="uint8",
crs="EPSG:4326",
transform=from_origin(4.0, 51.0, 0.001, 0.001),
) as dataset:
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
valid = {
"type": "Feature",
"id": "native-1",
"properties": {"TYPE": "main building"},
"geometry": {"type": "Polygon", "coordinates": [[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]]},
}
duplicate = json.loads(json.dumps(valid))
duplicate["id"] = "native-2"
canopy = json.loads(json.dumps(valid))
canopy["id"] = "native-3"
canopy["properties"]["TYPE"] = "canopy"
tiny = json.loads(json.dumps(valid))
tiny["id"] = "native-4"
tiny["geometry"] = {"type": "Polygon", "coordinates": [[[4.03, 50.97], [4.031, 50.97], [4.031, 50.969], [4.03, 50.969], [4.03, 50.97]]]}
reference_path = tmp_path / "reference.geojson"
reference_path.write_text(json.dumps({"type": "FeatureCollection", "features": [valid, duplicate, canopy, tiny]}), encoding="utf-8")
normalized, audit = module.normalize(
reference_path=reference_path,
raster_path=raster_path,
source_name="urbis",
min_label_px=3,
imagery_observed_at="2026-01-01T00:00:00Z",
reference_observed_at="2025-12-01T00:00:00Z",
)
assert len(normalized["features"]) == 1
properties = normalized["features"][0]["properties"]
assert properties["canonical_class"] == "building"
assert properties["source_name"] == "urbis"
assert properties["source_feature_id"] == "native-1"
assert properties["source_class"] == "main building"
assert audit["decision_counts"] == {
"accepted": 1,
"below_resolvable_pixel_size": 1,
"duplicate_geometry": 1,
"excluded_canopy": 1,
}
assert audit["temporal_mismatch_days"] == 31
def test_spatial_leakage_audit_fails_cross_split_neighbors() -> None:
samples = [
{"sample_slug": "train-a", "split": "train", "bbox_epsg4326": [4.0, 50.0, 4.01, 50.01]},
{"sample_slug": "val-a", "split": "val", "bbox_epsg4326": [4.005, 50.005, 4.02, 50.02]},
{"sample_slug": "test-far", "split": "test", "bbox_epsg4326": [5.0, 51.0, 5.01, 51.01]},
]
audit = assembler.audit_spatial_leakage(samples)
assert audit["status"] == "failed"
assert audit["findings"][0]["left"] == "train-a"
assert audit["findings"][0]["right"] == "val-a"
def test_normalizer_rejects_features_created_after_dated_imagery(tmp_path: Path) -> None:
raster_path = tmp_path / "image.tif"
with rasterio.open(
raster_path,
"w",
driver="GTiff",
width=100,
height=100,
count=3,
dtype="uint8",
crs="EPSG:4326",
transform=from_origin(4.0, 51.0, 0.001, 0.001),
) as dataset:
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
geometry = {
"type": "Polygon",
"coordinates": [[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]],
}
reference_path = tmp_path / "reference.geojson"
reference_path.write_text(
json.dumps(
{
"type": "FeatureCollection",
"features": [
{"type": "Feature", "id": "old", "properties": {"BEGINDATUM": "2024-01-01"}, "geometry": geometry},
{"type": "Feature", "id": "new", "properties": {"BEGINDATUM": "2026-01-01"}, "geometry": geometry},
],
}
),
encoding="utf-8",
)
normalized, audit = module.normalize(
reference_path=reference_path,
raster_path=raster_path,
source_name="grb",
min_label_px=3,
imagery_observed_at="2025-01-01T00:00:00Z",
imagery_valid_to="2025-12-31T23:59:59Z",
reference_observed_at="2026-07-01T00:00:00Z",
)
assert len(normalized["features"]) == 1
assert audit["decision_counts"] == {"accepted": 1, "created_after_imagery_period": 1}
def test_normalizer_rejects_same_year_feature_after_annual_mosaic_start(tmp_path: Path) -> None:
raster_path = tmp_path / "image.tif"
with rasterio.open(
raster_path, "w", driver="GTiff", width=100, height=100, count=3,
dtype="uint8", crs="EPSG:4326", transform=from_origin(4.0, 51.0, 0.001, 0.001),
) as dataset:
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
geometry = {
"type": "Polygon",
"coordinates": [[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]],
}
reference_path = tmp_path / "reference.geojson"
reference_path.write_text(json.dumps({
"type": "FeatureCollection",
"features": [{"type": "Feature", "properties": {"BEGINDATUM": "2025-08-18"}, "geometry": geometry}],
}), encoding="utf-8")
normalized, audit = module.normalize(
reference_path=reference_path,
raster_path=raster_path,
source_name="grb",
min_label_px=3,
imagery_observed_at="2025-01-01T00:00:00Z",
imagery_valid_to="2025-12-31T23:59:59Z",
reference_observed_at="2026-07-01T00:00:00Z",
)
assert normalized["features"] == []
assert audit["decision_counts"] == {"created_after_imagery_period": 1}
assert audit["imagery_feature_creation_cutoff"] == "2025-01-01T00:00:00+00:00"
def test_normalizer_applies_provider_native_source_class_allowlist(tmp_path: Path) -> None:
raster_path = tmp_path / "image.tif"
with rasterio.open(
raster_path, "w", driver="GTiff", width=100, height=100, count=3,
dtype="uint8", crs="EPSG:4326", transform=from_origin(4.0, 51.0, 0.001, 0.001),
) as dataset:
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
def feature(feature_id: str, source_type: int, left: float) -> dict:
return {
"type": "Feature", "id": feature_id, "properties": {"TYPE": source_type},
"geometry": {"type": "Polygon", "coordinates": [[[left, 50.99], [left + .01, 50.99], [left + .01, 50.98], [left, 50.98], [left, 50.99]]]},
}
reference_path = tmp_path / "reference.geojson"
reference_path.write_text(json.dumps({"type": "FeatureCollection", "features": [feature("main", 1, 4.01), feature("annex", 2, 4.03)]}), encoding="utf-8")
normalized, audit = module.normalize(
reference_path=reference_path, raster_path=raster_path, source_name="grb",
min_label_px=3, imagery_observed_at=None, reference_observed_at=None,
allowed_source_classes={"1"},
)
assert [item["id"] for item in normalized["features"]] == ["grb:main"]
assert audit["allowed_source_classes"] == ["1"]
assert audit["decision_counts"] == {"accepted": 1, "source_class_not_allowed": 1}
def test_normalizer_merges_only_touching_visible_roof_instances(tmp_path: Path) -> None:
raster_path = tmp_path / "image.tif"
with rasterio.open(
raster_path,
"w",
driver="GTiff",
width=100,
height=100,
count=3,
dtype="uint8",
crs="EPSG:4326",
transform=from_origin(4.0, 51.0, 0.001, 0.001),
) as dataset:
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
polygons = [
[[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]],
[[[4.02, 50.99], [4.03, 50.99], [4.03, 50.98], [4.02, 50.98], [4.02, 50.99]]],
[[[4.04, 50.99], [4.05, 50.99], [4.05, 50.98], [4.04, 50.98], [4.04, 50.99]]],
]
reference_path = tmp_path / "reference.geojson"
reference_path.write_text(
json.dumps(
{
"type": "FeatureCollection",
"features": [
{"type": "Feature", "id": str(index), "properties": {}, "geometry": {"type": "Polygon", "coordinates": coordinates}}
for index, coordinates in enumerate(polygons)
],
}
),
encoding="utf-8",
)
normalized, audit = module.normalize(
reference_path=reference_path,
raster_path=raster_path,
source_name="urbis",
min_label_px=3,
imagery_observed_at=None,
reference_observed_at=None,
merge_touching_roofs=True,
)
assert len(normalized["features"]) == 2
assert sorted(item["properties"]["source_feature_count"] for item in normalized["features"]) == [1, 2]
assert audit["accepted_source_feature_count"] == 3
assert audit["accepted_feature_count"] == 2
def test_visible_roof_merge_retains_large_touching_chains() -> None:
features = []
for index in range(13):
left = float(index)
features.append(
{
"type": "Feature",
"id": str(index),
"properties": {"source_feature_id": str(index)},
"geometry": {
"type": "Polygon",
"coordinates": [[[left, 0], [left + 1, 0], [left + 1, 1], [left, 1], [left, 0]]],
},
}
)
merged = module.merge_touching_roof_instances(features, "grb")
assert len(merged) == 13
assert {item["properties"]["label_semantics"] for item in merged} == {
"native_instance_complex_touch_group"
}
@@ -0,0 +1,26 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
SCRIPT = Path(__file__).parents[2] / "scripts" / "train_building_proposal_filter.py"
SPEC = importlib.util.spec_from_file_location("building_proposal_filter", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def test_evenly_limited_is_deterministic() -> None:
assert MODULE.evenly_limited(list(range(10)), 3) == [0, 3, 6]
def test_validation_threshold_is_selected_without_test_data() -> None:
threshold, metrics = MODULE.choose_threshold([0.9, 0.8, 0.2, 0.1], [1, 1, 0, 0])
assert 0.2 < threshold <= 0.8
assert metrics["f1"] == 1.0
def test_negative_match_iou() -> None:
assert MODULE.iou((0, 0, 10, 10), (0, 0, 10, 10)) == 1.0
assert MODULE.iou((0, 0, 10, 10), (20, 20, 30, 30)) == 0.0
@@ -0,0 +1,128 @@
"""Change detection must answer the question the operator actually asked.
Every other analysis in the workbench is bounded by the drawn selection.
Change detection was not: it compared two datasets in full, loaded every
feature of both into Python, and — with ``include_unchanged`` defaulting to
true — returned a FeatureCollection containing both datasets entire. For a
regional building layer that is the wrong answer to "what changed here" and a
response no browser should be asked to hold.
"""
from __future__ import annotations
import pytest
from shapely.geometry import box
from app.core.errors import AppError
from app.services.change_detection_service import ChangeDetectionService
def _feature(feature_id: str, geometry):
return {"feature_id": feature_id, "properties": {}, "geometry": geometry}
INSIDE = box(0.0, 0.0, 1.0, 1.0)
OUTSIDE = box(50.0, 50.0, 51.0, 51.0)
SELECTION = box(-1.0, -1.0, 2.0, 2.0)
def test_features_outside_the_selection_are_not_compared() -> None:
kept = ChangeDetectionService.restrict_to_selection(
[_feature("in", INSIDE), _feature("out", OUTSIDE)],
SELECTION,
)
assert [item["feature_id"] for item in kept] == ["in"]
def test_a_feature_crossing_the_selection_edge_is_kept_and_flagged() -> None:
crossing = box(1.5, 1.5, 3.0, 3.0)
kept = ChangeDetectionService.restrict_to_selection(
[_feature("crossing", crossing)],
SELECTION,
)
assert len(kept) == 1
assert kept[0]["partially_covered"] is True
# The geometry is not clipped: a change class describes a whole object, and
# comparing a clipped 2020 footprint with an unclipped 2024 one would
# invent change at the selection edge.
assert kept[0]["geometry"].equals(crossing)
def test_a_feature_wholly_inside_is_not_flagged() -> None:
kept = ChangeDetectionService.restrict_to_selection([_feature("in", INSIDE)], SELECTION)
assert kept[0]["partially_covered"] is False
def test_no_selection_leaves_the_population_untouched() -> None:
features = [_feature("in", INSIDE), _feature("out", OUTSIDE)]
assert ChangeDetectionService.restrict_to_selection(features, None) == features
def test_an_empty_intersection_is_an_explicit_error_not_a_silent_zero() -> None:
with pytest.raises(AppError) as exc_info:
ChangeDetectionService.restrict_to_selection([_feature("out", OUTSIDE)], SELECTION, label="Source")
assert exc_info.value.code == "CHANGE_DETECTION_SELECTION_EMPTY"
assert "Source" in exc_info.value.message
def test_the_preview_is_capped_while_the_counts_stay_complete() -> None:
features = [
{
"change_type": "added" if index % 2 else "removed",
"geometry": box(index, 0, index + 1, 1),
"source_feature_id": None,
"target_feature_id": f"t{index}",
"iou": None,
"properties": {},
}
for index in range(250)
]
preview, truncated = ChangeDetectionService.limit_preview(features, limit=100)
assert len(preview) == 100
assert truncated is True
def test_a_short_result_is_not_reported_as_truncated() -> None:
features = [
{
"change_type": "added",
"geometry": box(0, 0, 1, 1),
"source_feature_id": None,
"target_feature_id": "t",
"iou": None,
"properties": {},
}
]
preview, truncated = ChangeDetectionService.limit_preview(features, limit=100)
assert len(preview) == 1
assert truncated is False
def test_the_preview_prefers_changes_over_unchanged_features() -> None:
"""A cap must not spend its budget on the least interesting class."""
features = [
{"change_type": "unchanged", "geometry": box(index, 0, index + 1, 1), "source_feature_id": f"s{index}",
"target_feature_id": f"t{index}", "iou": 1.0, "properties": {}}
for index in range(100)
] + [
{"change_type": "added", "geometry": box(0, 5, 1, 6), "source_feature_id": None,
"target_feature_id": "new", "iou": None, "properties": {}},
{"change_type": "modified", "geometry": box(0, 7, 1, 8), "source_feature_id": "s",
"target_feature_id": "t", "iou": 0.6, "properties": {}},
]
preview, truncated = ChangeDetectionService.limit_preview(features, limit=3)
assert truncated is True
assert sorted(item["change_type"] for item in preview[:2]) == ["added", "modified"]
@@ -0,0 +1,114 @@
"""An extended building is a change, not a deletion plus a new building.
With only added/removed/unchanged, a footprint that grew by an annexe drops
below the IoU threshold and is reported twice: once as removed and once as
added. That hides exactly the category a change-detection product exists to
show, and inflates both counts.
"""
from __future__ import annotations
import pytest
from shapely.geometry import box
from app.services.change_detection_service import ChangeDetectionService
def _feature(feature_id: str, geometry):
return {"feature_id": feature_id, "properties": {}, "geometry": geometry}
def _classify(source, target, *, iou_threshold=0.8, modified_threshold=0.3):
return ChangeDetectionService._classify_features(
source,
target,
iou_threshold=iou_threshold,
modified_threshold=modified_threshold,
)
def test_an_extended_footprint_is_reported_as_modified() -> None:
source = [_feature("b1", box(0, 0, 10, 10))]
target = [_feature("b1-new", box(0, 0, 10, 14))] # IoU 100/140 = 0.71
result = _classify(source, target)
assert [item["change_type"] for item in result] == ["modified"]
assert result[0]["source_feature_id"] == "b1"
assert result[0]["target_feature_id"] == "b1-new"
assert result[0]["iou"] == pytest.approx(100 / 140)
def test_a_nearly_identical_footprint_is_unchanged() -> None:
source = [_feature("b1", box(0, 0, 10, 10))]
target = [_feature("b1", box(0, 0, 10, 10.2))]
result = _classify(source, target)
assert [item["change_type"] for item in result] == ["unchanged"]
def test_a_genuinely_new_building_stays_added() -> None:
source = [_feature("b1", box(0, 0, 10, 10))]
target = [_feature("b1", box(0, 0, 10, 10)), _feature("b2", box(50, 50, 60, 60))]
result = _classify(source, target)
assert sorted(item["change_type"] for item in result) == ["added", "unchanged"]
def test_a_demolished_building_stays_removed() -> None:
source = [_feature("b1", box(0, 0, 10, 10)), _feature("b2", box(50, 50, 60, 60))]
target = [_feature("b1", box(0, 0, 10, 10))]
result = _classify(source, target)
assert sorted(item["change_type"] for item in result) == ["removed", "unchanged"]
def test_barely_overlapping_footprints_are_not_called_modified() -> None:
"""Below the modified floor the two are separate objects, not one changed."""
source = [_feature("b1", box(0, 0, 10, 10))]
target = [_feature("b2", box(9, 9, 19, 19))] # IoU ~0.005
result = _classify(source, target)
assert sorted(item["change_type"] for item in result) == ["added", "removed"]
def test_each_target_is_claimed_at_most_once() -> None:
source = [_feature("a", box(0, 0, 10, 10)), _feature("b", box(0, 0, 10, 12))]
target = [_feature("t", box(0, 0, 10, 10))]
result = _classify(source, target)
claimed = [item for item in result if item["target_feature_id"] == "t"]
assert len(claimed) == 1
def test_classification_is_independent_of_input_order() -> None:
source = [_feature("a", box(0, 0, 10, 10)), _feature("b", box(30, 30, 40, 40))]
target = [_feature("a2", box(0, 0, 10, 14)), _feature("c", box(70, 70, 80, 80))]
forward = _classify(source, target)
reverse = _classify(list(reversed(source)), list(reversed(target)))
def signature(items):
return sorted(
(item["change_type"], item["source_feature_id"], item["target_feature_id"]) for item in items
)
assert signature(forward) == signature(reverse)
def test_large_populations_do_not_use_a_full_cross_product() -> None:
"""A spatial index keeps a city-sized comparison tractable."""
source = [_feature(f"s{i}", box(i * 10, 0, i * 10 + 8, 8)) for i in range(400)]
target = [_feature(f"t{i}", box(i * 10, 0, i * 10 + 8, 8)) for i in range(400)]
result = _classify(source, target)
assert all(item["change_type"] == "unchanged" for item in result)
assert len(result) == 400
@@ -0,0 +1,449 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from hashlib import sha256
import json
from pathlib import Path
import pytest
from shapely.geometry import box
from app.core.errors import AppError
from app.services.data_contract_validation import (
AttributeRule,
BoundingBox,
ContractKind,
DataAssetValidationInput,
DataContract,
DataContractRegistry,
DataContractValidator,
FreshnessRules,
GeometryRecord,
GeometryRules,
LineageEvidence,
LineageRules,
RasterRules,
RequirementLevel,
Resolution,
ResolutionRules,
TransformationEvidence,
ValidationStatus,
build_default_data_contract_registry,
build_label_validation_input,
build_model_validation_input,
build_raster_ingest_input,
build_vector_ingest_input,
validate_registered_asset,
)
from app.services.data_quarantine_service import AssetUse, DataQuarantineService
FIXTURE_ROOT = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "data-contracts"
NOW = datetime(2026, 8, 1, 12, 0, tzinfo=timezone.utc)
CHECKSUM_A = "a" * 64
def _fixture_json(name: str) -> tuple[bytes, object]:
raw = (FIXTURE_ROOT / name).read_bytes()
return raw, json.loads(raw)
def _checksum(raw: bytes) -> str:
return sha256(raw).hexdigest()
def _lineage_with_transform() -> LineageEvidence:
return LineageEvidence(
transformations=(
TransformationEvidence(
name="epsg31370-to-epsg4326",
version="1.0.0",
checksum_sha256=CHECKSUM_A,
),
),
)
def _vector_input_from_fixture(name: str, *, source_crs: str = "EPSG:31370", storage_crs: str = "EPSG:4326") -> DataAssetValidationInput:
raw, payload = _fixture_json(name)
assert isinstance(payload, dict)
return build_vector_ingest_input(
asset_id=f"fixture:{name}",
source_crs=source_crs,
storage_crs=storage_crs,
feature_collection=payload,
checksum_sha256=_checksum(raw),
computed_checksum_sha256=_checksum(raw),
content=raw,
source_registry_id="source:digitaal-vlaanderen:grb",
source_snapshot_id="snapshot:grb:2026-07-31",
imported_at=NOW,
metadata={"license": "Open Data Licence", "provider": "Digitaal Vlaanderen"},
observed_at=NOW - timedelta(days=1),
source_version="2026.07.31",
lineage=_lineage_with_transform() if source_crs != storage_crs else LineageEvidence(),
)
def _issue_codes(report) -> set[str]:
return {issue.code for issue in report.issues}
def test_default_vector_contract_accepts_transformed_geojson_with_complete_provenance() -> None:
report = validate_registered_asset(_vector_input_from_fixture("vector-building-valid.geojson"), now=NOW)
assert report.validation_status == ValidationStatus.PASSED
assert report.quarantine_status == "not_quarantined"
assert report.provenance_status == "complete"
assert report.lineage_status == "complete"
persisted = report.persistence_fields()
assert persisted["data_contract_key"] == "geointel.vector.geojson"
assert persisted["data_contract_version"] == "1.0.0"
assert persisted["validation_report_json"]["report_sha256"] == report.report_sha256
def test_default_vector_contract_quarantines_lambert_coordinates_mislabelled_as_epsg4326() -> None:
report = validate_registered_asset(
_vector_input_from_fixture(
"vector-lambert-mislabelled-as-4326.geojson",
source_crs="EPSG:4326",
storage_crs="EPSG:4326",
),
now=NOW,
)
assert report.validation_status == ValidationStatus.FAILED
assert report.quarantine_status == "quarantined"
assert "CRS_COORDINATE_DOMAIN_VIOLATION" in _issue_codes(report)
def test_vector_contract_checks_geometry_attributes_bounds_and_topology_fail_closed() -> None:
contract = DataContract(
key="test.vector.buildings",
version="1.0.0",
kind=ContractKind.VECTOR,
accepted_source_crs=frozenset({"EPSG:4326"}),
canonical_storage_crs="EPSG:4326",
spatial_domain=BoundingBox(2.0, 49.0, 7.0, 52.0),
require_bounds=True,
geometry_rules=GeometryRules(
allowed_geometry_types=frozenset({"Polygon"}),
attribute_rules=(AttributeRule("native_id", accepted_types=("integer",)),),
forbid_shared_area=True,
),
)
raw = b"overlapping-vector"
asset = DataAssetValidationInput(
asset_id="vector:bad",
data_contract_key=contract.key,
data_contract_version=contract.version,
kind=ContractKind.VECTOR,
source_crs="EPSG:4326",
storage_crs="EPSG:4326",
bounds=BoundingBox(4.0, 51.0, 4.1, 51.1),
checksum_sha256=_checksum(raw),
computed_checksum_sha256=_checksum(raw),
content=raw,
geometry_records=(
GeometryRecord(box(4.0, 51.0, 4.05, 51.05), {"native_id": "wrong-type"}),
GeometryRecord(box(4.025, 51.025, 4.075, 51.075), {}),
),
source_registry_id="source:test",
source_snapshot_id="snapshot:test",
imported_at=NOW,
)
report = DataContractValidator.validate(contract, asset, now=NOW)
assert report.validation_status == ValidationStatus.FAILED
assert {"ATTRIBUTE_TYPE_INVALID", "ATTRIBUTE_REQUIRED", "TOPOLOGY_SHARED_AREA"} <= _issue_codes(report)
assert "BOUNDS_GEOMETRY_MISMATCH" in _issue_codes(report)
def test_default_vector_contract_validates_replayable_large_partition_stream_without_materialising_geometry_list() -> None:
"""Regional imports may be large but remain fully schema/domain checked.
The default contract has no source-specific shared-area rule, so the
validator must make its bounds/schema passes over a replayable stream
without accumulating every Shapely geometry in memory. A stricter
source-specific contract can still opt into a bounded topology batch.
"""
class ReplayableRecords:
def __init__(self, count: int) -> None:
self.count = count
self.iterations = 0
def __iter__(self):
self.iterations += 1
for index in range(self.count):
yield GeometryRecord(
box(4.69, 51.09, 4.70, 51.10),
{"partition_feature": index},
)
raw = b"partitioned-vector-stream"
records = ReplayableRecords(12_000)
asset = DataAssetValidationInput(
asset_id="vector:partitioned-stream",
data_contract_key="geointel.vector.geojson",
data_contract_version="1.0.0",
kind=ContractKind.VECTOR,
source_crs="EPSG:4326",
storage_crs="EPSG:4326",
bounds=BoundingBox(4.69, 51.09, 4.70, 51.10),
checksum_sha256=_checksum(raw),
computed_checksum_sha256=_checksum(raw),
content=raw,
metadata={"license": "Open Data"},
geometry_records=records,
source_registry_id="source:grb",
source_snapshot_id="snapshot:grb:partitioned",
imported_at=NOW,
observed_at=NOW,
source_version="2026-08-01",
)
report = validate_registered_asset(asset, now=NOW)
assert report.validation_status == ValidationStatus.PASSED
assert records.iterations >= 2
def test_raster_contract_accepts_explicit_units_and_quarantines_stale_bad_profile() -> None:
raw = b"raster-stage"
valid = build_raster_ingest_input(
asset_id="raster:valid",
source_crs="EPSG:31370",
storage_crs="EPSG:31370",
raster_profile={"width": 512, "height": 512, "band_count": 3, "dtype": ["uint8"]},
bounds=BoundingBox(193_277.5, 205_708.3, 193_777.5, 206_208.3),
resolution=Resolution(0.9765625, 0.9765625, "m"),
checksum_sha256=_checksum(raw),
computed_checksum_sha256=_checksum(raw),
content=raw,
source_registry_id="source:orthophoto",
source_snapshot_id="snapshot:orthophoto:2026.01",
imported_at=NOW,
metadata={"license": "Open Data"},
observed_at=None,
temporal_unknown_reason="latest mosaic has no per-pixel observation date",
source_version=None,
source_version_unknown_reason="provider did not publish an edition",
)
assert validate_registered_asset(valid, now=NOW).validation_status == ValidationStatus.PASSED
strict = DataContract(
key="test.raster.strict",
version="1.0.0",
kind=ContractKind.RASTER,
accepted_source_crs=frozenset({"EPSG:31370"}),
require_bounds=True,
raster_rules=RasterRules(allowed_band_counts=frozenset({3}), allowed_dtypes=frozenset({"uint8"})),
resolution_rules=ResolutionRules(allowed_units=frozenset({"m"}), min_x=0.2, max_x=1.0, min_y=0.2, max_y=1.0),
freshness_rules=FreshnessRules(observed_at=RequirementLevel.REQUIRED, max_age=timedelta(days=30)),
lineage_rules=LineageRules(require_transformation_when_crs_changes=False),
)
invalid = DataAssetValidationInput(
asset_id="raster:bad",
data_contract_key=strict.key,
data_contract_version=strict.version,
kind=ContractKind.RASTER,
source_crs="EPSG:31370",
storage_crs="EPSG:31370",
bounds=BoundingBox(100.0, 100.0, 200.0, 200.0),
checksum_sha256=_checksum(raw),
computed_checksum_sha256=_checksum(raw),
content=raw,
raster_profile={"width": 0, "height": 10, "band_count": 2, "dtype": ["float32"]},
resolution=Resolution(2.0, 0.1, "degree"),
source_registry_id="source:raster",
source_snapshot_id="snapshot:raster",
imported_at=NOW,
observed_at=NOW - timedelta(days=31),
)
report = DataContractValidator.validate(strict, invalid, now=NOW)
assert report.validation_status == ValidationStatus.FAILED
assert {
"RASTER_PROFILE_VALUE_INVALID",
"RASTER_BAND_COUNT_NOT_ALLOWED",
"RASTER_DTYPE_NOT_ALLOWED",
"RESOLUTION_UNIT_NOT_ALLOWED",
"RESOLUTION_OUT_OF_RANGE",
"FRESHNESS_EXCEEDED",
} <= _issue_codes(report)
def test_default_label_and_model_contracts_validate_good_and_bad_fixtures() -> None:
valid_raw, valid_labels = _fixture_json("labels-valid.json")
invalid_raw, invalid_labels = _fixture_json("labels-invalid.json")
assert isinstance(valid_labels, list)
assert isinstance(invalid_labels, list)
lineage = LineageEvidence(upstream_asset_ids=("image:1",), upstream_checksums_sha256=(CHECKSUM_A,))
valid_label = build_label_validation_input(
asset_id="label:valid",
label_records=valid_labels,
checksum_sha256=_checksum(valid_raw),
computed_checksum_sha256=_checksum(valid_raw),
content=valid_raw,
source_registry_id="source:labels",
source_snapshot_id="snapshot:labels:1",
imported_at=NOW,
metadata={
"image_checksum_sha256": CHECKSUM_A,
"class_ontology_version": "buildings-v1",
"source_corpus_manifest_sha256": CHECKSUM_A,
},
temporal_unknown_reason="labels inherit image observation handling",
source_version_unknown_reason="label release is represented by its snapshot",
lineage=lineage,
)
valid_report = validate_registered_asset(valid_label, now=NOW)
assert valid_report.validation_status == ValidationStatus.PASSED
invalid_label = build_label_validation_input(
asset_id="label:invalid",
label_records=invalid_labels,
checksum_sha256=_checksum(invalid_raw),
computed_checksum_sha256=_checksum(invalid_raw),
content=invalid_raw,
source_registry_id="source:labels",
source_snapshot_id="snapshot:labels:1",
imported_at=NOW,
metadata={
"image_checksum_sha256": "not-a-sha256",
"class_ontology_version": "buildings-v1",
"source_corpus_manifest_sha256": CHECKSUM_A,
},
temporal_unknown_reason="labels inherit image observation handling",
source_version_unknown_reason="label release is represented by its snapshot",
lineage=lineage,
)
invalid_report = validate_registered_asset(invalid_label, now=NOW)
assert invalid_report.validation_status == ValidationStatus.FAILED
assert {
"LABEL_CLASS_ID_NOT_ALLOWED",
"LABEL_NORMALIZED_COORDINATE_INVALID",
"METADATA_CHECKSUM_INVALID",
} <= _issue_codes(invalid_report)
pure_background_raw = b""
pure_background_metadata = {
"image_checksum_sha256": CHECKSUM_A,
"class_ontology_version": "buildings-v1",
"source_corpus_manifest_sha256": CHECKSUM_A,
"label_mode": "pure_background",
"sample_slug": "forest-background-aoi",
"split": "train",
"raster_dataset_id": "dataset:raster:1",
"reference_dataset_id": "dataset:reference:1",
"review_decision": "accepted",
"reviewer_id": "reviewer@example.test",
"reviewed_at": "2026-08-01T11:00:00+00:00",
"review_artifact_sha256": CHECKSUM_A,
}
pure_background = build_label_validation_input(
asset_id="label:pure-background",
label_records=(),
label_mode="pure_background",
checksum_sha256=_checksum(pure_background_raw),
computed_checksum_sha256=_checksum(pure_background_raw),
content=pure_background_raw,
source_registry_id="source:labels",
source_snapshot_id="snapshot:labels:1",
imported_at=NOW,
metadata=pure_background_metadata,
temporal_unknown_reason="labels inherit image observation handling",
source_version_unknown_reason="label release is represented by its snapshot",
lineage=LineageEvidence(
upstream_asset_ids=("dataset:raster:1", "dataset:reference:1"),
upstream_checksums_sha256=(CHECKSUM_A, CHECKSUM_A),
),
)
assert validate_registered_asset(pure_background, now=NOW).validation_status == ValidationStatus.PASSED
unmarked_empty = build_label_validation_input(
asset_id="label:unmarked-empty",
label_records=(),
checksum_sha256=_checksum(pure_background_raw),
computed_checksum_sha256=_checksum(pure_background_raw),
content=pure_background_raw,
source_registry_id="source:labels",
source_snapshot_id="snapshot:labels:1",
imported_at=NOW,
metadata={
"image_checksum_sha256": CHECKSUM_A,
"class_ontology_version": "buildings-v1",
"source_corpus_manifest_sha256": CHECKSUM_A,
},
temporal_unknown_reason="labels inherit image observation handling",
source_version_unknown_reason="label release is represented by its snapshot",
lineage=LineageEvidence(
upstream_asset_ids=("dataset:raster:1", "dataset:reference:1"),
upstream_checksums_sha256=(CHECKSUM_A, CHECKSUM_A),
),
)
assert "PURE_BACKGROUND_MODE_REQUIRED" in _issue_codes(validate_registered_asset(unmarked_empty, now=NOW))
model_raw = b"model-asset"
model = build_model_validation_input(
asset_id="model:valid",
model_metadata={"model_format": "pytorch", "framework": "torch", "class_mapping": {"0": "building"}},
checksum_sha256=_checksum(model_raw),
computed_checksum_sha256=_checksum(model_raw),
content=model_raw,
source_registry_id="source:model-registry",
source_snapshot_id="snapshot:model:1",
imported_at=NOW,
source_version="candidate-1",
metadata={"training_manifest_sha256": CHECKSUM_A, "runtime_manifest_sha256": CHECKSUM_A},
lineage=lineage,
)
assert validate_registered_asset(model, now=NOW).validation_status == ValidationStatus.PASSED
def test_unknown_contract_and_quarantine_gate_are_deterministic_and_fail_closed() -> None:
unknown = DataAssetValidationInput(
asset_id="asset:unknown",
data_contract_key="does.not.exist",
data_contract_version="9.9.9",
kind=ContractKind.VECTOR,
)
report = DataContractRegistry().validate(unknown, now=NOW)
assert report.validation_status == ValidationStatus.FAILED
assert report.quarantine_status == "quarantined"
assert _issue_codes(report) == {"DATA_CONTRACT_UNKNOWN"}
first = DataQuarantineService.decide(report)
second = DataQuarantineService.decide(report)
assert first.idempotency_key == second.idempotency_key
assert first.reason_codes == ("DATA_CONTRACT_UNKNOWN",)
with pytest.raises(AppError, match="cannot enter this pipeline") as exc_info:
DataQuarantineService.require_eligible(first, use=AssetUse.PRODUCTION_INFERENCE)
assert exc_info.value.code == "DATASET_QUARANTINED"
assert exc_info.value.details["use"] == "production_inference"
clean_report = validate_registered_asset(_vector_input_from_fixture("vector-building-valid.geojson"), now=NOW)
release_request = DataQuarantineService.decide(clean_report, previous=first)
assert release_request.quarantine_status == "quarantined"
assert release_request.requires_explicit_release is True
assert release_request.reason_codes == ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",)
def test_registry_requires_exact_contract_version_and_fingerprints_schema() -> None:
registry = build_default_data_contract_registry()
version_mismatch = _vector_input_from_fixture("vector-building-valid.geojson")
mismatched = DataAssetValidationInput(
**{**version_mismatch.__dict__, "data_contract_version": "2.0.0"},
)
report = registry.validate(mismatched, now=NOW)
assert report.validation_status == ValidationStatus.FAILED
assert "DATA_CONTRACT_UNKNOWN" in _issue_codes(report)
contract = registry.resolve("geointel.vector.geojson", "1.0.0")
assert contract is not None
direct_report = DataContractValidator.validate(contract, mismatched, now=NOW)
assert direct_report.validation_status == ValidationStatus.FAILED
assert "DATA_CONTRACT_IDENTITY_MISMATCH" in _issue_codes(direct_report)
@@ -0,0 +1,327 @@
from __future__ import annotations
from types import SimpleNamespace
from uuid import uuid4
import pytest
from shapely.geometry import box
from app.core.errors import AppError
from app.models import Dataset, SourceRegistry, SourceSnapshot
from app.services.coverage_registry_service import CoverageRegistryService, SOURCE_DEFINITIONS
import app.services.dataset_consumption_gate_service as gate_module
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.export_service import ExportService
def _governed_dataset(
*,
source_key: str = "grb",
classification: str = "authoritative",
snapshot_freshness_status: str = "current",
) -> Dataset:
source_id = uuid4()
snapshot_id = uuid4()
checksum = "a" * 64
source = SourceRegistry(
id=source_id,
source_key=source_key,
display_name=f"{source_key} test source",
classification=classification,
authority_name="GeoIntel test authority",
authority_scope_json={"scope": "test"},
usage_policy_json={
"ground_truth_allowed": classification == "authoritative",
"validation_authority": {"building_validation": "primary"}
if classification == "authoritative"
else {},
},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key="test-snapshot",
checksum_sha256=checksum,
freshness_status=snapshot_freshness_status,
ingest_status="ingested",
)
dataset = Dataset(
id=uuid4(),
project_id=uuid4(),
name="governed.tif",
dataset_type="raster",
source=source_key,
source_name=source_key,
dataset_role="source",
checksum_sha256=checksum,
source_registry_id=source_id,
source_snapshot_id=snapshot_id,
data_contract_key="geointel.raster.geotiff",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
status="ready",
)
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
def test_governed_dataset_passes_production_inference_and_authoritative_coverage() -> None:
dataset = _governed_dataset()
inference = DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
coverage = DatasetConsumptionGate.assert_eligible(dataset, purpose="authoritative_coverage")
assert inference.eligible is True
assert coverage.eligible is True
@pytest.mark.parametrize(
("field", "value", "error_code"),
(
("provenance_status", "incomplete", "DATASET_PROVENANCE_INCOMPLETE"),
("validation_status", "failed", "DATASET_QUARANTINED"),
("quarantine_status", "quarantined", "DATASET_QUARANTINED"),
),
)
def test_explicit_unsafe_states_can_never_be_relaxed(field: str, value: str, error_code: str) -> None:
dataset = _governed_dataset()
setattr(dataset, field, value)
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(
dataset,
purpose="production_inference",
fixture_mode=True,
)
assert exc_info.value.code == error_code
assert field.replace("_status", "") in " ".join(exc_info.value.details["reasons"])
def test_legacy_fixture_can_support_fixture_qa_but_never_authoritative_coverage() -> None:
fixture = Dataset(
id=uuid4(),
project_id=uuid4(),
name="fixture.tif",
dataset_type="raster",
source="fixture",
)
qa = DatasetConsumptionGate.assert_eligible(fixture, purpose="quality_assessment")
with pytest.raises(AppError) as inference_error:
DatasetConsumptionGate.assert_eligible(
fixture,
purpose="production_inference",
fixture_mode=True,
)
with pytest.raises(AppError) as export_error:
DatasetConsumptionGate.assert_eligible(fixture, purpose="export")
with pytest.raises(AppError) as fixture_export_error:
DatasetConsumptionGate.assert_eligible(fixture, purpose="export", fixture_mode=True)
coverage = DatasetConsumptionGate.evaluate(fixture, purpose="authoritative_coverage")
assert qa.fixture_legacy_exception is True
assert inference_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "fixture_qa_only" in inference_error.value.details["reasons"]
assert export_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert fixture_export_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "fixture_qa_only" in fixture_export_error.value.details["reasons"]
assert coverage.eligible is False
assert "fixture_not_authoritative_coverage" in coverage.reasons
def test_unprovenanced_persistent_dataset_is_blocked(monkeypatch) -> None:
dataset = Dataset(
id=uuid4(),
project_id=uuid4(),
name="manual.tif",
dataset_type="raster",
source="manual_upload",
)
monkeypatch.setattr(gate_module, "sa_inspect", lambda _dataset: SimpleNamespace(transient=False))
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(
dataset,
purpose="production_inference",
fixture_mode=True,
)
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "phase2_provenance_missing" in exc_info.value.details["reasons"]
assert "fixture_source_required" in exc_info.value.details["reasons"]
def test_transient_orm_test_double_can_only_bypass_missing_legacy_fields_for_qa() -> None:
transient = Dataset(
id=uuid4(),
project_id=uuid4(),
name="transient-test.tif",
dataset_type="raster",
source="manual_upload",
)
decision = DatasetConsumptionGate.assert_eligible(transient, purpose="quality_assessment")
coverage = DatasetConsumptionGate.evaluate(transient, purpose="authoritative_coverage")
with pytest.raises(AppError) as production_error:
DatasetConsumptionGate.assert_eligible(transient, purpose="production_inference")
assert decision.fixture_legacy_exception is True
assert production_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert coverage.eligible is False
assert "phase2_provenance_missing" in coverage.reasons
@pytest.mark.parametrize("purpose", ("production_inference", "derived_processing", "export"))
def test_passed_manual_or_experimental_dataset_cannot_cross_production_boundary(purpose: str) -> None:
"""A syntactically valid manual upload remains experimental, never production-ready."""
manual = _governed_dataset(source_key="manual", classification="experimental")
manual.source = "manual_upload"
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(manual, purpose=purpose) # type: ignore[arg-type]
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
def test_fully_governed_demo_fixture_still_cannot_enter_production_inference() -> None:
fixture = _governed_dataset(source_key="fixture", classification="experimental")
fixture.source_metadata = {"fixture": True, "usage": "offline demo raster workflow only"}
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(fixture, purpose="production_inference")
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
def test_reference_validation_requires_authoritative_ground_truth_reference() -> None:
reference = _governed_dataset()
reference.dataset_type = "vector"
reference.dataset_role = "reference"
decision = DatasetConsumptionGate.assert_eligible(
reference,
purpose="reference_validation",
reference_task="building_validation",
)
assert decision.eligible is True
reference.source_registry.classification = "corroborative"
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(
reference,
purpose="reference_validation",
reference_task="building_validation",
)
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "reference_source_not_authoritative" in exc_info.value.details["reasons"]
def test_pending_regional_building_authority_cannot_become_truth_without_approval() -> None:
reference = _governed_dataset(source_key="spw_picc", classification="authoritative")
reference.dataset_type = "vector"
reference.dataset_role = "reference"
reference.source_registry.authority_scope_json = {"zone": "Wallonia"}
reference.source_registry.usage_policy_json = {
"ground_truth_allowed": True,
"validation_authority": {"building_validation": "regional_primary_pending_contract"},
}
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(
reference,
purpose="reference_validation",
reference_task="building_validation",
)
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "reference_task_authority_not_approved" in exc_info.value.details["reasons"]
def test_source_snapshot_must_belong_to_the_dataset_source_registry() -> None:
dataset = _governed_dataset()
dataset.source_snapshot.source_registry_id = uuid4()
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "source_snapshot_registry_mismatch" in exc_info.value.details["reasons"]
@pytest.mark.parametrize("freshness_status", ("unknown", "review_required", "due", "stale"))
def test_non_consumable_source_snapshot_freshness_is_blocked_at_production_boundaries(
freshness_status: str,
) -> None:
dataset = _governed_dataset(snapshot_freshness_status=freshness_status)
for purpose in ("production_inference", "authoritative_coverage"):
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(dataset, purpose=purpose) # type: ignore[arg-type]
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "source_snapshot_freshness_not_eligible" in exc_info.value.details["reasons"]
def test_coverage_registry_ignores_explicitly_incomplete_materialization() -> None:
definition = next(item for item in SOURCE_DEFINITIONS if item.contract.source_name == "digitaal_vlaanderen")
unsafe_materialization = SimpleNamespace(
id=uuid4(),
status="ready",
source_name="grb",
validation_status="passed",
provenance_status="incomplete",
lineage_status="complete",
quarantine_status="not_quarantined",
)
matches, fully_covered = CoverageRegistryService._matching_datasets(
[unsafe_materialization],
definition,
"buildings",
"flanders",
box(4.0, 50.8, 4.1, 50.9),
)
assert matches == []
assert fully_covered is False
def test_vector_export_is_fail_closed_before_selection(monkeypatch) -> None:
dataset = _governed_dataset()
dataset.dataset_type = "vector"
dataset.status = "quarantined"
queried = False
class _Session:
@staticmethod
def get(model, item_id):
return dataset if model is Dataset and item_id == dataset.id else None
def _unexpected_selection(*_args, **_kwargs):
nonlocal queried
queried = True
raise AssertionError("unsafe dataset must be rejected before querying vector features")
monkeypatch.setattr(
"app.services.export_service.VectorFeatureService.select_features_by_bbox",
_unexpected_selection,
)
with pytest.raises(AppError) as exc_info:
ExportService.export_vector_selection_geojson(
_Session(),
dataset.id,
{"min_x": 4.0, "min_y": 50.0, "max_x": 4.1, "max_y": 50.1, "crs": "EPSG:4326"},
)
assert exc_info.value.code == "DATASET_QUARANTINED"
assert queried is False
@@ -0,0 +1,125 @@
"""A precise failure must not be reported as a generic one.
``get_dataset_geojson`` wrapped the JSON parse, the metadata read, the CRS
resolution and the canonicalisation in one ``try``, and reported everything as
"Stored dataset is not valid JSON" with a 500. An operator whose dataset has an
unusable CRS was sent to inspect a file that parses perfectly well, and the
specific AppError the canonicaliser raised — with its own code and status —
never reached them.
"""
from __future__ import annotations
import json
from pathlib import Path
from uuid import uuid4
import pytest
from app.core.errors import AppError
from app.models import Dataset
from app.services.dataset_service import DatasetService
from app.services.vector_feature_service import VectorFeatureService
class FakeSession:
def __init__(self, dataset: Dataset) -> None:
self.dataset = dataset
def get(self, _model, item_id):
return self.dataset if item_id == self.dataset.id else None
def _dataset(tmp_path: Path, payload: str) -> Dataset:
path = tmp_path / "features.geojson"
path.write_text(payload, encoding="utf-8")
return Dataset(
id=uuid4(),
project_id=uuid4(),
name="features.geojson",
dataset_type="geojson",
source="test",
status="ready",
storage_path=str(path),
crs="EPSG:4326",
)
def test_a_valid_dataset_is_returned(tmp_path: Path) -> None:
dataset = _dataset(
tmp_path,
json.dumps(
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
}
],
}
),
)
result = DatasetService.get_dataset_geojson(FakeSession(dataset), dataset.id)
assert result["type"] == "FeatureCollection"
def test_unparseable_bytes_are_reported_as_invalid_json(tmp_path: Path) -> None:
dataset = _dataset(tmp_path, "{ not json")
with pytest.raises(AppError) as exc_info:
DatasetService.get_dataset_geojson(FakeSession(dataset), dataset.id)
assert exc_info.value.code == "INVALID_GEOJSON"
assert "JSON" in exc_info.value.message
def test_a_parseable_file_that_is_not_a_feature_collection_says_so(tmp_path: Path) -> None:
"""Valid JSON, wrong shape. Telling the operator it is not JSON sends them
to inspect a file that parses perfectly well."""
dataset = _dataset(tmp_path, json.dumps({"type": "Feature", "geometry": None}))
with pytest.raises(AppError) as exc_info:
DatasetService.get_dataset_geojson(FakeSession(dataset), dataset.id)
assert exc_info.value.code == "INVALID_GEOJSON"
assert "FeatureCollection" in exc_info.value.message
# The canonicaliser's own status survives; it is a bad request, not a
# server fault.
assert exc_info.value.status_code == 400
def test_a_canonicalisation_failure_keeps_its_own_code(tmp_path: Path, monkeypatch) -> None:
dataset = _dataset(tmp_path, json.dumps({"type": "FeatureCollection", "features": []}))
def failing(*_args, **_kwargs):
raise AppError(code="INVALID_CRS", message="Unusable source CRS", status_code=409)
monkeypatch.setattr(VectorFeatureService, "canonicalize_geojson_payload", staticmethod(failing))
with pytest.raises(AppError) as exc_info:
DatasetService.get_dataset_geojson(FakeSession(dataset), dataset.id)
assert exc_info.value.code == "INVALID_CRS"
assert exc_info.value.status_code == 409
def test_an_unexpected_failure_still_becomes_a_server_error(tmp_path: Path, monkeypatch) -> None:
"""A genuine bug must not masquerade as a client mistake."""
dataset = _dataset(tmp_path, json.dumps({"type": "FeatureCollection", "features": []}))
def exploding(*_args, **_kwargs):
raise RuntimeError("pyproj blew up")
monkeypatch.setattr(VectorFeatureService, "canonicalize_geojson_payload", staticmethod(exploding))
with pytest.raises(AppError) as exc_info:
DatasetService.get_dataset_geojson(FakeSession(dataset), dataset.id)
assert exc_info.value.status_code == 500
assert exc_info.value.code == "DATASET_GEOJSON_UNREADABLE"
@@ -0,0 +1,119 @@
"""Calibrating a confidence threshold does not need one inference run per value.
The workbench ran the model over every tile once per threshold — three GPU
passes to compare 0.50, 0.25 and 0.15. The answer is already in a single run at
the lowest value: detections above a higher cut are a subset of it, and
suppression walks candidates in descending confidence, so a lower-confidence
box can never displace a higher-confidence one. The kept set above any cut is
therefore identical whichever threshold the run used.
One matching pass produces every operating point exactly, so the sweep is free
rather than N times the cost.
"""
from __future__ import annotations
import pytest
from shapely.geometry import box
from app.services.detection_metrics_service import DetectionMetricsService
def _candidate(name: str, geometry, confidence: float):
return ({"id": name, "confidence": confidence}, geometry)
def _reference(name: str, geometry):
return ({"id": name}, geometry)
REFERENCES = [
_reference("r1", box(0, 0, 1, 1)),
_reference("r2", box(5, 5, 6, 6)),
_reference("r3", box(10, 10, 11, 11)),
]
CANDIDATES = [
_candidate("hit-high", box(0, 0, 1, 1), 0.90),
_candidate("hit-mid", box(5, 5, 6, 6), 0.40),
_candidate("junk-low", box(30, 30, 31, 31), 0.20),
]
def _curve():
return DetectionMetricsService.precision_recall_curve(CANDIDATES, REFERENCES, iou_threshold=0.5)
class TestOperatingPoints:
def test_a_strict_cut_keeps_only_the_confident_detection(self) -> None:
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.5)
assert point["true_positives"] == 1
assert point["false_positives"] == 0
assert point["false_negatives"] == 2
assert point["precision"] == pytest.approx(1.0)
assert point["recall"] == pytest.approx(1 / 3)
def test_a_looser_cut_finds_more_and_stays_exact(self) -> None:
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.3)
assert point["true_positives"] == 2
assert point["false_positives"] == 0
assert point["recall"] == pytest.approx(2 / 3)
def test_the_loosest_cut_admits_the_false_positive(self) -> None:
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.1)
assert point["true_positives"] == 2
assert point["false_positives"] == 1
assert point["precision"] == pytest.approx(2 / 3)
def test_a_cut_above_every_detection_finds_nothing_but_still_reports(self) -> None:
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.99)
assert point["true_positives"] == 0
assert point["false_negatives"] == 3
assert point["recall"] == pytest.approx(0.0)
assert point["precision"] is None
def test_the_requested_threshold_is_echoed_back(self) -> None:
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.42)
assert point["min_confidence"] == pytest.approx(0.42)
# The nearest actual operating point sits at the detection's own
# confidence, which is what the numbers describe.
assert point["confidence_threshold"] == pytest.approx(0.9)
class TestSweep:
def test_a_sweep_returns_one_row_per_requested_threshold(self) -> None:
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.5, 0.3, 0.1])
assert [row["min_confidence"] for row in rows] == [0.5, 0.3, 0.1]
def test_the_sweep_is_ordered_from_strict_to_loose(self) -> None:
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.1, 0.5, 0.3])
assert [row["min_confidence"] for row in rows] == [0.5, 0.3, 0.1]
def test_a_repeated_threshold_is_asked_once(self) -> None:
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.3, 0.3])
assert len(rows) == 1
def test_recall_never_falls_as_the_cut_loosens(self) -> None:
"""The monotonicity that makes one run sufficient."""
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.9, 0.5, 0.3, 0.1])
recalls = [row["recall"] for row in rows]
assert recalls == sorted(recalls)
def test_an_empty_sweep_is_not_an_error(self) -> None:
assert DetectionMetricsService.calibration_sweep(_curve(), thresholds=[]) == []
def test_the_sweep_marks_the_f1_optimal_row(self) -> None:
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.5, 0.3, 0.1])
best = [row for row in rows if row["best_f1_in_sweep"]]
assert len(best) == 1
assert best[0]["f1_score"] == max(row["f1_score"] for row in rows)
@@ -0,0 +1,73 @@
"""Inference must refuse to guess a CRS.
Detection QA rejects a tile without explicit CRS metadata, but the inference
side silently assumed EPSG:4326. That produced geometry that renders as a
plausible polygon in the wrong place, which is worse than a clear failure:
"fail closed" is the stated rule for the runtime.
"""
from __future__ import annotations
import pytest
from app.core.errors import AppError
from app.services.detection_georeferencing import (
pixel_bbox_to_epsg4326_polygon,
pixel_points_to_epsg4326_polygon,
)
from app.services.detection_service import DetectionService
TILE_WITHOUT_CRS = {
"bounds": [4.0, 51.0, 5.0, 52.0],
"pixel_window": [0, 0, 100, 100],
}
def test_manifest_without_crs_is_rejected() -> None:
with pytest.raises(AppError) as exc_info:
DetectionService._require_manifest_crs({"tiles": [{"bounds": [0, 0, 1, 1]}]})
assert exc_info.value.code == "DETECTION_TILE_MANIFEST_INVALID"
def test_manifest_crs_is_read_from_any_of_the_documented_keys() -> None:
assert DetectionService._require_manifest_crs({"crs": "EPSG:31370"}) == "EPSG:31370"
assert DetectionService._require_manifest_crs({"source_crs": "EPSG:31370"}) == "EPSG:31370"
assert DetectionService._require_manifest_crs({"dataset_crs": "EPSG:3812"}) == "EPSG:3812"
def test_bbox_georeferencing_requires_an_explicit_crs() -> None:
with pytest.raises(AppError) as exc_info:
pixel_bbox_to_epsg4326_polygon(bbox=[0.0, 0.0, 10.0, 10.0], tile=TILE_WITHOUT_CRS)
assert exc_info.value.code == "DETECTION_TILE_CRS_REQUIRED"
def test_mask_georeferencing_requires_an_explicit_crs() -> None:
with pytest.raises(AppError) as exc_info:
pixel_points_to_epsg4326_polygon(
points=[[0.0, 0.0], [10.0, 0.0], [10.0, 10.0]], tile=TILE_WITHOUT_CRS
)
assert exc_info.value.code == "DETECTION_TILE_CRS_REQUIRED"
def test_explicit_crs_on_the_tile_is_used() -> None:
tile = {**TILE_WITHOUT_CRS, "crs": "EPSG:4326"}
polygon = pixel_bbox_to_epsg4326_polygon(bbox=[0.0, 0.0, 50.0, 50.0], tile=tile)
assert polygon.bounds == pytest.approx((4.0, 51.5, 4.5, 52.0))
def test_projected_bounds_are_reprojected_as_a_whole_rectangle() -> None:
# Lambert 72 around Mol. All four corners must be transformed, otherwise a
# rotated footprint is understated.
bounds = DetectionService._bounds_to_epsg4326([200000.0, 200000.0, 201000.0, 201000.0], "EPSG:31370")
assert bounds is not None
min_x, min_y, max_x, max_y = bounds
assert 4.0 < min_x < 6.0
assert 50.0 < min_y < 52.0
assert max_x > min_x and max_y > min_y
@@ -0,0 +1,254 @@
"""Comparing two models must not compare two different questions.
The workbench ranks model variants by a stored F1, each measured at that
model's own confidence threshold. A conservatively calibrated model then looks
worse than a liberal one without detecting anything differently — the number
says as much about the threshold as about the model.
It also says nothing about whether the two runs are comparable at all. Two runs
over different rasters, or with different inference coverage, produce numbers
that cannot be placed side by side however they were measured.
"""
from __future__ import annotations
from uuid import uuid4
import pytest
from app.core.errors import AppError
from app.services.detection_comparison_service import DetectionComparisonService
def _entry(
*,
dataset_id,
model_asset_id: str = "asset-a",
coverage_mode: str = "persisted_tile_manifest_union",
reference_dataset_id=None,
reference_evaluated: int = 100,
):
return {
"analysis_run_id": uuid4(),
"dataset_id": dataset_id,
"model_id": "yolo-configured",
"model_asset_id": model_asset_id,
"reference_dataset_id": reference_dataset_id or uuid4(),
"coverage_mode": coverage_mode,
"reference_evaluated_count": reference_evaluated,
}
class TestComparability:
def test_runs_over_the_same_raster_and_reference_are_comparable(self) -> None:
dataset_id, reference_id = uuid4(), uuid4()
entries = [
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a"),
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="b"),
]
report = DetectionComparisonService.assess_comparability(entries)
assert report["comparable"] is True
assert report["blocking_reasons"] == []
def test_runs_over_different_rasters_are_not_comparable(self) -> None:
reference_id = uuid4()
entries = [
_entry(dataset_id=uuid4(), reference_dataset_id=reference_id),
_entry(dataset_id=uuid4(), reference_dataset_id=reference_id),
]
report = DetectionComparisonService.assess_comparability(entries)
assert report["comparable"] is False
assert "different_source_raster" in report["blocking_reasons"]
def test_runs_scored_against_different_references_are_not_comparable(self) -> None:
dataset_id = uuid4()
entries = [_entry(dataset_id=dataset_id), _entry(dataset_id=dataset_id)]
report = DetectionComparisonService.assess_comparability(entries)
assert report["comparable"] is False
assert "different_reference_dataset" in report["blocking_reasons"]
def test_a_run_without_proven_coverage_is_flagged(self) -> None:
dataset_id, reference_id = uuid4(), uuid4()
entries = [
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a"),
_entry(
dataset_id=dataset_id,
reference_dataset_id=reference_id,
model_asset_id="b",
coverage_mode="unbounded_no_manifest",
),
]
report = DetectionComparisonService.assess_comparability(entries)
assert report["comparable"] is False
assert "coverage_not_proven" in report["blocking_reasons"]
def test_a_differing_evaluated_population_is_flagged(self) -> None:
"""Same raster and reference, but the runs did not see the same ground."""
dataset_id, reference_id = uuid4(), uuid4()
entries = [
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a", reference_evaluated=100),
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="b", reference_evaluated=60),
]
report = DetectionComparisonService.assess_comparability(entries)
assert report["comparable"] is False
assert "different_evaluated_population" in report["blocking_reasons"]
def test_one_run_is_never_a_comparison(self) -> None:
with pytest.raises(AppError) as exc_info:
DetectionComparisonService.assess_comparability([_entry(dataset_id=uuid4())])
assert exc_info.value.code == "DETECTION_COMPARISON_NEEDS_TWO_RUNS"
class TestRanking:
def _row(self, name: str, *, ap: float, best_f1: float, threshold_f1: float):
return {
"model_asset_id": name,
"average_precision": ap,
"best_f1": best_f1,
"best_f1_threshold": 0.3,
"f1_at_run_threshold": threshold_f1,
}
def test_ranking_uses_average_precision_not_the_run_threshold_f1(self) -> None:
rows = [
self._row("liberal", ap=0.55, best_f1=0.60, threshold_f1=0.61),
self._row("conservative", ap=0.72, best_f1=0.71, threshold_f1=0.44),
]
ranked = DetectionComparisonService.rank(rows)
# The conservative model detects better; its stored F1 only looked worse
# because it was measured at a stricter cut.
assert [row["model_asset_id"] for row in ranked] == ["conservative", "liberal"]
assert ranked[0]["rank"] == 1
def test_every_row_states_how_far_behind_the_leader_it_is(self) -> None:
rows = [
self._row("a", ap=0.72, best_f1=0.71, threshold_f1=0.44),
self._row("b", ap=0.55, best_f1=0.60, threshold_f1=0.61),
]
ranked = DetectionComparisonService.rank(rows)
assert ranked[0]["average_precision_gap"] == pytest.approx(0.0)
assert ranked[1]["average_precision_gap"] == pytest.approx(0.17)
def test_only_the_leader_carries_a_lead(self) -> None:
"""A follower's gap must never be readable as an advantage."""
rows = [
self._row("a", ap=0.72, best_f1=0.71, threshold_f1=0.44),
self._row("b", ap=0.55, best_f1=0.60, threshold_f1=0.61),
]
ranked = DetectionComparisonService.rank(rows)
assert ranked[0]["lead_over_next"] == pytest.approx(0.17)
assert ranked[1]["lead_over_next"] is None
def test_a_tied_leader_claims_no_lead(self) -> None:
rows = [
self._row("a", ap=0.6, best_f1=0.6, threshold_f1=0.6),
self._row("b", ap=0.6, best_f1=0.5, threshold_f1=0.5),
]
assert all(row["lead_over_next"] is None for row in DetectionComparisonService.rank(rows))
def test_a_tie_is_reported_as_a_tie_rather_than_an_arbitrary_winner(self) -> None:
rows = [
self._row("a", ap=0.6, best_f1=0.6, threshold_f1=0.6),
self._row("b", ap=0.6, best_f1=0.5, threshold_f1=0.5),
]
ranked = DetectionComparisonService.rank(rows)
assert [row["rank"] for row in ranked] == [1, 1]
assert all(row["tied"] for row in ranked)
class TestComparingRealRuns:
"""Through the same QA path the workbench uses, so the two cannot drift."""
def test_two_runs_are_scored_ranked_and_judged_comparable(self, monkeypatch) -> None:
from app.services.detection_comparison_service import DetectionComparisonService as Service
from app.services.detection_service import DetectionService
dataset_id, reference_id = uuid4(), uuid4()
run_a, run_b = uuid4(), uuid4()
class _Run:
def __init__(self, asset: str, threshold: float) -> None:
self.dataset_id = dataset_id
self.model_name = "yolo-configured"
self.parameters_json = {"model_asset_id": asset, "confidence_threshold": threshold}
# Both runs post-processed identically, so they stay comparable.
self.result_json = {
"containment_suppression_threshold": 0.85,
"duplicate_iou_threshold": 0.5,
}
runs = {run_a: _Run("liberal", 0.15), run_b: _Run("conservative", 0.45)}
scores = {
run_a: {"average_precision": 0.55, "best_f1": 0.60, "f1": 0.61},
run_b: {"average_precision": 0.72, "best_f1": 0.71, "f1": 0.44},
}
class _Session:
def get(self, _model, item_id):
return runs.get(item_id)
def fake_qa(_db, *, analysis_run_id, reference_dataset_id, iou_threshold, **_kwargs):
score = scores[analysis_run_id]
return {
"quality_check_id": str(uuid4()),
"f1_score": score["f1"],
"precision": 0.6,
"recall": 0.6,
"coverage": {"mode": "persisted_tile_manifest_union", "reference_evaluated_count": 100},
"precision_recall_curve": {
"average_precision": score["average_precision"],
"best_f1": score["best_f1"],
"best_f1_threshold": 0.3,
},
}
monkeypatch.setattr(DetectionService, "compare_detections_with_reference", staticmethod(fake_qa))
report = Service.compare_runs(
_Session(),
analysis_run_ids=[run_a, run_b],
reference_dataset_id=reference_id,
iou_threshold=0.5,
)
assert report["comparability"]["comparable"] is True
assert report["ranking_metric"] == "average_precision"
# Ranked on AP, so the conservative model leads despite the lower F1 at
# its own threshold — which stays visible next to it.
assert [row["model_asset_id"] for row in report["rows"]] == ["conservative", "liberal"]
assert report["rows"][0]["f1_at_run_threshold"] == pytest.approx(0.44)
assert report["rows"][1]["f1_at_run_threshold"] == pytest.approx(0.61)
def test_one_run_twice_is_refused(self) -> None:
run_id = uuid4()
with pytest.raises(AppError) as exc_info:
DetectionComparisonService.compare_runs(
object(),
analysis_run_ids=[run_id, run_id],
reference_dataset_id=uuid4(),
)
assert exc_info.value.code == "DETECTION_COMPARISON_NEEDS_TWO_RUNS"
@@ -0,0 +1,116 @@
"""A single F1 at one arbitrary confidence cut cannot compare two models.
Reporting F1 at whichever threshold the operator happened to type makes two
models look better or worse depending on their calibration rather than their
detection quality. The sweep produces the standard curve instead: precision
and recall at every operating point, average precision, and the threshold
where F1 actually peaks.
"""
from __future__ import annotations
import pytest
from shapely.geometry import box
from app.services.detection_metrics_service import DetectionMetricsService
def _candidate(name: str, geometry, confidence: float):
return ({"id": name, "confidence": confidence}, geometry)
def _reference(name: str, geometry):
return ({"id": name}, geometry)
def test_perfect_detector_reaches_average_precision_one() -> None:
references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(5, 5, 6, 6))]
candidates = [
_candidate("c1", box(0, 0, 1, 1), 0.9),
_candidate("c2", box(5, 5, 6, 6), 0.8),
]
curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
assert curve["average_precision"] == pytest.approx(1.0)
assert curve["best_f1"] == pytest.approx(1.0)
assert curve["reference_count"] == 2
def test_low_confidence_false_positive_is_only_penalised_below_its_threshold() -> None:
references = [_reference("r1", box(0, 0, 1, 1))]
candidates = [
_candidate("hit", box(0, 0, 1, 1), 0.9),
_candidate("junk", box(20, 20, 21, 21), 0.2),
]
curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
# Cutting at 0.2 admits the junk box, so precision there is 0.5.
low = next(point for point in curve["points"] if point["confidence_threshold"] == pytest.approx(0.2))
assert low["precision"] == pytest.approx(0.5)
assert low["recall"] == pytest.approx(1.0)
# The optimum simply drops it.
assert curve["best_f1"] == pytest.approx(1.0)
assert curve["best_f1_threshold"] == pytest.approx(0.9)
# AP is computed over the ranking, so one trailing false positive after
# full recall does not reduce it.
assert curve["average_precision"] == pytest.approx(1.0)
def test_ranking_quality_is_visible_in_average_precision() -> None:
"""A detector that ranks its mistake above its hit scores worse."""
references = [_reference("r1", box(0, 0, 1, 1))]
good = DetectionMetricsService.precision_recall_curve(
[_candidate("hit", box(0, 0, 1, 1), 0.9), _candidate("junk", box(20, 20, 21, 21), 0.1)],
references,
iou_threshold=0.5,
)
bad = DetectionMetricsService.precision_recall_curve(
[_candidate("hit", box(0, 0, 1, 1), 0.1), _candidate("junk", box(20, 20, 21, 21), 0.9)],
references,
iou_threshold=0.5,
)
assert good["average_precision"] > bad["average_precision"]
assert bad["average_precision"] == pytest.approx(0.5)
def test_missed_reference_caps_recall_and_average_precision() -> None:
references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(9, 9, 10, 10))]
candidates = [_candidate("hit", box(0, 0, 1, 1), 0.9)]
curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
assert curve["points"][0]["recall"] == pytest.approx(0.5)
assert curve["average_precision"] == pytest.approx(0.5)
assert curve["best_f1"] == pytest.approx(2 / 3)
def test_curve_is_independent_of_input_order() -> None:
references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(5, 5, 6, 6))]
candidates = [
_candidate("c1", box(0, 0, 1, 1), 0.9),
_candidate("c2", box(5, 5, 6, 6), 0.4),
_candidate("c3", box(30, 30, 31, 31), 0.6),
]
forward = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
reverse = DetectionMetricsService.precision_recall_curve(
list(reversed(candidates)), list(reversed(references)), iou_threshold=0.5
)
assert forward == reverse
def test_empty_candidate_population_is_reported_not_crashed() -> None:
curve = DetectionMetricsService.precision_recall_curve(
[], [_reference("r1", box(0, 0, 1, 1))], iou_threshold=0.5
)
assert curve["average_precision"] == 0.0
assert curve["best_f1"] == 0.0
assert curve["best_f1_threshold"] is None
assert curve["points"] == []
@@ -0,0 +1,78 @@
"""Every accuracy figure shown to an operator must exist in the evidence record.
The recommended profile published precision 0.6140895327792112, recall
0.6062221049337548 and F1 0.6068607646002744. Those three numbers appear
nowhere in this repository except the file that publishes them and the test
that pinned them as literal strings. The only recorded evaluation of that
model at that operating point — tile 512, overlap 64, threshold 0.15 — reported
0.5898197518, 0.5769921004 and 0.5824578632, so the published figures were
about two and a half points more flattering than anything that was measured,
and a test guaranteed nobody would correct them.
An operator cannot check a number that has no source. This test refuses one.
"""
from __future__ import annotations
import re
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
PROFILES = ROOT / "frontend" / "src" / "components" / "detection" / "detectionProfiles.ts"
EVIDENCE = ROOT / "docs" / "CODEX_EXECUTION_LOG.md"
METRIC_FIELDS = ("precision", "recall", "f1")
# The log rounds; the source file may carry more digits of the same value.
TOLERANCE = 1e-9
def _published_metrics() -> list[tuple[str, str, float]]:
source = PROFILES.read_text(encoding="utf-8")
profiles = re.findall(r"id: '([^']+)',(.*?)\n \},", source, re.S)
assert profiles, "no operator profiles found; the file shape changed"
published: list[tuple[str, str, float]] = []
for profile_id, body in profiles:
for field in METRIC_FIELDS:
match = re.search(rf"^\s*{field}: ([0-9.]+),", body, re.M)
assert match, f"{profile_id} publishes no {field}"
published.append((profile_id, field, float(match.group(1))))
return published
def _recorded_values() -> list[float]:
text = EVIDENCE.read_text(encoding="utf-8")
return [float(value) for value in re.findall(r"\b0\.\d{4,}\b", text)]
def test_every_published_accuracy_figure_appears_in_the_evidence_record() -> None:
recorded = _recorded_values()
untraceable = [
f"{profile_id}.{field} = {value}"
for profile_id, field, value in _published_metrics()
if not any(abs(value - candidate) <= TOLERANCE for candidate in recorded)
]
assert not untraceable, (
"These figures are shown to operators but were never recorded in "
f"docs/CODEX_EXECUTION_LOG.md: {untraceable}. Publish the measurement "
"that was taken, or record the evaluation that produced these."
)
def test_each_profile_names_the_measurement_behind_its_numbers() -> None:
source = PROFILES.read_text(encoding="utf-8")
profile_count = source.count("modelAssetId:")
assert source.count("evidenceReference:") == profile_count
assert source.count("backgroundGate:") == profile_count
assert source.count("backgroundSampleCount:") == profile_count
def test_the_check_would_notice_an_invented_figure() -> None:
"""Without this the test could pass because nothing ever matches."""
recorded = _recorded_values()
assert any(abs(0.5898197518 - value) <= TOLERANCE for value in recorded)
assert not any(abs(0.6140895327792112 - value) <= TOLERANCE for value in recorded)
@@ -0,0 +1,86 @@
"""Loading a run's results must not depend on the run being small.
``/detection/runs/{id}/detections`` and its GeoJSON sibling returned every
persisted detection. A regional run holds tens of thousands, so the endpoints
the map and the results table call after every run grew without bound. The
counts stay complete; what is transferred does not.
"""
from __future__ import annotations
import uuid
import pytest
from app.services.detection_service import DetectionService
class _Detection:
def __init__(self, index: int) -> None:
self.id = uuid.uuid4()
self.index = index
def _rows(count: int) -> list[_Detection]:
return [_Detection(index) for index in range(count)]
def test_a_page_is_returned_with_the_complete_total() -> None:
page, total, truncated = DetectionService.paginate(_rows(1_000), limit=100, offset=0)
assert len(page) == 100
assert total == 1_000
assert truncated is True
def test_the_offset_walks_the_population() -> None:
page, total, _ = DetectionService.paginate(_rows(10), limit=3, offset=6)
assert [row.index for row in page] == [6, 7, 8]
assert total == 10
def test_an_offset_past_the_end_yields_an_empty_page_not_an_error() -> None:
page, total, truncated = DetectionService.paginate(_rows(5), limit=10, offset=50)
assert page == []
assert total == 5
assert truncated is True
def test_a_population_inside_one_page_is_not_reported_as_truncated() -> None:
page, total, truncated = DetectionService.paginate(_rows(7), limit=100, offset=0)
assert len(page) == 7
assert total == 7
assert truncated is False
def test_a_zero_limit_returns_everything_for_callers_that_need_it() -> None:
page, total, truncated = DetectionService.paginate(_rows(2_500), limit=0, offset=0)
assert len(page) == 2_500
assert total == 2_500
assert truncated is False
def test_a_negative_offset_is_treated_as_the_start() -> None:
page, _, _ = DetectionService.paginate(_rows(4), limit=2, offset=-5)
assert [row.index for row in page] == [0, 1]
@pytest.mark.parametrize("limit", [1, 2, 3])
def test_paging_covers_the_population_exactly_once(limit: int) -> None:
rows = _rows(7)
seen: list[int] = []
offset = 0
while True:
page, total, _ = DetectionService.paginate(rows, limit=limit, offset=offset)
if not page:
break
seen.extend(row.index for row in page)
offset += limit
assert seen == list(range(7))
assert total == 7
@@ -0,0 +1,125 @@
"""Detections that straddle a tile seam must not become two half buildings.
Tiling uses a fixed overlap. An object wider than that overlap is truncated by
both tiles, so the two boxes barely intersect and plain IoU suppression keeps
them both: two false positives plus one missed footprint for every seam
building. The suppressor therefore also compares overlap against the smaller
box, and truncated boxes that sit against an interior tile edge are dropped in
favour of the neighbouring tile's complete view.
"""
from __future__ import annotations
from shapely.geometry import box
from app.services.detection_service import DetectionService
def _candidate(name: str, geometry, confidence: float, *, tile_index: int = 0, tile_bounds=None):
return {
"class_name": "building",
"confidence": confidence,
"geometry": geometry,
"bbox": [0.0, 0.0, 1.0, 1.0],
"source_tile_path": f"/tiles/tile_{tile_index:04d}.tif",
"properties": {"tile_index": tile_index, "name": name},
"tile_bounds": tile_bounds,
}
def test_identical_overlapping_predictions_are_still_suppressed() -> None:
kept = DetectionService._suppress_duplicate_candidates(
[
_candidate("a", box(0.0, 0.0, 1.0, 1.0), 0.7),
_candidate("b", box(0.02, 0.02, 1.02, 1.02), 0.9),
],
iou_threshold=0.5,
)
assert [item["properties"]["name"] for item in kept] == ["b"]
def test_a_box_contained_in_a_larger_one_is_suppressed() -> None:
"""A truncated seam half sits inside the complete box from the next tile."""
complete = box(0.0, 0.0, 10.0, 10.0)
truncated_half = box(0.0, 0.0, 4.0, 10.0) # IoU with ``complete`` is 0.4
kept = DetectionService._suppress_duplicate_candidates(
[
_candidate("complete", complete, 0.88),
_candidate("truncated", truncated_half, 0.61),
],
iou_threshold=0.5,
)
assert [item["properties"]["name"] for item in kept] == ["complete"]
def test_genuinely_adjacent_buildings_are_both_kept() -> None:
"""Terraced houses touch but do not contain one another."""
kept = DetectionService._suppress_duplicate_candidates(
[
_candidate("left", box(0.0, 0.0, 10.0, 10.0), 0.9),
_candidate("right", box(10.0, 0.0, 20.0, 10.0), 0.85),
],
iou_threshold=0.5,
)
assert sorted(item["properties"]["name"] for item in kept) == ["left", "right"]
def test_different_classes_are_never_merged() -> None:
first = _candidate("a", box(0.0, 0.0, 10.0, 10.0), 0.9)
second = _candidate("b", box(0.0, 0.0, 10.0, 10.0), 0.8)
second["class_name"] = "solar_panel"
kept = DetectionService._suppress_duplicate_candidates([first, second], iou_threshold=0.5)
assert len(kept) == 2
def test_boxes_clipped_by_an_interior_tile_edge_are_dropped() -> None:
"""The overlapping neighbour tile still sees the whole object."""
tile = box(0.0, 0.0, 10.0, 10.0)
raster = box(0.0, 0.0, 30.0, 10.0)
candidates = [
# Sits against the tile's right edge: truncated by the tile, not real.
_candidate("edge", box(9.0, 2.0, 10.0, 4.0), 0.8, tile_bounds=tile.bounds),
# Comfortably inside the tile.
_candidate("interior", box(2.0, 2.0, 4.0, 4.0), 0.8, tile_bounds=tile.bounds),
]
kept = DetectionService._drop_tile_edge_truncations(
candidates, raster_bounds=raster.bounds, tolerance=0.001
)
assert [item["properties"]["name"] for item in kept] == ["interior"]
def test_boxes_against_the_raster_edge_are_kept() -> None:
"""No neighbouring tile exists there, so the box is all the evidence there is."""
tile = box(0.0, 0.0, 10.0, 10.0)
raster = box(0.0, 0.0, 10.0, 10.0)
candidates = [_candidate("edge", box(9.0, 2.0, 10.0, 4.0), 0.8, tile_bounds=tile.bounds)]
kept = DetectionService._drop_tile_edge_truncations(
candidates, raster_bounds=raster.bounds, tolerance=0.001
)
assert [item["properties"]["name"] for item in kept] == ["edge"]
def test_edge_filter_keeps_candidates_without_tile_bounds() -> None:
candidates = [_candidate("unknown", box(2.0, 2.0, 4.0, 4.0), 0.8, tile_bounds=None)]
kept = DetectionService._drop_tile_edge_truncations(
candidates, raster_bounds=(0.0, 0.0, 30.0, 10.0), tolerance=0.001
)
assert len(kept) == 1
+512
View File
@@ -0,0 +1,512 @@
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def test_backend_dockerfile_copies_package_sources_before_pip_install() -> None:
dockerfile = ROOT / "backend" / "Dockerfile"
lines = dockerfile.read_text(encoding="utf-8").splitlines()
pip_install_index = lines.index('RUN extras=".[gis]" \\')
preceding = "\n".join(lines[:pip_install_index])
assert "COPY pyproject.toml README.md /app/" in preceding
assert "COPY app /app/app" in preceding
def test_backend_dockerfile_installs_approved_gis_runtime_stack() -> None:
dockerfile = (ROOT / "backend" / "Dockerfile").read_text(encoding="utf-8")
assert "ARG GEOINTEL_INSTALL_AI=false" in dockerfile
assert 'extras=".[gis]"' in dockerfile
assert 'extras=".[gis,ai]"' in dockerfile
assert "RUN python scripts/gis_import_smoke.py" in dockerfile
assert "gdal-bin" in dockerfile
assert "libgdal-dev" in dockerfile
assert "libgeos-dev" in dockerfile
assert "libproj-dev" in dockerfile
assert "proj-bin" in dockerfile
assert "libxcb1" in dockerfile
assert "libgl1" in dockerfile
assert "libglib2.0-0" in dockerfile
def test_backend_pyproject_exposes_gis_optional_dependency_group() -> None:
pyproject = (ROOT / "backend" / "pyproject.toml").read_text(encoding="utf-8")
assert "gis = [" in pyproject
assert '"rasterio>=1.4.3"' in pyproject
assert '"geopandas>=1.0.1"' in pyproject
assert '"pyogrio>=0.10.0"' in pyproject
assert '"ultralytics>=8.3,<9"' not in pyproject.split("gis = [", 1)[1].split("]", 1)[0]
def test_all_in_one_dockerfile_can_opt_into_ai_dependencies_without_base_install() -> None:
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
assert "ARG GEOINTEL_INSTALL_AI=false" in dockerfile
assert "COPY backend/pyproject.toml /app/" in dockerfile
assert "COPY backend/requirements-runtime.lock /app/" in dockerfile
assert "COPY backend/requirements-ai-linux.lock /app/" in dockerfile
assert "COPY backend/requirements-build-tools.lock /app/" in dockerfile
assert "COPY backend/pyproject.toml backend/README.md /app/" not in dockerfile
assert "GeoIntel backend package metadata" in dockerfile
assert "--require-hashes -r requirements-runtime.lock" in dockerfile
assert "--require-hashes" in dockerfile
assert "-r requirements-ai-linux.lock" in dockerfile
assert "-r requirements-build-tools.lock" in dockerfile
assert "python scripts/gis_import_smoke.py" in dockerfile
assert "yolo_preflight.py" in dockerfile
assert "libxcb1" in dockerfile
assert "libgl1" in dockerfile
assert "libglib2.0-0" in dockerfile
def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None:
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
for line in dockerfile.splitlines():
if line.startswith("COPY scripts/"):
source_path = line.split()[1]
assert (ROOT / source_path).is_file()
required_runtime_scripts = {
"prepare_operator_real_data_samples.py",
"export_operator_yolo_tile_dataset.py",
"audit_operator_yolo_dataset_quality.py",
"render_operator_yolo_label_qa_contact_sheets.py",
"train_operator_yolo_detector.sh",
"training_dataset_eligibility.py",
"training_release_manifest.py",
"verify_real_data_detection_qa_workflow.sh",
"run_detection_quality_matrix.sh",
"run_multi_sample_detection_quality_matrix.sh",
"run_mol_operational_validation.sh",
"export_detection_calibration_evidence.sh",
"assemble_detection_calibration_evidence_portfolio.sh",
"build_fixed_threshold_evidence_portfolio_inputs.py",
"audit_detection_false_negative_evidence.py",
"audit_detection_false_positive_evidence.py",
"render_detection_false_positive_review_contact_sheets.py",
"render_detection_false_negative_review_contact_sheets.py",
"validate_detection_false_positive_review_decisions.py",
"validate_detection_false_negative_review_decisions.py",
"run_operator_hard_negative_detection_matrix.sh",
"run_background_corpus_split_matrix.sh",
"build_background_corpus_split_report.py",
"build_detection_model_promotion_report.py",
"run_split_background_promotion_workflow.sh",
"activate_promoted_yolo_candidate.py",
"migrate_runtime_model_provenance.py",
"manage_grb_refresh.py",
"orthophoto_release_preflight.py",
"provision_walous_sources.py",
"provision_spw_terrain_source.py",
}
for script_name in required_runtime_scripts:
assert f"COPY scripts/{script_name} /app/scripts/{script_name}" in dockerfile
def test_all_in_one_dockerfile_copies_operator_scripts_after_dependency_install() -> None:
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
dependency_install_index = dockerfile.index('/usr/bin/python3.11 -m venv /opt/geointel/venv \\')
operator_copy_index = dockerfile.index(
"COPY scripts/render_operator_yolo_label_qa_contact_sheets.py "
"/app/scripts/render_operator_yolo_label_qa_contact_sheets.py"
)
assert operator_copy_index > dependency_install_index
def test_compose_does_not_require_missing_root_env_file() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
assert "env_file:" not in compose
assert "DATABASE_URL: postgresql+psycopg://${GEOINTEL_POSTGRES_USER:-geointel}" in compose
def test_compose_exposes_frontend_on_configurable_host_port_with_cors_origin() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
env_example = (ROOT / ".env.example").read_text(encoding="utf-8")
assert '"${GEOINTEL_BIND_ADDRESS:-127.0.0.1}:${GEOINTEL_FRONTEND_PORT:-1202}:80"' in compose
assert '"${GEOINTEL_BIND_ADDRESS:-127.0.0.1}:${GEOINTEL_BACKEND_PORT:-8000}:8000"' in compose
assert "CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202}" in compose
assert "GEOINTEL_FRONTEND_PORT=1202" in env_example
assert "GEOINTEL_BACKEND_PORT=8000" in env_example
assert "http://localhost:1202" in env_example
assert "http://127.0.0.1:1202" in env_example
def test_packaged_runtime_uses_fail_closed_authentication_defaults() -> None:
compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
env_example = (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(encoding="utf-8")
assert "GEOINTEL_AUTH_ENABLED: ${GEOINTEL_AUTH_ENABLED:-true}" in compose
assert "GEOINTEL_AUTH_REQUIRE_HTTPS: ${GEOINTEL_AUTH_REQUIRE_HTTPS:-true}" in compose
assert "GEOINTEL_GUEST_ACCESS_ENABLED: ${GEOINTEL_GUEST_ACCESS_ENABLED:-false}" in compose
assert "GEOINTEL_AUTH_ENABLED=true" in env_example
assert "GEOINTEL_AUTH_REQUIRE_HTTPS=true" in env_example
assert "GEOINTEL_GUEST_ACCESS_ENABLED=false" in env_example
def test_env_example_uses_runtime_env_names_read_by_backend_and_frontend() -> None:
env_example = (ROOT / ".env.example").read_text(encoding="utf-8")
assert "GEOINTEL_INSTALL_AI=false" in env_example
assert "YOLO_ENABLED=false" in env_example
assert "YOLO_MODELS_DIR=/app/models" in env_example
assert "YOLO_MODEL_PATH=" in env_example
assert "YOLO_CONFIG_DIR=./storage/ultralytics" in env_example
assert "YOLO_MAX_TILES=100" in env_example
assert "YOLO_MAX_DETECTIONS=1000" in env_example
assert "YOLO_DUPLICATE_IOU_THRESHOLD=0.5" in env_example
assert "ENABLE_YOLO" not in env_example
assert "ENABLE_SAM" not in env_example
assert "VITE_API_BASE_URL=" in env_example
assert "VITE_API_PROXY_TARGET=http://localhost:8000" in env_example
def test_walloon_runtime_settings_are_editable_in_compose_and_unraid() -> None:
files = [
(ROOT / "docker-compose.yml").read_text(encoding="utf-8"),
(ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8"),
(ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(encoding="utf-8"),
(ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8"),
(ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(encoding="utf-8"),
]
for content in files:
assert "SPW_FLOOD_HAZARD_ENABLED" in content
assert "SPW_FLOOD_HAZARD_MAPSERVER_URL" in content
assert "WALOUS_ENABLED" in content
assert "WALOUS_SOURCE_DIR" in content
assert "WALOUS_ANALYSIS_RESOLUTION_M" in content
assert "WALOUS_MAX_SIDE_M" in content
assert "WALOUS_MAX_PIXELS" in content
def test_in_memory_vector_limit_is_propagated_and_validated_in_every_runtime() -> None:
expected = "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB"
for path in (
ROOT / ".env.example",
ROOT / "docker-compose.yml",
ROOT / "docker-compose.unraid.yml",
ROOT / "deploy" / "unraid" / "geointel.env.example",
ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml",
):
assert expected in path.read_text(encoding="utf-8"), path
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(
encoding="utf-8"
)
assert 'GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}"' in run_script
assert "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB must be between 1 and 256." in run_script
assert '-e GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB"' in run_script
def test_frontend_uses_same_origin_api_proxy_by_default() -> None:
api_client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8")
nginx_config = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8")
dockerfile = (ROOT / "frontend" / "Dockerfile").read_text(encoding="utf-8")
assert '?? ""' in api_client
assert "http://localhost:8000" not in api_client
assert "FROM nginx:" in dockerfile
assert "COPY --from=build /app/dist /usr/share/nginx/html" in dockerfile
assert "location /api/" in nginx_config
assert "proxy_pass http://backend:8000/api/" in nginx_config
assert "location = /health" in nginx_config
assert 'add_header Cache-Control "no-cache"' in nginx_config
assert "location /assets/" in nginx_config
assert "try_files $uri $uri/ /index.html" in nginx_config
def test_nginx_runtime_allows_real_gis_upload_payloads() -> None:
frontend_nginx = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8")
all_in_one_nginx = (ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(encoding="utf-8")
start_script = (ROOT / "deploy" / "unraid" / "all-in-one-start.sh").read_text(encoding="utf-8")
assert "client_max_body_size 250m;" in frontend_nginx
assert "client_max_body_size __GEOINTEL_MAX_UPLOAD_MB__m;" in all_in_one_nginx
assert 'sed -i "s/__GEOINTEL_MAX_UPLOAD_MB__/${MAX_UPLOAD_MB}/g"' in start_script
def test_nginx_runtime_allows_long_ai_and_qa_requests() -> None:
frontend_nginx = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8")
all_in_one_nginx = (ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(encoding="utf-8")
for config in (frontend_nginx, all_in_one_nginx):
assert "proxy_read_timeout 600s;" in config
assert "proxy_send_timeout 600s;" in config
def test_nginx_preserves_outer_https_scheme_for_secure_session_cookies() -> None:
configs = (
(ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8"),
(ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(encoding="utf-8"),
)
for config in configs:
assert "geo $geointel_trusted_forwarder" in config
assert "default 0;" in config
assert "172.16.0.0/12 1;" in config
assert 'map "$geointel_trusted_forwarder:$http_x_forwarded_proto"' in config
assert '"1:https" https;' in config
assert "proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;" in config
assert "proxy_set_header X-Forwarded-Proto $scheme;" not in config
def test_nginx_runtime_sets_security_headers_on_all_cached_locations() -> None:
configs = (
(ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8"),
(ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(
encoding="utf-8"
),
)
required = (
'Content-Security-Policy "frame-ancestors \'none\'" always;',
'X-Frame-Options "DENY" always;',
'X-Content-Type-Options "nosniff" always;',
'Referrer-Policy "strict-origin-when-cross-origin" always;',
'Permissions-Policy "camera=(), microphone=(), geolocation=()" always;',
)
for config in configs:
cached_locations = config.count("add_header Cache-Control")
assert cached_locations >= 2
for header in required:
# Nginx 1.27 locations with Cache-Control do not inherit server-level
# add_header directives, so every cached location repeats the policy.
assert config.count(f"add_header {header}") == cached_locations + 1
def test_compose_does_not_publish_postgis_on_default_host_port() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
assert '"5432:5432"' not in compose
def test_compose_waits_for_healthy_database_and_applies_migrations() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
assert "pg_isready -U ${GEOINTEL_POSTGRES_USER:-geointel} -d ${GEOINTEL_POSTGRES_DB:-geointel}" in compose
assert "condition: service_healthy" in compose
assert "sh /app/docker_start.sh" in compose
def test_compose_mounts_demo_fixtures_for_backend_runtime() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
assert "./fixtures:/app/fixtures:ro" in compose
def test_compose_has_backend_and_frontend_healthchecks() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
assert "http://127.0.0.1:8000/health/ready" in compose
assert "urllib.request.urlopen" in compose
assert "http://127.0.0.1/health/ready" in compose
assert "wget -q -O -" in compose
assert "start_period: 30s" in compose
assert "start_period: 10s" in compose
def test_frontend_waits_for_healthy_backend_in_compose() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
frontend_section = compose.split(" frontend:", 1)[1]
assert "backend:" in frontend_section
assert "condition: service_healthy" in frontend_section
def test_backend_docker_start_script_waits_for_sql_connection_before_migrations() -> None:
script = (ROOT / "backend" / "docker_start.sh").read_text(encoding="utf-8")
assert "Waiting for database connection" in script
assert "create_engine(settings.database_url" in script
assert "SELECT 1" in script
assert "python -m alembic upgrade head" in script
assert "uvicorn app.main:app --host 0.0.0.0 --port 8000" in script
def test_runtime_sets_writable_ultralytics_config_directory() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
unraid_compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
start_script = (ROOT / "deploy" / "unraid" / "all-in-one-start.sh").read_text(encoding="utf-8")
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
unraid_env = (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(encoding="utf-8")
assert "YOLO_CONFIG_DIR: ${YOLO_CONFIG_DIR:-/app/storage/ultralytics}" in compose
assert "YOLO_CONFIG_DIR: ${YOLO_CONFIG_DIR:-/app/storage/ultralytics}" in unraid_compose
assert 'export YOLO_CONFIG_DIR="${YOLO_CONFIG_DIR:-$STORAGE_ROOT/ultralytics}"' in start_script
assert 'mkdir -p "$PGDATA" "$STORAGE_ROOT" "$YOLO_CONFIG_DIR"' in start_script
assert 'YOLO_CONFIG_DIR="${YOLO_CONFIG_DIR:-/app/storage/ultralytics}"' in run_script
assert '-e YOLO_CONFIG_DIR="$YOLO_CONFIG_DIR"' in run_script
assert "YOLO_CONFIG_DIR=/app/storage/ultralytics" in unraid_env
def test_regional_official_vector_sources_are_configurable_in_every_runtime() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
unraid_compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(
encoding="utf-8"
)
env_example = (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(
encoding="utf-8"
)
template = (ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(
encoding="utf-8"
)
for key in (
"SPW_PICC_ENABLED",
"SPW_PICC_MAPSERVER_URL",
"URBIS_ENABLED",
"URBIS_WFS_URL",
):
assert key in compose
assert key in unraid_compose
assert f'{key}="${{{key}:-' in run_script
assert f'-e {key}="${key}"' in run_script
assert f"{key}=" in env_example
assert f'Target="{key}"' in template
def test_segmentation_and_mdk_acquisition_are_configurable_in_every_runtime() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
unraid_compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
env_example = (ROOT / ".env.example").read_text(encoding="utf-8")
unraid_env = (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(encoding="utf-8")
template = (ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(encoding="utf-8")
for key in (
"YOLO_SEG_ENABLED",
"YOLO_SEG_MODEL_PATH",
"SAM_ENABLED",
"SAM_MODEL_PATH",
"SEGMENTATION_MAX_MASKS_PER_TILE",
"SEGMENTATION_DUPLICATE_IOU_THRESHOLD",
"MDK_BATHYMETRY_ACQUISITION_ENABLED",
"MDK_BATHYMETRY_COVERAGE_ID",
"MDK_BATHYMETRY_MAX_BBOX_DEG2",
):
assert key in compose, key
assert key in unraid_compose, key
assert f'{key}="${{{key}:-' in run_script, key
assert f'-e {key}="${key}"' in run_script, key
assert f"{key}=" in env_example, key
assert f"{key}=" in unraid_env, key
assert f'Target="{key}"' in template, key
def test_compose_reconciles_interrupted_runs_after_restart_like_unraid_runtime() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
start_script = (ROOT / "deploy" / "unraid" / "all-in-one-start.sh").read_text(encoding="utf-8")
assert (
"GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP: "
"${GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true}"
) in compose
assert (
'GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP='
'"${GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true}"'
) in start_script
def test_docker_build_contexts_exclude_vendor_build_and_cache_outputs() -> None:
required_patterns = {
"node_modules",
"dist",
"__pycache__",
"*.pyc",
".pytest_cache",
}
for relative_path in ("backend/.dockerignore", "frontend/.dockerignore"):
content = (ROOT / relative_path).read_text(encoding="utf-8")
for pattern in required_patterns:
assert pattern in content
def test_browser_runtime_verification_script_detects_proxy_contract() -> None:
script = (ROOT / "scripts" / "verify_browser_runtime.sh").read_text(encoding="utf-8")
assert "/api/v1/projects" in script
assert "<!doctype html" in script
assert "canonical GeoIntel envelope" in script
assert '"status":"ok"' in script
def test_gis_runtime_verification_script_detects_required_capabilities() -> None:
script = (ROOT / "scripts" / "verify_gis_runtime.sh").read_text(encoding="utf-8")
assert "/api/v1/system/capabilities" in script
assert '"postgis":true' in script
assert '"rasterio":true' in script
assert '"geopandas":true' in script
assert "<!doctype html" in script
def test_gis_import_smoke_script_checks_runtime_imports() -> None:
docker_script = (ROOT / "backend" / "scripts" / "gis_import_smoke.py").read_text(encoding="utf-8")
root_wrapper = (ROOT / "scripts" / "gis_import_smoke.py").read_text(encoding="utf-8")
assert 'REQUIRED_MODULES = ("rasterio", "geopandas", "pyogrio")' in docker_script
assert "importlib.import_module" in docker_script
assert '"gis_imports"' in docker_script
assert 'ROOT / "backend" / "scripts"' in root_wrapper
assert "from gis_import_smoke import main" in root_wrapper
def test_backend_docker_context_contains_gis_import_smoke_script() -> None:
assert (ROOT / "backend" / "scripts" / "gis_import_smoke.py").exists()
def test_all_in_one_dockerfile_caches_dependencies_and_pins_driver_compatible_cuda_torch() -> None:
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
metadata_copy_index = dockerfile.index("COPY backend/pyproject.toml /app/")
placeholder_readme_index = dockerfile.index("GeoIntel backend package metadata")
dependency_install_index = dockerfile.index('/usr/bin/python3.11 -m venv /opt/geointel/venv \\')
backend_copy_index = dockerfile.index("COPY backend/ /app/")
smoke_index = dockerfile.index("RUN python scripts/gis_import_smoke.py")
assert metadata_copy_index < placeholder_readme_index < dependency_install_index < backend_copy_index < smoke_index
assert "GEOINTEL_TORCH_INDEX_URL=https://download.pytorch.org/whl/cu128" in dockerfile
assert '--extra-index-url "$GEOINTEL_TORCH_INDEX_URL"' in dockerfile
ai_lock = (ROOT / "backend" / "requirements-ai-linux.lock").read_text(encoding="utf-8")
assert "torch==2.11.0+cu128 --hash=sha256:" in ai_lock
assert "torchvision==0.26.0+cu128 --hash=sha256:" in ai_lock
def test_unraid_deploy_passes_ai_build_arg_and_yolo_runtime_env() -> None:
deploy_ps1 = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8")
deploy_sh = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8")
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
assert 'DEPLOY_GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-}"' in deploy_sh
assert "--build-arg GEOINTEL_INSTALL_AI=" in release_script
assert "DEPLOY_GEOINTEL_INSTALL_AI" in deploy_ps1
assert "[string]$InstallAi" in deploy_ps1
assert 'YOLO_ENABLED="${YOLO_ENABLED:-false}"' in run_script
assert '-e YOLO_ENABLED="$YOLO_ENABLED"' in run_script
assert 'YOLO_MODELS_DIR="${YOLO_MODELS_DIR:-/app/models}"' in run_script
assert '-e YOLO_MODELS_DIR="$YOLO_MODELS_DIR"' in run_script
assert '-e YOLO_MODEL_PATH="$YOLO_MODEL_PATH"' in run_script
assert '-e YOLO_MAX_TILES="$YOLO_MAX_TILES"' in run_script
assert '-e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS"' in run_script
assert '-e YOLO_DUPLICATE_IOU_THRESHOLD="$YOLO_DUPLICATE_IOU_THRESHOLD"' in run_script
assert "-v \"${GEOINTEL_MODELS_PATH}:/app/models\"" in run_script
def test_unraid_ai_runtime_requests_nvidia_and_fails_closed() -> None:
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
assert "--gpus all" in run_script
assert 'YOLO_DEVICE="${YOLO_DEVICE:-cuda:0}"' in run_script
assert 'YOLO_REQUIRE_CUDA="${YOLO_REQUIRE_CUDA:-true}"' in run_script
assert '-e YOLO_REQUIRE_CUDA="$YOLO_REQUIRE_CUDA"' in run_script
assert "https://download.pytorch.org/whl/cu128" in dockerfile
@@ -0,0 +1,88 @@
"""A broad handler must not overwrite a precise diagnosis with a generic one.
``except Exception: raise AppError(...)`` also catches ``AppError``, so a
service that raised INVALID_CRS with a 409 comes out as whatever generic code
the outer handler chose. For a product whose whole claim is that a result can
be traced back to its cause, that is the wrong direction in which to lose
information.
Wrapping a narrow call whose failure the outer message describes better is
legitimate — coercing a CRS string, parsing one geometry, calling one's own
private helper. Relabelling what *another* component reported is not.
"""
from __future__ import annotations
import re
from pathlib import Path
APP = Path(__file__).resolve().parents[1] / "app"
GENERIC_HANDLER = re.compile(r"except Exception as \w+:\s*\n\s*raise AppError\(", re.M)
OTHER_SERVICE = re.compile(r"\b([A-Z]\w*Service)\.")
# Cross-component entry points that carry their own considered diagnosis.
GOVERNED_CALLS = re.compile(r"assert_eligible|assert_within_storage_root|canonicalize_|_load_dataset_payload")
def _enclosing_service(text: str, handler_line: int) -> str | None:
"""The class the handler sits in, not merely the first one in the file."""
enclosing = None
for match in re.finditer(r"^class (\w+)[:(]", text, re.M):
if text[: match.start()].count(chr(10)) > handler_line:
break
enclosing = match.group(1)
return enclosing
def _try_body(lines: list[str], handler_line: int) -> str:
index = handler_line
while index > 0 and lines[index].strip() != "try:":
index -= 1
return "\n".join(lines[index:handler_line])
def calls_another_component(body: str, own_service: str | None) -> bool:
if GOVERNED_CALLS.search(body):
return True
return any(name != own_service for name in OTHER_SERVICE.findall(body))
def _offenders() -> list[str]:
found: list[str] = []
for path in sorted(APP.rglob("*.py")):
text = path.read_text(encoding="utf-8")
lines = text.splitlines()
for match in GENERIC_HANDLER.finditer(text):
preceding = text[max(0, match.start() - 240):match.start() + 40]
if "except AppError:" in preceding:
continue
handler_line = text[:match.start()].count("\n")
own = _enclosing_service(text, handler_line)
if calls_another_component(_try_body(lines, handler_line), own):
found.append(f"{path.relative_to(APP)}:{handler_line + 1}")
return found
def test_a_call_into_another_component_keeps_the_error_it_raised() -> None:
offenders = _offenders()
assert not offenders, (
"These catch Exception around a call into another component and relabel "
"whatever it raised, including its AppError. Add `except AppError: "
f"raise` before the generic handler: {offenders}"
)
def test_the_check_recognises_the_shape_it_guards_against() -> None:
"""Without this the guard could pass because its pattern never matches."""
other = "try:\n DatasetService.get_dataset(db, dataset_id)"
own = "try:\n GrbAcquisitionService._extract_dimension(geometry)"
governed = "try:\n DatasetConsumptionGate.assert_eligible(dataset)"
assert calls_another_component(other, "GrbAcquisitionService")
assert calls_another_component(governed, "GrbAcquisitionService")
# Wrapping one's own private helper is the legitimate case.
assert not calls_another_component(own, "GrbAcquisitionService")
assert GENERIC_HANDLER.search("except Exception as exc:\n raise AppError(")
@@ -0,0 +1,49 @@
from __future__ import annotations
from fastapi import HTTPException
from fastapi.testclient import TestClient
from app.main import app, create_app
def assert_contract_error(payload: dict, code: str) -> None:
assert payload["error"] == code
assert isinstance(payload["message"], str)
assert "details" in payload
assert "request_id" in payload
def test_app_error_uses_top_level_api_error_contract() -> None:
client = TestClient(app)
response = client.get("/api/v1/external/providers/unknown")
assert response.status_code == 404
assert_contract_error(response.json(), "PROVIDER_NOT_FOUND")
assert response.json()["message"] == "Provider not found"
def test_http_exception_uses_top_level_api_error_contract() -> None:
test_app = create_app()
@test_app.get("/__test__/http-error")
def raise_http_error() -> None:
raise HTTPException(status_code=404, detail="Project not found")
client = TestClient(test_app)
response = client.get("/__test__/http-error")
assert response.status_code == 404
assert_contract_error(response.json(), "HTTP_ERROR")
assert response.json()["message"] == "Project not found"
def test_validation_error_uses_top_level_api_error_contract() -> None:
client = TestClient(app)
response = client.get("/api/v1/projects/not-a-uuid")
assert response.status_code == 422
assert_contract_error(response.json(), "VALIDATION_ERROR")
assert response.json()["message"] == "Validation failed"
assert isinstance(response.json()["details"], list)
@@ -0,0 +1,125 @@
"""A downloaded export must carry its own provenance and its own limits.
Counterpart to ``test_export_provenance_member``: this reads the file the
service actually writes, so it proves the member survives serialisation rather
than only that the helper builds it.
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
from app.models import Dataset, Export
from app.services.export_service import ExportService
from app.services.storage_service import StorageService
from app.services.vector_feature_service import VectorFeatureService
from tests.test_sprint107_map_selection_export import FakeSession, _govern_fixture_dataset
BBOX = {"min_x": 4.9, "min_y": 50.9, "max_x": 5.2, "max_y": 51.2, "crs": "EPSG:4326"}
def _dataset(dataset_id):
dataset = _govern_fixture_dataset(
Dataset(
id=dataset_id,
project_id=uuid4(),
name="grb-buildings.geojson",
dataset_type="vector",
source="grb",
source_name="grb",
source_version="2024-06",
observed_at=datetime(2024, 6, 1, tzinfo=timezone.utc),
status="ready",
)
)
return dataset
def _selection(*, feature_count: int, total: int, truncated: bool, warning: str | None = None):
return {
"selection_bbox": BBOX,
"feature_count": feature_count,
"total_feature_count": total,
"limit": 250,
"truncated": truncated,
"summary": {"selection_edge_warning": warning} if warning else {},
"geojson": {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {"vector_feature_id": f"vf-{index}"},
}
for index in range(feature_count)
],
},
}
def _export(monkeypatch, tmp_path: Path, selection) -> dict:
dataset_id = uuid4()
dataset = _dataset(dataset_id)
db = FakeSession({(Dataset, dataset_id): dataset})
export_path = tmp_path / "selection.geojson"
monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path))
monkeypatch.setattr(VectorFeatureService, "select_features_by_bbox", lambda *_args, **_kwargs: selection)
ExportService.export_vector_selection_geojson(db, dataset_id, BBOX, limit=250, name="selection")
assert [item for item in db.added if isinstance(item, Export)]
return json.loads(export_path.read_text(encoding="utf-8"))
def test_a_truncated_export_file_states_that_it_is_partial(monkeypatch, tmp_path: Path) -> None:
written = _export(monkeypatch, tmp_path, _selection(feature_count=2, total=1_400, truncated=True))
provenance = written["geointel_provenance"]
assert provenance["complete"] is False
assert "1400" in provenance["completeness_note"].replace(".", "")
# The features are still there; the file simply no longer implies it holds
# everything the selection contains.
assert len(written["features"]) == 2
def test_a_complete_export_file_says_so(monkeypatch, tmp_path: Path) -> None:
written = _export(monkeypatch, tmp_path, _selection(feature_count=2, total=2, truncated=False))
provenance = written["geointel_provenance"]
assert provenance["complete"] is True
assert provenance["completeness_note"] is None
def test_the_file_identifies_its_source_edition(monkeypatch, tmp_path: Path) -> None:
written = _export(monkeypatch, tmp_path, _selection(feature_count=1, total=1, truncated=False))
provenance = written["geointel_provenance"]
assert provenance["source_name"] == "grb"
assert provenance["source_version"] == "2024-06"
assert provenance["observed_at"].startswith("2024-06-01")
assert provenance["selection_bbox"] == BBOX
def test_selection_caveats_travel_into_the_file(monkeypatch, tmp_path: Path) -> None:
written = _export(
monkeypatch,
tmp_path,
_selection(feature_count=1, total=1, truncated=False, warning="22 objecten liggen deels buiten de selectie."),
)
assert written["geointel_provenance"]["warnings"] == ["22 objecten liggen deels buiten de selectie."]
def test_the_file_remains_a_valid_feature_collection(monkeypatch, tmp_path: Path) -> None:
"""The provenance is a foreign member, not a change to the GeoJSON shape."""
written = _export(monkeypatch, tmp_path, _selection(feature_count=1, total=1, truncated=False))
assert written["type"] == "FeatureCollection"
assert isinstance(written["features"], list)
assert written["features"][0]["type"] == "Feature"
@@ -0,0 +1,129 @@
"""An exported GeoJSON must say what it is and what it leaves out.
The vector selection export caps its features and records ``truncated`` in the
export *record*. The file itself said nothing: an operator downloads
``mol-selection.geojson``, opens it in QGIS and sees 250 buildings where the
workbench said 1.400, with nothing in the file to indicate the difference.
For a product whose promise is that an export is a reproducible result, a file
that looks complete and is not is the sharpest possible violation. RFC 7946
allows foreign members on a FeatureCollection, and the detection export already
uses one; this makes that convention uniform.
"""
from __future__ import annotations
from datetime import datetime, timezone
from uuid import uuid4
from app.services.export_service import ExportService
def test_provenance_names_the_source_and_the_moment() -> None:
dataset_id = uuid4()
project_id = uuid4()
member = ExportService.provenance_member(
source="vector_selection",
project_id=project_id,
dataset_id=dataset_id,
source_name="grb",
source_version="2024-06",
observed_at=datetime(2024, 6, 1, tzinfo=timezone.utc),
)
assert member["source"] == "vector_selection"
assert member["dataset_id"] == str(dataset_id)
assert member["project_id"] == str(project_id)
assert member["source_name"] == "grb"
assert member["source_version"] == "2024-06"
assert member["observed_at"].startswith("2024-06-01")
assert member["exported_at"]
def test_a_truncated_export_says_so_in_words() -> None:
member = ExportService.provenance_member(
source="vector_selection",
project_id=uuid4(),
dataset_id=uuid4(),
feature_count=250,
total_feature_count=1_400,
truncated=True,
)
assert member["complete"] is False
assert member["feature_count"] == 250
assert member["total_feature_count"] == 1_400
assert "1400" in member["completeness_note"].replace(".", "").replace(",", "")
assert "250" in member["completeness_note"]
def test_a_complete_export_is_stated_as_complete() -> None:
member = ExportService.provenance_member(
source="detection_run",
project_id=uuid4(),
dataset_id=uuid4(),
feature_count=12,
total_feature_count=12,
truncated=False,
)
assert member["complete"] is True
assert member["completeness_note"] is None
def test_counts_that_disagree_are_treated_as_incomplete() -> None:
"""A caller that forgets the flag must not produce a file claiming completeness."""
member = ExportService.provenance_member(
source="vector_selection",
project_id=uuid4(),
dataset_id=uuid4(),
feature_count=100,
total_feature_count=140,
truncated=False,
)
assert member["complete"] is False
def test_selection_context_travels_with_the_file() -> None:
area_id = uuid4()
bbox = {"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3, "crs": "EPSG:4326"}
member = ExportService.provenance_member(
source="vector_selection",
project_id=uuid4(),
dataset_id=uuid4(),
selection_bbox=bbox,
selection_area_id=area_id,
warnings=["22 van de 100 objecten liggen deels buiten de selectie."],
)
assert member["selection_bbox"] == bbox
assert member["selection_area_id"] == str(area_id)
assert member["warnings"] == ["22 van de 100 objecten liggen deels buiten de selectie."]
def test_empty_context_is_omitted_rather_than_written_as_null_noise() -> None:
member = ExportService.provenance_member(
source="dataset",
project_id=uuid4(),
dataset_id=uuid4(),
)
assert "selection_bbox" not in member
assert "warnings" not in member
assert "source_version" not in member
def test_the_member_attaches_under_a_reserved_key() -> None:
collection = {"type": "FeatureCollection", "features": []}
ExportService.attach_provenance(
collection,
ExportService.provenance_member(source="dataset", project_id=uuid4(), dataset_id=uuid4()),
)
assert collection["type"] == "FeatureCollection"
assert collection["geointel_provenance"]["source"] == "dataset"
@@ -0,0 +1,369 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
import pytest
SCRIPT = Path(__file__).parents[2] / "scripts" / "build_failure_driven_yolo_sampling.py"
SPEC = importlib.util.spec_from_file_location("failure_sampling", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def test_dataset_validation_source_preserves_manifest_path(tmp_path: Path):
source = tmp_path / "dataset.yaml"
source.write_text(
"path: /data/source\ntrain: /data/source/train.txt\n"
"val: /data/source/internal-val.txt\nnames:\n 0: building\n",
encoding="utf-8",
)
assert MODULE.dataset_validation_source(source) == "/data/source/internal-val.txt"
def test_sampling_rejects_protected_test_and_background_feedback() -> None:
manifest = {
"samples": [
{"sample_slug": "train-fl", "split": "train", "region": "flanders"},
{"sample_slug": "train-wa", "split": "train", "region": "wallonia"},
{"sample_slug": "test-fl", "split": "test", "region": "flanders"},
]
}
summary = {
"tiles": [
{"sample_slug": "train-fl", "split": "train", "label_count": 2, "image_path": "/tmp/fl-pos.png"},
{"sample_slug": "train-fl", "split": "train", "label_count": 0, "image_path": "/tmp/fl-neg.png"},
{"sample_slug": "train-wa", "split": "train", "label_count": 1, "image_path": "/tmp/wa-pos.png"},
{"sample_slug": "test-fl", "split": "val", "label_count": 1, "image_path": "/tmp/protected.png"},
]
}
assessment = {
"status": "continue_training_loop",
"gates": {
"min_region_f1": 0.45,
"min_region_precision": 0.5,
"min_region_recall": 0.4,
"max_pure_empty_false_positives": 0,
},
"test": {
"regions": {
"flanders": {"f1": 0.2, "precision": 0.3, "recall": 0.2},
"wallonia": {"f1": 0.6, "precision": 0.6, "recall": 0.6},
}
},
"background": {"pure_empty_false_positives": 2},
}
with pytest.raises(ValueError, match="protected test/background evidence"):
MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
)
def test_sampling_can_use_calibration_before_test_is_opened() -> None:
manifest = {"samples": [{"sample_slug": "train-fl", "split": "train", "region": "flanders"}]}
summary = {
"tiles": [
{"sample_slug": "train-fl", "split": "train", "label_count": 1, "image_path": "/tmp/fl.png"}
]
}
assessment = {
"status": "continue_training_loop",
"gates": {
"min_region_f1": 0.45,
"min_region_precision": 0.5,
"min_region_recall": 0.4,
"max_pure_empty_false_positives": 0,
},
"calibration": {
"regions": {"flanders": {"f1": 0.4, "precision": 0.6, "recall": 0.35}}
},
"test": None,
"background": None,
}
paths, metadata = MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
)
assert len(paths) == 3
assert metadata["failure_evidence_source"] == "calibration"
def test_precision_correction_can_balance_positive_and_negative_tiles() -> None:
manifest = {"samples": [{"sample_slug": "train-fl", "split": "train", "region": "flanders"}]}
summary = {
"tiles": [
{"sample_slug": "train-fl", "split": "train", "label_count": 2, "image_path": "/tmp/fl-pos.png"},
{"sample_slug": "train-fl", "split": "train", "label_count": 0, "image_path": "/tmp/fl-neg.png"},
]
}
assessment = {
"status": "continue_training_loop",
"gates": {
"min_region_f1": 0.45,
"min_region_precision": 0.5,
"min_region_recall": 0.4,
"max_pure_empty_false_positives": 0,
},
"calibration": {
"regions": {"flanders": {"f1": 0.46, "precision": 0.45, "recall": 0.46}}
},
}
paths, metadata = MODULE.build_sampling(
summary=summary,
manifest=manifest,
assessment=assessment,
precision_positive_repeat=2,
negative_repeat=3,
max_region_share=1.0,
)
assert paths.count(str(Path("/tmp/fl-pos.png").resolve())) == 2
assert paths.count(str(Path("/tmp/fl-neg.png").resolve())) == 3
assert metadata["precision_positive_repeat"] == 2
def test_sampling_targets_failed_calibration_contexts_without_using_protected_tiles() -> None:
manifest = {
"samples": [
{"sample_slug": "train-industry", "split": "train", "region": "flanders", "context": "industrial"},
{"sample_slug": "train-suburban", "split": "train", "region": "flanders", "context": "suburban"},
{"sample_slug": "cal-industry", "split": "calibration", "region": "flanders", "context": "industrial"},
]
}
summary = {
"tiles": [
{"sample_slug": "train-industry", "split": "train", "label_count": 2, "image_path": "/tmp/industry-pos.png"},
{"sample_slug": "train-industry", "split": "train", "label_count": 0, "image_path": "/tmp/industry-neg.png"},
{"sample_slug": "train-suburban", "split": "train", "label_count": 2, "image_path": "/tmp/suburban-pos.png"},
{"sample_slug": "cal-industry", "split": "val", "label_count": 2, "image_path": "/tmp/protected.png"},
]
}
assessment = {
"status": "continue_training_loop",
"gates": {
"min_region_f1": 0.45,
"min_region_precision": 0.5,
"min_region_recall": 0.4,
"max_pure_empty_false_positives": 0,
},
"calibration": {
"regions": {"flanders": {"f1": 0.3, "precision": 0.35, "recall": 0.27}},
"samples": {"cal-industry": {"f1": 0.2, "precision": 0.3, "recall": 0.15}},
},
}
paths, metadata = MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
)
assert paths.count(str(Path("/tmp/industry-pos.png").resolve())) == 5
assert paths.count(str(Path("/tmp/industry-neg.png").resolve())) == 1
assert paths.count(str(Path("/tmp/suburban-pos.png").resolve())) == 3
assert not any("protected" in path for path in paths)
assert metadata["weak_recall_contexts"] == ["flanders:industrial"]
assert metadata["weak_precision_contexts"] == ["flanders:industrial"]
assert metadata["recall_dominant_regions"] == ["flanders"]
def test_sampling_rejects_background_feedback_after_a_protected_background_opening() -> None:
manifest = {"samples": [
{"sample_slug": "positive", "split": "train", "region": "flanders", "context": "industrial"},
{"sample_slug": "negative", "split": "train", "region": "flanders", "context": "industrial-hard-negative"},
]}
summary = {"tiles": [
{"sample_slug": "positive", "split": "train", "label_count": 1, "image_path": "/tmp/positive.png"},
{"sample_slug": "negative", "split": "train", "label_count": 0, "image_path": "/tmp/negative.png"},
]}
assessment = {
"status": "continue_training_loop",
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
"max_pure_empty_false_positives": 0},
"calibration": {"regions": {
"flanders": {"f1": .25, "precision": .4, "recall": .2},
}},
"background": {"pure_empty_false_positives": 1},
}
with pytest.raises(ValueError, match="protected background evidence"):
MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0,
)
def test_region_cap_drops_only_repeats_and_preserves_every_unique_tile() -> None:
manifest = {"samples": [
{"sample_slug": "fl", "split": "train", "region": "flanders", "context": "industrial"},
{"sample_slug": "wa", "split": "train", "region": "wallonia", "context": "rural-town"},
{"sample_slug": "br", "split": "train", "region": "brussels", "context": "dense-urban"},
]}
summary = {"tiles": [
{"sample_slug": "fl", "split": "train", "label_count": 2, "image_path": f"/tmp/fl-{index}.png"}
for index in range(4)
] + [
{"sample_slug": "wa", "split": "train", "label_count": 2, "image_path": f"/tmp/wa-{index}.png"}
for index in range(2)
] + [
{"sample_slug": "br", "split": "train", "label_count": 2, "image_path": f"/tmp/br-{index}.png"}
for index in range(2)
]}
assessment = {
"status": "continue_training_loop",
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
"max_pure_empty_false_positives": 0},
"calibration": {"regions": {
"flanders": {"f1": .2, "precision": .3, "recall": .2},
"wallonia": {"f1": .6, "precision": .6, "recall": .6},
"brussels": {"f1": .6, "precision": .6, "recall": .6},
}},
}
paths, metadata = MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment,
positive_repeat=5, max_region_share=.65,
)
assert all(str(Path(f"/tmp/fl-{index}.png").resolve()) in paths for index in range(4))
assert metadata["pre_cap_entries_by_region"]["flanders"] == 20
assert metadata["sampled_entries_by_region"]["flanders"] == 7
assert metadata["dropped_region_repeat_count"] == 13
assert metadata["sampled_entries_by_region"]["flanders"] / len(paths) <= .65
def test_region_cap_rotates_repeats_between_sampling_rounds() -> None:
manifest = {"samples": [
{"sample_slug": "fl", "split": "train", "region": "flanders", "context": "industrial"},
{"sample_slug": "wa", "split": "train", "region": "wallonia", "context": "rural-town"},
{"sample_slug": "br", "split": "train", "region": "brussels", "context": "dense-urban"},
]}
summary = {"tiles": [
{"sample_slug": "fl", "split": "train", "label_count": 2, "image_path": f"/tmp/fl-{index}.png"}
for index in range(4)
] + [
{"sample_slug": "wa", "split": "train", "label_count": 2, "image_path": f"/tmp/wa-{index}.png"}
for index in range(2)
] + [
{"sample_slug": "br", "split": "train", "label_count": 2, "image_path": f"/tmp/br-{index}.png"}
for index in range(2)
]}
assessment = {
"status": "continue_training_loop",
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
"max_pure_empty_false_positives": 0},
"calibration": {"regions": {
"flanders": {"f1": .2, "precision": .3, "recall": .2},
"wallonia": {"f1": .6, "precision": .6, "recall": .6},
"brussels": {"f1": .6, "precision": .6, "recall": .6},
}},
}
first, first_metadata = MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment,
positive_repeat=5, max_region_share=.65, sampling_round=1,
)
second, second_metadata = MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment,
positive_repeat=5, max_region_share=.65, sampling_round=2,
)
assert first != second
assert set(first) == set(second)
assert first_metadata["sampling_round"] == 1
assert second_metadata["sampling_round"] == 2
def test_region_cap_preserves_failed_context_positive_before_hard_negative() -> None:
manifest = {"samples": [
{"sample_slug": "target", "split": "train", "region": "flanders", "context": "industrial"},
{"sample_slug": "negative", "split": "train", "region": "flanders", "context": "industrial-hard-negative"},
{"sample_slug": "wa", "split": "train", "region": "wallonia", "context": "rural-town"},
{"sample_slug": "br", "split": "train", "region": "brussels", "context": "dense-urban"},
]}
summary = {"tiles": [
{"sample_slug": "target", "split": "train", "label_count": 2, "image_path": "/tmp/target.png"},
{"sample_slug": "negative", "split": "train", "label_count": 0, "image_path": "/tmp/negative.png"},
{"sample_slug": "wa", "split": "train", "label_count": 1, "image_path": "/tmp/wa.png"},
{"sample_slug": "br", "split": "train", "label_count": 1, "image_path": "/tmp/br.png"},
]}
assessment = {
"status": "continue_training_loop",
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
"max_pure_empty_false_positives": 0},
"calibration": {
"regions": {
"flanders": {"f1": .2, "precision": .2, "recall": .3},
"wallonia": {"f1": .6, "precision": .6, "recall": .6},
"brussels": {"f1": .6, "precision": .6, "recall": .6},
},
"samples": {"target": {"f1": .2, "precision": .2, "recall": .3}},
},
}
paths, metadata = MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment, max_region_share=.65,
)
assert paths.count(str(Path("/tmp/target.png").resolve())) == 2
assert paths.count(str(Path("/tmp/negative.png").resolve())) == 1
assert metadata["priority_positive_repeat_count"] == 4
def test_precision_guard_band_keeps_near_gate_region_stabilized() -> None:
manifest = {"samples": [
{"sample_slug": "wa-positive", "split": "train", "region": "wallonia", "context": "rural-town"},
{"sample_slug": "wa-negative", "split": "train", "region": "wallonia", "context": "farmland-hard-negative"},
]}
summary = {"tiles": [
{"sample_slug": "wa-positive", "split": "train", "label_count": 1, "image_path": "/tmp/wa-positive.png"},
{"sample_slug": "wa-negative", "split": "train", "label_count": 0, "image_path": "/tmp/wa-negative.png"},
]}
assessment = {
"status": "continue_training_loop",
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
"max_pure_empty_false_positives": 0},
"calibration": {"regions": {
"wallonia": {"f1": .6, "precision": .52, "recall": .7},
}},
}
paths, metadata = MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0,
)
assert "wallonia" in metadata["weak_precision_regions"]
assert paths.count(str(Path("/tmp/wa-negative.png").resolve())) == 4
assert metadata["precision_guard_band"] == .03
def test_coastal_precision_failure_targets_port_and_dunes_negatives() -> None:
manifest = {"samples": [
{"sample_slug": "coastal-train", "split": "train", "region": "flanders", "context": "coastal-urban"},
{"sample_slug": "port-negative", "split": "train", "region": "flanders", "context": "port-hard-negative"},
{"sample_slug": "dunes-negative", "split": "train", "region": "flanders", "context": "dunes-negative"},
{"sample_slug": "coastal-cal", "split": "calibration", "region": "flanders", "context": "coastal-urban"},
]}
summary = {"tiles": [
{"sample_slug": "coastal-train", "split": "train", "label_count": 2, "image_path": "/tmp/coastal.png"},
{"sample_slug": "port-negative", "split": "train", "label_count": 0, "image_path": "/tmp/port.png"},
{"sample_slug": "dunes-negative", "split": "train", "label_count": 0, "image_path": "/tmp/dunes.png"},
]}
assessment = {
"status": "continue_training_loop",
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
"max_pure_empty_false_positives": 0},
"calibration": {
"regions": {"flanders": {"f1": .3, "precision": .2, "recall": .4}},
"samples": {"coastal-cal": {"f1": .1, "precision": .05, "recall": .2}},
},
}
paths, metadata = MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
)
assert paths.count(str(Path("/tmp/port.png").resolve())) == 6
assert paths.count(str(Path("/tmp/dunes.png").resolve())) == 6
assert "flanders:port-hard-negative" in metadata["targeted_negative_contexts"]
assert "flanders:dunes-negative" in metadata["targeted_negative_contexts"]
@@ -0,0 +1,105 @@
"""Flood risk must be a share of what was modelled, not of what was drawn.
The share and fraction divided the inundated cells by every cell whose centre
fell inside the selection, including cells where the VMM raster holds nodata
because the area lies outside the modelled extent. An operator drawing a
rectangle that reaches past the model coverage read "3% at risk" where the
honest answer is "of the 40% we have a model for, 7.5% is at risk, and for the
rest there is no model at all".
Terrain, bathymetry and thematic raster analysis already divide by valid cells
and report a coverage ratio; this brings flood hazard in line.
"""
from __future__ import annotations
import pytest
np = pytest.importorskip("numpy")
from app.services.flood_hazard_analysis_service import ( # noqa: E402 - optional NumPy gate precedes service import
FloodHazardCellStatistics,
)
NODATA = -9999.0
def _stats(values, selected) -> FloodHazardCellStatistics:
return FloodHazardCellStatistics.from_cells(
np.asarray(values, dtype="float64"),
np.asarray(selected, dtype=bool),
nodata=NODATA,
)
def test_share_ignores_cells_the_model_does_not_cover() -> None:
# Ten selected cells: four modelled (one of them wet), six nodata.
values = [1.5, 0.0, 0.0, 0.0] + [NODATA] * 6
selected = [True] * 10
stats = _stats(values, selected)
assert stats.selected_cell_count == 10
assert stats.valid_cell_count == 4
assert stats.no_data_cell_count == 6
assert stats.inundated_cell_count == 1
# 1 of 4 modelled cells, not 1 of 10 drawn cells.
assert stats.inundated_fraction == pytest.approx(0.25)
assert stats.data_coverage_ratio == pytest.approx(0.4)
def test_cells_outside_the_drawn_selection_are_not_counted() -> None:
values = [1.5, 1.5, 0.0, 0.0]
selected = [True, False, True, False]
stats = _stats(values, selected)
assert stats.selected_cell_count == 2
assert stats.valid_cell_count == 2
assert stats.inundated_cell_count == 1
assert stats.inundated_fraction == pytest.approx(0.5)
def test_a_selection_without_any_model_data_reports_zero_coverage() -> None:
stats = _stats([NODATA] * 4, [True] * 4)
assert stats.valid_cell_count == 0
assert stats.no_data_cell_count == 4
assert stats.data_coverage_ratio == 0.0
# No model, so no risk figure may be invented.
assert stats.inundated_fraction is None
def test_nan_is_treated_as_missing_model_data() -> None:
stats = _stats([float("nan"), 2.0], [True, True])
assert stats.valid_cell_count == 1
assert stats.no_data_cell_count == 1
assert stats.inundated_cell_count == 1
def test_negative_depths_are_data_but_not_inundation() -> None:
"""A modelled zero or negative depth means dry, not unknown."""
stats = _stats([0.0, 0.0, 3.0], [True, True, True])
assert stats.valid_cell_count == 3
assert stats.inundated_cell_count == 1
assert stats.inundated_fraction == pytest.approx(1 / 3)
def test_depth_statistics_use_only_inundated_cells() -> None:
stats = _stats([0.0, 2.0, 4.0, NODATA], [True] * 4)
assert stats.depth_values.tolist() == [2.0, 4.0]
assert stats.depth_values.mean() == pytest.approx(3.0)
def test_areas_are_derived_from_the_matching_cell_populations() -> None:
stats = _stats([1.0, 1.0, 0.0, NODATA], [True] * 4)
# 100 m2 cells: 2 inundated, 3 modelled, 4 drawn.
assert stats.inundated_area_ha(100.0) == pytest.approx(2 * 100.0 / 10_000.0)
assert stats.analysed_area_ha(100.0) == pytest.approx(3 * 100.0 / 10_000.0)
assert stats.selected_area_ha(100.0) == pytest.approx(4 * 100.0 / 10_000.0)
@@ -0,0 +1,22 @@
from __future__ import annotations
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def test_frontend_api_client_accepts_top_level_error_contract() -> None:
client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8")
assert 'typeof payload?.error === "string"' in client
assert "payload?.message" in client
assert "payload?.details" in client
def test_frontend_api_client_remains_legacy_error_tolerant() -> None:
client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8")
assert "legacyError?.code" in client
assert "legacyError?.message" in client
assert "legacyError?.details" in client
@@ -0,0 +1,120 @@
"""Keep the frontend-contract tests from drifting back to formatting checks.
211 of the backend test files read frontend sources and assert on literal
substrings. That reds the suite on renames and copy changes without proving
anything about behaviour, and it is why the suite was failing on main.
This guard blocks the two patterns that caused the failures: counting
occurrences of a code fragment (a formatting rule) and asserting an exact
number of hook calls. New wiring assertions belong in
``tests/frontend_contract.py``; anything about what a user sees belongs in a
frontend component test, where the rendered output can be checked.
"""
from __future__ import annotations
import re
from pathlib import Path
TESTS_DIR = Path(__file__).resolve().parent
# Counting a quoted *code* fragment — one containing a bracket, an arrow or a
# semicolon — is the pattern to block. Counting values in a list, or counting
# how often a URL was requested, are ordinary behavioural assertions.
COUNT_ASSERTION = re.compile(
r"""assert\s+\w+\.count\(\s*["'][^"']*[(){}=;][^"']*["']\s*\)\s*[=!<>]=""",
)
# Tests that predate the guard and still count call sites. The list must only
# ever shrink; a new entry means a new formatting test was written.
KNOWN_COUNT_ASSERTIONS: set[str] = set()
def _test_sources() -> list[tuple[Path, str]]:
return [
(path, path.read_text(encoding="utf-8"))
for path in sorted(TESTS_DIR.glob("test_*.py"))
if path.name != Path(__file__).name
]
def test_no_test_counts_occurrences_of_a_source_fragment() -> None:
offenders = {
path.name
for path, source in _test_sources()
if COUNT_ASSERTION.search(source)
}
unexpected = sorted(offenders - KNOWN_COUNT_ASSERTIONS)
assert not unexpected, (
"These tests assert on how often a code fragment appears, which is a "
f"formatting rule rather than a contract: {unexpected}. Use "
"tests/frontend_contract.py to assert on wiring instead."
)
def test_the_known_offender_list_does_not_grow_stale() -> None:
"""A file listed as an exception must still exist and still offend."""
offenders = {
path.name
for path, source in _test_sources()
if COUNT_ASSERTION.search(source)
}
stale = sorted(KNOWN_COUNT_ASSERTIONS - offenders)
assert not stale, f"Remove these from KNOWN_COUNT_ASSERTIONS; they are clean now: {stale}"
def test_frontend_contract_helpers_are_available() -> None:
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired
source = "const total = analyzeSelection(bbox, areaIdForSelection(bbox))"
assert_wired(source, "analyzeSelection")
assert_calls(source, "analyzeSelection", first_argument="bbox")
assert_mentions(source, "AREAIDFORSELECTION")
FRONTEND_READ = re.compile(
r'(?P<var>\w+)\s*=\s*\(?\s*(?:ROOT|root|REPO_ROOT)\s*/\s*'
r'(?P<path>"[^"]+"(?:\s*/\s*"[^"]+")*)\s*\)?\.read_text\('
)
NEGATIVE_ASSERT = re.compile(r"assert\s+[^\n]*not in\s+(\w+)")
def _feature_owner() -> dict[str, str]:
from tests.frontend_contract import FEATURE_SOURCES
return {source: feature for feature, sources in FEATURE_SOURCES.items() for source in sources}
def test_a_positive_contract_reads_the_feature_not_one_file() -> None:
"""Moving code between sibling modules must not red the suite.
A single-file read is right for a *negative* contract — "this component
performs no transport" is a statement about that file, and widening it
would quietly weaken the check. For a positive contract it pins the
contract to whichever file happens to hold it today.
"""
owner = _feature_owner()
offenders: list[str] = []
for path, source in _test_sources():
if "frontend" not in source:
continue
negatives = set(NEGATIVE_ASSERT.findall(source))
for match in FRONTEND_READ.finditer(source):
if match.group("var") in negatives:
continue
joined = re.sub(r'["\s/]+', "/", match.group("path")).strip("/")
if "frontend/src/" not in joined:
continue
relative = joined.split("frontend/src/", 1)[1]
if relative in owner:
offenders.append(f"{path.name}: {match.group('var')} -> {relative}")
assert not offenders, (
"These read one file of a multi-module feature for a positive contract. "
f"Use read_feature() from tests/frontend_contract.py instead: {sorted(offenders)}"
)
@@ -0,0 +1,188 @@
from __future__ import annotations
import json
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
from app.core.errors import AppError
from app.services import geojson_service
from app.services.dataset_service import DatasetService
def test_parse_geojson_payload_extracts_metadata() -> None:
payload = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [4.5, 51.3],
},
}
],
}
metadata = geojson_service.parse_geojson_payload(payload)
assert metadata["feature_count"] == 1
assert metadata["feature_geometry_count"] == 1
assert metadata["bounds_json"] == {
"min_x": 4.5,
"min_y": 51.3,
"max_x": 4.5,
"max_y": 51.3,
}
assert metadata["geometry_types"] == ["Point"]
def test_parse_geojson_payload_rejects_non_feature_collection() -> None:
payload = {"type": "Feature", "features": []}
try:
geojson_service.parse_geojson_payload(payload)
except ValueError as exc:
assert "FeatureCollection" in str(exc)
else:
raise AssertionError("Invalid GeoJSON should raise ValueError")
def test_get_dataset_geojson_reads_stored_payload(tmp_path, monkeypatch) -> None:
file_path = tmp_path / "dataset.geojson"
file_path.write_text(
json.dumps({"type": "FeatureCollection", "features": []}),
encoding="utf-8",
)
dataset = SimpleNamespace(dataset_type="vector", storage_path=str(file_path))
monkeypatch.setattr(DatasetService, "get_dataset", lambda _db, _id: dataset)
payload = DatasetService.get_dataset_geojson(Path("."), uuid4())
assert payload["type"] == "FeatureCollection"
def test_get_dataset_geojson_rejects_invalid_stored_json(tmp_path, monkeypatch) -> None:
file_path = tmp_path / "invalid.geojson"
file_path.write_text("not-json", encoding="utf-8")
dataset = SimpleNamespace(dataset_type="vector", storage_path=str(file_path))
monkeypatch.setattr(DatasetService, "get_dataset", lambda _db, _id: dataset)
try:
DatasetService.get_dataset_geojson(Path("."), uuid4())
except AppError as exc:
assert exc.code == "INVALID_GEOJSON"
else:
raise AssertionError("Invalid stored payload should raise AppError")
def test_parse_geojson_payload_returns_vector_metadata() -> None:
payload = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {
"type": "Polygon",
"coordinates": [
[
[4.3, 51.2],
[4.4, 51.2],
[4.4, 51.3],
[4.3, 51.3],
[4.3, 51.2],
]
],
},
}
],
"crs": {"type": "name", "properties": {"name": "EPSG:31370"}},
}
metadata = geojson_service.parse_geojson_payload(payload)
assert metadata["feature_count"] == 1
assert metadata["feature_geometry_count"] == 1
assert metadata["geometry_types"] == ["Polygon"]
assert metadata["bounds_json"] == {
"min_x": 4.3,
"min_y": 51.2,
"max_x": 4.4,
"max_y": 51.3,
}
assert metadata["crs"] == "EPSG:31370"
assert metadata["approximate_area_m2"] is not None
assert metadata["approximate_area_m2"] >= 0.0
def test_parse_geojson_payload_reports_z_dimension_for_canonical_2d_storage() -> None:
metadata = geojson_service.parse_geojson_payload(
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.08, 51.18, 0.0]},
"properties": {},
}
],
}
)
assert metadata["z_dimension_feature_count"] == 1
assert metadata["canonical_storage_dimension"] == "2D"
def test_parse_geojson_payload_rejects_invalid_geometry() -> None:
payload = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {
"type": "Polygon",
"coordinates": "invalid",
},
}
],
}
try:
geojson_service.parse_geojson_payload(payload)
except ValueError as exc:
assert "Invalid feature geometry" in str(exc)
else:
raise AssertionError("Invalid geometry should raise ValueError")
def test_get_dataset_geojson_accepts_legacy_geojson_type(tmp_path, monkeypatch) -> None:
file_path = tmp_path / "legacy.geojson"
file_path.write_text(
json.dumps({"type": "FeatureCollection", "features": []}),
encoding="utf-8",
)
dataset = SimpleNamespace(dataset_type="geojson", storage_path=str(file_path))
monkeypatch.setattr(DatasetService, "get_dataset", lambda _db, _id: dataset)
payload = DatasetService.get_dataset_geojson(Path("."), uuid4())
assert payload["type"] == "FeatureCollection"
def test_vector_summary_supports_legacy_geojson_type(monkeypatch) -> None:
dataset = SimpleNamespace(
dataset_type="geojson",
metadata_json={
"feature_count": 7,
"geometry_types": ["Point"],
"bounds_json": {"min_x": 0.0, "min_y": 0.0, "max_x": 1.0, "max_y": 1.0},
},
storage_path="",
)
monkeypatch.setattr(DatasetService, "get_dataset", lambda _db, _id: dataset)
summary = DatasetService.vector_summary(Path("."), uuid4())
assert summary["feature_count"] == 7
assert summary["geometry_types"] == ["Point"]
@@ -0,0 +1,765 @@
from __future__ import annotations
from datetime import UTC, datetime
from hashlib import sha256
import json
from pathlib import Path
from uuid import UUID, uuid4
import pytest
from pyproj import Transformer
from app.core.errors import AppError
from app.models import (
Area,
Dataset,
DatasetQuarantine,
DatasetVersion,
Project,
SourceRegistry,
SourceSnapshot,
VectorFeature,
)
from app.services.dataset_service import DatasetService, _PartitionedGeoJsonRecords
from app.services.vector_operations_service import VectorOperationsService
class _Query:
def __init__(self, session: "_Session", model: type) -> None:
self.session = session
self.model = model
self.predicates = []
def filter(self, *predicates):
self.predicates.extend(predicates)
return self
def one_or_none(self):
matches = self._matches()
if len(matches) > 1:
raise AssertionError(
f"expected one {self.model.__name__}, found {len(matches)}"
)
return matches[0] if matches else None
def all(self):
return self._matches()
def _matches(self):
matches = list(self.session.rows.get(self.model, []))
for predicate in self.predicates:
field_name = predicate.left.key
expected = predicate.right.value
operator_name = getattr(predicate.operator, "__name__", "")
if operator_name == "in_op":
matches = [
item for item in matches if getattr(item, field_name) in expected
]
else:
matches = [
item for item in matches if getattr(item, field_name) == expected
]
return matches
class _Session:
"""Small ORM-shaped harness that exercises the real governed path."""
def __init__(self, project: Project) -> None:
self.rows: dict[type, list[object]] = {Project: [project]}
self.commits = 0
self.rollbacks = 0
self.flushes = 0
def get(self, model: type, item_id: UUID):
return next(
(
item
for item in self.rows.get(model, [])
if getattr(item, "id", None) == item_id
),
None,
)
def query(self, model: type) -> _Query:
return _Query(self, model)
def add(self, item: object) -> None:
if getattr(item, "id", None) is None:
setattr(item, "id", uuid4())
self.rows.setdefault(type(item), []).append(item)
def flush(self) -> None:
self.flushes += 1
def commit(self) -> None:
self.commits += 1
def rollback(self) -> None:
self.rollbacks += 1
def refresh(self, _item: object) -> None:
return None
def expunge(self, _item: object) -> None:
return None
def _storage_info(tmp_path: Path, content: bytes) -> dict[str, object]:
path = tmp_path / "grb-buildings.geojson"
path.write_bytes(content)
return {
"storage_path": str(path),
"original_filename": path.name,
"stored_filename": path.name,
"content_type": "application/geo+json",
"size_bytes": len(content),
"checksum_sha256": sha256(content).hexdigest(),
}
def _valid_payload() -> bytes:
return json.dumps(
{
"type": "FeatureCollection",
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
"features": [
{
"type": "Feature",
"id": "gbg-1",
"properties": {"id": "gbg-1"},
"geometry": {
"type": "Polygon",
"coordinates": [
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
],
},
}
],
}
).encode("utf-8")
def _grb_payload_without_required_id() -> bytes:
return json.dumps(
{
"type": "FeatureCollection",
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
"features": [
{
"type": "Feature",
"properties": {"unrelated": "not a GRB identity"},
"geometry": {
"type": "Polygon",
"coordinates": [
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
],
},
}
],
}
).encode("utf-8")
def _lambert_grb_payload() -> tuple[bytes, tuple[float, float, float, float]]:
"""Create a valid GRB-shaped source artifact in its declared native CRS."""
longitude, latitude = 4.70, 51.10
max_longitude, max_latitude = 4.7001, 51.1001
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
lambert_ring = [
to_lambert.transform(longitude, latitude),
to_lambert.transform(max_longitude, latitude),
to_lambert.transform(max_longitude, max_latitude),
to_lambert.transform(longitude, latitude),
]
return (
json.dumps(
{
"type": "FeatureCollection",
"crs": {"type": "name", "properties": {"name": "EPSG:31370"}},
"features": [
{
"type": "Feature",
"id": "GBG.lambert.1",
"properties": {"id": "GBG.lambert.1"},
"geometry": {"type": "Polygon", "coordinates": [lambert_ring]},
}
],
}
).encode("utf-8"),
(longitude, latitude, max_longitude, max_latitude),
)
def test_governed_vector_import_persists_snapshot_contract_and_queryable_features(
monkeypatch, tmp_path: Path
) -> None:
project = Project(id=uuid4(), name="Phase 2 governed ingest")
db = _Session(project)
raw = _valid_payload()
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
lambda **_kwargs: _storage_info(tmp_path, raw),
)
result = DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="grb-buildings.geojson",
content=raw,
source="grb_wfs",
source_name="grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={
"license": "Open data",
"source_url": "https://example.invalid/grb",
},
provenance_metadata={"adapter": "test"},
temporal_series_key="grb:2026-08",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01",
temporal_granularity="snapshot",
)
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
snapshot = db.rows[SourceSnapshot][0]
source = db.rows[SourceRegistry][0]
assert result.status == "ready"
assert dataset.source_name == "grb"
assert dataset.source_registry_id == source.id
assert dataset.source_snapshot_id == snapshot.id
assert dataset.validation_status == "passed"
assert dataset.provenance_status == "complete"
assert dataset.lineage_status == "complete"
assert dataset.quarantine_status == "not_quarantined"
assert dataset.crs == "EPSG:4326"
assert snapshot.checksum_sha256 == sha256(raw).hexdigest()
assert len(db.rows[VectorFeature]) == 1
assert db.commits == 1
# A retry with identical governed evidence is idempotent and does not
# create a second source snapshot, dataset or vector feature.
repeated = DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="grb-buildings.geojson",
content=raw,
source="grb_wfs",
source_name="grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={"license": "Open data"},
provenance_metadata={"adapter": "test"},
temporal_series_key="grb:2026-08",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01",
temporal_granularity="snapshot",
)
assert repeated.id == result.id
assert len(db.rows[Dataset]) == 1
assert len(db.rows[SourceSnapshot]) == 1
assert len(db.rows[VectorFeature]) == 1
def test_governed_lambert_geojson_persists_canonical_consumption_bytes_and_provenance_evidence(
monkeypatch, tmp_path: Path
) -> None:
"""Projected source bytes must never be the file that vector operations consume."""
project = Project(id=uuid4(), name="Canonical GeoJSON storage")
db = _Session(project)
raw, (longitude, latitude, max_longitude, max_latitude) = _lambert_grb_payload()
consumption_path = tmp_path / "consumption" / "grb-buildings.geojson"
provenance_path = tmp_path / "provenance" / "grb-buildings.geojson"
def _persist_dataset_file(**kwargs):
stored = kwargs["content"]
consumption_path.parent.mkdir(parents=True, exist_ok=True)
consumption_path.write_bytes(stored)
return _storage_info(consumption_path.parent, stored)
def _persist_file(storage_path, content, original_filename, content_type):
del storage_path, original_filename, content_type
provenance_path.parent.mkdir(parents=True, exist_ok=True)
provenance_path.write_bytes(content)
return _storage_info(provenance_path.parent, content)
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
_persist_dataset_file,
)
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_file",
_persist_file,
)
result = DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="grb-lambert.geojson",
content=raw,
source="grb_wfs",
source_name="grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={"license": "Open data"},
provenance_metadata={"adapter": "test"},
temporal_series_key="grb:lambert:2026-08",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01-lambert",
temporal_granularity="snapshot",
)
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
dataset_version = db.rows[DatasetVersion][0]
snapshot = db.rows[SourceSnapshot][0]
canonical_bytes = Path(str(dataset.storage_path)).read_bytes()
canonical_payload = json.loads(canonical_bytes)
source_artifact = dataset.provenance_metadata["source_artifact"]
assert result.status == "ready"
assert dataset.crs == "EPSG:4326"
assert canonical_payload["crs"]["properties"]["name"] == "EPSG:4326"
assert canonical_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
[longitude, latitude], abs=0.000001
)
assert sha256(canonical_bytes).hexdigest() == dataset.checksum_sha256
assert dataset_version.checksum_sha256 == dataset.checksum_sha256
assert snapshot.checksum_sha256 == dataset.checksum_sha256
assert source_artifact["retention"] == "provenance_evidence_only"
assert source_artifact["checksum_sha256"] == sha256(raw).hexdigest()
assert source_artifact["storage_path"] != dataset.storage_path
assert Path(source_artifact["storage_path"]).read_bytes() == raw
assert dataset.provenance_metadata["canonical_consumption_artifact"] == {
"checksum_sha256": dataset.checksum_sha256,
"crs": "EPSG:4326",
"storage_role": "dataset_consumption",
}
inspection = VectorOperationsService.inspect(db, dataset.id)
assert inspection.crs == "EPSG:4326"
assert inspection.bounds_json == {
"min_x": pytest.approx(longitude, abs=0.000001),
"min_y": pytest.approx(latitude, abs=0.000001),
"max_x": pytest.approx(max_longitude, abs=0.000001),
"max_y": pytest.approx(max_latitude, abs=0.000001),
}
response_payload = DatasetService.get_dataset_geojson(db, dataset.id)
assert response_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
[longitude, latitude], abs=0.000001
)
# The storage identity is enforced at the operation boundary too; a
# replacement with different canonical bytes is not silently processed.
Path(str(dataset.storage_path)).write_bytes(canonical_bytes + b"\n")
with pytest.raises(AppError) as exc_info:
VectorOperationsService.inspect(db, dataset.id)
assert exc_info.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH"
def test_metadata_refresh_refuses_mutated_governed_artifact(
monkeypatch, tmp_path: Path
) -> None:
"""A passed snapshot cannot be silently re-described from mutable storage."""
project = Project(id=uuid4(), name="Phase 2 immutable refresh")
db = _Session(project)
raw = _valid_payload()
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
lambda **_kwargs: _storage_info(tmp_path, raw),
)
result = DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="grb-buildings.geojson",
content=raw,
source="grb_wfs",
source_name="grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={"license": "Open data"},
provenance_metadata={"adapter": "test"},
temporal_series_key="grb:2026-08",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01",
temporal_granularity="snapshot",
)
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
original_checksum = dataset.checksum_sha256
original_metadata = dict(dataset.metadata_json or {})
original_commit_count = db.commits
# Simulate an out-of-band storage replacement at the same path. The
# refresh endpoint must not parse it into an already-passed contract row.
Path(str(dataset.storage_path)).write_bytes(_grb_payload_without_required_id())
with pytest.raises(AppError) as exc_info:
DatasetService.refresh_metadata(db, dataset.id)
assert exc_info.value.code == "GOVERNED_DATASET_REINGEST_REQUIRED"
assert dataset.status == "ready"
assert dataset.validation_status == "passed"
assert dataset.checksum_sha256 == original_checksum
assert dataset.metadata_json == original_metadata
assert db.commits == original_commit_count
def test_governed_import_quarantines_bad_artifacts_and_refuses_unknown_source(
monkeypatch, tmp_path: Path
) -> None:
project = Project(id=uuid4(), name="Phase 2 quarantine")
db = _Session(project)
raw = b'{"type":"FeatureCollection","features":[]}'
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
lambda **_kwargs: _storage_info(tmp_path, raw),
)
quarantined = DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="empty.geojson",
content=raw,
source="grb_wfs",
source_name="grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={"license": "Open data"},
provenance_metadata={},
temporal_series_key="grb:2026-08-empty",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01-empty",
temporal_granularity="snapshot",
)
assert quarantined.status == "quarantined"
assert quarantined.validation_status == "failed"
assert quarantined.quarantine_status == "quarantined"
assert len(db.rows[DatasetQuarantine]) == 1
assert db.rows[SourceSnapshot][0].ingest_status == "quarantined"
with pytest.raises(AppError) as exc_info:
DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="unregistered.geojson",
content=_valid_payload(),
source="caller_controlled",
source_name="caller_claimed_grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={"license": "Open data"},
provenance_metadata={},
)
assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND"
def test_governed_grb_vector_quarantines_missing_server_owned_required_attribute(
monkeypatch, tmp_path: Path
) -> None:
project = Project(id=uuid4(), name="Phase 2 source schema")
db = _Session(project)
raw = _grb_payload_without_required_id()
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
lambda **kwargs: _storage_info(tmp_path, kwargs["content"]),
)
quarantined = DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="grb-missing-id.geojson",
content=raw,
source="grb_wfs",
source_name="grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={"license": "Open data"},
provenance_metadata={"adapter": "test"},
temporal_series_key="grb:missing-id",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01-missing-id",
temporal_granularity="snapshot",
)
dataset = next(item for item in db.rows[Dataset] if item.id == quarantined.id)
assert quarantined.status == "quarantined"
assert dataset.validation_status == "failed"
assert dataset.quarantine_status == "quarantined"
issue = dataset.validation_report_json["issues"][0]
assert issue["code"] == "SOURCE_SCHEMA_REQUIRED_ATTRIBUTE_MISSING"
assert issue["category"] == "source_schema"
assert len(db.rows[DatasetQuarantine]) == 1
def test_partitioned_vector_ingest_is_idempotent_and_quarantines_noncanonical_partition_coordinates(
monkeypatch,
tmp_path: Path,
) -> None:
project = Project(id=uuid4(), name="Partitioned governed ingest")
area = Area(id=uuid4(), project_id=project.id, name="Partitioned AOI")
db = _Session(project)
db.rows[Area] = [area]
feature = {
"type": "Feature",
"id": "GBG.1",
"properties": {"id": "GBG.1", "source_feature_id": "GBG.1"},
"geometry": {
"type": "Polygon",
"coordinates": [
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
],
},
}
partition_payload = {"type": "FeatureCollection", "features": [feature]}
partition_path = tmp_path / "partition-01.geojson"
partition_path.write_text(json.dumps(partition_payload), encoding="utf-8")
artifact_payload = {
"type": "FeatureCollection",
"crs": "EPSG:4326",
"features": [feature],
}
artifact_path = tmp_path / "grb-partitioned.geojson"
artifact_raw = json.dumps(artifact_payload).encode("utf-8")
artifact_path.write_bytes(artifact_raw)
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
lambda **_kwargs: _storage_info(tmp_path, artifact_raw),
)
result = DatasetService.import_partitioned_vector_artifact(
db,
project_id=project.id,
area_id=area.id,
artifact_path=artifact_path,
partition_paths=[partition_path],
original_filename="grb-partitioned.geojson",
source="operator_official_import",
dataset_role="reference",
source_name="grb",
reference_layer_name="buildings",
metadata_json={
"feature_count": 1,
"crs": "EPSG:4326",
"bounds_json": {
"min_x": 4.69,
"min_y": 51.09,
"max_x": 4.70,
"max_y": 51.10,
},
},
source_metadata={"license": "Open data"},
provenance_metadata={
"artifact_sha256": sha256(artifact_raw).hexdigest(),
"partition_checksums": {
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
},
},
temporal_series_key="grb:partitioned:test",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01",
)
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
assert result.status == "ready"
assert dataset.validation_status == "passed"
assert dataset.provenance_status == "complete"
assert dataset.source_name == "grb"
assert len(db.rows[SourceSnapshot]) == 1
assert len(db.rows[VectorFeature]) == 1
assert dataset.metadata_json["partitioned_geometry_audit"][
"partition_checksums_sha256"
] == {partition_path.name: sha256(partition_path.read_bytes()).hexdigest()}
assert dataset.provenance_metadata["partition_checksum_manifest_sha256"]
assert dataset.provenance_metadata["partitioned_artifact_binding_sha256"]
repeated = DatasetService.import_partitioned_vector_artifact(
db,
project_id=project.id,
area_id=area.id,
artifact_path=artifact_path,
partition_paths=[partition_path],
original_filename="grb-partitioned.geojson",
source="operator_official_import",
dataset_role="reference",
source_name="grb",
reference_layer_name="buildings",
metadata_json={
"feature_count": 1,
"crs": "EPSG:4326",
"bounds_json": {
"min_x": 4.69,
"min_y": 51.09,
"max_x": 4.70,
"max_y": 51.10,
},
},
source_metadata={"license": "Open data"},
provenance_metadata={
"artifact_sha256": sha256(artifact_raw).hexdigest(),
"partition_checksums": {
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
},
},
temporal_series_key="grb:partitioned:test",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01",
)
assert repeated.id == result.id
assert len(db.rows[Dataset]) == 1
assert len(db.rows[VectorFeature]) == 1
lambert_feature = {
**feature,
"id": "GBG.lambert",
"properties": {"id": "GBG.lambert"},
"geometry": {
"type": "Polygon",
"coordinates": [
[[150000, 170000], [150010, 170000], [150010, 170010], [150000, 170000]]
],
},
}
lambert_partition = tmp_path / "partition-lambert.geojson"
lambert_partition.write_text(
json.dumps({"type": "FeatureCollection", "features": [lambert_feature]}),
encoding="utf-8",
)
lambert_artifact = tmp_path / "grb-lambert.geojson"
lambert_raw = json.dumps(
{"type": "FeatureCollection", "features": [lambert_feature]}
).encode("utf-8")
lambert_artifact.write_bytes(lambert_raw)
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
lambda **_kwargs: _storage_info(tmp_path, lambert_raw),
)
quarantined = DatasetService.import_partitioned_vector_artifact(
db,
project_id=project.id,
area_id=area.id,
artifact_path=lambert_artifact,
partition_paths=[lambert_partition],
original_filename="grb-lambert.geojson",
source="operator_official_import",
dataset_role="reference",
source_name="grb",
reference_layer_name="buildings",
metadata_json={
"feature_count": 1,
"crs": "EPSG:4326",
"bounds_json": {
"min_x": 150000,
"min_y": 170000,
"max_x": 150010,
"max_y": 170010,
},
},
source_metadata={"license": "Open data"},
provenance_metadata={
"artifact_sha256": sha256(lambert_raw).hexdigest(),
"partition_checksums": {
lambert_partition.name: sha256(
lambert_partition.read_bytes()
).hexdigest()
},
},
temporal_series_key="grb:partitioned:lambert",
observed_at=datetime(2026, 8, 2, tzinfo=UTC),
source_version="2026-08-02",
)
assert quarantined.status == "quarantined"
assert quarantined.validation_status == "failed"
assert quarantined.quarantine_status == "quarantined"
missing_manifest_partition = tmp_path / "partition-missing-manifest.geojson"
missing_manifest_partition.write_text(
json.dumps(partition_payload), encoding="utf-8"
)
missing_manifest_artifact = tmp_path / "grb-missing-manifest.geojson"
missing_manifest_raw = json.dumps(
{"type": "FeatureCollection", "features": [feature]}
).encode("utf-8")
missing_manifest_artifact.write_bytes(missing_manifest_raw)
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
lambda **_kwargs: _storage_info(tmp_path, missing_manifest_raw),
)
missing_manifest = DatasetService.import_partitioned_vector_artifact(
db,
project_id=project.id,
area_id=area.id,
artifact_path=missing_manifest_artifact,
partition_paths=[missing_manifest_partition],
original_filename="grb-missing-manifest.geojson",
source="operator_official_import",
dataset_role="reference",
source_name="grb",
reference_layer_name="buildings",
metadata_json={
"feature_count": 1,
"crs": "EPSG:4326",
"bounds_json": {
"min_x": 4.69,
"min_y": 51.09,
"max_x": 4.70,
"max_y": 51.10,
},
},
source_metadata={"license": "Open data"},
provenance_metadata={
"artifact_sha256": sha256(missing_manifest_raw).hexdigest()
},
temporal_series_key="grb:partitioned:missing-manifest",
observed_at=datetime(2026, 8, 3, tzinfo=UTC),
source_version="2026-08-03",
)
assert missing_manifest.status == "quarantined"
assert (
missing_manifest.validation_report_json["issues"][0]["code"]
== "PARTITION_CHECKSUM_MANIFEST_REQUIRED"
)
def test_partitioned_geometry_audit_handles_more_than_generic_topology_limit_without_materializing_geometries(
tmp_path: Path,
) -> None:
feature_count = 10_001
partition_path = tmp_path / "large-partition.geojson"
partition_path.write_text(
json.dumps(
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": f"GBG.{index}",
"properties": {"id": f"GBG.{index}"},
"geometry": {
"type": "Point",
"coordinates": [4.0 + index / 10_000_000, 51.0],
},
}
for index in range(feature_count)
],
}
),
encoding="utf-8",
)
audit = _PartitionedGeoJsonRecords(
[partition_path],
expected_feature_count=feature_count,
declared_partition_checksums={
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
},
).audit()
assert audit.feature_count == feature_count
assert audit.bounds_json["min_x"] == 4.0
assert audit.bounds_json["max_x"] > audit.bounds_json["min_x"]
assert audit.representative_record.geometry.geom_type == "MultiPoint"
@@ -0,0 +1,127 @@
from __future__ import annotations
import json
import subprocess
import sys
from hashlib import sha256
from pathlib import Path
from PIL import Image
SCRIPT = Path(__file__).parents[2] / "scripts" / "build_grayscale_yolo_dataset.py"
def test_grayscale_builder_preserves_labels_and_split(tmp_path: Path) -> None:
source = tmp_path / "source"
entries = []
for split, sample_slug, colour in (("train", "fixture-train", (255, 0, 0)), ("val", "fixture-val", (0, 255, 0))):
image = source / "images" / split / f"{sample_slug}.png"
label = source / "labels" / split / f"{sample_slug}.txt"
image.parent.mkdir(parents=True, exist_ok=True)
label.parent.mkdir(parents=True, exist_ok=True)
Image.new("RGB", (8, 8), colour).save(image)
label.write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8")
entries.append((split, sample_slug, image, label))
manifest = source / "operator_samples_manifest.json"
policy = "geointel-training-source-eligibility/v1"
manifest.write_text(
json.dumps(
{
"immutable": True,
"training_eligibility": {"policy_version": policy, "status": "eligible", "fixture_mode": True},
"samples": [
{
"sample_slug": sample_slug,
"split": split,
"raster_dataset_id": f"raster:{sample_slug}",
"reference_dataset_id": f"reference:{sample_slug}",
"training_eligibility": {
"policy_version": policy,
"eligible": True,
"fixture_mode": True,
"raster": {"eligible": True, "reasons": [], "evidence": {"dataset_id": f"raster:{sample_slug}", "checksum_sha256": "a" * 64, "source_registry_id": "fixture-raster", "source_snapshot_id": "fixture-raster-snapshot"}},
"reference": {"eligible": True, "reasons": [], "evidence": {"dataset_id": f"reference:{sample_slug}", "checksum_sha256": "b" * 64, "source_registry_id": "fixture-reference", "source_snapshot_id": "fixture-reference-snapshot"}},
},
}
for split, sample_slug, _image, _label in entries
],
}
),
encoding="utf-8",
)
(source / "corpus-freeze.json").write_text(
json.dumps(
{
"schema_version": 2,
"immutable": True,
"fixture_mode": True,
"training_eligibility_policy": policy,
"manifest_sha256": sha256(manifest.read_bytes()).hexdigest(),
}
),
encoding="utf-8",
)
dataset_yaml = source / "dataset.yaml"
dataset_yaml.write_text(
f"path: {source}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
encoding="utf-8",
)
release_script = SCRIPT.parent / "training_release_manifest.py"
subprocess.run(
[
sys.executable,
str(release_script),
"create",
"--train-yaml",
str(dataset_yaml),
"--corpus-manifest",
str(manifest),
"--fixture-mode",
],
check=True,
)
release_path = dataset_yaml.with_name(dataset_yaml.name + ".geointel-training-release.json")
asset_path = dataset_yaml.with_name(dataset_yaml.name + ".geointel-training-assets.json")
assets = json.loads(asset_path.read_text(encoding="utf-8"))
summary = source / "yolo_tile_dataset_summary.json"
summary.write_text(
json.dumps(
{
"dataset_yaml": str(dataset_yaml.resolve()),
"training_release_manifest": str(release_path.resolve()),
"training_release_manifest_sha256": sha256(release_path.read_bytes()).hexdigest(),
"training_asset_manifest": str(asset_path.resolve()),
"source_manifest_sha256": sha256(manifest.read_bytes()).hexdigest(),
"tiles": [
{"split": entry["split"], "image_path": entry["image_path"], "label_path": entry["label_path"]}
for entry in assets["entries"]
],
}
),
encoding="utf-8",
)
output = tmp_path / "gray"
subprocess.run(
[
sys.executable,
str(SCRIPT),
"--summary",
str(summary),
"--train-yaml",
str(dataset_yaml),
"--corpus-manifest",
str(manifest),
"--fixture-mode",
"--output-dir",
str(output),
],
check=True,
)
converted = Image.open(output / "images" / "train" / "fixture-train.png")
r, g, b = converted.getpixel((0, 0))
assert r == g == b
assert (output / "labels" / "train" / "fixture-train.txt").read_text() == "0 0.5 0.5 0.5 0.5\n"
evidence = json.loads((output / "grayscale-dataset-evidence.json").read_text())
assert evidence["converted_tile_count"] == 2
assert evidence["training_eligible"] is False
+348
View File
@@ -0,0 +1,348 @@
from __future__ import annotations
from pathlib import Path
from uuid import UUID
import pytest
from fastapi.testclient import TestClient
from app.core.config import get_settings
from app.core.public_demo import PUBLIC_DEMO_PROJECT_ID
from app.db.session import get_db
from app.main import create_app
from app.models import AnalysisRun, Dataset, Detection, Export, Job, Segmentation
from app.schemas import (
DetectionRunListResponse,
DetectionRunResponse,
SegmentationRunListResponse,
SegmentationRunResponse,
)
from app.services.auth_service import AuthService
from app.services.detection_service import DetectionService
from app.services.segmentation_service import SegmentationService
GUEST_PROJECT_ID = PUBLIC_DEMO_PROJECT_ID
OTHER_PROJECT_ID = UUID("00000000-0000-0000-0000-000000000999")
DATASET_ID = UUID("00000000-0000-0000-0000-000000000201")
DETECTION_RUN_ID = UUID("00000000-0000-0000-0000-000000000202")
SEGMENTATION_RUN_ID = UUID("00000000-0000-0000-0000-000000000203")
DETECTION_ID = UUID("00000000-0000-0000-0000-000000000204")
SEGMENTATION_ID = UUID("00000000-0000-0000-0000-000000000205")
EXPORT_ID = UUID("00000000-0000-0000-0000-000000000206")
JOB_ID = UUID("00000000-0000-0000-0000-000000000207")
class FakeSession:
def __init__(self, objects: dict[tuple[type, UUID], object]) -> None:
self.objects = objects
def get(self, model, row_id):
return self.objects.get((model, row_id))
def _guest_client(monkeypatch, db: FakeSession) -> TestClient:
password_hash = AuthService.hash_password(
"operator-password",
salt=b"guest-scope-test-salt",
iterations=100_000,
)
monkeypatch.setenv("GEOINTEL_AUTH_ENABLED", "true")
monkeypatch.setenv("GEOINTEL_AUTH_USERNAME", "operator")
monkeypatch.setenv("GEOINTEL_AUTH_PASSWORD_HASH", password_hash)
monkeypatch.setenv(
"GEOINTEL_AUTH_SESSION_SECRET",
"guest-scope-test-session-secret-value",
)
monkeypatch.setenv("GEOINTEL_GUEST_ACCESS_ENABLED", "true")
monkeypatch.setenv("GEOINTEL_GUEST_DISPLAY_NAME", "Gast")
client = TestClient(create_app())
def fake_db():
yield db
client.app.dependency_overrides[get_db] = fake_db
token = AuthService.create_session_token(
"Gast",
get_settings(),
role="guest",
project_id=GUEST_PROJECT_ID,
)
client.cookies.set("geointel_session", token)
return client
def _project_objects(project_id: UUID, export_path: Path) -> dict[tuple[type, UUID], object]:
return {
(Dataset, DATASET_ID): Dataset(
id=DATASET_ID,
project_id=project_id,
name="scope-test.tif",
dataset_type="raster",
source="fixture",
),
(AnalysisRun, DETECTION_RUN_ID): AnalysisRun(
id=DETECTION_RUN_ID,
project_id=project_id,
dataset_id=DATASET_ID,
analysis_type="detection",
status="success",
parameters_json={},
),
(AnalysisRun, SEGMENTATION_RUN_ID): AnalysisRun(
id=SEGMENTATION_RUN_ID,
project_id=project_id,
dataset_id=DATASET_ID,
analysis_type="segmentation",
status="success",
parameters_json={},
),
(Detection, DETECTION_ID): Detection(
id=DETECTION_ID,
project_id=project_id,
dataset_id=DATASET_ID,
analysis_run_id=DETECTION_RUN_ID,
model_name="fixture-detector",
class_name="building",
confidence=0.9,
geometry="SRID=4326;POINT (5 51)",
),
(Segmentation, SEGMENTATION_ID): Segmentation(
id=SEGMENTATION_ID,
project_id=project_id,
dataset_id=DATASET_ID,
analysis_run_id=SEGMENTATION_RUN_ID,
model_name="fixture-segmenter",
class_name="building",
confidence=0.9,
geometry="SRID=4326;MULTIPOLYGON (((5 51, 5.1 51, 5.1 51.1, 5 51)))",
),
(Export, EXPORT_ID): Export(
id=EXPORT_ID,
project_id=project_id,
export_type="dataset_geojson",
storage_path=str(export_path),
metadata_json={},
),
}
@pytest.mark.parametrize(
"path",
[
f"/api/v1/detection/runs/{DETECTION_RUN_ID}",
f"/api/v1/detection/runs/{DETECTION_RUN_ID}/detections",
f"/api/v1/detection/runs/{DETECTION_RUN_ID}/geojson",
f"/api/v1/detection/datasets/{DATASET_ID}/detections",
f"/api/v1/detection/datasets/{DATASET_ID}/geojson",
f"/api/v1/detection/detections/{DETECTION_ID}",
f"/api/v1/segmentation/runs/{SEGMENTATION_RUN_ID}",
f"/api/v1/segmentation/runs/{SEGMENTATION_RUN_ID}/segmentations",
f"/api/v1/segmentation/runs/{SEGMENTATION_RUN_ID}/geojson",
f"/api/v1/segmentation/datasets/{DATASET_ID}/segmentations",
f"/api/v1/segmentation/datasets/{DATASET_ID}/geojson",
f"/api/v1/segmentation/segmentations/{SEGMENTATION_ID}",
f"/api/v1/exports/{EXPORT_ID}",
f"/api/v1/exports/{EXPORT_ID}/content",
f"/api/v1/exports/{EXPORT_ID}/download",
f"/api/v1/exports/projects/{OTHER_PROJECT_ID}/exports",
],
)
def test_matching_guest_query_cannot_authorize_another_projects_resource(
path: str,
tmp_path: Path,
monkeypatch,
) -> None:
artifact = tmp_path / "other-project.geojson"
artifact.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
client = _guest_client(monkeypatch, FakeSession(_project_objects(OTHER_PROJECT_ID, artifact)))
response = client.get(f"{path}?project_id={GUEST_PROJECT_ID}")
assert response.status_code == 403
assert response.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
@pytest.mark.parametrize(
("path", "payload"),
[
(
"/api/v1/detection/run",
{"project_id": str(OTHER_PROJECT_ID), "dataset_id": str(DATASET_ID), "model_id": "fixture"},
),
(
"/api/v1/detection/run-async",
{"project_id": str(OTHER_PROJECT_ID), "dataset_id": str(DATASET_ID), "model_id": "fixture"},
),
(
"/api/v1/segmentation/run",
{"project_id": str(OTHER_PROJECT_ID), "dataset_id": str(DATASET_ID), "model_id": "fixture"},
),
(
"/api/v1/segmentation/run-async",
{"project_id": str(OTHER_PROJECT_ID), "dataset_id": str(DATASET_ID), "model_id": "fixture"},
),
(
"/api/v1/detection/runs/{run_id}/qa/reference".format(run_id=DETECTION_RUN_ID),
{"reference_dataset_id": str(DATASET_ID)},
),
(
"/api/v1/segmentation/runs/{run_id}/qa/reference".format(run_id=SEGMENTATION_RUN_ID),
{"reference_dataset_id": str(DATASET_ID)},
),
(
"/api/v1/exports/geojson",
{"export_kind": "dataset", "dataset_id": str(DATASET_ID)},
),
(
"/api/v1/exports/geojson",
{"export_kind": "detection_run", "analysis_run_id": str(DETECTION_RUN_ID)},
),
(
"/api/v1/exports/geojson",
{"export_kind": "segmentation_run", "analysis_run_id": str(SEGMENTATION_RUN_ID)},
),
(
"/api/v1/exports/metadata",
{"project_id": str(OTHER_PROJECT_ID)},
),
(
"/api/v1/exports/report",
{"project_id": str(OTHER_PROJECT_ID)},
),
(
"/api/v1/exports/map-result",
{
"project_id": str(OTHER_PROJECT_ID),
"mode": "current",
"dataset_id": str(DATASET_ID),
"bbox": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.1, "max_y": 51.1, "crs": "EPSG:4326"},
},
),
],
)
def test_matching_guest_query_cannot_override_post_body_or_target_scope(
path: str,
payload: dict,
tmp_path: Path,
monkeypatch,
) -> None:
artifact = tmp_path / "other-project.geojson"
artifact.write_text("{}", encoding="utf-8")
client = _guest_client(monkeypatch, FakeSession(_project_objects(OTHER_PROJECT_ID, artifact)))
response = client.post(f"{path}?project_id={GUEST_PROJECT_ID}", json=payload)
assert response.status_code == 403
assert response.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
def test_guest_can_still_read_and_download_its_own_resources(
tmp_path: Path,
monkeypatch,
) -> None:
artifact = tmp_path / "demo.geojson"
artifact.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
client = _guest_client(monkeypatch, FakeSession(_project_objects(GUEST_PROJECT_ID, artifact)))
suffix = f"?project_id={GUEST_PROJECT_ID}"
detection = client.get(f"/api/v1/detection/runs/{DETECTION_RUN_ID}{suffix}")
segmentation = client.get(f"/api/v1/segmentation/runs/{SEGMENTATION_RUN_ID}{suffix}")
export = client.get(f"/api/v1/exports/{EXPORT_ID}{suffix}")
download = client.get(f"/api/v1/exports/{EXPORT_ID}/download{suffix}")
assert detection.status_code == 200
assert segmentation.status_code == 200
assert export.status_code == 200
assert download.status_code == 200
assert download.json()["type"] == "FeatureCollection"
def test_guest_run_lists_and_new_runs_remain_bound_to_the_session_project(
tmp_path: Path,
monkeypatch,
) -> None:
artifact = tmp_path / "demo.geojson"
artifact.write_text("{}", encoding="utf-8")
client = _guest_client(monkeypatch, FakeSession(_project_objects(GUEST_PROJECT_ID, artifact)))
observed: list[UUID] = []
def detection_list(_db, *, project_id, **_kwargs):
observed.append(project_id)
return DetectionRunListResponse(items=[], total=0, limit=50, offset=0, truncated=False)
def segmentation_list(_db, *, project_id, **_kwargs):
observed.append(project_id)
return SegmentationRunListResponse(items=[], total=0, limit=50, offset=0, truncated=False)
def detection_run(**kwargs):
observed.append(kwargs["project_id"])
return DetectionRunResponse(
analysis_run_id=DETECTION_RUN_ID,
job_id=JOB_ID,
project_id=kwargs["project_id"],
dataset_id=kwargs["dataset_id"],
model_id=kwargs["model_id"],
status="success",
detection_count=0,
message="Demo run completed",
)
def segmentation_run(**kwargs):
observed.append(kwargs["project_id"])
return SegmentationRunResponse(
analysis_run_id=SEGMENTATION_RUN_ID,
job_id=JOB_ID,
project_id=kwargs["project_id"],
dataset_id=kwargs["dataset_id"],
model_id=kwargs["model_id"],
status="success",
segmentation_count=0,
message="Demo run completed",
)
monkeypatch.setattr(DetectionService, "list_runs", detection_list)
monkeypatch.setattr(SegmentationService, "list_runs", segmentation_list)
monkeypatch.setattr(DetectionService, "run_detection", detection_run)
monkeypatch.setattr(SegmentationService, "run_segmentation", segmentation_run)
def enqueue_detection(**kwargs):
observed.append(kwargs["project_id"])
return Job(
id=JOB_ID,
job_type="detection.run",
status="queued",
project_id=kwargs["project_id"],
dataset_id=kwargs["dataset_id"],
parameters_json={},
)
monkeypatch.setattr(DetectionService, "enqueue_detection", enqueue_detection)
def enqueue_segmentation(**kwargs):
observed.append(kwargs["project_id"])
return Job(
id=JOB_ID,
job_type="segmentation.run",
status="queued",
project_id=kwargs["project_id"],
dataset_id=kwargs["dataset_id"],
parameters_json={},
)
monkeypatch.setattr(SegmentationService, "enqueue_segmentation", enqueue_segmentation)
query = f"?project_id={GUEST_PROJECT_ID}"
payload = {"project_id": str(GUEST_PROJECT_ID), "dataset_id": str(DATASET_ID), "model_id": "fixture"}
responses = [
client.get(f"/api/v1/detection/runs{query}"),
client.get(f"/api/v1/segmentation/runs{query}"),
client.post(f"/api/v1/detection/run{query}", json=payload),
client.post(f"/api/v1/detection/run-async{query}", json=payload),
client.post(f"/api/v1/segmentation/run{query}", json=payload),
client.post(f"/api/v1/segmentation/run-async{query}", json=payload),
]
assert all(response.status_code == 200 for response in responses)
assert observed == [GUEST_PROJECT_ID] * 6
+100
View File
@@ -0,0 +1,100 @@
from __future__ import annotations
from types import SimpleNamespace
from fastapi.testclient import TestClient
from app.api.routes import health
from app.main import app
READY_DATABASE = {
"database": "ok",
"postgis": "ok:3.4 USE_GEOS=1 USE_PROJ=1",
"migration": "ok:202607160001",
}
def test_liveness_is_independent_from_database(monkeypatch) -> None:
monkeypatch.setattr(
health,
"_database_checks",
lambda: (_ for _ in ()).throw(AssertionError("must not query DB")),
)
response = TestClient(app).get("/health/live")
assert response.status_code == 200
assert response.json()["status"] == "ok"
def test_readiness_returns_ok_only_when_all_checks_pass(monkeypatch) -> None:
monkeypatch.setattr(health, "_database_checks", lambda: READY_DATABASE.copy())
monkeypatch.setattr(health, "_storage_check", lambda _: "ok")
response = TestClient(app).get("/health/ready")
assert response.status_code == 200
payload = response.json()
assert payload["status"] == "ok"
assert payload["service"] == "geointel-backend"
assert payload["database"] == "ok"
assert payload["postgis"].startswith("ok:")
assert payload["migration"] == "ok:202607160001"
assert payload["storage"] == "ok"
def test_compatibility_health_is_fail_closed(monkeypatch) -> None:
monkeypatch.setattr(
health,
"_database_checks",
lambda: {
"database": "degraded",
"postgis": "degraded",
"migration": "degraded",
},
)
monkeypatch.setattr(health, "_storage_check", lambda _: "ok")
response = TestClient(app).get("/health")
assert response.status_code == 503
assert response.json()["status"] == "degraded"
def test_system_capabilities_report_runtime_state(monkeypatch) -> None:
monkeypatch.setattr(
health,
"_dependency_enabled",
lambda module_name: module_name in {"rasterio", "geopandas"},
)
monkeypatch.setattr(health, "_database_checks", lambda: READY_DATABASE.copy())
monkeypatch.setattr(
health.ModelRegistryService,
"get_model_capability",
lambda *args, **kwargs: SimpleNamespace(
configured=True,
status="configured",
),
)
response = TestClient(app).get("/api/v1/system/capabilities")
assert response.status_code == 200
payload = response.json()["data"]
assert payload["postgis"] is True
assert payload["rasterio"] is True
assert payload["geopandas"] is True
assert payload["yolo"] is True
assert payload["yolo_status"] == "configured"
assert payload["version"]
def test_requests_receive_a_correlation_id() -> None:
client = TestClient(app)
generated = client.get("/health/live")
retained = client.get("/health/live", headers={"x-request-id": "test-request"})
assert generated.headers["x-request-id"]
assert retained.headers["x-request-id"] == "test-request"
@@ -0,0 +1,34 @@
from pathlib import Path
def test_live_migration_smoke_checks_postgis_after_migrations() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "live_migration_smoke.sh"
content = script.read_text(encoding="utf-8")
upgrade_index = content.index("-m alembic upgrade head")
postgis_index = content.index("PostGIS_Version()")
assert upgrade_index < postgis_index
def test_live_migration_smoke_checks_required_runtime_schema_objects() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "live_migration_smoke.sh"
content = script.read_text(encoding="utf-8")
assert "to_regclass(:object_name)" in content
assert '"public.projects"' in content
assert '"public.datasets"' in content
assert '"public.vector_features"' in content
assert '"public.detections"' in content
assert '"public.segmentations"' in content
assert '"public.ix_segmentations_geometry"' in content
def test_live_migration_smoke_reports_collation_version_mismatch_without_failing() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "live_migration_smoke.sh"
content = script.read_text(encoding="utf-8")
assert "pg_database_collation_actual_version(oid)" in content
assert "COLLATION_VERSION_MISMATCH" in content
assert "REFRESH COLLATION VERSION" in content
assert "Database collation version: ok" in content
@@ -0,0 +1,153 @@
from __future__ import annotations
import io
from pathlib import Path
from uuid import uuid4
import pytest
from app.core.config import Settings
from app.schemas.bathymetry import MdkBathymetryAcquireRequest
from app.schemas.operations import VectorSelectionBBox
from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisitionService
CAPABILITIES_XML = b"""<?xml version="1.0" encoding="UTF-8"?>
<WCS_Capabilities version="1.0.0" xmlns="http://www.opengis.net/wcs">
<ContentMetadata>
<CoverageOfferingBrief>
<name>depth_model_20m_lat</name>
<label>Belgian Continental Shelf depth model</label>
</CoverageOfferingBrief>
</ContentMetadata>
</WCS_Capabilities>
"""
class FakeResponse:
def __init__(self, content: bytes, content_type: str = "application/xml") -> None:
self._stream = io.BytesIO(content)
self.headers = {"Content-Type": content_type}
def read(self, limit: int = -1) -> bytes:
return self._stream.read(limit)
def __enter__(self):
return self
def __exit__(self, *args):
return False
def _payload(**overrides) -> MdkBathymetryAcquireRequest:
values = {
"bbox": VectorSelectionBBox(min_x=2.5, min_y=51.3, max_x=2.6, max_y=51.4),
"force_refresh": True,
}
values.update(overrides)
return MdkBathymetryAcquireRequest(**values)
def _settings(**overrides) -> Settings:
values = {
"mdk_bathymetry_acquisition_enabled": True,
"mdk_bathymetry_coverage_id": "depth_model_20m_lat",
}
values.update(overrides)
return Settings(**values)
def test_acquisition_fails_closed_when_disabled() -> None:
settings = _settings(mdk_bathymetry_acquisition_enabled=False)
with pytest.raises(Exception) as exc_info:
MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings)
assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_ACQUISITION_DISABLED"
def test_acquisition_fails_closed_without_coverage_id() -> None:
settings = _settings(mdk_bathymetry_coverage_id=None)
with pytest.raises(Exception) as exc_info:
MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings)
assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_COVERAGE_NOT_CONFIGURED"
def test_acquisition_rejects_oversized_bbox() -> None:
settings = _settings(mdk_bathymetry_max_bbox_deg2=0.001)
with pytest.raises(Exception) as exc_info:
MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings)
assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_BBOX_TOO_LARGE"
def test_acquisition_requires_reachable_probe() -> None:
settings = _settings()
def failing_opener(request, timeout=None):
raise OSError("connection refused")
with pytest.raises(Exception) as exc_info:
MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings, opener=failing_opener)
assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_ENDPOINT_NOT_READY"
def test_acquisition_requires_advertised_coverage_id() -> None:
settings = _settings(mdk_bathymetry_coverage_id="not_advertised_coverage")
def opener(request, timeout=None):
return FakeResponse(CAPABILITIES_XML)
with pytest.raises(Exception) as exc_info:
MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings, opener=opener)
assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_COVERAGE_NOT_ADVERTISED"
def test_acquisition_rejects_non_geotiff_coverage_response() -> None:
settings = _settings()
responses = []
def opener(request, timeout=None):
url = request.full_url if hasattr(request, "full_url") else str(request)
responses.append(url)
if "GetCapabilities" in url:
return FakeResponse(CAPABILITIES_XML)
return FakeResponse(b"<ServiceExceptionReport>boom</ServiceExceptionReport>", "application/xml")
with pytest.raises(Exception) as exc_info:
MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings, opener=opener)
assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_INVALID_RESPONSE"
assert any("GetCoverage" in url for url in responses)
coverage_urls = [url for url in responses if "GetCoverage" in url]
assert "coverage=depth_model_20m_lat" in coverage_urls[0]
assert "format=GeoTIFF" in coverage_urls[0]
def test_get_coverage_url_is_bounded_and_pinned() -> None:
settings = _settings()
bbox = [2.5, 51.3, 2.6, 51.4]
url = MdkBathymetryAcquisitionService._get_coverage_url(settings, "depth_model_20m_lat", bbox)
assert url.startswith("https://")
assert "request=GetCoverage" in url
assert "version=1.0.0" in url
assert "crs=EPSG%3A4326" in url or "crs=EPSG:4326" in url
width, height = MdkBathymetryAcquisitionService._pixel_dimensions(bbox)
assert 1 <= width <= MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE
assert 1 <= height <= MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE
def test_source_module_never_disables_tls_verification() -> None:
source = (
Path(__file__).resolve().parents[1] / "app" / "services" / "mdk_bathymetry_acquisition_service.py"
).read_text(encoding="utf-8")
assert "_create_unverified_context" not in source
assert "CERT_NONE" not in source
assert "check_hostname = False" not in source
+355
View File
@@ -0,0 +1,355 @@
from __future__ import annotations
from hashlib import sha256
import json
from pathlib import Path
from uuid import uuid4
import pytest
from fastapi.testclient import TestClient
from app.core.config import Settings
from app.core.errors import AppError
from app.main import app
from app.models import (
AnalysisRun,
Dataset,
DatasetVersion,
Detection,
Job,
Project,
SourceRegistry,
SourceSnapshot,
)
from app.services.detection_service import DetectionService
from app.services.model_asset_catalog_service import ModelAssetCatalogService
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.tile_manifest_service import TileManifestService
class FakeSession:
def __init__(self, objects=None) -> None:
self.objects = objects or {}
self.added = []
self.commits = 0
self.refreshes = []
def get(self, model, item_id):
return self.objects.get((model, item_id))
def add(self, item) -> None:
self.added.append(item)
if getattr(item, "id", None) is not None:
self.objects[(item.__class__, item.id)] = item
def commit(self) -> None:
self.commits += 1
def refresh(self, item) -> None:
self.refreshes.append(item)
class MockYoloAdapter:
def __init__(self, settings: Settings) -> None:
self.settings = settings
@staticmethod
def dependencies_available() -> bool:
return True
def load_model(self, model_path: Path):
return {"model_path": str(model_path)}
def predict_tiles(self, model, tile_paths, confidence_threshold: float) -> list[list[dict]]:
# The service batches tiles; this double still answers per tile.
return [self.predict_tile(model, tile_path, confidence_threshold) for tile_path in tile_paths]
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
assert model["model_path"].endswith("building-detector.pt")
return [
{
"class_name": "building",
"confidence": 0.9,
"bbox": [10.0, 20.0, 30.0, 40.0],
"properties": {"adapter": "mock"},
}
]
def _project_and_raster_dataset():
project_id = uuid4()
dataset_id = uuid4()
source_registry_id = uuid4()
source_snapshot_id = uuid4()
checksum = "a" * 64
project = Project(id=project_id, name="Geel")
source_registry = SourceRegistry(
id=source_registry_id,
source_key="test-derived-raster",
display_name="Governed test-derived raster",
classification="derived",
authority_name="GeoIntel test fixture",
usage_policy_json={"ground_truth_allowed": False},
)
source_snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key="test-derived-raster-v1",
checksum_sha256=checksum,
freshness_status="current",
ingest_status="ingested",
)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="source.tif",
dataset_type="raster",
source="test-derived-raster",
source_name="test-derived-raster",
storage_path="storage/uploads/source.tif",
checksum_sha256=checksum,
crs="EPSG:4326",
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
data_contract_key="geointel.raster.geotiff",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
status="ready",
)
dataset.source_registry = source_registry
dataset.source_snapshot = source_snapshot
dataset.versions.append(
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
)
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
return db, project_id, dataset_id
def _manifest(tmp_path: Path, db: FakeSession, dataset: Dataset) -> Path:
tile_path = tmp_path / "tile_0000.tif"
tile_path.write_bytes(b"tile")
binding = TileManifestService.dataset_binding(db, dataset)
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text(
json.dumps(
{
**binding,
"tile_set_id": "tiles-fixture",
"count": 1,
"crs": "EPSG:4326",
"bounds": [4.0, 51.0, 5.0, 52.0],
"tiles": [
{
"path": str(tile_path),
"pixel_window": [0, 0, 100, 100],
"bounds": [4.0, 51.0, 5.0, 52.0],
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
"crs": "EPSG:4326",
"index": 0,
**TileManifestService.tile_integrity(tile_path),
}
],
}
),
encoding="utf-8",
)
return manifest_path
def _write_model_sidecar(
model_path: Path,
settings: Settings,
*,
db: FakeSession | None = None,
) -> None:
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
source_registry_id = uuid4()
source_snapshot_id = uuid4()
source_version = settings.yolo_model_version or "test-v1"
if db is not None:
source_registry = SourceRegistry(
id=source_registry_id,
source_key="model",
display_name="Governed test model artifact",
classification="experimental",
authority_name="GeoIntel test fixture",
freshness_status="current",
ingest_status="configured",
)
source_snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key=f"model-{source_version}",
source_version=source_version,
checksum_sha256=model_sha256,
freshness_status="current",
ingest_status="ingested",
)
db.objects[(SourceRegistry, source_registry_id)] = source_registry
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
payload = {
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
"model": {
"model_id": settings.yolo_model_id,
"task_type": "object_detection",
"sha256": model_sha256,
"model_format": "pytorch",
"framework": "ultralytics/pytorch",
"class_mapping": {"0": "building"},
"source_version": source_version,
},
"source": {
"source_registry_id": str(source_registry_id),
"source_snapshot_id": str(source_snapshot_id),
"source_registry_key": "model",
"source_snapshot_checksum_sha256": model_sha256,
},
"lineage": {
"upstream_asset_ids": ["test-training-corpus"],
"upstream_checksums_sha256": ["a" * 64],
"transformations": [
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
],
},
"metadata": {"training_manifest_sha256": "c" * 64},
"imported_at": "2026-08-01T10:00:00+00:00",
}
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
json.dumps(payload, sort_keys=True),
encoding="utf-8",
)
def test_model_asset_catalog_lists_supported_local_model_files(tmp_path: Path) -> None:
model_file = tmp_path / "building-detector.pt"
model_file.write_bytes(b"local model")
ignored_file = tmp_path / "notes.txt"
ignored_file.write_text("ignore me", encoding="utf-8")
settings = Settings(yolo_models_dir=str(tmp_path), yolo_model_path=str(model_file), yolo_enabled=True)
response = ModelAssetCatalogService.list_assets(settings=settings)
assert response.total == 1
asset = response.items[0]
assert asset.model_asset_id == "building-detector-pt"
assert asset.filename == "building-detector.pt"
assert asset.display_name == "building-detector"
assert asset.model_path == str(model_file)
assert asset.size_bytes == len(b"local model")
assert len(asset.sha256) == 64
assert asset.active is True
assert asset.runtime_available is True
assert asset.runtime_status == "active"
assert asset.governed_validation_status == "not_verified_by_catalog"
assert asset.promotion_status == "not_verified_by_catalog"
assert asset.status == "runtime_active"
assert asset.will_download_models is False
def test_model_asset_catalog_resolves_known_asset(tmp_path: Path) -> None:
model_file = tmp_path / "building-detector.pt"
model_file.write_bytes(b"local model")
settings = Settings(yolo_models_dir=str(tmp_path), yolo_enabled=True)
asset = ModelAssetCatalogService.resolve_asset("building-detector-pt", settings=settings)
assert asset.filename == "building-detector.pt"
assert asset.model_path == str(model_file)
def test_model_asset_catalog_only_exposes_explicit_active_asset_in_runtime(tmp_path: Path) -> None:
active_file = tmp_path / "approved-building-detector.pt"
active_file.write_bytes(b"approved")
(tmp_path / "training-smoke.pt").write_bytes(b"experiment")
(tmp_path / "partial-checkpoint.pt").write_bytes(b"partial")
settings = Settings(
yolo_models_dir=str(tmp_path),
yolo_model_path=str(active_file),
yolo_enabled=True,
)
response = ModelAssetCatalogService.list_assets(settings=settings)
assert response.total == 1
assert response.items[0].filename == active_file.name
assert response.items[0].active is True
assert response.items[0].runtime_status == "active"
assert response.items[0].governed_validation_status == "not_verified_by_catalog"
assert response.items[0].promotion_status == "not_verified_by_catalog"
assert response.items[0].status == "runtime_active"
def test_model_asset_catalog_rejects_unknown_asset(tmp_path: Path) -> None:
settings = Settings(yolo_models_dir=str(tmp_path), yolo_enabled=True)
with pytest.raises(AppError) as exc_info:
ModelAssetCatalogService.resolve_asset("missing-model", settings=settings)
assert exc_info.value.code == "DETECTION_MODEL_ASSET_NOT_FOUND"
assert exc_info.value.status_code == 404
def test_model_assets_api_returns_canonical_envelope(monkeypatch, tmp_path: Path) -> None:
model_file = tmp_path / "building-detector.pt"
model_file.write_bytes(b"local model")
monkeypatch.setenv("YOLO_MODELS_DIR", str(tmp_path))
monkeypatch.setenv("YOLO_MODEL_PATH", str(model_file))
response = TestClient(app).get("/api/v1/detection/model-assets")
assert response.status_code == 200
payload = response.json()
assert set(payload) == {"data"}
assert payload["data"]["total"] == 1
assert payload["data"]["items"][0]["model_asset_id"] == "building-detector-pt"
assert payload["data"]["items"][0]["active"] is True
assert payload["data"]["items"][0]["runtime_status"] == "active"
assert payload["data"]["items"][0]["governed_validation_status"] == "not_verified_by_catalog"
assert payload["data"]["items"][0]["promotion_status"] == "not_verified_by_catalog"
assert payload["data"]["items"][0]["will_download_models"] is False
def test_detection_run_persists_selected_model_asset_parameters(tmp_path, monkeypatch: Path) -> None:
# A manifest written into tmp_path is only a governed artifact if
# tmp_path is the storage root.
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
model_file = tmp_path / "building-detector.pt"
model_file.write_bytes(b"local model")
db, project_id, dataset_id = _project_and_raster_dataset()
settings = Settings(
yolo_enabled=True,
yolo_model_path=str(tmp_path / "default.pt"),
yolo_models_dir=str(tmp_path),
yolo_max_tiles=4,
)
_write_model_sidecar(model_file, settings, db=db)
result = DetectionService.run_detection(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-configured",
model_asset_id="building-detector-pt",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path, db, db.get(Dataset, dataset_id))),
settings=settings,
yolo_adapter_class=MockYoloAdapter,
)
jobs = [item for item in db.added if isinstance(item, Job)]
runs = [item for item in db.added if isinstance(item, AnalysisRun)]
detections = [item for item in db.added if isinstance(item, Detection)]
assert result.status == "success"
assert result.detection_count == 1
assert jobs[0].parameters_json["model_asset_id"] == "building-detector-pt"
assert jobs[0].parameters_json["model_asset_path"] == str(model_file)
assert len(jobs[0].parameters_json["model_asset_sha256"]) == 64
assert runs[0].parameters_json["model_asset_id"] == "building-detector-pt"
assert detections[0].model_name == "yolo-configured"
@@ -0,0 +1,22 @@
from scripts.render_operator_polygon_label_qa import geometry_rings
def test_geometry_rings_yields_polygon_exterior_and_hole() -> None:
exterior = [[0, 0], [1, 0], [1, 1], [0, 0]]
hole = [[0.2, 0.2], [0.4, 0.2], [0.2, 0.2]]
assert list(geometry_rings({"type": "Polygon", "coordinates": [exterior, hole]})) == [
exterior,
hole,
]
def test_geometry_rings_flattens_multipolygon_rings() -> None:
first = [[0, 0], [1, 0], [0, 0]]
second = [[2, 2], [3, 2], [2, 2]]
assert list(
geometry_rings({"type": "MultiPolygon", "coordinates": [[first], [second]]})
) == [first, second]
def test_geometry_rings_ignores_non_polygon_geometry() -> None:
assert list(geometry_rings({"type": "Point", "coordinates": [0, 0]})) == []
@@ -0,0 +1,303 @@
"""Bounded acquisition must stay bounded to the official host.
Every acquisition service builds its URL from configured settings, so the
request payload cannot point the runtime anywhere. The redirect chain can:
``urlopen`` follows redirects by default, so a misconfigured or compromised
upstream can send the runtime to ``127.0.0.1``, to the container network, or to
a cloud metadata endpoint — and the response is then persisted as if it were
official source data.
The product's stated rule is that acquisition fails closed and never
substitutes fabricated data for official data. A redirect off the configured
host is exactly that substitution.
"""
from __future__ import annotations
import pytest
from app.core.errors import AppError
from app.services.outbound_request_guard import (
_ValidatedRedirects,
assert_public_http_url,
assert_same_origin_redirect,
validated_redirect_opener,
)
class TestUrlShape:
def test_an_official_https_endpoint_is_accepted(self) -> None:
assert_public_http_url("https://geo.api.vlaanderen.be/dhmv/wcs?SERVICE=WCS")
def test_a_non_http_scheme_is_refused(self) -> None:
with pytest.raises(AppError) as exc_info:
assert_public_http_url("file:///etc/passwd")
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
@pytest.mark.parametrize(
"url",
[
"http://127.0.0.1:8000/internal",
"http://localhost/internal",
"http://10.1.2.3/internal",
"http://192.168.123.45/internal",
"http://172.16.0.9/internal",
"http://169.254.169.254/latest/meta-data/",
"http://[::1]/internal",
],
)
def test_private_and_loopback_destinations_are_refused(self, url: str) -> None:
with pytest.raises(AppError) as exc_info:
assert_public_http_url(url)
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
def test_a_url_without_a_host_is_refused(self) -> None:
with pytest.raises(AppError):
assert_public_http_url("https:///no-host")
class TestRedirects:
def test_a_redirect_within_the_same_origin_is_allowed(self) -> None:
assert_same_origin_redirect(
"https://geo.api.vlaanderen.be/dhmv/wcs",
"https://geo.api.vlaanderen.be/dhmv/wcs/v2?x=1",
)
def test_a_redirect_to_another_host_is_refused(self) -> None:
with pytest.raises(AppError) as exc_info:
assert_same_origin_redirect(
"https://geo.api.vlaanderen.be/dhmv/wcs",
"https://cdn.example.net/payload.tif",
)
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
assert "cdn.example.net" in str(exc_info.value.details)
def test_a_downgrade_to_plain_http_is_refused(self) -> None:
with pytest.raises(AppError) as exc_info:
assert_same_origin_redirect(
"https://geo.api.vlaanderen.be/wcs",
"http://geo.api.vlaanderen.be/wcs",
)
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
def test_a_redirect_to_the_loopback_is_refused_even_on_the_same_scheme(self) -> None:
with pytest.raises(AppError):
assert_same_origin_redirect("https://geo.api.vlaanderen.be/wcs", "https://127.0.0.1/wcs")
def test_an_upgrade_to_https_stays_allowed(self) -> None:
assert_same_origin_redirect("http://geo.example.be/wcs", "https://geo.example.be/wcs")
def test_a_redirect_to_another_port_is_refused(self) -> None:
with pytest.raises(AppError) as exc_info:
assert_same_origin_redirect(
"https://geo.api.vlaanderen.be/wcs",
"https://geo.api.vlaanderen.be:8443/wcs",
)
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
def test_embedded_credentials_are_refused(self) -> None:
with pytest.raises(AppError) as exc_info:
assert_public_http_url("https://operator:secret@geo.example.be/wcs")
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
def test_a_redirect_with_an_invalid_port_fails_closed(self) -> None:
with pytest.raises(AppError) as exc_info:
assert_same_origin_redirect(
"https://geo.api.vlaanderen.be/wcs",
"https://geo.api.vlaanderen.be:not-a-port/wcs",
)
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
def test_the_guard_opener_refuses_a_cross_host_redirect() -> None:
"""The opener is what the acquisition services actually call."""
from app.services.outbound_request_guard import guarded_opener
opener = guarded_opener("https://geo.api.vlaanderen.be/wcs")
class _Redirecting:
def __init__(self, location: str) -> None:
self.url = location
def __enter__(self):
return self
def __exit__(self, *_args):
return False
with pytest.raises(AppError) as exc_info:
with opener(
type("Req", (), {"full_url": "https://geo.api.vlaanderen.be/wcs"})(),
timeout=1,
_transport=lambda *_a, **_k: _Redirecting("https://evil.example.net/x"),
):
pass
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
class TestTheGuardIsWiredIntoAcquisition:
"""Behavioural, not a grep: each service is called on its real fetch path.
Every existing acquisition test injects an ``opener``, which bypasses the
guard by design — that is how those tests stub the network. These call the
production default instead.
"""
def _settings(self):
from app.core.config import Settings
return Settings(_env_file=None)
def test_dhmv_refuses_a_loopback_endpoint(self) -> None:
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
with pytest.raises(AppError) as exc_info:
DhmvAcquisitionService._fetch("http://127.0.0.1:9/wcs", self._settings())
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
def test_flood_hazard_refuses_a_link_local_endpoint(self) -> None:
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
with pytest.raises(AppError) as exc_info:
FloodHazardAcquisitionService._fetch("http://169.254.169.254/latest/", self._settings())
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
def test_thematic_raster_refuses_a_private_endpoint(self) -> None:
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
with pytest.raises(AppError) as exc_info:
ThematicRasterAcquisitionService._fetch("http://10.0.0.5/product.tif", self._settings())
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
def test_orthophoto_refuses_a_private_endpoint(self) -> None:
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
with pytest.raises(AppError) as exc_info:
OrthophotoAcquisitionService._fetch("http://192.168.123.45/wms", self._settings())
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
class TestOneRedirectPolicy:
"""Two acquisition services rejected every redirect through their own
opener while eight allowed a same-origin one through this guard. Two
policies with no stated reason, and only one of them checked where the
response actually came from."""
def test_the_strict_policy_refuses_any_redirect(self) -> None:
from app.services.outbound_request_guard import guarded_opener
opener = guarded_opener("https://geo.api.vlaanderen.be/GRB/wfs", allow_redirect=False)
class _Redirected:
url = "https://geo.api.vlaanderen.be/GRB/wfs/v2"
def __enter__(self):
return self
def __exit__(self, *_args):
return False
with pytest.raises(AppError) as exc_info:
with opener(object(), timeout=1, _transport=lambda *_a, **_k: _Redirected()):
pass
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
def test_the_strict_policy_still_allows_the_response_it_asked_for(self) -> None:
from app.services.outbound_request_guard import guarded_opener
url = "https://geo.api.vlaanderen.be/GRB/wfs"
opener = guarded_opener(url, allow_redirect=False)
class _Direct:
def __init__(self) -> None:
self.url = url
def __enter__(self):
return self
def __exit__(self, *_args):
return False
with opener(object(), timeout=1, _transport=lambda *_a, **_k: _Direct()) as response:
assert response.url == url
def test_both_policies_refuse_a_private_destination(self) -> None:
from app.services.outbound_request_guard import guarded_opener
for allow_redirect in (True, False):
with pytest.raises(AppError) as exc_info:
guarded_opener("http://10.0.0.5/wfs", allow_redirect=allow_redirect)
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
def test_the_strict_services_use_the_shared_guard(self) -> None:
"""Behavioural: their own fetch paths refuse a private endpoint, which
the hand-rolled opener never checked."""
from app.core.config import Settings
from app.services.grb_acquisition_service import GrbAcquisitionService
from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService
settings = Settings(_env_file=None)
for service in (GrbAcquisitionService, OfficialVectorAcquisitionService):
with pytest.raises(AppError) as exc_info:
service._read_page("http://127.0.0.1:9/wfs", settings, None)
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
def test_a_refused_redirect_is_never_requested() -> None:
"""Rejecting after the fact still sends the request.
Checking ``response.url`` means urllib has already followed the chain: the
connection to the redirect target was opened and the response read. For a
destination like a metadata endpoint that is the whole attack. The strict
policy must refuse to follow, not refuse afterwards.
"""
from app.services.outbound_request_guard import no_redirect_opener
opener = no_redirect_opener()
handlers = [type(handler).__name__ for handler in opener.handlers]
assert "_RejectRedirects" in handlers
def test_the_default_guard_validates_before_following_a_redirect() -> None:
opener = validated_redirect_opener("https://geo.api.vlaanderen.be/wcs")
handlers = [type(handler).__name__ for handler in opener.handlers]
assert "_ValidatedRedirects" in handlers
handler = _ValidatedRedirects("https://geo.api.vlaanderen.be/wcs")
with pytest.raises(AppError) as exc_info:
handler.redirect_request(
None,
None,
302,
"Found",
{},
"http://169.254.169.254/latest/meta-data/",
)
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
def test_the_rejecting_handler_returns_no_new_request() -> None:
from app.services.outbound_request_guard import _RejectRedirects
handler = _RejectRedirects()
assert handler.redirect_request(None, None, 302, "Found", {}, "http://169.254.169.254/") is None
@@ -0,0 +1,120 @@
"""The paged readers' loop protections, exercised rather than assumed.
GRB and official vector both refuse a repeated page URL and bound the page
count, and GRB deduplicates on feature identity. None of that had a test, so
none of it was known to work — the same category as the redirect handler that
turned out to be dead code while looking like protection.
A provider that answers every page with a "next" link pointing back at itself
is not hypothetical: it is what a misconfigured cursor or a caching proxy in
front of an OGC endpoint produces.
"""
from __future__ import annotations
import pytest
from shapely.geometry import Polygon
from app.core.config import Settings
from app.core.errors import AppError
from app.services.grb_acquisition_service import GrbAcquisitionService
from tests.test_sprint239_bounded_grb_acquisition import JsonResponse, polygon_feature
COLLECTION_ITEMS = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
SCOPE = Polygon([(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)])
def _building(feature_id: str) -> dict:
return polygon_feature(
feature_id,
[(5.155, 51.185), (5.175, 51.185), (5.175, 51.195), (5.155, 51.195), (5.155, 51.185)],
)
def _fetch(opener):
return GrbAcquisitionService._fetch_features(
GrbAcquisitionService._product("buildings"),
SCOPE,
SCOPE.bounds,
"bounded_selection",
Settings(_env_file=None),
opener,
)
def test_a_next_link_pointing_at_itself_is_refused() -> None:
requests: list[str] = []
def opener(request, timeout): # noqa: ARG001
requests.append(request.full_url)
return JsonResponse(
{
"type": "FeatureCollection",
"features": [_building("GBG.1")],
"links": [{"rel": "next", "href": f"{COLLECTION_ITEMS}?cursor=stuck"}],
}
)
with pytest.raises(AppError) as exc_info:
_fetch(opener)
assert exc_info.value.code == "GRB_PROVIDER_PAGINATION_LOOP"
# Refused on the second sighting, not after exhausting the page budget.
assert len(requests) == 2
def test_an_endless_chain_of_fresh_pages_stops_at_the_page_limit() -> None:
"""Distinct URLs defeat the loop check, so the page budget is the backstop."""
settings = Settings(_env_file=None)
requests: list[str] = []
def opener(request, timeout): # noqa: ARG001
requests.append(request.full_url)
cursor = len(requests)
return JsonResponse(
{
"type": "FeatureCollection",
"features": [_building(f"GBG.{cursor}")],
"links": [{"rel": "next", "href": f"{COLLECTION_ITEMS}?cursor=p{cursor}"}],
}
)
with pytest.raises(AppError) as exc_info:
_fetch(opener)
assert exc_info.value.code == "GRB_SELECTION_TOO_LARGE"
assert exc_info.value.status_code == 422
assert len(requests) == settings.grb_max_pages
def test_a_repeated_feature_across_pages_is_counted_once() -> None:
"""Two pages, distinct URLs, overlapping content.
Unlike a repeated URL this is not necessarily provider misbehaviour — a
cursor over a changing table can hand back a record twice — so the reader
keeps it once rather than failing the acquisition.
"""
def opener(request, timeout): # noqa: ARG001
if "cursor=next" in request.full_url:
return JsonResponse(
{
"type": "FeatureCollection",
"features": [_building("GBG.1"), _building("GBG.2")],
"links": [],
}
)
return JsonResponse(
{
"type": "FeatureCollection",
"features": [_building("GBG.1")],
"links": [{"rel": "next", "href": f"{COLLECTION_ITEMS}?cursor=next"}],
}
)
features, transfer = _fetch(opener)
assert {feature["id"] for feature in features} == {"GBG:GBG.1", "GBG:GBG.2"}
assert transfer["candidate_feature_count"] == 3
assert transfer["feature_count"] == 2
@@ -0,0 +1,61 @@
"""A rectangle across a municipal boundary must not return the same object twice.
Partitioned selection de-duplicated ``total_feature_count`` on
``source_feature_id`` but returned the raw rows. A feature present in two
municipal partitions was therefore drawn twice on the map and counted once in
the headline, so the number on the panel disagreed with the geometry beside it.
"""
from __future__ import annotations
from uuid import uuid4
from app.services.vector_feature_service import VectorFeatureService
class _Row:
def __init__(self, source_feature_id, row_id=None, dataset_id=None):
self.source_feature_id = source_feature_id
self.id = row_id or uuid4()
self.dataset_id = dataset_id or uuid4()
def _ids(rows):
return [row.source_feature_id or str(row.id) for row in rows]
def test_a_feature_in_two_partitions_is_returned_once() -> None:
shared = "grb-building-42"
rows = [_Row(shared), _Row("grb-building-7"), _Row(shared)]
kept = VectorFeatureService.deduplicate_rows(rows)
assert _ids(kept) == [shared, "grb-building-7"]
def test_the_first_occurrence_wins_so_the_result_is_stable() -> None:
first = _Row("dup")
second = _Row("dup")
assert VectorFeatureService.deduplicate_rows([first, second])[0] is first
assert VectorFeatureService.deduplicate_rows([second, first])[0] is second
def test_rows_without_a_source_id_fall_back_to_their_own_identity() -> None:
"""Two distinct rows with no source id are two distinct features."""
rows = [_Row(None), _Row(None)]
assert len(VectorFeatureService.deduplicate_rows(rows)) == 2
def test_an_empty_source_id_is_not_treated_as_a_shared_identity() -> None:
rows = [_Row(""), _Row("")]
assert len(VectorFeatureService.deduplicate_rows(rows)) == 2
def test_deduplication_leaves_a_clean_population_untouched() -> None:
rows = [_Row("a"), _Row("b"), _Row("c")]
assert VectorFeatureService.deduplicate_rows(rows) == rows
@@ -0,0 +1,644 @@
from __future__ import annotations
from datetime import datetime, timezone
from hashlib import sha256
import json
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
from app.models import Dataset, DatasetVersion, SourceRegistry, SourceSnapshot
from app.services.data_contract_validation import (
build_vector_ingest_input,
validate_registered_asset,
)
from app.services.demo_workflow_service import DemoWorkflowService
from app.services.derived_dataset_governance_service import (
DerivedDatasetGovernanceService,
)
from app.services.raster_operations_service import RasterOperationsService
from app.services.storage_service import StorageService
from app.services.vector_feature_service import VectorFeatureService
from app.services.vector_operations_service import VectorOperationsService
_CHECKSUM = "a" * 64
class _FakeSession:
def __init__(self, rows=None) -> None:
self.rows = rows or {}
self.added = []
def get(self, model, row_id):
row = self.rows.get((model, row_id))
if row is not None:
return row
return next(
(
item
for item in self.added
if isinstance(item, model) and item.id == row_id
),
None,
)
def add(self, row) -> None:
self.added.append(row)
def commit(self) -> None:
return None
def refresh(self, row) -> None:
return None
class _GovernedSession:
"""Small ORM-shaped session for the real governance branch.
Registry persistence is monkeypatched below; the test exercises the
service's orchestration and report decisions without needing PostGIS.
"""
def __init__(self) -> None:
self.added = []
self.flushes = 0
class _EmptyQuery:
def filter(self, *_args, **_kwargs):
return self
@staticmethod
def all():
return []
@staticmethod
def one_or_none():
return None
def query(self, *_args, **_kwargs):
# Governance now performs a bounded lineage traversal during quarantine.
# This focused harness intentionally has no persisted siblings/edges.
return self._EmptyQuery()
def add(self, row) -> None:
self.added.append(row)
def flush(self) -> None:
self.flushes += 1
def _governed_parent() -> Dataset:
source_id = uuid4()
snapshot_id = uuid4()
source = SourceRegistry(
id=source_id,
source_key="grb",
display_name="GRB parent fixture",
classification="authoritative",
authority_name="Digitaal Vlaanderen",
authority_scope_json={"zone": "Flanders"},
usage_policy_json={"ground_truth_allowed": True},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key="governed-parent",
checksum_sha256=_CHECKSUM,
freshness_status="current",
ingest_status="ingested",
)
dataset = Dataset(
id=uuid4(),
project_id=uuid4(),
name="governed.geojson",
dataset_type="vector",
source="grb",
source_name="grb",
status="ready",
checksum_sha256=_CHECKSUM,
data_contract_key="geointel.vector.geojson",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
source_registry_id=source_id,
source_snapshot_id=snapshot_id,
)
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
def test_lineage_evidence_quarantines_ungoverned_parent_without_inventing_a_checksum() -> (
None
):
parent = _governed_parent()
valid_lineage = DerivedDatasetGovernanceService._lineage_evidence(
parent, "vector.clip", {"area_id": "a"}
)
valid_report = validate_registered_asset(
build_vector_ingest_input(
asset_id="derived-valid",
source_crs="EPSG:4326",
storage_crs="EPSG:4326",
feature_collection={
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
}
],
},
checksum_sha256=_CHECKSUM,
computed_checksum_sha256=_CHECKSUM,
source_registry_id="derived-source",
source_snapshot_id="derived-snapshot",
imported_at=datetime.now(timezone.utc),
metadata={
"license": "internal derived artifact",
"bounds_json": {
"min_x": 5.0,
"min_y": 51.0,
"max_x": 5.0,
"max_y": 51.0,
},
},
temporal_unknown_reason="derived input has no precise observation timestamp",
source_version_unknown_reason="transform version is recorded separately",
lineage=valid_lineage,
)
)
assert valid_report.validation_status.value == "passed"
assert valid_lineage.upstream_asset_ids == (str(parent.id),)
assert valid_lineage.upstream_checksums_sha256 == (_CHECKSUM,)
parent.validation_status = "not_validated"
rejected_lineage = DerivedDatasetGovernanceService._lineage_evidence(
parent, "vector.clip", {}
)
rejected_report = validate_registered_asset(
build_vector_ingest_input(
asset_id="derived-rejected",
source_crs="EPSG:4326",
storage_crs="EPSG:4326",
feature_collection={
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
}
],
},
checksum_sha256=_CHECKSUM,
computed_checksum_sha256=_CHECKSUM,
source_registry_id="derived-source",
source_snapshot_id="derived-snapshot",
imported_at=datetime.now(timezone.utc),
metadata={
"license": "internal derived artifact",
"bounds_json": {
"min_x": 5.0,
"min_y": 51.0,
"max_x": 5.0,
"max_y": 51.0,
},
},
temporal_unknown_reason="derived input has no precise observation timestamp",
source_version_unknown_reason="transform version is recorded separately",
lineage=rejected_lineage,
)
)
assert rejected_lineage.upstream_checksums_sha256 == (
"parent_dataset_not_governed",
)
assert rejected_report.validation_status.value == "failed"
assert rejected_report.quarantine_status.value == "quarantined"
assert any(
issue.code == "UPSTREAM_CHECKSUM_FORMAT_INVALID"
for issue in rejected_report.issues
)
def test_govern_vector_binds_snapshot_contract_and_lineage_before_marking_ready(
monkeypatch,
) -> None:
from app.services.source_registry_service import SourceRegistryService
db = _GovernedSession()
parent = _governed_parent()
parent_version_id = uuid4()
dataset = Dataset(
id=uuid4(),
project_id=parent.project_id,
name="derived.geojson",
dataset_type="vector",
source="operation:clip",
source_name="derived",
checksum_sha256=_CHECKSUM,
imported_at=datetime.now(timezone.utc),
crs="EPSG:4326",
metadata_json={
"bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0}
},
status="validating",
)
version = DatasetVersion(
id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM
)
source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact")
snapshot = SimpleNamespace(id=uuid4(), source_registry_id=source.id)
edges = []
monkeypatch.setattr(
SourceRegistryService,
"ensure_server_owned_source",
lambda *_args, **_kwargs: source,
)
monkeypatch.setattr(
SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot
)
monkeypatch.setattr(
DerivedDatasetGovernanceService,
"_latest_parent_version_id",
lambda *_args: parent_version_id,
)
def _bind(target, **kwargs):
target.source_registry_id = kwargs["source"].id
target.source_snapshot_id = kwargs["snapshot"].id
target.data_contract_key = kwargs["data_contract_key"]
target.data_contract_version = kwargs["data_contract_version"]
target.validation_status = kwargs["validation_status"]
target.provenance_status = kwargs["provenance_status"]
target.lineage_status = kwargs["lineage_status"]
return target
monkeypatch.setattr(SourceRegistryService, "bind_dataset_provenance", _bind)
monkeypatch.setattr(SourceRegistryService, "bind_dataset_version_provenance", _bind)
monkeypatch.setattr(
SourceRegistryService,
"record_lineage_edge",
lambda *_args, **kwargs: edges.append(kwargs),
)
ready = DerivedDatasetGovernanceService.govern_vector(
db,
dataset=dataset,
dataset_version=version,
feature_collection={
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
}
],
},
source_key="derived",
operation="vector.clip",
parent_dataset=parent,
operation_parameters={"area_id": "a"},
)
assert ready is True
assert dataset.status == "ready"
assert dataset.validation_status == "passed"
assert dataset.provenance_status == "complete"
assert dataset.source_registry_id == source.id
assert version.source_snapshot_id == snapshot.id
assert db.flushes >= 1
assert edges[0]["parent_dataset_id"] == parent.id
assert edges[0]["parent_dataset_version_id"] == parent_version_id
assert edges[0]["child_dataset_version_id"] == version.id
def test_govern_vector_quarantines_output_when_parent_is_manual_or_experimental(
monkeypatch,
) -> None:
from app.services.source_registry_service import SourceRegistryService
db = _GovernedSession()
parent = _governed_parent()
parent.source = "manual"
parent.source_name = "manual"
parent.source_registry.source_key = "manual"
parent.source_registry.classification = "experimental"
parent.source_registry.usage_policy_json = {"ground_truth_allowed": False}
dataset = Dataset(
id=uuid4(),
project_id=parent.project_id,
name="manual-derived.geojson",
dataset_type="vector",
source="operation:clip",
source_name="derived",
checksum_sha256=_CHECKSUM,
imported_at=datetime.now(timezone.utc),
crs="EPSG:4326",
metadata_json={
"bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0}
},
status="validating",
)
version = DatasetVersion(
id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM
)
source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact")
snapshot = SimpleNamespace(
id=uuid4(), source_registry_id=source.id, ingest_status="ingested"
)
monkeypatch.setattr(
SourceRegistryService,
"ensure_server_owned_source",
lambda *_args, **_kwargs: source,
)
monkeypatch.setattr(
SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot
)
monkeypatch.setattr(
DerivedDatasetGovernanceService,
"_latest_parent_version_id",
lambda *_args: None,
)
monkeypatch.setattr(
SourceRegistryService, "record_lineage_edge", lambda *_args, **_kwargs: None
)
ready = DerivedDatasetGovernanceService.govern_vector(
db,
dataset=dataset,
dataset_version=version,
feature_collection={
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
}
],
},
source_key="derived",
operation="vector.clip",
parent_dataset=parent,
)
assert ready is False
assert dataset.status == "quarantined"
assert dataset.quarantine_status == "quarantined"
assert dataset.validation_status == "failed"
assert any(
issue["code"] == "PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING"
for issue in dataset.validation_report_json["issues"]
)
assert snapshot.ingest_status == "quarantined"
def test_vector_selection_uses_map_selection_registry_and_skips_features_when_quarantined(
monkeypatch, tmp_path
) -> None:
source = _governed_parent()
source.area_id = None
source.storage_path = str(tmp_path / "source.geojson")
db = _FakeSession({(Dataset, source.id): source})
output_path = tmp_path / "selection.geojson"
calls = []
persisted_features = []
monkeypatch.setattr(
VectorFeatureService,
"select_features_by_bbox",
lambda *_args, **_kwargs: {
"selection_bbox": {
"min_x": 4.9,
"min_y": 50.9,
"max_x": 5.2,
"max_y": 51.2,
"crs": "EPSG:4326",
},
"feature_count": 1,
"limit": 250,
"truncated": False,
"geojson": {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": "source-feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {
"vector_feature_id": "source-feature",
"dataset_id": str(source.id),
},
}
],
},
},
)
def _persist_dataset_file(**kwargs):
output_path.write_bytes(kwargs["content"])
return {
"original_filename": kwargs["original_filename"],
"stored_filename": output_path.name,
"content_type": kwargs["content_type"],
"size_bytes": len(kwargs["content"]),
"checksum_sha256": _CHECKSUM,
"storage_path": str(output_path),
}
monkeypatch.setattr(StorageService, "persist_dataset_file", _persist_dataset_file)
def _quarantine(db, **kwargs):
calls.append(kwargs)
kwargs["dataset"].status = "quarantined"
return False
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine)
monkeypatch.setattr(
VectorFeatureService,
"persist_geojson_features",
lambda **kwargs: persisted_features.append(kwargs),
)
response = VectorOperationsService.derive_selection_dataset(
db=db,
dataset_id=source.id,
bbox={
"min_x": 4.9,
"min_y": 50.9,
"max_x": 5.2,
"max_y": 51.2,
"crs": "EPSG:4326",
},
)
assert response.status == "quarantined"
assert calls[0]["source_key"] == "map_selection"
assert calls[0]["parent_dataset"] is source
assert calls[0]["operation"] == "vector.selection"
assert persisted_features == []
def test_vector_buffer_uses_projected_metres_instead_of_wgs84_degrees(
monkeypatch, tmp_path
) -> None:
source = _governed_parent()
source.storage_path = str(tmp_path / "source.geojson")
source.crs = "EPSG:4326"
Path(source.storage_path).write_text(
json.dumps(
{
"type": "FeatureCollection",
"crs": "EPSG:4326",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
},
],
}
),
encoding="utf-8",
)
# A governed consumption artifact must carry the checksum of these exact
# bytes; vector operations deliberately refuse a stale fixture checksum.
source.checksum_sha256 = sha256(Path(source.storage_path).read_bytes()).hexdigest()
source.source_snapshot.checksum_sha256 = source.checksum_sha256
db = _FakeSession({(Dataset, source.id): source})
captured = {}
def _persist(**kwargs):
captured.update(kwargs)
return uuid4()
monkeypatch.setattr(VectorOperationsService, "_persist_derived_dataset", _persist)
VectorOperationsService.buffer(
db, source.id, distance_m=100.0, dissolve=False, output_name=None
)
coordinates = captured["feature_collection"]["features"][0]["geometry"][
"coordinates"
][0]
longitudes = [coordinate[0] for coordinate in coordinates]
latitudes = [coordinate[1] for coordinate in coordinates]
assert max(longitudes) - min(longitudes) < 0.01
assert max(latitudes) - min(latitudes) < 0.01
def test_raster_operation_uses_derived_registry_before_commit(
monkeypatch, tmp_path
) -> None:
source = _governed_parent()
source.dataset_type = "raster"
source.storage_path = str(tmp_path / "source.tif")
output_path = tmp_path / "derived.tif"
output_path.write_bytes(b"derived-raster")
db = _FakeSession()
calls = []
def _govern(db, **kwargs):
calls.append(kwargs)
kwargs["dataset"].status = "quarantined"
return False
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_raster", _govern)
result = RasterOperationsService._persist_derived_dataset(
db,
source_dataset=source,
source_dataset_id=source.id,
operation="ndvi",
output_path=str(output_path),
output_name="derived.tif",
metadata={
"crs": "EPSG:31370",
"bounds": [100000.0, 100000.0, 100001.0, 100001.0],
"resolution": [1.0, 1.0],
"width": 1,
"height": 1,
"band_count": 1,
"dtype": ["float32"],
"operation_parameters": {"nir_band": 4, "red_band": 3},
},
)
derived = next(item for item in db.added if isinstance(item, Dataset))
assert result == derived.id
assert derived.status == "quarantined"
assert derived.source_name == "derived"
assert calls[0]["source_key"] == "derived"
assert calls[0]["parent_dataset"] is source
assert calls[0]["operation"] == "raster.ndvi"
def test_demo_fixture_creation_is_governed_and_does_not_persist_features_when_rejected(
monkeypatch, tmp_path
) -> None:
project_id = uuid4()
area_id = uuid4()
db = _FakeSession()
calls = []
features = []
monkeypatch.setattr(
StorageService,
"persist_dataset_file",
lambda **kwargs: {
"storage_path": str(tmp_path / kwargs["original_filename"]),
"original_filename": kwargs["original_filename"],
"stored_filename": kwargs["original_filename"],
"content_type": kwargs["content_type"],
"size_bytes": len(kwargs["content"]),
"checksum_sha256": _CHECKSUM,
},
)
def _quarantine(db, **kwargs):
calls.append(kwargs)
kwargs["dataset"].status = "quarantined"
return False
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine)
monkeypatch.setattr(
VectorFeatureService,
"persist_geojson_features",
lambda **kwargs: features.append(kwargs),
)
payload = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
}
],
}
dataset = DemoWorkflowService._create_dataset(
db,
project_id=project_id,
area_id=area_id,
filename="fixture.geojson",
payload=payload,
raw=json.dumps(payload).encode("utf-8"),
role="source",
source_name="fixture",
reference_layer_name=None,
)
assert dataset.status == "quarantined"
assert calls[0]["source_key"] == "fixture"
assert calls[0]["operation"] == "demo.fixture_vector"
assert features == []
@@ -0,0 +1,453 @@
from __future__ import annotations
import json
from types import SimpleNamespace
from urllib.parse import parse_qs, urlparse
from uuid import uuid4
import pytest
from geoalchemy2.shape import from_shape
from pyproj import Transformer
from shapely.geometry import MultiPolygon, Polygon
from app.core.config import Settings
from app.core.errors import AppError
from app.models import Area, Dataset, Project
from app.schemas.official_vector import OfficialVectorAcquireRequest
from app.services.dataset_service import DatasetService
from app.services.official_vector_acquisition_service import (
OfficialVectorAcquisitionService,
_TO_LAMBERT72,
)
class FakeQuery:
def __init__(self, result=None):
self.result = result
def filter(self, *_args):
return self
def order_by(self, *_args):
return self
def all(self):
return self.result if isinstance(self.result, list) else []
class FakeSession:
def __init__(self, rows=None, query_result=None):
self.rows = rows or {}
self.query_result = query_result
def get(self, model, row_id):
return self.rows.get((model, row_id))
def query(self, _model):
return FakeQuery(self.query_result)
class JsonResponse:
def __init__(self, payload, content_type="application/geo+json"):
self.content = json.dumps(payload).encode("utf-8")
self.content_type = content_type
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self, size=-1):
return self.content if size < 0 else self.content[:size]
def getheader(self, name):
return self.content_type if name.lower() == "content-type" else None
def request(product_key: str, bbox: tuple[float, float, float, float], area_id=None):
return OfficialVectorAcquireRequest(
bbox={
"min_x": bbox[0],
"min_y": bbox[1],
"max_x": bbox[2],
"max_y": bbox[3],
"crs": "EPSG:4326",
},
area_id=area_id,
product_key=product_key,
force_refresh=True,
)
def area(project_id, name: str, bounds: tuple[float, float, float, float]):
min_x, min_y, max_x, max_y = bounds
geometry = MultiPolygon(
[
Polygon(
[
(min_x, min_y),
(max_x, min_y),
(max_x, max_y),
(min_x, max_y),
(min_x, min_y),
]
)
]
)
return Area(
id=uuid4(),
project_id=project_id,
name=name,
geometry=from_shape(geometry, srid=4326),
)
def test_regional_product_registry_is_explicit_and_source_specific() -> None:
products = {
item["key"]: item
for item in OfficialVectorAcquisitionService.list_products()
}
assert products["spw_picc_buildings"]["coverage_zones"] == ["wallonia"]
assert products["spw_picc_roads"]["geometry_types"] == [
"LineString",
"MultiLineString",
]
assert products["spw_picc_waterways"]["collection"] == "28"
assert products["spw_picc_water_surfaces"]["collection"] == "30"
assert products["spw_flood_hazard_2021"]["collection"] == "2"
assert products["spw_flood_hazard_2021"]["theme"] == "flood_hazard"
assert products["spw_flood_hazard_2021"]["coverage_zones"] == ["wallonia"]
assert products["urbis_buildings"]["coverage_zones"] == ["brussels"]
assert products["urbis_buildings"]["license_note"] == "Buildings are published under CC0."
assert "FPS Finance" in products["urbis_cadastral_parcels"]["license_note"]
# urbis_street_axes is live-validated against the UrbIS WFS capabilities:
# urbisvector:StreetAxes exposes INSPIRE_ID and LineString geometry. The
# same capabilities document advertises no hydrography feature type, so
# Brussels surface water intentionally stays not_configured.
assert products["urbis_street_axes"]["coverage_zones"] == ["brussels"]
assert products["urbis_street_axes"]["collection"] == "urbisvector:StreetAxes"
assert products["urbis_street_axes"]["geometry_types"] == [
"LineString",
"MultiLineString",
]
assert products["urbis_street_axes"]["theme"] == "roads"
assert products["urbis_land_cover_blocks"]["collection"] == "urbisvector:Blocks"
assert products["urbis_land_cover_blocks"]["theme"] == "space_occupation"
assert products["urbis_forest_parks"]["theme"] == "forest"
assert products["urbis_water_surfaces"]["theme"] == "water"
def test_urbis_land_cover_products_filter_only_documented_block_classes() -> None:
scope_wgs84 = Polygon(
[(4.35, 50.84), (4.36, 50.84), (4.36, 50.85), (4.35, 50.85), (4.35, 50.84)]
)
scope_metric = Polygon([_TO_LAMBERT72.transform(x, y) for x, y in scope_wgs84.exterior.coords])
min_x, min_y, max_x, max_y = scope_metric.bounds
def block(block_type: str):
return {
"type": "Feature",
"id": f"Blocks.{block_type}",
"geometry": {
"type": "Polygon",
"coordinates": [[
[min_x + 10, min_y + 10],
[min_x + 100, min_y + 10],
[min_x + 100, min_y + 100],
[min_x + 10, min_y + 100],
[min_x + 10, min_y + 10],
]],
},
"properties": {
"INSPIRE_ID": f"https://databrussels.be/id/block/{block_type}",
"TYPE": block_type,
},
}
forest_product = OfficialVectorAcquisitionService._product("urbis_forest_parks")
water_product = OfficialVectorAcquisitionService._product("urbis_water_surfaces")
land_cover_product = OfficialVectorAcquisitionService._product("urbis_land_cover_blocks")
assert OfficialVectorAcquisitionService._normalize_regional_feature(
forest_product, block("FO"), scope_metric, "brussels"
) is not None
assert OfficialVectorAcquisitionService._normalize_regional_feature(
forest_product, block("CB"), scope_metric, "brussels"
) is None
assert OfficialVectorAcquisitionService._normalize_regional_feature(
water_product, block("WB"), scope_metric, "brussels"
) is not None
assert OfficialVectorAcquisitionService._normalize_regional_feature(
water_product, block("GB"), scope_metric, "brussels"
) is None
normalized = OfficialVectorAcquisitionService._normalize_regional_feature(
land_cover_product, block("CB"), scope_metric, "brussels"
)
assert normalized is not None
assert normalized["properties"]["TYPE"] == "CB"
assert normalized["properties"]["clipped_area_ha"] > 0
def test_spw_arcgis_paging_is_bounded_stable_and_clipped() -> None:
product = OfficialVectorAcquisitionService._product("spw_picc_buildings")
scope = Polygon(
[(4.55, 50.58), (4.56, 50.58), (4.56, 50.59), (4.55, 50.59), (4.55, 50.58)]
)
scope_metric = Polygon(
[
_TO_LAMBERT72.transform(x, y)
for x, y in scope.exterior.coords
]
)
offsets = []
def feature(object_id: int, min_x: float):
return {
"type": "Feature",
"id": object_id,
"geometry": {
"type": "Polygon",
"coordinates": [[
[min_x, 50.581],
[min_x + 0.002, 50.581],
[min_x + 0.002, 50.583],
[min_x, 50.583],
[min_x, 50.581],
]],
},
"properties": {"OBJECTID": object_id, "GEOREF_ID": f"wallonia-{object_id}"},
}
def opener(raw_request, timeout):
assert timeout == 180
query = parse_qs(urlparse(raw_request.full_url).query)
assert query["orderByFields"] == ["OBJECTID"]
assert query["f"] == ["geojson"]
offset = int(query["resultOffset"][0])
offsets.append(offset)
return JsonResponse(
{
"type": "FeatureCollection",
"features": [feature(offset + 1, 4.551 + offset * 0.0001)],
"exceededTransferLimit": offset == 0,
}
)
features, transfer = OfficialVectorAcquisitionService._fetch_features(
product,
scope,
scope_metric,
"wallonia",
Settings(_env_file=None, OFFICIAL_VECTOR_PAGE_SIZE=1),
opener,
)
assert offsets == [0, 1]
assert transfer["page_count"] == 2
assert transfer["reference_truncated"] is False
assert {item["properties"]["source_feature_id"] for item in features} == {
"11:wallonia-1",
"11:wallonia-2",
}
assert all(item["properties"]["coverage_scope"] == "wallonia" for item in features)
assert all(item["properties"]["clipped_area_ha"] > 0 for item in features)
def test_spw_flood_hazard_uses_separate_governed_endpoint_and_persists_classification() -> None:
product = OfficialVectorAcquisitionService._product("spw_flood_hazard_2021")
scope = Polygon(
[(4.55, 50.58), (4.56, 50.58), (4.56, 50.59), (4.55, 50.59), (4.55, 50.58)]
)
scope_metric = Polygon(
[_TO_LAMBERT72.transform(x, y) for x, y in scope.exterior.coords]
)
def opener(raw_request, timeout):
assert timeout == 180
parsed = urlparse(raw_request.full_url)
assert parsed.path.endswith("/EAU/ALEA_INOND/MapServer/2/query")
query = parse_qs(parsed.query)
assert query["outSR"] == ["4326"]
return JsonResponse(
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": 7,
"geometry": {
"type": "Polygon",
"coordinates": [[
[4.551, 50.581],
[4.559, 50.581],
[4.559, 50.589],
[4.551, 50.589],
[4.551, 50.581],
]],
},
"properties": {
"OBJECTID": 7,
"LOCALID": "ALEA-7",
"TYPEALEA": "Debordement",
"CLASSEMENT": 130,
"MILLESIME": 2021,
},
}
],
"exceededTransferLimit": False,
}
)
features, transfer = OfficialVectorAcquisitionService._fetch_features(
product,
scope,
scope_metric,
"wallonia",
Settings(_env_file=None),
opener,
)
assert transfer["feature_count"] == 1
assert features[0]["id"] == "2:ALEA-7"
assert features[0]["properties"]["CLASSEMENT"] == 130
assert features[0]["properties"]["source_name"] == "spw_flood_hazard"
assert features[0]["properties"]["clipped_area_ha"] > 0
def test_spw_flood_hazard_can_be_disabled_independently() -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
with pytest.raises(AppError) as exc_info:
OfficialVectorAcquisitionService.acquire(
db,
project_id,
request("spw_flood_hazard_2021", (4.55, 50.58, 4.56, 50.59)),
settings=Settings(_env_file=None, SPW_FLOOD_HAZARD_ENABLED=False),
)
assert exc_info.value.code == "SPW_FLOOD_HAZARD_NOT_CONFIGURED"
def test_regional_products_require_the_persisted_authoritative_coverage_area() -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
with pytest.raises(AppError) as exc_info:
OfficialVectorAcquisitionService.acquire(
db,
project_id,
request("spw_picc_buildings", (4.55, 50.58, 4.56, 50.59)),
settings=Settings(_env_file=None),
)
assert exc_info.value.code == "OFFICIAL_VECTOR_COVERAGE_NOT_READY"
def test_urbis_wfs_transforms_lambert72_and_persists_through_dataset_service(
monkeypatch,
) -> None:
project_id, dataset_id = uuid4(), uuid4()
brussels = area(project_id, "Brussels-Capital Region", (4.25, 50.75, 4.5, 50.95))
db = FakeSession(
{
(Project, project_id): Project(id=project_id, name="Belgium"),
(Area, brussels.id): brussels,
},
query_result=[brussels],
)
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
min_x, min_y = to_lambert.transform(4.35, 50.84)
max_x, max_y = to_lambert.transform(4.351, 50.841)
captured = {}
def opener(raw_request, timeout):
assert timeout == 180
query = parse_qs(urlparse(raw_request.full_url).query)
assert query["typeNames"] == ["urbisvector:Buildings"]
assert query["srsName"] == ["EPSG:31370"]
assert query["sortBy"] == ["INSPIRE_ID"]
return JsonResponse(
{
"type": "FeatureCollection",
"numberMatched": 1,
"numberReturned": 1,
"features": [
{
"type": "Feature",
"id": "Buildings.1",
"geometry": {
"type": "MultiPolygon",
"coordinates": [[[
[min_x, min_y],
[max_x, min_y],
[max_x, max_y],
[min_x, max_y],
[min_x, min_y],
]]],
},
"properties": {
"INSPIRE_ID": "https://databrussels.be/id/building/1",
"AREA": 75,
},
}
],
},
"application/json",
)
def persist(_db, **kwargs):
captured.update(kwargs)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
area_id=brussels.id,
name=kwargs["filename"],
dataset_type="vector",
source=kwargs["source"],
dataset_role=kwargs["dataset_role"],
source_name=kwargs["source_name"],
reference_layer_name=kwargs["reference_layer_name"],
temporal_series_key=kwargs["temporal_series_key"],
observed_at=kwargs["observed_at"],
source_version=kwargs["source_version"],
source_metadata=kwargs["source_metadata"],
provenance_metadata=kwargs["provenance_metadata"],
metadata_json={"feature_count": 1},
status="ready",
)
db.rows[(Dataset, dataset_id)] = dataset
return SimpleNamespace(id=dataset_id)
monkeypatch.setattr(DatasetService, "import_vector_bytes", persist)
result = OfficialVectorAcquisitionService.acquire(
db,
project_id,
request(
"urbis_buildings",
(4.349, 50.839, 4.352, 50.842),
area_id=brussels.id,
),
settings=Settings(_env_file=None),
opener=opener,
)
assert result["output_dataset_id"] == str(dataset_id)
assert captured["source_name"] == "urbis"
assert captured["reference_layer_name"] == "buildings"
assert captured["source_metadata"]["coverage_zones"] == ["brussels"]
assert captured["source_metadata"]["selection_aggregation"]["metric_key"] == (
"building_footprint_area"
)
collection = json.loads(captured["content"])
geometry = collection["features"][0]["geometry"]
assert geometry["type"] in {"Polygon", "MultiPolygon"}
first_coordinate = (
geometry["coordinates"][0][0][0]
if geometry["type"] == "MultiPolygon"
else geometry["coordinates"][0][0]
)
assert 4.34999 <= first_coordinate[0] <= 4.35101
assert 50.83999 <= first_coordinate[1] <= 50.84101
@@ -0,0 +1,144 @@
"""What is promoted must be what the workbench then runs.
The candidate evaluation freezes its post-processing before the protected test
— NMS IoU and a containment threshold selected during calibration. The runtime
applied its own hardcoded containment value, so a model gated at one setting
was served at another and suppressed detections the gate had counted. The
difference is invisible in both reports.
Containment is therefore configuration, recorded with every run, and two runs
that post-processed differently are not comparable however good their numbers
look.
"""
from __future__ import annotations
import pytest
from shapely.geometry import box
from app.core.config import Settings
from app.services.detection_comparison_service import DetectionComparisonService
from app.services.detection_service import DetectionService
def _candidate(name: str, geometry, confidence: float):
return {
"class_name": "building",
"confidence": confidence,
"geometry": geometry,
"bbox": [0.0, 0.0, 1.0, 1.0],
"source_tile_path": f"/tiles/{name}.tif",
"properties": {"name": name},
}
class TestConfigurableContainment:
def test_the_runtime_threshold_comes_from_settings(self) -> None:
assert Settings(_env_file=None).yolo_containment_nms_threshold == pytest.approx(0.85)
assert Settings(
_env_file=None, yolo_containment_nms_threshold=1.0
).yolo_containment_nms_threshold == pytest.approx(1.0)
def test_a_strict_threshold_suppresses_only_a_fully_nested_box(self) -> None:
outer = _candidate("outer", box(0, 0, 10, 10), 0.9)
# 90% of the smaller box lies inside the larger one, but their IoU is
# only 0.09 — so only the containment rule can act on this pair.
mostly_nested = _candidate("mostly", box(8.2, 1, 10.2, 6), 0.5)
kept = DetectionService._suppress_duplicate_candidates(
[outer, mostly_nested], iou_threshold=0.5, containment_threshold=1.0
)
assert [item["properties"]["name"] for item in kept] == ["outer", "mostly"]
def test_a_looser_threshold_suppresses_it(self) -> None:
outer = _candidate("outer", box(0, 0, 10, 10), 0.9)
mostly_nested = _candidate("mostly", box(8.2, 1, 10.2, 6), 0.5)
kept = DetectionService._suppress_duplicate_candidates(
[outer, mostly_nested], iou_threshold=0.5, containment_threshold=0.7
)
assert [item["properties"]["name"] for item in kept] == ["outer"]
def test_the_default_matches_the_documented_runtime_value(self) -> None:
outer = _candidate("outer", box(0, 0, 10, 10), 0.9)
nested = _candidate("nested", box(1, 1, 9, 9), 0.5)
kept = DetectionService._suppress_duplicate_candidates([outer, nested], iou_threshold=0.5)
assert [item["properties"]["name"] for item in kept] == ["outer"]
class TestComparabilityOfPostProcessing:
def _entry(self, *, dataset_id, reference_id, containment: float, duplicate_iou: float = 0.5):
from uuid import uuid4
return {
"analysis_run_id": uuid4(),
"dataset_id": dataset_id,
"model_id": "yolo-configured",
"model_asset_id": "asset",
"reference_dataset_id": reference_id,
"coverage_mode": "persisted_tile_manifest_union",
"reference_evaluated_count": 100,
"containment_suppression_threshold": containment,
"duplicate_iou_threshold": duplicate_iou,
}
def test_runs_with_the_same_post_processing_stay_comparable(self) -> None:
from uuid import uuid4
dataset_id, reference_id = uuid4(), uuid4()
report = DetectionComparisonService.assess_comparability(
[
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85),
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85),
]
)
assert report["comparable"] is True
def test_a_different_containment_threshold_blocks_the_comparison(self) -> None:
from uuid import uuid4
dataset_id, reference_id = uuid4(), uuid4()
report = DetectionComparisonService.assess_comparability(
[
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85),
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=1.0),
]
)
assert report["comparable"] is False
assert "different_post_processing" in report["blocking_reasons"]
def test_a_different_duplicate_iou_blocks_the_comparison(self) -> None:
from uuid import uuid4
dataset_id, reference_id = uuid4(), uuid4()
report = DetectionComparisonService.assess_comparability(
[
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85, duplicate_iou=0.5),
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85, duplicate_iou=0.7),
]
)
assert report["comparable"] is False
assert "different_post_processing" in report["blocking_reasons"]
def test_runs_from_before_the_setting_existed_do_not_block(self) -> None:
"""Older runs recorded no threshold; absence is not a difference."""
from uuid import uuid4
dataset_id, reference_id = uuid4(), uuid4()
entries = [
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85),
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85),
]
for entry in entries:
entry.pop("containment_suppression_threshold")
entry.pop("duplicate_iou_threshold")
assert DetectionComparisonService.assess_comparability(entries)["comparable"] is True
@@ -0,0 +1,29 @@
from __future__ import annotations
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def test_publication_hygiene_gate_passes_for_tracked_tree() -> None:
result = subprocess.run(
[sys.executable, str(ROOT / "scripts" / "check_repository_hygiene.py")],
cwd=ROOT,
check=False,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
def test_checkpoint_inspector_uses_weights_only_deserialization() -> None:
inspector = (ROOT / "scripts" / "inspect_torch_checkpoint.py").read_text(
encoding="utf-8"
)
assert "weights_only=True" in inspector
assert "weights_only=False" not in inspector
@@ -0,0 +1,88 @@
"""QA matching must be reproducible and must credit the best candidate.
The greedy IoU matcher decides which candidate is reported as a match and
which becomes false-positive evidence for an operator. If that decision
depends on database row order, the same run produces different scores and
points reviewers at the wrong geometry.
"""
from __future__ import annotations
from shapely.geometry import box
from app.services.qa_service import QaService
REFERENCE = [({"id": "R1"}, box(0.0, 0.0, 10.0, 10.0))]
# Two detections of the same building. ``sloppy`` is far too tall (IoU 0.51),
# ``accurate`` is nearly exact (IoU 0.96).
SLOPPY = box(0.0, 0.0, 10.0, 19.5)
ACCURATE = box(0.0, 0.0, 10.0, 10.4)
def _candidates(order: list[tuple[str, float]]) -> list[tuple[dict, object]]:
geometries = {"sloppy": SLOPPY, "accurate": ACCURATE}
return [
({"id": name, "confidence": confidence}, geometries[name])
for name, confidence in order
]
def test_matching_is_independent_of_candidate_row_order() -> None:
forward = QaService._match_io_u_evidence(
_candidates([("sloppy", 0.42), ("accurate", 0.91)]), REFERENCE, 0.5
)
reverse = QaService._match_io_u_evidence(
_candidates([("accurate", 0.91), ("sloppy", 0.42)]), REFERENCE, 0.5
)
assert forward.matches == reverse.matches
assert forward.false_positives == reverse.false_positives
assert forward.false_negatives == reverse.false_negatives
assert forward.match_iou_values == reverse.match_iou_values
assert forward.match_evidence == reverse.match_evidence
assert forward.false_positive_evidence == reverse.false_positive_evidence
def test_highest_confidence_candidate_claims_the_reference() -> None:
evidence = QaService._match_io_u_evidence(
_candidates([("sloppy", 0.42), ("accurate", 0.91)]), REFERENCE, 0.5
)
assert evidence.matches == 1
assert evidence.match_evidence[0]["candidate_feature_id"] == "accurate"
assert evidence.false_positive_evidence == [{"candidate_feature_id": "sloppy"}]
assert evidence.match_iou_values[0] > 0.9
def test_matching_without_confidence_is_still_deterministic() -> None:
"""Vector-vs-vector QA has no confidence; identity keeps it reproducible."""
left = [
({"id": "b-second"}, SLOPPY),
({"id": "a-first"}, ACCURATE),
]
right = list(reversed(left))
assert QaService._match_io_u_evidence(left, REFERENCE, 0.5).match_evidence == (
QaService._match_io_u_evidence(right, REFERENCE, 0.5).match_evidence
)
def test_evidence_is_ordered_by_confidence_for_review() -> None:
evidence = QaService._match_io_u_evidence(
[
({"id": "low", "confidence": 0.30}, box(30.0, 30.0, 31.0, 31.0)),
({"id": "high", "confidence": 0.95}, box(40.0, 40.0, 41.0, 41.0)),
({"id": "mid", "confidence": 0.60}, box(50.0, 50.0, 51.0, 51.0)),
],
REFERENCE,
0.5,
)
assert [item["candidate_feature_id"] for item in evidence.false_positive_evidence] == [
"high",
"mid",
"low",
]
+205
View File
@@ -0,0 +1,205 @@
from __future__ import annotations
import json
from hashlib import sha256
from pathlib import Path
from uuid import uuid4
from app.models import Dataset, SourceRegistry, SourceSnapshot
from app.services.qa_service import QaService
class FakeSession:
def __init__(self, datasets=None, areas=None):
self.datasets = {item.id: item for item in (datasets or [])}
self.areas = {item.id: item for item in (areas or [])}
def get(self, model, item_id):
if model.__name__ == "Dataset":
return self.datasets.get(item_id)
if model.__name__ == "Area":
return self.areas.get(item_id)
return None
def _feature(feature_id: str, coordinates: list[list[list[float]]]) -> dict:
return {
"type": "Feature",
"id": feature_id,
"properties": {"source_feature_id": feature_id},
"geometry": {
"type": "Polygon",
"coordinates": coordinates,
},
}
def _write_dataset(path: Path, coordinates: list[list[list[float]]]) -> None:
payload = {
"type": "FeatureCollection",
"features": [_feature("feature-1", coordinates)],
}
path.write_text(json.dumps(payload), encoding="utf-8")
def _write_features(path: Path, features: list[dict]) -> None:
path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8")
def _authoritative_reference(dataset: Dataset) -> Dataset:
"""Give QA reference fixtures the same durable authority proof as GRB."""
source_id = uuid4()
snapshot_id = uuid4()
checksum = sha256(Path(str(dataset.storage_path)).read_bytes()).hexdigest()
source = SourceRegistry(
id=source_id,
source_key="grb",
display_name="GRB test reference",
classification="authoritative",
authority_name="Digitaal Vlaanderen",
authority_scope_json={"zone": "Flanders"},
usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key=f"qa-grb-{dataset.id}",
checksum_sha256=checksum,
ingest_status="ingested",
freshness_status="current",
)
dataset.source = "grb"
dataset.source_name = "grb"
dataset.dataset_role = "reference"
dataset.status = "ready"
dataset.checksum_sha256 = checksum
dataset.source_registry_id = source_id
dataset.source_snapshot_id = snapshot_id
dataset.data_contract_key = "geointel.vector.geojson"
dataset.data_contract_version = "1.0.0"
dataset.validation_status = "passed"
dataset.provenance_status = "complete"
dataset.lineage_status = "complete"
dataset.quarantine_status = "not_quarantined"
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
project_id = uuid4()
candidate_id = uuid4()
reference_id = uuid4()
candidate_path = tmp_path / "candidate.geojson"
reference_path = tmp_path / "reference.geojson"
polygon = [[[4.0, 51.0], [4.1, 51.0], [4.1, 51.1], [4.0, 51.1], [4.0, 51.0]]]
_write_dataset(candidate_path, polygon)
_write_dataset(reference_path, polygon)
candidate = Dataset(
id=candidate_id,
project_id=project_id,
name="candidate.geojson",
dataset_type="vector",
source="test",
storage_path=str(candidate_path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
)
reference = _authoritative_reference(Dataset(
id=reference_id,
project_id=project_id,
name="reference.geojson",
dataset_type="vector",
source="test",
storage_path=str(reference_path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
))
result = QaService.compare_candidate_with_reference(
db=FakeSession([candidate, reference]),
project_id=project_id,
candidate_dataset_id=candidate_id,
reference_dataset_id=reference_id,
iou_threshold=0.5,
)
assert result.status == "ok"
assert result.matches == 1
assert result.false_positives == 0
assert result.false_negatives == 0
assert result.precision == 1.0
assert result.recall == 1.0
assert result.f1_score == 1.0
def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_path) -> None:
project_id = uuid4()
candidate_id = uuid4()
reference_id = uuid4()
candidate_path = tmp_path / "candidate.geojson"
reference_path = tmp_path / "reference.geojson"
matched_candidate = [[[4.0, 51.0], [4.1, 51.0], [4.1, 51.1], [4.0, 51.1], [4.0, 51.0]]]
matched_reference = [[[4.0, 51.0], [4.1, 51.0], [4.1, 51.1], [4.0, 51.1], [4.0, 51.0]]]
false_positive = [[[4.5, 51.5], [4.6, 51.5], [4.6, 51.6], [4.5, 51.6], [4.5, 51.5]]]
false_negative = [[[4.8, 51.8], [4.9, 51.8], [4.9, 51.9], [4.8, 51.9], [4.8, 51.8]]]
_write_features(candidate_path, [_feature("candidate-match", matched_candidate), _feature("candidate-extra", false_positive)])
_write_features(reference_path, [_feature("reference-match", matched_reference), _feature("reference-missing", false_negative)])
candidate = Dataset(
id=candidate_id,
project_id=project_id,
name="candidate.geojson",
dataset_type="vector",
source="test",
storage_path=str(candidate_path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
)
reference = _authoritative_reference(Dataset(
id=reference_id,
project_id=project_id,
name="reference.geojson",
dataset_type="vector",
source="test",
storage_path=str(reference_path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
))
result = QaService.compare_candidate_with_reference(
db=FakeSession([candidate, reference]),
project_id=project_id,
candidate_dataset_id=candidate_id,
reference_dataset_id=reference_id,
iou_threshold=0.5,
)
assert result.matches == 1
assert result.false_positives == 1
assert result.false_negatives == 1
assert result.match_evidence == [
{
"candidate_feature_id": "candidate-match",
"reference_feature_id": "reference-match",
"iou": 1.0,
}
]
assert result.false_positive_evidence == [{"candidate_feature_id": "candidate-extra"}]
assert result.false_negative_evidence == [{"reference_feature_id": "reference-missing"}]
def test_dataset_reference_metadata_migration_declares_required_columns() -> None:
migration_path = Path(__file__).parents[1] / "alembic" / "versions" / "202606120001_add_dataset_reference_metadata.py"
migration_text = migration_path.read_text(encoding="utf-8")
for column_name in (
"dataset_role",
"source_name",
"reference_layer_name",
"source_metadata",
"provenance_metadata",
"imported_at",
):
assert column_name in migration_text
@@ -0,0 +1,127 @@
"""Evidence review must stay usable on a regional run.
evidence_geojson emitted one feature per false positive, one per false
negative and *two* per match, with no limit. A regional QA run of 40k
detections against 45k reference footprints produced well over a hundred
thousand features in a single response, plus one warning string per
unresolvable identifier. The endpoint the whole review workflow depends on
therefore stopped working exactly where review matters most.
The budget goes to what a reviewer must act on — misses and false positives —
before confirmations, and the response says what it left out.
"""
from __future__ import annotations
from app.services.quality_evidence_service import QualityEvidenceService
def _features(role: str, count: int) -> list[dict]:
return [{"properties": {"evidence_role": role}, "id": f"{role}-{index}"} for index in range(count)]
def test_missing_identifiers_collapse_into_one_statement() -> None:
warnings = QualityEvidenceService.summarize_missing(
candidate_ids=["a", "b", "c"],
reference_ids=["r1"],
)
assert len(warnings) == 1
assert "3" in warnings[0]
assert "1" in warnings[0]
def test_nothing_missing_produces_no_warning() -> None:
assert QualityEvidenceService.summarize_missing(candidate_ids=[], reference_ids=[]) == []
def test_the_plan_is_capped_before_any_geometry_is_fetched() -> None:
"""Resolving 130k geometries to draw 5k of them is work for nothing."""
findings = {
"match_evidence": [{"candidate_feature_id": f"c{i}", "reference_feature_id": f"r{i}"} for i in range(100)],
"false_positive_evidence": [{"candidate_feature_id": f"fp{i}"} for i in range(10)],
"false_negative_evidence": [{"reference_feature_id": f"fn{i}"} for i in range(10)],
}
plan = QualityEvidenceService.plan_evidence(findings, limit=8)
assert plan.truncated is True
assert len(plan.items) == 8
# Both error classes are represented; confirmations do not get a share
# while errors are still waiting.
assert {item.role for item in plan.items} == {"false_negative", "false_positive"}
# Only the identifiers that will actually be drawn need resolving.
assert len(plan.candidate_ids) + len(plan.reference_ids) == 8
assert plan.candidate_ids <= {f"fp{i}" for i in range(10)}
assert plan.reference_ids <= {f"fn{i}" for i in range(10)}
def test_a_rare_error_class_is_never_crowded_out() -> None:
"""50.000 misses must not hide the three false positives."""
findings = {
"false_negative_evidence": [{"reference_feature_id": f"fn{i}"} for i in range(5_000)],
"false_positive_evidence": [{"candidate_feature_id": f"fp{i}"} for i in range(3)],
}
plan = QualityEvidenceService.plan_evidence(findings, limit=100)
roles = [item.role for item in plan.items]
assert roles.count("false_positive") >= 1
assert roles.count("false_negative") >= 90
assert len(plan.items) == 100
def test_the_plan_reports_the_complete_population_not_the_capped_one() -> None:
findings = {
"match_evidence": [{"candidate_feature_id": f"c{i}", "reference_feature_id": f"r{i}"} for i in range(100)],
"false_negative_evidence": [{"reference_feature_id": "fn"}],
}
plan = QualityEvidenceService.plan_evidence(findings, limit=2)
assert plan.total_feature_count == 201
assert plan.role_counts == {"match_candidate": 100, "match_reference": 100, "false_negative": 1}
def test_an_uncapped_plan_keeps_everything() -> None:
findings = {"false_negative_evidence": [{"reference_feature_id": f"fn{i}"} for i in range(30)]}
plan = QualityEvidenceService.plan_evidence(findings, limit=0)
assert plan.truncated is False
assert len(plan.items) == 30
assert plan.reference_ids == {f"fn{i}" for i in range(30)}
def test_evidence_without_identifiers_is_skipped_not_planned() -> None:
findings = {
"false_positive_evidence": [{"candidate_feature_id": None}, {"candidate_feature_id": "fp"}],
"false_negative_evidence": [{}],
}
plan = QualityEvidenceService.plan_evidence(findings, limit=0)
assert [item.role for item in plan.items] == ["false_positive"]
assert plan.candidate_ids == {"fp"}
def test_a_planned_match_keeps_its_candidate_and_reference_together() -> None:
"""Half a match is not reviewable evidence."""
findings = {
"match_evidence": [
{"candidate_feature_id": "c1", "reference_feature_id": "r1"},
{"candidate_feature_id": "c2", "reference_feature_id": "r2"},
]
}
plan = QualityEvidenceService.plan_evidence(findings, limit=3)
assert plan.truncated is True
# An odd budget drops the second pair rather than showing one side of it.
assert len(plan.items) == 1
assert plan.candidate_ids == {"c1"}
assert plan.reference_ids == {"r1"}
@@ -0,0 +1,84 @@
"""A selection smaller than a raster cell must not silently read as zero.
``geometry_mask`` selects a cell when its *centre* falls inside the geometry.
A rectangle smaller than one cell, or one that lands between four centres,
therefore selects nothing at all — and the analysis returned zeros, which on
screen is indistinguishable from "we looked and there is nothing here". On a
100 m population raster a 40 m rectangle over a city block reported no
inhabitants.
Selection now falls back to every touched cell and says that it did, so the
value is readable as "at least one whole cell", not as an empty area.
"""
from __future__ import annotations
import pytest
np = pytest.importorskip("numpy")
rasterio = pytest.importorskip("rasterio")
from rasterio.transform import from_origin # noqa: E402 - optional rasterio gate precedes imports
from shapely.geometry import box # noqa: E402 - optional rasterio gate precedes imports
from app.services.raster_cell_selection import select_cells # noqa: E402 - optional rasterio gate precedes service import
# 100 m cells, origin at the top-left corner of a 3x3 grid.
TRANSFORM = from_origin(200_000, 210_000, 100.0, 100.0)
SHAPE = (3, 3)
def test_a_normal_selection_uses_cell_centres() -> None:
selection = select_cells(box(200_000, 209_700, 200_300, 210_000), out_shape=SHAPE, transform=TRANSFORM)
assert selection.mask.sum() == 9
assert selection.mode == "cell_centre"
assert selection.expanded_to_touched_cells is False
assert selection.warning is None
def test_a_rectangle_smaller_than_one_cell_still_returns_that_cell() -> None:
selection = select_cells(box(200_010, 209_960, 200_050, 209_990), out_shape=SHAPE, transform=TRANSFORM)
assert selection.mask.sum() == 1
assert selection.mode == "all_touched"
assert selection.expanded_to_touched_cells is True
assert "cel" in selection.warning
def test_a_rectangle_between_four_cell_centres_returns_all_four() -> None:
selection = select_cells(box(200_080, 209_880, 200_120, 209_920), out_shape=SHAPE, transform=TRANSFORM)
assert selection.mask.sum() == 4
assert selection.expanded_to_touched_cells is True
def test_a_selection_entirely_off_the_raster_selects_nothing() -> None:
"""Falling back must not invent coverage where the geometry does not reach."""
selection = select_cells(box(300_000, 300_000, 300_100, 300_100), out_shape=SHAPE, transform=TRANSFORM)
assert selection.mask.sum() == 0
assert selection.expanded_to_touched_cells is False
assert selection.mode == "cell_centre"
def test_the_warning_states_how_much_larger_the_analysed_area_is() -> None:
selection = select_cells(
box(200_010, 209_960, 200_050, 209_990),
out_shape=SHAPE,
transform=TRANSFORM,
cell_area_m2=100.0 * 100.0,
)
# One 100x100 m cell was analysed for a 40x30 m request.
assert "1 rastercel" in selection.warning
assert "1.0 ha" in selection.warning
def test_the_mask_shape_always_matches_the_raster_window() -> None:
selection = select_cells(box(200_010, 209_960, 200_050, 209_990), out_shape=SHAPE, transform=TRANSFORM)
assert selection.mask.shape == SHAPE
assert selection.mask.dtype == np.bool_
File diff suppressed because it is too large Load Diff
+67
View File
@@ -0,0 +1,67 @@
from app.core.errors import AppError
from app.services.raster_service import extract_raster_metadata
def test_extract_raster_metadata_returns_dependency_aware_error(monkeypatch, tmp_path) -> None:
monkeypatch.setattr("app.services.raster_service._import_rasterio", lambda: (_ for _ in ()).throw(ImportError("rasterio not installed")))
file_path = tmp_path / "missing.tif"
file_path.write_bytes(b"\x00\x01\x02")
try:
extract_raster_metadata(str(file_path))
except AppError as exc:
assert exc.code == "RASTER_PROCESSING_UNAVAILABLE"
else:
raise AssertionError("Missing rasterio should raise AppError code RASTER_PROCESSING_UNAVAILABLE")
def test_extract_raster_metadata_maps_basic_profile_fields(monkeypatch, tmp_path) -> None:
file_path = tmp_path / "sample.tif"
file_path.write_bytes(b"fake")
class FakeDataset:
width = 1024
height = 768
count = 4
driver = "GTiff"
crs = "EPSG:31370"
bounds = (100.0, 200.0, 500.0, 800.0)
res = (0.25, 0.25)
dtypes = ["uint16", "uint16", "uint16", "uint16"]
nodata = -9999
class transform:
@staticmethod
def to_gdal():
return (0.25, 0.0, 100.0, 0.0, -0.25, 800.0, 0.0, 0.0, 1.0)
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return None
class FakeRasterio:
class errors:
class RasterioIOError(Exception):
...
def open(self, *_):
return FakeDataset()
class FakeErrors:
RasterioIOError = FakeRasterio.errors.RasterioIOError
monkeypatch.setattr("app.services.raster_service._import_rasterio", lambda: (FakeRasterio(), FakeErrors()))
metadata = extract_raster_metadata(str(file_path))
assert metadata["driver"] == "GTiff"
assert metadata["width"] == 1024
assert metadata["height"] == 768
assert metadata["band_count"] == 4
assert metadata["crs"] == "EPSG:31370"
assert metadata["bounds"] == [100.0, 200.0, 500.0, 800.0]
assert metadata["resolution"] == [0.25, 0.25]
assert metadata["dtype"] == ["uint16", "uint16", "uint16", "uint16"]
assert metadata["nodata"] == -9999.0
+569
View File
@@ -0,0 +1,569 @@
from __future__ import annotations
import hashlib
import importlib.util
import json
import os
import subprocess
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
import pytest
ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
def load_script(name: str):
path = SCRIPTS / name
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha256") -> None:
root.mkdir(parents=True)
manifest = {
"schema_version": 1,
"release_id": "rc10-test",
"created_at": created_at.isoformat(),
"read_only_source": True,
"database_password_secure": True,
"inventory_mode": inventory_mode,
"storage_inventory_requested": True,
"storage_snapshot_requested": True,
"models_inventory_requested": False,
"models_snapshot_requested": False,
"git_commit": "0123456789abcdef",
}
files = {
"manifest.json": json.dumps(manifest),
"database.dump": "database",
"database.list": "list",
"database-metadata.tsv": "alembic_head\t202607160001",
"table-counts.tsv": "datasets\t1",
"storage-manifest.tsv": "relative_path\tsize_bytes\tmtime_ns\tsha256",
}
for name, content in files.items():
(root / name).write_text(content, encoding="utf-8")
(root / "storage-snapshot").mkdir()
checksums = []
for name in sorted(files):
digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
checksums.append(f"{digest} {name}")
(root / "CHECKSUMS.sha256").write_text("\n".join(checksums) + "\n", encoding="utf-8")
def test_backup_guard_requires_recent_complete_sha256_storage_backup(tmp_path: Path) -> None:
guard = load_script("release_backup_guard.py")
now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
backup = tmp_path / "backup"
write_backup(backup, created_at=now - timedelta(hours=2))
verified = guard.verify_current_backup(backup, now=now)
assert verified.release_id == "rc10-test"
assert verified.age_hours == pytest.approx(2)
def test_backup_guard_rejects_stale_or_tampered_backup(tmp_path: Path) -> None:
guard = load_script("release_backup_guard.py")
now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
stale = tmp_path / "stale"
write_backup(stale, created_at=now - timedelta(hours=30))
with pytest.raises(RuntimeError, match="maximum allowed age"):
guard.verify_current_backup(stale, now=now)
current = tmp_path / "tampered"
write_backup(current, created_at=now)
(current / "database.dump").write_text("tampered", encoding="utf-8")
with pytest.raises(RuntimeError, match="checksum mismatch"):
guard.verify_current_backup(current, now=now)
def test_storage_lifecycle_is_fail_closed_and_protects_release_evidence() -> None:
audit = load_script("audit_data_operations.py")
assert audit.classify_relative_path("release-evidence/rc11/manifest.json") == (
"release-evidence",
True,
False,
)
assert audit.classify_relative_path("operator-evidence/source/raw.json")[1:] == (True, False)
assert audit.classify_relative_path("uploads/project/data.geojson")[1:] == (True, False)
assert audit.classify_relative_path("exports/project/old.json")[1:] == (False, True)
assert audit.classify_relative_path("unknown/value.bin")[1:] == (True, False)
def test_storage_audit_only_selects_old_unreferenced_allowlisted_files(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
audit = load_script("audit_data_operations.py")
storage = tmp_path / "storage"
old_orphan = storage / "exports" / "project" / "old.json"
referenced = storage / "exports" / "project" / "kept.json"
protected = storage / "release-evidence" / "rc" / "manifest.json"
unknown = storage / "misc" / "unknown.bin"
for path in (old_orphan, referenced, protected, unknown):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(path.name, encoding="utf-8")
old_timestamp = (datetime.now(timezone.utc) - timedelta(days=30)).timestamp()
for path in (old_orphan, referenced, protected, unknown):
path.touch()
Path(path).chmod(0o644)
import os
os.utime(path, (old_timestamp, old_timestamp))
monkeypatch.setattr(
audit,
"collect_database_state",
lambda _db, _root: {
"references": {referenced.resolve()},
"counts": {},
"source_families": {"national": [], "regional": [], "maritime": []},
},
)
monkeypatch.setattr(
audit,
"disk_pressure",
lambda _root: {
"status": "ok",
"total_bytes": 100,
"used_bytes": 50,
"free_bytes": 50,
"free_percent": 50.0,
"acquisition_allowed": True,
},
)
report, candidates = audit.build_report(storage, SimpleNamespace(), minimum_age_days=7)
assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"]
assert report["cleanup"]["candidate_count"] == 1
assert "release-evidence" in report["cleanup"]["protected_prefixes"]
assert report["integrity"]["missing_referenced_path_count"] == 0
assert report["integrity"]["missing_manifest_artifact_count"] == 0
def test_referenced_tile_manifest_protects_its_tiles(tmp_path: Path) -> None:
audit = load_script("audit_data_operations.py")
storage = tmp_path / "storage"
manifest = storage / "tiles" / "dataset" / "set" / "manifest.json"
tile = manifest.parent / "tile_0000.tif"
tile.parent.mkdir(parents=True)
tile.write_bytes(b"tile")
manifest.write_text(json.dumps({"tiles": [{"path": "tile_0000.tif"}]}), encoding="utf-8")
expanded = audit.expand_manifest_references({manifest.resolve()}, storage)
assert manifest.resolve() in expanded
assert tile.resolve() in expanded
def test_ordinary_json_export_is_not_treated_as_an_artifact_manifest(tmp_path: Path) -> None:
audit = load_script("audit_data_operations.py")
storage = tmp_path / "storage"
export = storage / "exports" / "project" / "report.json"
export.parent.mkdir(parents=True)
export.write_text(json.dumps({"dataset_id": "not-a-file"}), encoding="utf-8")
expanded = audit.expand_manifest_references({export.resolve()}, storage)
assert expanded == {export.resolve()}
def test_manifest_ids_dates_and_labels_are_not_treated_as_paths(tmp_path: Path) -> None:
audit = load_script("audit_data_operations.py")
storage = tmp_path / "storage"
manifest = storage / "operator-data" / "scope" / "scope.manifest.json"
actual = manifest.parent / "scope.geojson"
manifest.parent.mkdir(parents=True)
actual.write_text("{}", encoding="utf-8")
manifest.write_text(
json.dumps(
{
"municipality_ids": ["13025", "11001"],
"generated_at": "2026-07-18T00:00:00Z",
"label": "Belgium",
"output_path": "scope.geojson",
"output_checksum_sha256": "a" * 64,
"output_crs": "EPSG:4326",
}
),
encoding="utf-8",
)
expanded = audit.expand_manifest_references({manifest.resolve()}, storage)
assert expanded == {manifest.resolve(), actual.resolve()}
def test_disk_pressure_uses_absolute_headroom_for_large_arrays(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
audit = load_script("audit_data_operations.py")
monkeypatch.setattr(
audit.shutil,
"disk_usage",
lambda _path: SimpleNamespace(
total=56 * 1024**4,
used=(56 * 1024**4) - (700 * 1024**3),
free=700 * 1024**3,
),
)
pressure = audit.disk_pressure(tmp_path)
assert pressure["free_percent"] < 2
assert pressure["status"] == "ok"
assert pressure["acquisition_allowed"] is True
def test_source_family_report_covers_national_regional_and_maritime(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
audit = load_script("audit_data_operations.py")
national_id = uuid4()
regional_id = uuid4()
now = datetime.now(timezone.utc)
rows = {
audit.Project: [
SimpleNamespace(id=national_id, name=audit.NATIONAL_PROJECT_NAME, status="active"),
SimpleNamespace(id=regional_id, name="Wallonia operator", status="active"),
],
audit.Dataset: [
SimpleNamespace(
project_id=national_id,
source_name="ngi_adminvector",
source="ngi",
name="Belgium boundary",
source_metadata={"coverage_zones": ["belgium"]},
source_version="2026",
imported_at=now,
status="ready",
storage_path=None,
metadata_json=None,
provenance_metadata=None,
),
SimpleNamespace(
project_id=national_id,
source_name="rbins_marine_reporting_units",
source="rbins",
name="Belgian North Sea",
source_metadata={"coverage_zones": ["belgian_north_sea"]},
source_version="2024",
imported_at=now,
status="ready",
storage_path=None,
metadata_json=None,
provenance_metadata=None,
),
SimpleNamespace(
project_id=regional_id,
source_name="wallonia_manual",
source="manual",
name="Wallonia source",
source_metadata={},
source_version="1",
imported_at=now,
status="ready",
storage_path=None,
metadata_json=None,
provenance_metadata=None,
),
],
audit.DatasetVersion: [],
audit.Export: [],
audit.Detection: [],
audit.Segmentation: [],
audit.Job: [],
audit.AnalysisRun: [],
}
class Query:
def __init__(self, values):
self.values = values
def all(self):
return self.values
class Session:
def query(self, model, *_fields):
if model in rows:
return Query(rows[model])
owner = getattr(model, "class_", None)
if owner in rows:
return Query(rows[owner])
raise AssertionError(f"Unexpected query entity: {model!r}")
monkeypatch.setattr(audit, "query_count", lambda _db, model, *_conditions: len(rows[model]))
monkeypatch.setattr(audit, "query_distinct_nonnull", lambda _db, _column: [])
state = audit.collect_database_state(Session(), tmp_path)
assert {item["source_name"] for item in state["source_families"]["national"]} == {
"ngi_adminvector",
"rbins_marine_reporting_units",
}
assert {item["source_name"] for item in state["source_families"]["maritime"]} == {
"rbins_marine_reporting_units"
}
assert {item["source_name"] for item in state["source_families"]["regional"]} == {
"wallonia_manual"
}
def test_national_and_maritime_sources_have_explicit_freshness_policies() -> None:
from app.services.source_freshness_service import SOURCE_POLICIES
for source_name in (
"ngi_adminvector",
"rbins_marine_reporting_units",
"rbins_msp_2026",
):
assert SOURCE_POLICIES[source_name].refresh_policy == "edition"
def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> None:
generic = (SCRIPTS / "cleanup_storage_artifacts.py").read_text(encoding="utf-8")
demo = (ROOT / "backend/scripts/cleanup_demo_artifacts.py").read_text(encoding="utf-8")
compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
dockerman = (ROOT / "deploy/unraid/run-dockerman-container.sh").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8")
assert "QUARANTINE_STORAGE_ARTIFACTS" in generic
assert "verify_current_backup" in generic
assert "os.link" in generic
assert 'entry["status"] = "linked"' in generic
assert "cleanup-quarantine" in generic
assert "DELETE_DEMO_EXPORTS" in demo
assert "verify_current_backup" in demo
assert "/app/backups:ro" in compose
assert '/app/backups:ro"' in dockerman
for name in (
"release_backup_guard.py",
"audit_data_operations.py",
"cleanup_storage_artifacts.py",
"restore_storage_quarantine.py",
"release_backup_snapshot.py",
):
assert f"COPY scripts/{name}" in dockerfile
assert f"py_compile scripts/{name}" in readiness
assert "bash -n scripts/run_rc10_data_operations_audit.sh" in readiness
assert "--apply" not in live_audit
assert "table-counts-before.tsv" in live_audit
assert "table-counts-after.tsv" in live_audit
assert "deleted_count" in live_audit
assert "missing_manifest_artifact_count" in live_audit
def test_cleanup_apply_moves_bytes_to_protected_traceable_quarantine(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
monkeypatch.syspath_prepend(str(SCRIPTS))
cleanup = load_script("cleanup_storage_artifacts.py")
storage = tmp_path / "storage"
source = storage / "derived" / "orphan.bin"
source.parent.mkdir(parents=True)
source.write_bytes(b"recoverable-derived-artifact")
candidate = SimpleNamespace(
path=source.resolve(),
relative_path="derived/orphan.bin",
size_bytes=source.stat().st_size,
)
now = datetime.now(timezone.utc)
class SessionContext:
def __enter__(self):
return SimpleNamespace()
def __exit__(self, *_args):
return False
monkeypatch.setattr(
cleanup,
"parse_args",
lambda: SimpleNamespace(
storage_root=storage,
minimum_age_days=7,
max_delete=1,
apply=True,
confirm="QUARANTINE_STORAGE_ARTIFACTS",
backup_dir=tmp_path / "backup",
backup_max_age_hours=24.0,
quarantine_root=None,
),
)
monkeypatch.setattr(cleanup, "SessionLocal", lambda: SessionContext())
monkeypatch.setattr(
cleanup,
"build_report",
lambda *_args, **_kwargs: ({"cleanup": {"protected_prefixes": []}}, [candidate]),
)
monkeypatch.setattr(
cleanup,
"verify_current_backup",
lambda *_args, **_kwargs: SimpleNamespace(
release_id="predeploy-test",
created_at=now,
age_hours=0.1,
backup_tool_revision="0123456789abcdef",
),
)
assert cleanup.main() == 0
payload = json.loads(capsys.readouterr().out)
manifest_path = Path(payload["quarantine_manifest"])
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
quarantined_path = storage / payload["quarantined"][0]["quarantine_relative_path"]
assert not source.exists()
assert quarantined_path.read_bytes() == b"recoverable-derived-artifact"
assert manifest["state"] == "complete"
assert manifest["backup_release_id"] == "predeploy-test"
assert manifest["entries"][0]["status"] == "quarantined"
assert payload["deleted_count"] == 0
restore = subprocess.run(
[
sys.executable,
str(SCRIPTS / "restore_storage_quarantine.py"),
"--storage-root",
str(storage),
"--manifest",
str(manifest_path),
"--confirm",
"RESTORE_QUARANTINED_ARTIFACTS",
],
cwd=ROOT,
capture_output=True,
text=True,
check=False,
)
assert restore.returncode == 0, restore.stderr
assert source.read_bytes() == b"recoverable-derived-artifact"
assert not quarantined_path.exists()
restored_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
assert restored_manifest["state"] == "restored"
assert restored_manifest["entries"][0]["status"] == "restored"
def _write_interrupted_quarantine(
storage: Path,
*,
original_exists: bool,
quarantine_exists: bool,
hard_linked: bool = False,
) -> tuple[Path, Path, Path]:
original = storage / "derived" / "interrupted.bin"
operation = storage / "operator-evidence" / "cleanup-quarantine" / "cleanup-interrupted"
quarantined = operation / "files" / "derived" / "interrupted.bin"
original.parent.mkdir(parents=True, exist_ok=True)
quarantined.parent.mkdir(parents=True, exist_ok=True)
retained = b"interrupted-retained-bytes"
if original_exists:
original.write_bytes(retained)
if quarantine_exists:
if hard_linked:
os.link(original, quarantined)
else:
quarantined.write_bytes(retained)
manifest = operation / "manifest.json"
manifest.write_text(
json.dumps(
{
"schema_version": 1,
"state": "in_progress",
"entries": [
{
"relative_path": "derived/interrupted.bin",
"quarantine_relative_path": quarantined.relative_to(storage).as_posix(),
"size_bytes": len(retained),
"sha256": hashlib.sha256(retained).hexdigest(),
"status": "linked" if hard_linked else "planned",
}
],
}
),
encoding="utf-8",
)
return manifest, original, quarantined
@pytest.mark.parametrize(
("original_exists", "quarantine_exists", "hard_linked"),
((False, True, False), (True, True, True)),
)
def test_quarantine_restore_recovers_each_interrupted_move_window(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
original_exists: bool,
quarantine_exists: bool,
hard_linked: bool,
) -> None:
restore = load_script("restore_storage_quarantine.py")
storage = tmp_path / "storage"
manifest, original, quarantined = _write_interrupted_quarantine(
storage,
original_exists=original_exists,
quarantine_exists=quarantine_exists,
hard_linked=hard_linked,
)
monkeypatch.setattr(
restore,
"parse_args",
lambda: SimpleNamespace(
storage_root=storage,
manifest=manifest,
confirm="RESTORE_QUARANTINED_ARTIFACTS",
),
)
assert restore.main() == 0
assert original.read_bytes() == b"interrupted-retained-bytes"
assert not quarantined.exists()
assert json.loads(manifest.read_text(encoding="utf-8"))["state"] == "restored"
def test_quarantine_restore_never_clobbers_recreated_destination(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
restore = load_script("restore_storage_quarantine.py")
storage = tmp_path / "storage"
manifest, original, quarantined = _write_interrupted_quarantine(
storage,
original_exists=False,
quarantine_exists=True,
)
original.write_bytes(b"new-runtime-bytes")
monkeypatch.setattr(
restore,
"parse_args",
lambda: SimpleNamespace(
storage_root=storage,
manifest=manifest,
confirm="RESTORE_QUARANTINED_ARTIFACTS",
),
)
with pytest.raises(RuntimeError, match="different bytes"):
restore.main()
assert original.read_bytes() == b"new-runtime-bytes"
assert quarantined.read_bytes() == b"interrupted-retained-bytes"
+135
View File
@@ -0,0 +1,135 @@
from __future__ import annotations
import importlib.util
import json
import shutil
import subprocess
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "build_release_package.py"
def load_script():
spec = importlib.util.spec_from_file_location("build_release_package", SCRIPT)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_release_version_is_consistent_across_runtime_packages() -> None:
version = (ROOT / "VERSION").read_text(encoding="utf-8").strip()
config = (ROOT / "backend" / "app" / "core" / "config.py").read_text(
encoding="utf-8"
)
pyproject = (ROOT / "backend" / "pyproject.toml").read_text(encoding="utf-8")
frontend = json.loads(
(ROOT / "frontend" / "package.json").read_text(encoding="utf-8")
)
package_lock = json.loads(
(ROOT / "frontend" / "package-lock.json").read_text(encoding="utf-8")
)
assert version == "1.0.0"
assert f'default="{version}"' in config
assert "GEOINTEL_APP_VERSION" in config
assert 'version = "1.0.0"' in pyproject
assert frontend["version"] == version
assert package_lock["version"] == version
assert package_lock["packages"][""]["version"] == version
def test_release_image_carries_semantic_version_identity() -> None:
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(
encoding="utf-8"
)
deploy = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(
encoding="utf-8"
)
assert "ARG GEOINTEL_APP_VERSION=1.0.0" in dockerfile
assert 'org.opencontainers.image.version="${GEOINTEL_APP_VERSION}"' in dockerfile
assert "GEOINTEL_APP_VERSION=\"$(tr -d '[:space:]' < VERSION)\"" in deploy
assert "--build-arg GEOINTEL_APP_VERSION=" in deploy
assert "stored_version" in deploy
@pytest.mark.skipif(shutil.which("ssh-keygen") is None, reason="ssh-keygen unavailable")
def test_release_package_signature_and_checksums_fail_closed(tmp_path: Path) -> None:
module = load_script()
key = tmp_path / "release-key"
result = subprocess.run(
["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(key)],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
package = tmp_path / "package"
package.mkdir()
evidence = package / "readiness.txt"
evidence.write_text("passed\n", encoding="utf-8")
nested_checksums = package / "backup" / module.CHECKSUMS_NAME
nested_checksums.parent.mkdir()
nested_checksums.write_text("backup evidence\n", encoding="utf-8")
identity = "geointel-release"
namespace = "geointel-release"
(package / module.SIGNERS_NAME).write_text(
f"{identity} {module.public_key(key)}\n",
encoding="utf-8",
)
manifest = {
"schema_version": 1,
"release_id": "v1.0.0",
"version": "1.0.0",
"scope": "Belgium and the Belgian North Sea",
"signature": {"identity": identity, "namespace": namespace},
"evidence": [
{
"path": evidence.name,
"size_bytes": evidence.stat().st_size,
"sha256": module.sha256(evidence),
}
],
}
manifest_path = package / module.MANIFEST_NAME
manifest_path.write_text(json.dumps(manifest) + "\n", encoding="utf-8")
module.run(
(
"ssh-keygen",
"-Y",
"sign",
"-f",
str(key),
"-n",
namespace,
str(manifest_path),
)
)
module.write_checksums(package)
verified = module.verify_package(package)
assert verified["release_id"] == "v1.0.0"
evidence.write_text("tampered\n", encoding="utf-8")
with pytest.raises(RuntimeError, match="Checksum mismatch"):
module.verify_package(package)
def test_release_package_cli_requires_tagged_clean_revision() -> None:
source = SCRIPT.read_text(encoding="utf-8")
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
encoding="utf-8"
)
assert 'run(("git", "status", "--porcelain=v1"))' in source
assert 'run(("git", "rev-list", "-n", "1", release_id))' in source
assert "Image revision must equal the tagged Git commit" in source
assert "ssh-keygen" in source
assert "verify_checksums(package_dir)" in source
assert "py_compile scripts/build_release_package.py" in readiness
+488
View File
@@ -0,0 +1,488 @@
from __future__ import annotations
from types import SimpleNamespace
from pathlib import Path
from uuid import uuid4
from fastapi.testclient import TestClient
from shapely.geometry import box
from app.core.errors import AppError
from app.main import app
from app.models import Area, Dataset, Project
from app.schemas.coverage import CoverageBBox
from app.services.coverage_registry_service import CoverageRegistryService, THEMES, ZONES
from app.services.vector_feature_service import VectorFeatureService
from tests.frontend_contract import read_map_workspace, read_feature
class FakeQuery:
def __init__(self, rows):
self.rows = rows
def filter(self, *_args, **_kwargs):
return self
def all(self):
return list(self.rows)
class FakeSession:
def __init__(self, *, project, areas, datasets):
self.project = project
self.areas = areas
self.datasets = datasets
def get(self, model, object_id):
if model is Project and str(self.project.id) == str(object_id):
return self.project
return None
def query(self, model):
if model is Area:
return FakeQuery(self.areas)
if model is Dataset:
return FakeQuery(self.datasets)
raise AssertionError(f"Unexpected query model: {model}")
def scope_area(name: str, geometry):
return SimpleNamespace(name=name, geometry=geometry)
def governed_materialization(
*,
source_name: str,
reference_layer_name: str | None,
source_metadata: dict[str, object],
dataset_id=None,
) -> SimpleNamespace:
"""Build a complete authoritative materialization for coverage tests.
Coverage is a production-facing statement. These fixtures must therefore
carry the same registry, immutable snapshot, checksum and freshness state
that a materialized official dataset needs in production.
"""
source_registry_id = uuid4()
source_snapshot_id = uuid4()
checksum_sha256 = "a" * 64
return SimpleNamespace(
id=dataset_id or uuid4(),
status="ready",
source=source_name,
source_name=source_name,
reference_layer_name=reference_layer_name,
source_metadata=dict(source_metadata),
checksum_sha256=checksum_sha256,
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
data_contract_key="geointel.vector.geojson",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
source_registry=SimpleNamespace(
source_key=source_name,
classification="authoritative",
authority_scope_json={"scope": "coverage test"},
usage_policy_json={},
),
source_snapshot=SimpleNamespace(
source_registry_id=source_registry_id,
checksum_sha256=checksum_sha256,
freshness_status="current",
ingest_status="ingested",
),
)
def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider_registry() -> None:
catalog = CoverageRegistryService.catalog()
assert set(catalog.themes) == set(THEMES)
assert set(catalog.zones) == set(ZONES)
assert catalog.statuses == ["unsupported", "not_configured", "partial", "operational"]
assert {source.source_name for source in catalog.sources} >= {
"ngi_adminvector",
"statbel",
"digitaal_vlaanderen",
"spw_geoportail",
"urbis",
"rbins_marine_reporting_units",
"rbins_msp_2026",
"mdk_bathymetry",
"vmm_vha_bathymetry_profiles",
}
assert next(source for source in catalog.sources if source.source_name == "ngi_adminvector").license_note == "CC BY 4.0"
assert next(source for source in catalog.sources if source.source_name == "mdk_bathymetry").integration_status == "not_configured"
assert next(
source for source in catalog.sources if source.source_name == "vmm_vha_bathymetry_profiles"
).integration_status == "operational"
response = TestClient(app).get("/api/v1/external/coverage/catalog")
assert response.status_code == 200
assert response.json()["data"]["themes"] == list(THEMES)
def test_national_and_maritime_reference_layers_are_selection_analyzable() -> None:
cases = (
("ngi_adminvector", "belgium_municipalities", "administrative"),
("rbins_marine_reporting_units", "marine_legal_scopes", "marine_environment"),
("rbins_msp_2026", "marine_spatial_plan_2026", "maritime_planning"),
)
for source_name, layer_name, expected_theme in cases:
dataset = Dataset(
id=uuid4(),
project_id=uuid4(),
name=f"{layer_name}.geojson",
dataset_type="vector",
source="operator_official_import",
source_name=source_name,
reference_layer_name=layer_name,
source_metadata={"authority_level": "authoritative"},
status="ready",
)
assert VectorFeatureService._dataset_theme(dataset) == expected_theme
assert VectorFeatureService.supports_selection_summary(dataset) is True
def test_national_scope_operator_assigns_explicit_map_themes() -> None:
root = Path(__file__).resolve().parents[2]
operator = (root / "scripts" / "provision_belgium_north_sea_scope.py").read_text(encoding="utf-8")
map_workspace = read_map_workspace()
assert '"belgium_municipalities": "administrative"' in operator
assert '"marine_legal_scopes": "marine_environment"' in operator
assert '"marine_spatial_plan_2026": "maritime_planning"' in operator
assert "id: 'administrative'" in map_workspace
assert "id: 'maritime_planning'" in map_workspace
assert "id: 'marine_environment'" in map_workspace
def test_coverage_resolver_only_reports_operational_for_materialized_ready_dataset() -> None:
project_id = uuid4()
project = SimpleNamespace(id=project_id)
areas = [
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)),
]
bbox = CoverageBBox(minx=4.9, miny=50.9, maxx=5.0, maxy=51.0)
without_materialized = CoverageRegistryService.resolve(
FakeSession(project=project, areas=areas, datasets=[]),
project_id,
bbox,
["admin"],
)
assert without_materialized.intersected_zones == ["flanders"]
assert without_materialized.items[0].status == "partial"
assert without_materialized.items[0].materialized_dataset_ids == []
dataset_id = uuid4()
materialized = governed_materialization(
dataset_id=dataset_id,
source_name="ngi_adminvector",
reference_layer_name="belgium_regions",
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
)
with_materialized = CoverageRegistryService.resolve(
FakeSession(project=project, areas=areas, datasets=[materialized]),
project_id,
bbox,
["admin"],
)
admin_item = next(item for item in with_materialized.items if item.zone == "flanders")
assert admin_item.status == "operational"
assert admin_item.materialized_dataset_ids == [dataset_id]
def test_statbel_population_materialization_does_not_masquerade_as_admin_data() -> None:
project_id = uuid4()
statbel_id = uuid4()
statbel = governed_materialization(
dataset_id=statbel_id,
source_name="statbel",
reference_layer_name="population",
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
)
result = CoverageRegistryService.resolve(
FakeSession(
project=SimpleNamespace(id=project_id),
areas=[
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)),
],
datasets=[statbel],
),
project_id,
CoverageBBox(minx=4.9, miny=50.9, maxx=5.0, maxy=51.0),
["admin", "population"],
)
admin = next(item for item in result.items if item.theme == "admin")
population = next(item for item in result.items if item.theme == "population")
assert admin.materialized_dataset_ids == []
assert population.status == "operational"
assert population.materialized_dataset_ids == [statbel_id]
def test_bounded_api_materialization_only_covers_its_persisted_bbox() -> None:
project_id = uuid4()
dataset_id = uuid4()
dataset = governed_materialization(
dataset_id=dataset_id,
source_name="spw_picc",
reference_layer_name="buildings",
source_metadata={
"coverage_zones": ["wallonia"],
"bbox_epsg4326": [4.55, 50.58, 4.56, 50.59],
},
)
session = FakeSession(
project=SimpleNamespace(id=project_id),
areas=[
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8)),
],
datasets=[dataset],
)
inside = CoverageRegistryService.resolve(
session,
project_id,
CoverageBBox(minx=4.551, miny=50.581, maxx=4.559, maxy=50.589),
["buildings"],
)
outside = CoverageRegistryService.resolve(
session,
project_id,
CoverageBBox(minx=4.7, miny=50.6, maxx=4.71, maxy=50.61),
["buildings"],
)
assert inside.items[0].status == "operational"
assert inside.items[0].materialized_dataset_ids == [dataset_id]
assert outside.items[0].status == "partial"
assert outside.items[0].materialized_dataset_ids == []
def test_bounded_partition_union_can_be_operational() -> None:
project_id = uuid4()
left_id = uuid4()
right_id = uuid4()
datasets = [
governed_materialization(
dataset_id=left_id,
source_name="spw_picc",
reference_layer_name="buildings",
source_metadata={
"coverage_zones": ["wallonia"],
"bbox_epsg4326": [4.50, 50.50, 4.60, 50.60],
},
),
governed_materialization(
dataset_id=right_id,
source_name="spw_picc",
reference_layer_name="buildings",
source_metadata={
"coverage_zones": ["wallonia"],
"bbox_epsg4326": [4.60, 50.50, 4.70, 50.60],
},
),
]
session = FakeSession(
project=SimpleNamespace(id=project_id),
areas=[scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)), scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8))],
datasets=datasets,
)
result = CoverageRegistryService.resolve(session, project_id, CoverageBBox(minx=4.51, miny=50.51, maxx=4.69, maxy=50.59), ["buildings"])
assert result.items[0].status == "operational"
assert result.items[0].materialized_dataset_ids == [left_id, right_id]
def test_spw_bathymetry_materialization_is_source_specific() -> None:
project_id = uuid4()
scope = [
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8)),
]
selection = CoverageBBox(minx=4.851, miny=50.451, maxx=4.869, maxy=50.469)
spw_picc = governed_materialization(
source_name="spw_picc",
reference_layer_name="buildings",
source_metadata={
"coverage_zones": ["wallonia"],
"bbox_epsg4326": [4.85, 50.45, 4.87, 50.47],
},
)
without_bathymetry = CoverageRegistryService.resolve(
FakeSession(project=SimpleNamespace(id=project_id), areas=scope, datasets=[spw_picc]),
project_id,
selection,
["bathymetry"],
)
assert without_bathymetry.items[0].status == "partial"
assert without_bathymetry.items[0].materialized_dataset_ids == []
bathymetry_id = uuid4()
bathymetry = governed_materialization(
dataset_id=bathymetry_id,
source_name="spw_bathymetry",
reference_layer_name=None,
source_metadata={
"coverage_zones": ["wallonia"],
"bbox_epsg4326": [4.85, 50.45, 4.87, 50.47],
},
)
with_bathymetry = CoverageRegistryService.resolve(
FakeSession(project=SimpleNamespace(id=project_id), areas=scope, datasets=[spw_picc, bathymetry]),
project_id,
selection,
["bathymetry"],
)
assert with_bathymetry.items[0].status == "operational"
assert with_bathymetry.items[0].materialized_dataset_ids == [bathymetry_id]
def test_vha_bathymetry_profiles_are_operational_only_inside_the_persisted_selection() -> None:
project_id = uuid4()
scope = [
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)),
]
selection = CoverageBBox(minx=5.101, miny=51.171, maxx=5.109, maxy=51.179)
without_profiles = CoverageRegistryService.resolve(
FakeSession(project=SimpleNamespace(id=project_id), areas=scope, datasets=[]),
project_id,
selection,
["bathymetry"],
)
assert without_profiles.items[0].status == "partial"
assert without_profiles.items[0].source_names == ["vmm_vha_bathymetry_profiles"]
profile_id = uuid4()
profiles = governed_materialization(
dataset_id=profile_id,
source_name="vmm_vha_bathymetry_profiles",
reference_layer_name="bathymetry_profile_points",
source_metadata={
"coverage_zones": ["flanders"],
"bbox_epsg4326": [5.1, 51.17, 5.11, 51.18],
},
)
with_profiles = CoverageRegistryService.resolve(
FakeSession(project=SimpleNamespace(id=project_id), areas=scope, datasets=[profiles]),
project_id,
selection,
["bathymetry"],
)
assert with_profiles.items[0].status == "operational"
assert with_profiles.items[0].materialized_dataset_ids == [profile_id]
def test_mixed_land_and_north_sea_selection_remains_split() -> None:
project_id = uuid4()
project = SimpleNamespace(id=project_id)
areas = [
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)),
scope_area("Belgian part of the North Sea", box(2.2, 51.1, 3.4, 51.9)),
scope_area("Belgian territorial sea (0-12 nautical miles)", box(2.7, 51.1, 3.4, 51.5)),
scope_area("Belgian exclusive economic zone beyond territorial sea", box(2.2, 51.5, 3.1, 51.9)),
scope_area("Belgian continental shelf beyond territorial sea", box(2.2, 51.5, 3.1, 51.9)),
]
result = CoverageRegistryService.resolve(
FakeSession(project=project, areas=areas, datasets=[]),
project_id,
CoverageBBox(minx=2.65, miny=51.05, maxx=2.85, maxy=51.2),
["admin", "bathymetry"],
)
assert result.intersected_zones == ["flanders", "territorial_sea"]
assert len(result.items) == 4
assert any("crosses coverage zones" in warning for warning in result.warnings)
bathymetry = next(item for item in result.items if item.zone == "territorial_sea" and item.theme == "bathymetry")
assert bathymetry.status == "not_configured"
assert bathymetry.materialized_dataset_ids == []
def test_flemish_materialization_is_theme_specific() -> None:
project_id = uuid4()
project = SimpleNamespace(id=project_id)
orthophoto_id = uuid4()
orthophoto = governed_materialization(
dataset_id=orthophoto_id,
source_name="digitaal_vlaanderen_orthophoto",
reference_layer_name="orthophoto",
source_metadata={"coverage_zones": ["flanders"]},
)
result = CoverageRegistryService.resolve(
FakeSession(
project=project,
areas=[scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5))],
datasets=[orthophoto],
),
project_id,
CoverageBBox(minx=4.9, miny=50.9, maxx=5.0, maxy=51.0),
["orthophoto", "roads"],
)
assert next(item for item in result.items if item.theme == "orthophoto").status == "operational"
assert next(item for item in result.items if item.theme == "roads").status == "partial"
def test_outside_scope_and_unknown_theme_are_explicit() -> None:
project_id = uuid4()
project = SimpleNamespace(id=project_id)
db = FakeSession(
project=project,
areas=[scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5))],
datasets=[],
)
result = CoverageRegistryService.resolve(
db,
project_id,
CoverageBBox(minx=7.0, miny=52.0, maxx=7.1, maxy=52.1),
["admin"],
)
assert result.intersected_zones == []
assert result.outside_supported_scope is True
assert result.items == []
try:
CoverageRegistryService.resolve(
db,
project_id,
CoverageBBox(minx=4.0, miny=50.0, maxx=4.1, maxy=50.1),
["invented_metric"],
)
except AppError as exc:
assert exc.code == "COVERAGE_THEME_UNSUPPORTED"
assert exc.status_code == 422
assert exc.details["unsupported_themes"] == ["invented_metric"]
else:
raise AssertionError("Unknown coverage theme was accepted")
def test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbox() -> None:
root = Path(__file__).parents[2]
focus = (root / "frontend" / "src" / "config" / "primaryFocus.ts").read_text(encoding="utf-8")
workspace_hook = read_feature("shell")
coverage_hook = read_feature("map_workspace")
map_workspace = read_map_workspace()
assert "Belgium and North Sea Workbench" in focus
assert "nationalProject" in workspace_hook
assert "return nationalProject.id" in workspace_hook
assert "NATIONAL_WORKSPACE_REGION" in workspace_hook
assert "externalApi.resolveCoverage" in coverage_hook
assert "coverage.outside_supported_scope" in map_workspace
assert "coverageStatusLabel" in map_workspace
assert "activeThemeSupportsCurrentSelection" in map_workspace
assert "activeThemeAvailable && !regionalPartitionedThemeActive" in map_workspace
@@ -0,0 +1,183 @@
from __future__ import annotations
import json
import sys
import zipfile
from pathlib import Path
import pytest
from shapely.geometry import box, mapping, shape
ROOT = Path(__file__).parents[2]
SCRIPTS = ROOT / "scripts"
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
import provision_belgium_north_sea_scope as operator # noqa: E402
from app.utils.geometry import normalize_to_multipolygon # noqa: E402
def marine_feature(identifier: str, geometry):
return {
"type": "Feature",
"id": identifier,
"geometry": mapping(geometry),
"properties": {"MarineReportingUnitId": identifier},
}
def test_marine_legal_scopes_are_derived_from_official_reporting_units() -> None:
reporting_units = {
"type": "FeatureCollection",
"features": [
marine_feature("ANS-BE-MS-1", box(0, 0, 10, 10)),
marine_feature("ANS-BE-AA-CW", box(0, 0, 2, 2)),
marine_feature("ANS-BE-AA-TEW", box(0, 2, 2, 4)),
marine_feature("ANS-BE-AA-OFFSHORE", box(2, 0, 10, 10)),
],
}
payload = operator.derive_marine_scope_payload(reporting_units)
by_zone = {
feature["properties"]["coverage_zone"]: feature
for feature in payload["features"]
}
assert set(by_zone) == {
"belgian_north_sea",
"territorial_sea",
"exclusive_economic_zone",
"continental_shelf",
}
assert shape(by_zone["territorial_sea"]["geometry"]).area == pytest.approx(8.0)
assert shape(by_zone["exclusive_economic_zone"]["geometry"]).equals(
shape(by_zone["continental_shelf"]["geometry"])
)
assert (
by_zone["exclusive_economic_zone"]["properties"]["legal_domain"]
!= by_zone["continental_shelf"]["properties"]["legal_domain"]
)
assert by_zone["territorial_sea"]["properties"]["derived_from_reporting_unit_ids"] == [
"ANS-BE-AA-CW",
"ANS-BE-AA-TEW",
]
def test_marine_scope_derivation_fails_when_a_required_unit_is_missing() -> None:
with pytest.raises(RuntimeError, match="ANS-BE-AA-CW"):
operator.derive_marine_scope_payload(
{
"type": "FeatureCollection",
"features": [
marine_feature("ANS-BE-MS-1", box(0, 0, 10, 10)),
marine_feature("ANS-BE-AA-TEW", box(0, 2, 2, 4)),
marine_feature("ANS-BE-AA-OFFSHORE", box(2, 0, 10, 10)),
],
}
)
def test_area_geometry_normalization_drops_source_z_dimension() -> None:
geometry = normalize_to_multipolygon(
{
"type": "Polygon",
"coordinates": [
[
[2.5, 49.5, 0],
[6.4, 49.5, 0],
[6.4, 51.5, 0],
[2.5, 51.5, 0],
[2.5, 49.5, 0],
]
],
}
)
assert geometry.has_z is False
assert geometry.geom_type == "MultiPolygon"
def test_archive_extraction_accepts_one_safe_geopackage_and_rejects_traversal(tmp_path: Path) -> None:
archive = tmp_path / "adminvector.zip"
with zipfile.ZipFile(archive, "w") as handle:
handle.writestr("release/adminvector.gpkg", b"sqlite-bytes")
result = operator.extract_single_geopackage(archive, tmp_path / "output")
assert result.read_bytes() == b"sqlite-bytes"
unsafe = tmp_path / "unsafe.zip"
with zipfile.ZipFile(unsafe, "w") as handle:
handle.writestr("../adminvector.gpkg", b"unsafe")
with pytest.raises(RuntimeError, match="unsafe"):
operator.extract_single_geopackage(unsafe, tmp_path / "unsafe-output")
class FakeResponse:
def __init__(self, payload):
self.payload = payload
self.content = json.dumps(payload).encode("utf-8")
def raise_for_status(self):
return None
def json(self):
return self.payload
class FakeSession:
def __init__(self, pages):
self.pages = list(pages)
self.calls = []
def get(self, url, params, timeout):
self.calls.append({"url": url, "params": params, "timeout": timeout})
return FakeResponse(self.pages.pop(0))
def test_wfs_fetch_is_allowlisted_paginated_and_complete() -> None:
pages = [
{
"type": "FeatureCollection",
"numberMatched": 2,
"features": [{"type": "Feature", "id": "unit.1", "geometry": None, "properties": {}}],
},
{
"type": "FeatureCollection",
"numberMatched": 2,
"features": [{"type": "Feature", "id": "unit.2", "geometry": None, "properties": {}}],
},
]
session = FakeSession(pages)
payload = operator.fetch_wfs_layer(
session,
service_url=operator.RBINS_MRU_WFS_URL,
layer_name=operator.RBINS_MRU_LAYER,
timeout=30,
page_size=1,
)
assert [feature["id"] for feature in payload["features"]] == ["unit.1", "unit.2"]
assert [call["params"]["startIndex"] for call in session.calls] == [0, 1]
assert all(call["params"]["srsName"] == "EPSG:4326" for call in session.calls)
with pytest.raises(RuntimeError, match="allowlist"):
operator.fetch_wfs_layer(
FakeSession([]),
service_url=operator.RBINS_MSP_WFS_URL,
layer_name="untrusted:layer",
timeout=30,
)
def test_operator_is_packaged_and_guarded_by_readiness() -> None:
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
source = (ROOT / "scripts" / "provision_belgium_north_sea_scope.py").read_text(encoding="utf-8")
assert "COPY scripts/provision_belgium_north_sea_scope.py /app/scripts/" in dockerfile
assert "py_compile scripts/provision_belgium_north_sea_scope.py" in readiness
assert "/datasets/upload" in source
assert "frame.to_json(drop_id=False, default=str)" in source
assert "from app.models" not in source
assert "INSERT INTO vector_features" not in source
@@ -0,0 +1,219 @@
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def test_build_identity_does_not_invalidate_dependency_layers() -> None:
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
dependency_install = dockerfile.index("/usr/bin/python3.11 -m venv /opt/geointel/venv")
source_copy = dockerfile.index("COPY backend/ /app/")
build_identity = dockerfile.index("ARG GEOINTEL_BUILD_SHA=unknown")
assert build_identity > dependency_install
assert build_identity > source_copy
assert 'org.opencontainers.image.revision="${GEOINTEL_BUILD_SHA}"' in dockerfile
assert 'org.opencontainers.image.created="${GEOINTEL_BUILD_TIME}"' in dockerfile
assert 'io.geointel.ai.enabled="${GEOINTEL_INSTALL_AI}"' in dockerfile
def test_release_deploy_preserves_immutable_and_backup_specific_rollback_images() -> None:
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
assert 'GEOINTEL_RELEASE_VARIANT="ai"' in script
assert 'GEOINTEL_RELEASE_VARIANT="gis"' in script
assert 'GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"' in script
assert 'GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"' in script
assert 'docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script
assert '--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script
assert "GEOINTEL_PREVIOUS_IMAGE" not in script
assert 'if docker image inspect "$GEOINTEL_RELEASE_IMAGE"' in script
assert "Immutable release tag has conflicting metadata" in script
assert "Reusing existing immutable image" in script
assert "rollback_previous()" in script
assert "Deployed immutable image" in script
def test_release_creates_verified_backup_before_candidate_migrations() -> None:
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
backup_index = script.index("create_predeploy_backup\n")
scan_index = script.index("scan_release_image\n")
candidate_start_index = script.index('if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"')
assert scan_index < backup_index
assert backup_index < candidate_start_index
assert script.index("docker build") < backup_index
assert "/mnt/user/appdata/geointel/backups" in script
assert "--inventory-mode sha256" in script
assert "scripts/verify_release_backup.sh" in script
assert "refusing an unbacked migration" in script
assert "Quiescing the current backend" in script
assert "restarting the unchanged current release" in script
assert "preflight_backup_capacity" in script
assert "select_verified_link_dest" in script
assert "release_backup_snapshot.py" in script
assert "run_low_impact()" in script
assert "ionice -c 2 -n 7" in script
assert "nice -n 10" in script
assert "run_low_impact bash scripts/backup_release_state.sh" in script
assert "run_low_impact bash scripts/verify_release_backup.sh" in script
def test_gitea_deploy_waits_for_the_mandatory_large_snapshot() -> None:
workflow = (ROOT / ".gitea" / "workflows" / "release-gates.yml").read_text(encoding="utf-8")
assert "cancel-in-progress: false" in workflow
deploy_job = workflow.split("\n deploy:\n", maxsplit=1)[1]
assert "timeout-minutes: 720" in deploy_job
assert "docker exec gitea-deploy-control" in deploy_job
def test_release_starts_only_the_locally_attested_ai_image() -> None:
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"' in script
assert "Production release deployment requires the gated AI image" in script
assert 'bash scripts/generate_container_sbom.sh "$scanned_image_id"' in script
assert 'bash scripts/scan_container_image.sh "$scanned_image_id"' in script
assert 'running_image_id="$(docker inspect --format \'{{.Image}}\' geointel)"' in script
assert 'if [ "$running_image_id" != "$image" ]' in script
assert "deployment-attestation.json" in script
assert "artifacts/release-evidence/deploy" in script
assert "GITEA_COMMIT_SHA" in script
assert "GITHUB_SHA" in script
assert "must contain one full 40-character Git commit SHA" in script
assert 'marker_path="$ROOT/.gitea-deploy/revision"' in script
assert "git rev-parse --show-toplevel" in script
assert "Prepared source revision marker does not match" in script
assert "neither an exact Git checkout nor bound" in script
assert 'running_revision" != "$GEOINTEL_BUILD_SHA"' in script
assert 'running_ai" != "true"' in script
assert '"revision": revision' in script
def test_release_and_container_replacement_are_serialized() -> None:
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
rollback_script = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(
encoding="utf-8"
)
restore_script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
encoding="utf-8"
)
assert "GEOINTEL_DEPLOY_LOCK_FILE" in release_script
assert "flock -n 9" in release_script
assert "GEOINTEL_DEPLOY_LOCK_FILE" in rollback_script
assert "flock -n 9" in rollback_script
assert "GEOINTEL_DEPLOY_LOCK_FILE" in restore_script
assert "flock -n 9" in restore_script
assert "GEOINTEL_DEPLOY_LOCK_HELD=true" in release_script
assert "GEOINTEL_CONTAINER_LOCK_FILE" in run_script
assert "flock -w 300 8" in run_script
assert "GeoIntel container removal did not complete within 60 seconds" in run_script
def test_release_waits_for_large_postgis_volume_recovery() -> None:
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
start_script = (ROOT / "deploy" / "unraid" / "all-in-one-start.sh").read_text(encoding="utf-8")
assert "for attempt in $(seq 1 480)" in release_script
assert "for attempt in $(seq 1 450)" in start_script
assert "PostGIS did not become ready within 15 minutes." in start_script
assert 'chown postgres:postgres "$PGDATA"' in start_script
assert 'chown -R postgres:postgres "$PGDATA"' not in start_script
def test_runtime_configuration_is_validated_before_container_replacement() -> None:
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
validation_index = run_script.index("validate_runtime_config")
# The script replaces the container with "docker rm -f"; what matters is
# that no removal happens before the runtime config has been validated.
replacement_index = run_script.index("docker rm -f geointel")
assert validation_index < replacement_index
assert "known-default PostGIS password" in run_script
assert "GEOINTEL_MAX_UPLOAD_MB must be between 1 and 2048" in run_script
assert 'docker image inspect "$GEOINTEL_IMAGE"' in run_script
def test_fresh_install_smoke_is_isolated_and_cleans_only_its_temp_path() -> None:
script = (ROOT / "scripts" / "verify_release_fresh_install.sh").read_text(encoding="utf-8")
assert "mktemp -d" in script
assert "geointel-fresh-smoke.*" in script
assert "-p 127.0.0.1::80" in script
assert "GEOINTEL_POSTGRES_PASSWORD=" in script
assert "/health/ready" in script
assert "/api/v1/system/capabilities" in script
assert "docker exec" in script
assert "python -m alembic heads" in script
def test_manual_rollback_reuses_persistent_paths_and_requires_existing_image() -> None:
rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(encoding="utf-8")
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
assert "Backup manifest lacks an immutable rollback image ID" in rollback
assert 'get("image_id", "")' in rollback
assert 'docker image inspect "$GEOINTEL_ROLLBACK_IMAGE"' in rollback
assert 'GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE"' in rollback
assert "restore-predeploy-database.sh" in rollback
assert "--confirm-production-database-restore" in rollback
assert "Image-only rollback" in rollback
assert '-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data"' in run_script
assert '-v "${GEOINTEL_STORAGE_PATH}:/app/storage"' in run_script
def test_same_revision_redeploy_rolls_back_by_backup_bound_image_id() -> None:
release = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(
encoding="utf-8"
)
restore = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
encoding="utf-8"
)
backup = (ROOT / "scripts" / "backup_release_state.sh").read_text(encoding="utf-8")
assert "current_image_id" in release
assert "release_image_id" not in release
assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in backup
assert 'if [ -z "$RESTORE_IMAGE" ]; then\n RESTORE_IMAGE="$BACKUP_IMAGE_ID"' in restore
assert 'GEOINTEL_ROLLBACK_IMAGE="$(python3 - "$BACKUP_DIR/manifest.json"' in rollback
assert "geointel-all-in-one:previous" not in release + rollback + restore
def test_readiness_checks_all_release_shell_entrypoints() -> None:
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
for path in (
"scripts/deploy_tower.sh",
"scripts/verify_release_fresh_install.sh",
"scripts/verify_release_upgrade_smoke.sh",
"deploy/unraid/all-in-one-start.sh",
"deploy/unraid/run-dockerman-container.sh",
"deploy/unraid/deploy-release.sh",
"deploy/unraid/rollback-dockerman-container.sh",
"deploy/unraid/restore-predeploy-database.sh",
):
assert f"bash -n {path}" in readiness
def test_upgrade_smoke_restores_and_upgrades_only_an_isolated_database() -> None:
script = (ROOT / "scripts" / "verify_release_upgrade_smoke.sh").read_text(encoding="utf-8")
assert "--confirm-isolated-upgrade" in script
assert "restore_release_backup_smoke.sh" in script
assert "--keep-database" in script
assert "geointel_restore_verify_" in script
assert "from sqlalchemy import URL" in script
assert 'username=os.environ["GEOINTEL_POSTGRES_USER"]' in script
assert 'password=os.environ["GEOINTEL_POSTGRES_PASSWORD"]' in script
assert 'database=os.environ["TARGET_DB"]' in script
assert '"$CONTAINER" sh -c' in script
assert '"$CONTAINER" sh -lc' not in script
assert "python -m alembic upgrade head" in script
assert "production_database_untouched" in script
assert 'dropdb --if-exists -U "$db_user" "$TARGET_DB"' in script
+198
View File
@@ -0,0 +1,198 @@
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def read(path: str) -> str:
return (ROOT / path).read_text(encoding="utf-8")
def test_python_ci_lock_is_hashed_linux_311_and_excludes_ai() -> None:
for lock_path in (
"backend/requirements-runtime.lock",
"backend/requirements-ci.lock",
):
lock = read(lock_path)
assert "pip-compile with Python 3.11" in lock
assert "# geointel-input-sha256: " in lock
assert "--generate-hashes" in lock
assert "\ntorch==" not in lock
assert "\nultralytics==" not in lock
def test_lock_generator_uses_pinned_linux_runtime_and_verifies_policy() -> None:
generator = read("scripts/generate_python_lock.sh")
assert "python:3.11-bookworm@sha256:" in generator
assert 'PIP_TOOLS_VERSION="7.5.3"' in generator
assert "--extra gis" in generator
assert "--extra dev" in generator
assert "requirements-runtime.lock" in generator
assert "requirements-ci.lock" in generator
assert "--generate-hashes" in generator
assert "verify_python_lock.py --stamp" in generator
def test_ci_runs_complete_release_and_supply_chain_gates() -> None:
for workflow_path, context in (
(".github/workflows/release-gates.yml", "github.sha"),
(".gitea/workflows/release-gates.yml", "gitea.sha"),
):
workflow = read(workflow_path)
assert "backend/requirements-ci.lock" in workflow
assert "scripts/verify_python_lock.py" in workflow
assert "scripts/run_readiness_check.sh" in workflow
assert "python -m alembic upgrade head --sql" in workflow
assert "docker compose config" in workflow
assert "pip-audit==2.10.1" in workflow
assert "audit_python_dependencies.sh" in workflow
assert "npm audit --audit-level=high" in workflow
assert "GEOINTEL_INSTALL_AI=true" in workflow
assert "geointel-ci:$RELEASE_SHA-ai" in workflow
assert "artifacts/image-id.txt" in workflow
assert 'scan_container_image.sh "$IMAGE_ID"' in workflow
assert "generate_container_sbom.sh" in workflow
assert "scan_container_image.sh" in workflow
assert "GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar" in workflow
assert "Remove temporary image archive" in workflow
assert context in workflow
assert (
"actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02"
in read(".github/workflows/release-gates.yml")
)
assert (
"actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de"
in read(".gitea/workflows/release-gates.yml")
)
def test_gitea_production_deploy_depends_on_every_release_gate() -> None:
release = read(".gitea/workflows/release-gates.yml")
legacy_deploy = ROOT / ".gitea" / "workflows" / "unraid-deploy.yml"
assert "pull_request:" in release
assert "needs: [quality, dependency-audit, container]" in release
assert "gitea.event_name == 'push'" in release
assert "gitea.ref == 'refs/heads/main'" in release
assert "/opt/gitea-deploy/deploy.py deploy" in release
assert not legacy_deploy.exists()
assert "workflow_dispatch:" in release
def test_release_workflows_pin_third_party_actions_to_reviewed_commits() -> None:
shared = (
"actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683",
"actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065",
"actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020",
)
for path in (".gitea/workflows/release-gates.yml", ".github/workflows/release-gates.yml"):
workflow = read(path)
for action in shared:
assert action in workflow
assert (
"actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02"
in read(".github/workflows/release-gates.yml")
)
assert (
"actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de"
in read(".gitea/workflows/release-gates.yml")
)
def test_managed_validation_targets_the_actual_backend_and_frontend_projects() -> None:
workflow = read(".gitea/workflows/managed-validation.yml")
assert "backend/requirements-ci.lock" in workflow
assert "frontend/package-lock.json" in workflow
assert "python -m pytest -W error::DeprecationWarning" in workflow
assert "cd frontend && npm run test:unit" in workflow
assert "python -m ruff check backend scripts tests" in workflow
assert "python scripts/verify_repository_layout.py" in workflow
assert "[[ -f pyproject.toml" not in workflow
def test_scanner_images_are_versioned_and_digest_pinned() -> None:
sbom = read("scripts/generate_container_sbom.sh")
scan = read("scripts/scan_container_image.sh")
assert "anchore/syft:v1.44.0@sha256:" in sbom
assert "--user 0:0" in sbom
assert 'docker save "$IMAGE_ID"' in sbom
assert '"docker-archive:$WORKDIR/$IMAGE_ARCHIVE"' in sbom
assert 'SYFT_PARALLELISM=${SYFT_PARALLELISM:-1}' in sbom
assert 'GOMEMLIMIT=${SYFT_GOMEMLIMIT:-4GiB}' in sbom
assert 'GOGC=${SYFT_GOGC:-25}' in sbom
assert "--select-catalogers=-binary" in sbom
assert '--volumes-from "$HOSTNAME"' in sbom
assert 'ARCHIVE_ID_FILE="${IMAGE_ARCHIVE}.image-id"' in sbom
assert "/var/run/docker.sock" not in sbom
assert "aquasec/trivy:0.70.0@sha256:" in scan
assert "--severity HIGH,CRITICAL" in scan
assert "--ignore-unfixed" in scan
assert "--timeout 20m" in scan
assert "--scanners vuln" in scan
assert 'docker save "$IMAGE_ID"' in scan
assert '--input "$WORKDIR/$IMAGE_ARCHIVE"' in scan
assert '--volumes-from "$HOSTNAME"' in scan
assert '--ignorefile "$CONTAINER_IGNORE_FILE"' in scan
assert "/var/run/docker.sock" not in scan
assert "--skip-files /usr/local/bin/gosu" in scan
assert "final filesystem replaces it with the audited setpriv shell wrapper" in scan
assert "geointel-container-vulnerabilities.json" in scan
def test_readiness_guards_lock_and_supply_chain_entrypoints() -> None:
readiness = read("scripts/run_readiness_check.sh")
assert "scripts/verify_python_lock.py" in readiness
assert "scripts/verify_security_exceptions.py" in readiness
for path in (
"scripts/generate_python_lock.sh",
"scripts/generate_container_sbom.sh",
"scripts/scan_container_image.sh",
"scripts/audit_python_dependencies.sh",
):
assert f"bash -n {path}" in readiness
def test_python_audit_policy_has_no_active_exceptions_and_keeps_full_evidence() -> None:
policy = read("security/pip-audit-exceptions.json")
audit_script = read("scripts/audit_python_dependencies.sh")
assert '"schema_version": 1' in policy
assert '"advisories": []' in policy
assert "pip-audit-full.json" in audit_script
assert "pip-audit-policy.json" in audit_script
assert "--ignore-vuln" in audit_script
assert "verify_security_exceptions.py" in audit_script
def test_release_image_uses_locked_non_ai_dependencies_and_npm_ci() -> None:
dockerfile = read("deploy/unraid/Dockerfile.all-in-one")
assert "RUN npm ci" in dockerfile
assert "COPY backend/requirements-runtime.lock /app/" in dockerfile
assert "COPY backend/requirements-ai-linux.lock /app/" in dockerfile
assert "COPY backend/requirements-build-tools.lock /app/" in dockerfile
assert "pip install --no-cache-dir --require-hashes -r requirements-runtime.lock" in dockerfile
assert "-r requirements-ai-linux.lock" in dockerfile
assert "-r requirements-build-tools.lock" in dockerfile
assert "--require-hashes" in dockerfile
assert "ultralytics==8.4.99 --hash=sha256:" in read("backend/requirements-ai-linux.lock")
assert "torch==2.11.0+cu128 --hash=sha256:" in read("backend/requirements-ai-linux.lock")
assert "COPY deploy/unraid/gosu-setpriv /usr/local/bin/gosu" in dockerfile
assert "&& pip check" in dockerfile
def test_gosu_compatibility_wrapper_uses_exec_and_setpriv() -> None:
wrapper = read("deploy/unraid/gosu-setpriv")
readiness = read("scripts/run_readiness_check.sh")
assert "exec setpriv" in wrapper
assert '--reuid="$target_user"' in wrapper
assert '--regid="$target_user"' in wrapper
assert "--init-groups" in wrapper
assert "bash -n deploy/unraid/gosu-setpriv" in readiness
@@ -0,0 +1,53 @@
from __future__ import annotations
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
BACKEND = ROOT / "backend"
def test_no_untyped_fastapi_response_models_remain() -> None:
route_root = BACKEND / "app" / "api" / "routes"
route_sources = "\n".join(
path.read_text(encoding="utf-8")
for path in sorted(route_root.glob("*.py"))
)
assert "response_model=dict" not in route_sources
def test_every_json_success_response_has_a_concrete_canonical_schema() -> None:
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(BACKEND))
from app.main import create_app
from scripts.audit_api_contracts import (
ALLOWED_NON_ENVELOPE_ENDPOINTS,
_validate_response_contracts,
)
openapi = create_app().openapi()
assert _validate_response_contracts(openapi) == []
untyped_successes: set[tuple[str, str]] = set()
for path, path_item in openapi["paths"].items():
for method, operation in path_item.items():
if method.upper() not in {"GET", "POST", "PATCH", "DELETE"}:
continue
has_json_schema = any(
response.get("content", {})
.get("application/json", {})
.get("schema")
for code, response in operation.get("responses", {}).items()
if str(code).startswith("2")
)
if not has_json_schema:
untyped_successes.add((method.upper(), path))
assert untyped_successes == ALLOWED_NON_ENVELOPE_ENDPOINTS - {
("GET", "/health"),
("GET", "/health/live"),
("GET", "/health/ready"),
}
@@ -0,0 +1,74 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SCRIPT_PATH = ROOT / "scripts" / "provision_release_golden_areas.py"
def load_operator():
spec = importlib.util.spec_from_file_location("provision_release_golden_areas_test", SCRIPT_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_release_golden_area_contract_covers_belgium_and_north_sea() -> None:
module = load_operator()
created_keys = {item["key"] for item in module.GOLDEN_AREAS}
source_keys = {item["key"] for item in module.SOURCE_AREAS}
assert created_keys == {
"wallonia_urban_rural",
"brussels_urban",
"language_boundary",
"coast_land_sea",
"north_sea_multi_zone",
}
assert source_keys == {"mol_municipality", "kempen_region"}
assert len(created_keys | source_keys) == 7
assert all(item["source_project"] != module.NATIONAL_PROJECT for item in module.SOURCE_AREAS)
expected_zones = {
zone
for definition in (*module.GOLDEN_AREAS, *module.SOURCE_AREAS)
for zone in definition["expected_zones"]
}
assert {
"flanders",
"wallonia",
"brussels",
"territorial_sea",
"exclusive_economic_zone",
"continental_shelf",
} <= expected_zones
def test_release_golden_area_geometries_are_bounded_and_fingerprintable() -> None:
module = load_operator()
hashes = set()
for definition in module.GOLDEN_AREAS:
bbox = module.geometry_bbox(definition["geometry"])
assert -180 <= bbox["minx"] < bbox["maxx"] <= 180
assert -90 <= bbox["miny"] < bbox["maxy"] <= 90
assert bbox["maxx"] - bbox["minx"] <= 0.25
assert bbox["maxy"] - bbox["miny"] <= 0.20
digest = module.canonical_hash(definition["geometry"])
assert len(digest) == 64
hashes.add(digest)
assert len(hashes) == len(module.GOLDEN_AREAS)
def test_rc8_runner_and_container_operator_are_release_wired() -> None:
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
package = (ROOT / "frontend" / "package.json").read_text(encoding="utf-8")
assert "npm run test:unit" in readiness
assert '--check frontend/e2e/releaseJourneys.mjs' in readiness
assert "bash -n scripts/run_rc8_release_journeys.sh" in readiness
assert "COPY scripts/provision_release_golden_areas.py" in dockerfile
assert '"test:e2e": "node e2e/releaseJourneys.mjs"' in package
@@ -0,0 +1,50 @@
from __future__ import annotations
import json
from pathlib import Path
from tests.frontend_contract import read_map_workspace, read_feature
ROOT = Path(__file__).resolve().parents[2]
def test_rc9_ux_audit_is_wired_into_frontend_and_readiness() -> None:
package = json.loads((ROOT / "frontend" / "package.json").read_text(encoding="utf-8"))
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
wrapper = ROOT / "scripts" / "run_rc9_ux_audit.sh"
assert package["scripts"]["test:e2e:ux"] == "node e2e/uxAudit.mjs"
assert '"${NODE_BIN}" --check frontend/e2e/uxAudit.mjs' in readiness
assert "bash -n scripts/run_rc9_ux_audit.sh" in readiness
assert wrapper.is_file()
def test_rc9_loading_and_accessibility_states_are_explicit() -> None:
app = read_feature("shell")
map_workspace = read_map_workspace()
geo_map = (ROOT / "frontend" / "src" / "components" / "GeoMap.tsx").read_text(
encoding="utf-8"
)
assert "workspaceDataLoading" in app
assert 'role="status" aria-live="polite"' in app
assert "Databronnen worden gecontroleerd" in map_workspace
# Copy changes; the requirement is that source availability is stated.
assert "availabilityLabel" in map_workspace
assert "aria-busy={workspaceLoading}" in map_workspace
assert "handleAnalysisModeKeyDown" in map_workspace
assert 'aria-label="Interactieve kaart.' in geo_map
def test_rc9_performance_budgets_are_documented_and_visible() -> None:
budget = (
ROOT / "frontend" / "src" / "lib" / "performanceBudget.ts"
).read_text(encoding="utf-8")
docs = (ROOT / "docs" / "UX_PERFORMANCE_BUDGETS.md").read_text(encoding="utf-8")
map_workspace = read_map_workspace()
assert "COVERAGE_RESPONSE_BUDGET_MS = 4_000" in budget
assert "MAP_ANALYSIS_BUDGET_MS = 15_000" in budget
assert "4 seconds" in docs
assert "15 seconds" in docs
assert "overschrijdt het releasebudget" in map_workspace
@@ -0,0 +1,142 @@
from __future__ import annotations
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
def read(name: str) -> str:
return (SCRIPTS / name).read_text(encoding="utf-8")
def test_backup_is_atomic_read_only_and_checksum_bound() -> None:
script = read("backup_release_state.sh")
assert "pg_dump" in script
assert "-Fc" in script
assert "--no-owner" in script
assert "CHECKSUMS.sha256" in script
assert "database-password" not in script.lower()
assert 'for required in docker python3 sha256sum; do' in script
assert 'for required in docker python3 sha256sum git; do' not in script
assert "GITEA_COMMIT_SHA" in script
assert "GITHUB_SHA" in script
assert "GEOINTEL_BUILD_SHA" in script
assert 'if command -v git >/dev/null 2>&1' in script
assert "mv \"$PARTIAL\" \"$FINAL\"" in script
assert "rm -rf -- \"$PARTIAL\"" in script
assert "DROP DATABASE" not in script
assert "pg_restore --clean" not in script
assert "/mnt/user/appdata/geointel/backups" in script
assert "release_backup_snapshot.py" in script
assert "storage-snapshot" not in script # labels are composed without unsafe path interpolation
assert "--link-dest-backup" in script
assert "--rollback-image-tag" in script
assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in script
def test_backup_binds_prepared_source_without_requiring_dot_git() -> None:
script = read("backup_release_state.sh")
controller_resolution = script.index('local gitea_sha="${GITEA_COMMIT_SHA:-}"')
optional_git_fallback = script.index('if command -v git >/dev/null 2>&1')
docker_access = script.index("docker inspect -f '{{.State.Running}}'")
assert controller_resolution < optional_git_fallback < docker_access
assert 'git -C "$ROOT" rev-parse --show-toplevel' in script
assert '"$(cd "$git_top" && pwd -P)" = "$(cd "$ROOT" && pwd -P)"' in script
assert 'SOURCE_REVISION="$explicit_sha"' in script
assert '"backup_tool_revision": ${SOURCE_REVISION@Q}' in script
assert '"running_image_revision": ${RUNNING_IMAGE_REVISION@Q}' in script
assert "Cannot bind backup to a source revision" in script
def test_backup_verification_is_read_only() -> None:
script = read("verify_release_backup.sh")
assert "sha256sum -c CHECKSUMS.sha256" in script
assert "pg_restore --list" in script
assert "createdb" not in script
assert "dropdb" not in script
assert "pg_restore --clean" not in script
def test_restore_smoke_is_forced_to_generated_isolated_database() -> None:
script = read("restore_release_backup_smoke.sh")
assert "--confirm-isolated-restore" in script
assert "geointel_restore_verify_" in script
assert 'if [ "$TARGET_DB" = "$DB_NAME" ]' in script
assert "createdb" in script
assert "dropdb --if-exists" in script
assert "pg_restore \\\n --clean" not in script
assert '"production_database_untouched": True' in script
def test_release_safety_scripts_have_valid_bash_syntax() -> None:
for name in (
"backup_release_state.sh",
"verify_release_backup.sh",
"restore_release_backup_smoke.sh",
"../deploy/unraid/restore-predeploy-database.sh",
):
script_path = f"scripts/{name}" if not name.startswith("../") else name.removeprefix("../")
result = subprocess.run(
["bash", "-n", script_path],
cwd=ROOT,
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, f"{name}: {result.stderr}"
def test_production_restore_is_explicit_bounded_and_verified() -> None:
script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
encoding="utf-8"
)
assert "--confirm-production-database-restore" in script
assert "/mnt/user/appdata/geointel/backups" in script
assert "backup.relative_to(root)" in script
assert "sha256sum -c CHECKSUMS.sha256" in script
assert '"$RESTORE_PROOF_DB"' in script
assert "pg_restore" in script
assert "Restored Alembic head" in script
assert "Restored count mismatch" in script
assert "pg_restore --clean" not in script
assert "geointel_restore_proof_" in script
assert "Isolated predeploy restore proof passed" in script
assert "ALTER DATABASE" in script
assert "Pre-restore production database retained" in script
drop_start = script.index("dropdb --if-exists --force")
drop_command = script[drop_start : script.index("\n fi", drop_start)]
assert '"$RESTORE_PROOF_DB"' in drop_command
assert '"$GEOINTEL_POSTGRES_DB"' not in drop_command
def test_readiness_gate_checks_release_safety_scripts() -> None:
readiness = read("run_readiness_check.sh")
for name in (
"backup_release_state.sh",
"verify_release_backup.sh",
"restore_release_backup_smoke.sh",
):
assert f"bash -n scripts/{name}" in readiness
def test_password_rotation_never_prints_or_persists_generated_secret() -> None:
script = read("rotate_postgres_password.sh")
assert "openssl rand -hex 32" in script
assert 'echo "$NEW_PASSWORD"' not in script
assert 'printf "%s" "$NEW_PASSWORD"' not in script
assert "GEOINTEL_ROTATED_DATABASE_PASSWORD" in script
assert "NamedTemporaryFile" in script
assert "temporary.replace(path)" in script
assert "ALTER ROLE %s PASSWORD" in script
assert "run-dockerman-container.sh" in script
assert "/health/ready" not in script
@@ -0,0 +1,137 @@
from __future__ import annotations
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
import pytest
from app.core.errors import AppError
from app.models import Dataset
from app.services.temporal_compatibility_service import TemporalCompatibilityService
ROOT = Path(__file__).parents[2]
def dataset(
*,
dataset_type: str,
source_name: str,
observed_at: datetime | None = None,
valid_from: datetime | None = None,
valid_to: datetime | None = None,
temporal_granularity: str | None = None,
source_metadata: dict | None = None,
) -> Dataset:
return Dataset(
id=uuid4(),
project_id=uuid4(),
name="temporal-source",
dataset_type=dataset_type,
source=source_name,
source_name=source_name,
observed_at=observed_at,
valid_from=valid_from,
valid_to=valid_to,
temporal_granularity=temporal_granularity,
source_metadata=source_metadata,
)
def test_historical_orthophoto_is_rejected_for_detection() -> None:
historical = dataset(
dataset_type="raster",
source_name="digitaal_vlaanderen_orthophoto",
observed_at=datetime(2020, 1, 1, tzinfo=UTC),
valid_from=datetime(2020, 1, 1, tzinfo=UTC),
valid_to=datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC),
temporal_granularity="year",
source_metadata={"product_key": "2020", "supports_detection": False},
)
with pytest.raises(AppError) as exc_info:
TemporalCompatibilityService.ensure_detection_source_supported(historical)
assert exc_info.value.code == "DETECTION_SOURCE_TEMPORALLY_UNSUPPORTED"
assert exc_info.value.status_code == 422
def test_historical_detection_qa_rejects_current_reference() -> None:
historical = dataset(
dataset_type="raster",
source_name="digitaal_vlaanderen_orthophoto",
valid_from=datetime(2020, 1, 1, tzinfo=UTC),
valid_to=datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC),
temporal_granularity="year",
source_metadata={"product_key": "2020", "supports_detection": False},
)
current_reference = dataset(
dataset_type="vector",
source_name="grb",
valid_from=datetime(2026, 7, 1, tzinfo=UTC),
valid_to=datetime(2026, 7, 31, 23, 59, 59, tzinfo=UTC),
temporal_granularity="month",
)
with pytest.raises(AppError) as exc_info:
TemporalCompatibilityService.assess_detection_qa(historical, current_reference)
assert exc_info.value.code == "DETECTION_QA_TEMPORAL_MISMATCH"
assert exc_info.value.status_code == 422
def test_historical_detection_qa_accepts_overlapping_reference_edition() -> None:
historical = dataset(
dataset_type="raster",
source_name="digitaal_vlaanderen_orthophoto",
valid_from=datetime(2020, 1, 1, tzinfo=UTC),
valid_to=datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC),
temporal_granularity="year",
source_metadata={"product_key": "2020", "supports_detection": False},
)
historical_reference = dataset(
dataset_type="vector",
source_name="manual",
valid_from=datetime(2020, 6, 1, tzinfo=UTC),
valid_to=datetime(2020, 6, 30, 23, 59, 59, tzinfo=UTC),
temporal_granularity="month",
)
result = TemporalCompatibilityService.assess_detection_qa(historical, historical_reference)
assert result["status"] == "compatible"
assert result["candidate_historical"] is True
assert result["candidate_interval"]["start"].startswith("2020-01-01")
assert result["reference_interval"]["start"].startswith("2020-06-01")
def test_current_source_with_unbounded_current_reference_remains_supported() -> None:
current = dataset(
dataset_type="raster",
source_name="digitaal_vlaanderen_orthophoto",
observed_at=datetime(2026, 7, 17, tzinfo=UTC),
valid_from=datetime(2026, 7, 17, tzinfo=UTC),
temporal_granularity="snapshot",
source_metadata={"product_key": "most_recent", "supports_detection": True},
)
current_reference = dataset(
dataset_type="vector",
source_name="grb",
observed_at=datetime(2026, 7, 16, tzinfo=UTC),
temporal_granularity="snapshot",
)
TemporalCompatibilityService.ensure_detection_source_supported(current)
result = TemporalCompatibilityService.assess_detection_qa(current, current_reference)
assert result["status"] == "compatible"
assert result["candidate_historical"] is False
def test_detection_frontend_has_no_implicit_first_raster_fallback() -> None:
source = (ROOT / "frontend" / "src" / "hooks" / "useDetectionWorkflow.ts").read_text(encoding="utf-8")
assert "rasterDatasets[0]" not in source
assert "setSelectedDetectionDatasetId(rasterDatasets" not in source
assert "const datasetId = selectedDetectionDatasetId" in source
+85
View File
@@ -0,0 +1,85 @@
from __future__ import annotations
import importlib.util
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "capture_release_evidence.py"
def load_script():
spec = importlib.util.spec_from_file_location("capture_release_evidence", SCRIPT)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_release_evidence_manifest_is_secret_free_and_read_only(tmp_path: Path) -> None:
module = load_script()
args = module.parse_args(
[
"--output",
str(tmp_path / "evidence.json"),
"--release-id",
"test-rc",
]
)
manifest = module.build_manifest(args)
assert manifest["schema_version"] == 1
assert manifest["release_id"] == "test-rc"
assert manifest["version"] == "1.0.0"
assert manifest["read_only"] is True
assert manifest["scope"] == "Belgium and the Belgian North Sea"
assert "DATABASE_URL" not in json.dumps(manifest).replace(
'"DATABASE_URL": false',
"",
).replace(
'"DATABASE_URL": true',
"",
)
assert manifest["storage"] == {"requested": False}
assert manifest["live"] == {"requested": False}
def test_release_evidence_cli_writes_single_head_manifest(tmp_path: Path) -> None:
output = tmp_path / "baseline.json"
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--output",
str(output),
"--release-id",
"test-cli",
],
cwd=ROOT,
capture_output=True,
text=True,
check=False,
timeout=60,
)
assert result.returncode == 0, result.stderr
payload = json.loads(output.read_text(encoding="utf-8"))
assert payload["git"]["commit"]
assert payload["migrations"]["single_head"] is True
assert payload["files"]["docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md"]["sha256"]
assert payload["files"]["docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md"]["sha256"]
assert payload["files"]["docs/RELEASE_RUNBOOK.md"]["sha256"]
assert payload["files"]["docs/KNOWN_LIMITATIONS.md"]["sha256"]
def test_readiness_gate_compiles_release_evidence_command() -> None:
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
encoding="utf-8"
)
assert "py_compile scripts/capture_release_evidence.py" in readiness
@@ -0,0 +1,53 @@
from pathlib import Path
from fastapi.testclient import TestClient
from app.main import app
ROOT = Path(__file__).parents[2]
def test_valid_request_id_is_returned() -> None:
response = TestClient(app).get("/health/live", headers={"x-request-id": "rc3-check.123"})
assert response.status_code == 200
assert response.headers["x-request-id"] == "rc3-check.123"
def test_unsafe_request_id_is_replaced() -> None:
response = TestClient(app).get("/health/live", headers={"x-request-id": "unsafe request/id"})
assert response.status_code == 200
assert response.headers["x-request-id"] != "unsafe request/id"
assert " " not in response.headers["x-request-id"]
def test_runtime_report_is_read_only_by_default_and_requires_confirmation() -> None:
source = (ROOT / "scripts" / "runtime_state_report.py").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
assert '"mode": "read_only"' in source
assert 'IMPORT_ROOT = ROOT if (ROOT / "app").is_dir() else BACKEND' in source
assert "if args.reconcile and args.confirm != RECONCILE_CONFIRMATION" in source
assert "RuntimeReconciliationService.reconcile(db)" in source
assert "COPY scripts/runtime_state_report.py /app/scripts/runtime_state_report.py" in dockerfile
def test_all_in_one_deploy_embeds_immutable_build_identity() -> None:
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
deploy_powershell = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8")
deploy_shell = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8")
assert "ARG GEOINTEL_BUILD_SHA=unknown" in dockerfile
assert 'GEOINTEL_BUILD_SHA="${GEOINTEL_BUILD_SHA}"' in dockerfile
assert 'GEOINTEL_BUILD_TIME="${GEOINTEL_BUILD_TIME}"' in dockerfile
assert 'org.opencontainers.image.revision="${GEOINTEL_BUILD_SHA}"' in dockerfile
# However it is factored, the build SHA must come from git HEAD.
assert "git rev-parse HEAD" in release_script
assert "GEOINTEL_BUILD_SHA" in release_script
assert "--build-arg GEOINTEL_BUILD_SHA=" in release_script
assert "--build-arg GEOINTEL_BUILD_TIME=" in release_script
for deploy_source in (deploy_powershell, deploy_shell):
assert "bash deploy/unraid/deploy-release.sh" in deploy_source
+141
View File
@@ -0,0 +1,141 @@
from pathlib import Path
def test_backend_uses_patched_starlette_and_explicit_httpx2_test_client() -> None:
pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml"
content = pyproject.read_text(encoding="utf-8")
assert '"starlette>=1.3.1,<2.0.0"' in content
assert '"httpx2>=2.0.0"' in content
def test_readiness_gate_treats_deprecation_warnings_as_errors() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
content = script.read_text(encoding="utf-8")
assert "-W error::DeprecationWarning" in content
def test_readiness_gate_runs_contract_smoke() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
content = script.read_text(encoding="utf-8")
assert "scripts/smoke_contracts.py" in content
def test_readiness_gate_checks_demo_export_workflow_script_syntax() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
content = script.read_text(encoding="utf-8")
assert "bash -n scripts/verify_demo_export_workflow.sh" in content
assert "bash -n scripts/verify_workbench_default_state.sh" in content
def test_readiness_gate_runs_golden_qa_benchmark() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
content = script.read_text(encoding="utf-8")
assert "scripts/run_golden_qa_benchmark.py --json" in content
assert "bash -n scripts/verify_golden_qa_benchmark.sh" in content
def test_readiness_gate_compiles_demo_cleanup_script() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
content = script.read_text(encoding="utf-8")
assert "-m py_compile scripts/cleanup_demo_artifacts.py" in content
assert "-m py_compile backend/scripts/cleanup_demo_artifacts.py" in content
def test_readiness_gate_checks_demo_cleanup_dry_run_script_syntax() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
content = script.read_text(encoding="utf-8")
assert "bash -n scripts/verify_demo_cleanup_dry_run.sh" in content
def test_demo_cleanup_dry_run_script_is_dry_run_only() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "verify_demo_cleanup_dry_run.sh"
content = script.read_text(encoding="utf-8")
assert "--apply" not in content
assert "deleted_export_count=0" in content
assert "dry_run=true" in content
assert "candidate_exports" in content
assert "CLEANUP_MODE" in content
assert "project_report_html" in content
def test_readiness_gate_checks_workbench_screenshot_capture_syntax() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
content = script.read_text(encoding="utf-8")
assert "bash -n scripts/capture_workbench_screenshots.sh" in content
def test_workbench_screenshot_capture_is_artifact_based_and_non_mutating() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "capture_workbench_screenshots.sh"
content = script.read_text(encoding="utf-8")
assert "artifacts/screenshots" in content
assert "/api/v1/demo/workflow" in content
assert "['overview', 'Overview']" in content
assert "['system', 'System']" in content
assert "workspace-nav-${workspaceKey}" in content
assert "manifest.json" in content
assert "page.screenshot" in content
assert "fullPage: false" in content
assert "Playwright is required" in content
assert "--apply" not in content
def test_readiness_gate_compiles_yolo_preflight_script() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
content = script.read_text(encoding="utf-8")
assert "-m py_compile scripts/yolo_preflight.py" in content
assert "-m py_compile backend/scripts/yolo_preflight.py" in content
def test_demo_export_workflow_script_verifies_export_endpoints() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "verify_demo_export_workflow.sh"
content = script.read_text(encoding="utf-8")
assert "/api/v1/demo/workflow" in content
assert "/areas" in content
assert "/datasets" in content
assert "/content" in content
assert "/vector/summary" in content
assert "GeoJSON Polygon/MultiPolygon geometry" in content
assert "precision" in content
assert "false_negative_count" in content
assert "fixtures/golden/expected_qa_metrics.json" in content
assert "QA/QC metric {key} drifted" in content
assert "Seeded QA/QC match count does not match golden baseline" in content
assert "/api/v1/exports/metadata" in content
assert "/api/v1/exports/report" in content
assert "/api/v1/exports/geojson" in content
assert "/download" in content
def test_workbench_default_state_script_verifies_populated_demo_start_state() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "verify_workbench_default_state.sh"
content = script.read_text(encoding="utf-8")
assert "/api/v1/demo/workflow" in content
assert "GeoIntel Demo - Building QA" in content
assert "/areas" in content
assert "/datasets" in content
assert "/quality-checks" in content
assert "data.items" in content
assert "Demo AOI - Geel buildings" in content
assert "3/3 ready" in content
def test_pass_end_check_excludes_vendor_and_build_outputs() -> None:
script = Path(__file__).resolve().parents[2] / "scripts" / "codex_pass_end_check.sh"
content = script.read_text(encoding="utf-8")
assert "--exclude-dir=node_modules" in content
assert "--exclude-dir=dist" in content
assert "--exclude-dir=__pycache__" in content
@@ -0,0 +1,33 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
SCRIPT = Path(__file__).parents[2] / "scripts" / "build_regional_yolo_dataset.py"
SPEC = importlib.util.spec_from_file_location("regional_yolo_dataset", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def test_select_paths_is_region_and_split_safe() -> None:
manifest = {
"samples": [
{"sample_slug": "f-train", "region": "flanders", "split": "train"},
{"sample_slug": "f-val", "region": "flanders", "split": "val"},
{"sample_slug": "f-test", "region": "flanders", "split": "test"},
{"sample_slug": "w-train", "region": "wallonia", "split": "train"},
]
}
summary = {
"tiles": [
{"sample_slug": "f-train", "split": "train", "image_path": "/f-train.png"},
{"sample_slug": "f-val", "split": "val", "image_path": "/f-val.png"},
{"sample_slug": "f-test", "split": "val", "image_path": "/f-test.png"},
{"sample_slug": "w-train", "split": "train", "image_path": "/w-train.png"},
]
}
train, val = MODULE.select_paths(summary, manifest, "flanders")
assert train == ["/f-train.png"]
assert val == ["/f-val.png"]
@@ -0,0 +1,83 @@
from __future__ import annotations
import importlib.util
import os
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "release_backup_snapshot.py"
def load_snapshot_module():
spec = importlib.util.spec_from_file_location("release_backup_snapshot_test", SCRIPT)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_snapshot_is_byte_complete_and_reuses_only_verified_backup_bytes(tmp_path: Path) -> None:
snapshot = load_snapshot_module()
source = tmp_path / "source"
source.mkdir()
(source / "same.bin").write_bytes(b"unchanged")
(source / "changed.bin").write_bytes(b"before")
(source / "empty").mkdir()
prior = tmp_path / "prior"
prior_manifest = tmp_path / "prior.tsv"
snapshot.create_snapshot(source, prior, prior_manifest, label="storage")
snapshot.verify_snapshot(prior, prior_manifest)
(source / "changed.bin").write_bytes(b"after")
current = tmp_path / "current"
current_manifest = tmp_path / "current.tsv"
snapshot.create_snapshot(
source,
current,
current_manifest,
label="storage",
link_dest_snapshot=prior,
link_dest_manifest=prior_manifest,
)
snapshot.verify_snapshot(current, current_manifest)
assert os.path.samefile(prior / "same.bin", current / "same.bin")
assert not os.path.samefile(prior / "changed.bin", current / "changed.bin")
assert (current / "changed.bin").read_bytes() == b"after"
assert (current / "empty").is_dir()
def test_snapshot_rejects_symlinked_content(tmp_path: Path) -> None:
snapshot = load_snapshot_module()
source = tmp_path / "source"
source.mkdir()
target = source / "target.bin"
target.write_bytes(b"target")
try:
(source / "link.bin").symlink_to(target)
except OSError:
pytest.skip("Symlink creation is unavailable on this host")
with pytest.raises(RuntimeError, match="refuses symlinked content"):
snapshot.create_snapshot(source, tmp_path / "snapshot", tmp_path / "manifest.tsv", label="storage")
def test_snapshot_verification_rejects_changed_retained_bytes(tmp_path: Path) -> None:
snapshot = load_snapshot_module()
source = tmp_path / "source"
source.mkdir()
(source / "artifact.bin").write_bytes(b"retained")
retained = tmp_path / "snapshot"
manifest = tmp_path / "manifest.tsv"
snapshot.create_snapshot(source, retained, manifest, label="storage")
(retained / "artifact.bin").chmod(0o644)
(retained / "artifact.bin").write_bytes(b"tampered")
with pytest.raises(RuntimeError, match="checksum differs"):
snapshot.verify_snapshot(retained, manifest)
+25
View File
@@ -0,0 +1,25 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "verify_repository_layout.py"
SPEC = importlib.util.spec_from_file_location("verify_repository_layout", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def test_nested_repository_mirror_is_absent() -> None:
assert MODULE.nested_mirror_markers(ROOT) == []
def test_nested_repository_mirror_is_detected(tmp_path: Path) -> None:
for marker in MODULE.CANONICAL_MARKERS:
path = tmp_path / "geointel" / marker
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("fixture", encoding="utf-8")
assert MODULE.nested_mirror_markers(tmp_path) == list(MODULE.CANONICAL_MARKERS)
@@ -0,0 +1,55 @@
import pytest
from fastapi.testclient import TestClient
from app.main import app
client = TestClient(app)
@pytest.mark.parametrize(
"host",
[
"trusted.example/@admin",
"trusted.example?shadow=admin",
"trusted.example#shadow",
],
)
def test_invalid_host_request_target_is_rejected_canonically(host: str) -> None:
response = client.get("/health/live", headers={"host": host})
assert response.status_code == 400
assert response.headers["x-request-id"]
assert response.json()["error"] == "INVALID_REQUEST_TARGET"
assert response.json()["request_id"] == response.headers["x-request-id"]
@pytest.mark.parametrize(
"host",
["localhost:1202", "127.0.0.1:8000", "[::1]:8000", "testserver"],
)
def test_normal_host_forms_remain_available(host: str) -> None:
response = client.get("/health/live", headers={"host": host})
assert response.status_code == 200
def test_urlencoded_form_body_is_rejected_before_starlette_form_parsing() -> None:
response = client.post(
"/api/v1/datasets/upload",
headers={"content-type": "application/x-www-form-urlencoded"},
content="dataset_type=vector&field=" + ("x" * 10_000),
)
assert response.status_code == 415
assert response.json()["error"] == "UNSUPPORTED_CONTENT_TYPE"
def test_multipart_upload_contract_remains_available() -> None:
response = client.post(
"/health/live",
files={"file": ("empty.geojson", b"{}", "application/geo+json")},
data={"dataset_type": "vector"},
)
assert response.status_code == 405
+20
View File
@@ -0,0 +1,20 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
SCRIPT = Path(__file__).parents[2] / "scripts" / "retile_yolo_dataset.py"
SPEC = importlib.util.spec_from_file_location("retile_yolo", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def test_tile_starts_cover_edges_with_overlap() -> None:
assert MODULE.tile_starts(640, 384, 128) == [0, 256]
assert MODULE.tile_starts(700, 384, 128) == [0, 256, 316]
def test_tile_starts_reject_image_smaller_than_tile() -> None:
assert MODULE.tile_starts(320, 384, 128) == []
@@ -0,0 +1,254 @@
"""An operator's adjudication must reach the score.
The review vocabulary already distinguishes a model error from a reference gap
the product's own position is that official footprints are not automatically
perfect ground truth. But the reviews were only counted. An operator who
inspects forty false positives and establishes that twelve are buildings the
reference simply lacks still sees a precision that counts all forty against the
model, and that they have personally disproved.
Because part of the evidence is usually still unreviewed, the honest answer is
an interval, not a single corrected number: pessimistic assumes every
unreviewed item is a model error, optimistic assumes none is.
"""
from __future__ import annotations
import pytest
from app.services.reviewed_metrics_service import ReviewedMetricsService
def _counts(**decisions: int) -> dict[str, int]:
return decisions
class TestAdjudication:
def test_a_reference_gap_stops_counting_against_precision(self) -> None:
result = ReviewedMetricsService.adjudicate(
matches=80,
false_positives=20,
false_negatives=10,
false_positive_decisions=_counts(reference_gap_or_change=20),
false_negative_decisions={},
)
# Every false positive was the reference missing a real building.
assert result["adjudicated"]["false_positives"] == 0
assert result["adjudicated"]["precision"] == pytest.approx(1.0)
def test_a_confirmed_model_error_keeps_counting(self) -> None:
result = ReviewedMetricsService.adjudicate(
matches=80,
false_positives=20,
false_negatives=0,
false_positive_decisions=_counts(confirmed_model_false_positive=20),
false_negative_decisions={},
)
assert result["adjudicated"]["false_positives"] == 20
assert result["adjudicated"]["precision"] == pytest.approx(0.8)
def test_an_alignment_mismatch_is_not_a_model_error(self) -> None:
"""Both the detection and the footprint were right; the matching failed."""
result = ReviewedMetricsService.adjudicate(
matches=80,
false_positives=20,
false_negatives=0,
false_positive_decisions=_counts(qa_alignment_mismatch=20),
false_negative_decisions={},
)
assert result["adjudicated"]["false_positives"] == 0
def test_a_reference_gap_on_a_miss_stops_counting_against_recall(self) -> None:
result = ReviewedMetricsService.adjudicate(
matches=80,
false_positives=0,
false_negatives=20,
false_positive_decisions={},
false_negative_decisions=_counts(reference_gap_or_change=20),
)
# The reference held twenty footprints that are not there.
assert result["adjudicated"]["false_negatives"] == 0
assert result["adjudicated"]["recall"] == pytest.approx(1.0)
def test_an_uncertain_verdict_keeps_counting_against_the_model(self) -> None:
"""Not being able to judge is not evidence in the model's favour."""
result = ReviewedMetricsService.adjudicate(
matches=80,
false_positives=20,
false_negatives=0,
false_positive_decisions=_counts(uncertain=10, imagery_obscured_or_uncertain=10),
false_negative_decisions={},
)
assert result["adjudicated"]["false_positives"] == 20
class TestBounds:
def test_a_partly_reviewed_check_reports_an_interval(self) -> None:
result = ReviewedMetricsService.adjudicate(
matches=80,
false_positives=20,
false_negatives=0,
false_positive_decisions=_counts(reference_gap_or_change=10),
false_negative_decisions={},
)
# Ten unreviewed: pessimistically all model errors, optimistically none.
assert result["pessimistic"]["precision"] == pytest.approx(80 / 90)
assert result["optimistic"]["precision"] == pytest.approx(1.0)
assert result["review_complete"] is False
def test_a_fully_reviewed_check_collapses_the_interval(self) -> None:
result = ReviewedMetricsService.adjudicate(
matches=80,
false_positives=20,
false_negatives=5,
false_positive_decisions=_counts(reference_gap_or_change=12, confirmed_model_false_positive=8),
false_negative_decisions=_counts(confirmed_model_false_negative=5),
)
assert result["review_complete"] is True
assert result["pessimistic"]["precision"] == pytest.approx(result["optimistic"]["precision"])
assert result["adjudicated"]["precision"] == pytest.approx(80 / 88)
def test_an_unreviewed_check_reports_the_raw_numbers_unchanged(self) -> None:
result = ReviewedMetricsService.adjudicate(
matches=80,
false_positives=20,
false_negatives=10,
false_positive_decisions={},
false_negative_decisions={},
)
assert result["review_complete"] is False
assert result["adjudicated"]["precision"] == pytest.approx(result["raw"]["precision"])
assert result["adjudicated"]["recall"] == pytest.approx(result["raw"]["recall"])
def test_the_raw_score_is_always_reported_alongside(self) -> None:
result = ReviewedMetricsService.adjudicate(
matches=80,
false_positives=20,
false_negatives=0,
false_positive_decisions=_counts(reference_gap_or_change=20),
false_negative_decisions={},
)
assert result["raw"]["precision"] == pytest.approx(0.8)
assert result["adjudicated"]["precision"] == pytest.approx(1.0)
class TestEdges:
def test_a_check_without_findings_makes_no_claim(self) -> None:
result = ReviewedMetricsService.adjudicate(
matches=0,
false_positives=0,
false_negatives=0,
false_positive_decisions={},
false_negative_decisions={},
)
assert result["adjudicated"]["precision"] is None
assert result["adjudicated"]["recall"] is None
assert result["review_complete"] is True
def test_more_decisions_than_findings_cannot_invent_a_negative_count(self) -> None:
result = ReviewedMetricsService.adjudicate(
matches=10,
false_positives=2,
false_negatives=0,
false_positive_decisions=_counts(reference_gap_or_change=99),
false_negative_decisions={},
)
assert result["adjudicated"]["false_positives"] == 0
def test_an_unknown_decision_is_treated_as_no_judgement(self) -> None:
result = ReviewedMetricsService.adjudicate(
matches=10,
false_positives=5,
false_negatives=0,
false_positive_decisions=_counts(something_new_from_a_later_release=5),
false_negative_decisions={},
)
assert result["adjudicated"]["false_positives"] == 5
assert result["review_complete"] is False
class TestThroughTheReviewPanel:
"""The score the panel shows, not just the arithmetic behind it."""
def _quality_check(self, quality_check_id, project_id):
from app.models import QualityCheck
return QualityCheck(
id=quality_check_id,
project_id=project_id,
reference_dataset_id=__import__("uuid").uuid4(),
check_type="detections_vs_reference",
status="ok",
findings_json={
"matches": 80,
"false_positives": 20,
"false_negatives": 0,
"false_positive_evidence": [
{"candidate_feature_id": f"detection-{index}"} for index in range(20)
],
"false_negative_evidence": [],
},
)
def test_adjudicated_reference_gaps_raise_the_reported_precision(self) -> None:
import uuid
from app.models import DetectionReview, QualityCheck
from app.services.detection_review_service import DetectionReviewService
quality_check_id, project_id = uuid.uuid4(), uuid.uuid4()
quality_check = self._quality_check(quality_check_id, project_id)
reviews = [
DetectionReview(
id=uuid.uuid4(),
quality_check_id=quality_check_id,
evidence_role="false_positive",
evidence_feature_id=f"detection-{index}",
decision="reference_gap_or_change",
)
for index in range(12)
]
class _Query:
def __init__(self, rows):
self.rows = rows
def filter(self, *_args):
return self
def all(self):
return self.rows
class _Session:
def get(self, model, item_id):
return quality_check if model is QualityCheck and item_id == quality_check_id else None
def query(self, _model):
return _Query(reviews)
result = DetectionReviewService.list_reviews(
_Session(), project_id=project_id, quality_check_id=quality_check_id
)
metrics = result.summary.reviewed_metrics
assert metrics is not None
assert metrics["raw"]["precision"] == pytest.approx(0.8)
# Twelve of the twenty were the reference missing a building.
assert metrics["adjudicated"]["precision"] == pytest.approx(80 / 88)
assert metrics["review_complete"] is False
assert metrics["false_positive_breakdown"]["exonerated"] == 12
assert metrics["false_positive_breakdown"]["unreviewed"] == 8
@@ -0,0 +1,21 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
import pytest
SCRIPT = Path(__file__).parents[2] / "scripts" / "rotate_belgium_building_holdouts.py"
SPEC = importlib.util.spec_from_file_location("rotate_building_holdouts", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def test_rotated_corpus_version_is_explicit_and_canonical() -> None:
assert MODULE.validate_dataset_version("building-be-v31-rotated-holdouts-r1") == (
"building-be-v31-rotated-holdouts-r1"
)
with pytest.raises(ValueError, match="canonical slug"):
MODULE.validate_dataset_version("Building BE v31")
+144
View File
@@ -0,0 +1,144 @@
from __future__ import annotations
from uuid import uuid4
import pytest
from app.core.config import Settings
from app.core.errors import AppError
from app.models import AnalysisRun, Dataset, Job, Project
from app.services.detection_service import DetectionService
from app.services.job_service import JobService
from app.services.segmentation_service import SegmentationService
class FakeSession:
"""Minimal session double without rollback support, mirroring existing test doubles."""
def __init__(self, objects=None) -> None:
self.objects = objects or {}
self.added = []
self.commits = 0
def get(self, model, item_id):
return self.objects.get((model, item_id))
def add(self, item) -> None:
self.added.append(item)
if getattr(item, "id", None) is not None:
self.objects[(item.__class__, item.id)] = item
def commit(self) -> None:
self.commits += 1
def refresh(self, item) -> None:
pass
def _project_and_dataset():
project_id = uuid4()
dataset_id = uuid4()
project = Project(id=project_id, name="Mol")
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="ortho.tif",
dataset_type="raster",
source="fixture",
source_name="fixture",
storage_path="storage/uploads/ortho.tif",
source_metadata={"fixture": True},
)
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
return db, project_id, dataset_id
def _statuses(db: FakeSession) -> tuple[list[str], list[str]]:
runs = [item.status for item in db.added if isinstance(item, AnalysisRun)]
jobs = [item.status for item in db.added if isinstance(item, Job)]
return runs, jobs
def test_invalid_fixture_detections_mark_run_and_job_failed() -> None:
db, project_id, dataset_id = _project_and_dataset()
with pytest.raises(AppError) as exc_info:
DetectionService.run_detection(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="manual-fixture-detector",
confidence_threshold=0.5,
parameters_json={"fixture_mode": True, "fixture_detections": "not-a-list"},
settings=Settings(_env_file=None),
)
assert exc_info.value.code == "INVALID_FIXTURE_DETECTIONS"
run_statuses, job_statuses = _statuses(db)
assert run_statuses and all(status == "failed" for status in run_statuses)
assert job_statuses and all(status == "failed" for status in job_statuses)
def test_invalid_fixture_segmentations_mark_run_and_job_failed() -> None:
db, project_id, dataset_id = _project_and_dataset()
with pytest.raises(AppError) as exc_info:
SegmentationService.run_segmentation(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="fixture-segmenter",
confidence_threshold=0.5,
parameters_json={"fixture_mode": True, "fixture_segmentations": "not-a-list"},
settings=Settings(_env_file=None),
)
assert exc_info.value.code == "INVALID_FIXTURE_SEGMENTATIONS"
run_statuses, job_statuses = _statuses(db)
assert run_statuses and all(status == "failed" for status in run_statuses)
assert job_statuses and all(status == "failed" for status in job_statuses)
def test_unexpected_error_in_sync_job_marks_job_failed() -> None:
project_id = uuid4()
db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Mol")})
def exploding_operation():
raise RuntimeError("unexpected internal failure")
with pytest.raises(RuntimeError):
JobService.run_sync_job(
db=db,
project_id=project_id,
job_type="test.unexpected",
parameters={},
operation=exploding_operation,
)
jobs = [item for item in db.added if isinstance(item, Job)]
assert jobs
final_job = jobs[-1]
assert final_job.status == "failed"
assert "Unexpected internal error" in (final_job.error_message or "")
def test_app_error_in_sync_job_still_marks_job_failed() -> None:
project_id = uuid4()
db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Mol")})
def failing_operation():
raise AppError(code="SOME_DOMAIN_ERROR", message="Bounded failure", status_code=422)
with pytest.raises(AppError):
JobService.run_sync_job(
db=db,
project_id=project_id,
job_type="test.bounded",
parameters={},
operation=failing_operation,
)
jobs = [item for item in db.added if isinstance(item, Job)]
assert jobs
assert jobs[-1].status == "failed"
assert jobs[-1].error_message == "Bounded failure"
@@ -0,0 +1,217 @@
from __future__ import annotations
from hashlib import sha256
import importlib.util
import json
from pathlib import Path
from uuid import uuid4
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "migrate_runtime_model_provenance.py"
SPEC = importlib.util.spec_from_file_location("migrate_runtime_model_provenance", SCRIPT)
assert SPEC and SPEC.loader
module = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(module)
def _sha(value: bytes) -> str:
return sha256(value).hexdigest()
def _args(tmp_path: Path):
model = tmp_path / "active.pt"
checkpoint = tmp_path / "best.pt"
base_model = tmp_path / "base.pt"
training_args = tmp_path / "args.yaml"
training_results = tmp_path / "results.csv"
dataset_summary = tmp_path / "dataset-summary.json"
dataset_yaml = tmp_path / "dataset.yaml"
training_summary = tmp_path / "training-summary.json"
model.write_bytes(b"exact promoted model bytes")
checkpoint.write_bytes(model.read_bytes())
base_model.write_bytes(b"exact base model bytes")
training_args.write_text("epochs: 30\nseed: 0\n", encoding="utf-8")
training_results.write_text("epoch,metric\n1,0.1\n", encoding="utf-8")
dataset_yaml.write_text("names:\n 0: building\n", encoding="utf-8")
dataset_summary.write_text(
json.dumps(
{
"status": "ok",
"class_names": ["building"],
"tile_count": 198,
"train_tile_count": 180,
"val_tile_count": 18,
"label_count": 58_820,
}
),
encoding="utf-8",
)
training_summary.write_text(
json.dumps(
{
"status": "ok",
"trained_model_sha256": _sha(model.read_bytes()),
"base_model_sha256": _sha(base_model.read_bytes()),
"dataset_summary_sha256": _sha(dataset_summary.read_bytes()),
"dataset_yaml_sha256": _sha(dataset_yaml.read_bytes()),
}
),
encoding="utf-8",
)
return module.parse_args(
[
"--model-path",
str(model),
"--checkpoint-path",
str(checkpoint),
"--base-model-path",
str(base_model),
"--training-summary-path",
str(training_summary),
"--training-args-path",
str(training_args),
"--training-results-path",
str(training_results),
"--dataset-summary-path",
str(dataset_summary),
"--dataset-yaml-path",
str(dataset_yaml),
"--source-version",
"sprint174-smallbld-minpx3-img640-ft30",
"--framework-version",
"8.4.93",
]
)
def test_recovered_evidence_requires_byte_identical_checkpoint_and_recorded_hashes(tmp_path: Path) -> None:
args = _args(tmp_path)
evidence = module.inspect_evidence(args)
assert evidence["checksums"]["model"] == evidence["checksums"]["checkpoint"]
assert evidence["checksums"]["training_summary"] == _sha(
Path(args.training_summary_path).read_bytes()
)
assert evidence["class_mapping"] == {"0": "building"}
def test_recovered_evidence_rejects_changed_checkpoint(tmp_path: Path) -> None:
args = _args(tmp_path)
Path(args.checkpoint_path).write_bytes(b"other checkpoint")
exit_code, payload = module.migrate(args)
assert exit_code == 2
assert payload["status"] == "evidence_invalid"
assert "checkpoint/model SHA-256 mismatch" in payload["message"]
def test_generated_sidecar_passes_exact_runtime_contract(tmp_path: Path) -> None:
args = _args(tmp_path)
evidence = module.inspect_evidence(args)
payload = module._manifest_payload(
args=args,
evidence=evidence,
source_registry_id=str(uuid4()),
source_snapshot_id=str(uuid4()),
imported_at="2026-08-23T21:00:00+00:00",
)
manifest_path = RuntimeModelProvenanceService.manifest_path_for_model(args.model_path)
assert module._write_manifest_atomically(manifest_path, payload) is True
validated = RuntimeModelProvenanceService.validate_for_runtime(
model_path=args.model_path,
model_id="yolo-configured",
task_type="object_detection",
expected_model_version="sprint174-smallbld-minpx3-img640-ft30",
allowed_frameworks=("ultralytics/pytorch",),
)
assert validated.model_sha256 == evidence["checksums"]["model"]
assert validated.runtime_manifest_sha256 == payload["metadata"]["runtime_manifest_sha256"]
assert module._write_manifest_atomically(manifest_path, payload) is False
def test_model_registry_definition_is_runtime_ready_without_mutating_server_owned_row() -> None:
from app.services.source_registry_service import SourceRegistryService
definition = SourceRegistryService.definition_for("model")
assert definition.ingest_status == "configured"
assert definition.freshness_status == "current"
script = SCRIPT.read_text(encoding="utf-8")
assert 'source.ingest_status = "configured"' not in script
assert 'source.freshness_status = "current"' not in script
def test_model_registry_status_migration_is_narrow_and_restores_write_guard() -> None:
migration = (
ROOT
/ "backend"
/ "alembic"
/ "versions"
/ "202608230001_configure_model_source_registry.py"
).read_text(encoding="utf-8")
assert "WHERE source_key = 'model'" in migration
assert "registry_owner' = 'server'" in migration
assert migration.count("DISABLE TRIGGER trg_source_registry_write_guard") == 1
assert migration.count("ENABLE TRIGGER trg_source_registry_write_guard") == 1
assert 'down_revision = "202608010001"' in migration
def test_recovery_receipt_requires_hashed_backup_inventory(tmp_path: Path) -> None:
args = _args(tmp_path)
evidence = module.inspect_evidence(args)
receipt = tmp_path / "dry-run.json"
receipt.write_text(
json.dumps(
{
"status": "ready_to_apply",
"claim_boundary": module.CLAIM_BOUNDARY,
"evidence": evidence,
}
),
encoding="utf-8",
)
storage_manifest = tmp_path / "storage-manifest.tsv"
missing_names = ("checkpoint", "training_summary", "training_args", "training_results")
rows = ["relative_path\tsize_bytes\tmtime_ns\tsha256"]
for name in missing_names:
original = Path(evidence["paths"][name])
evidence["paths"][name] = f"/app/storage/training/{original.name}"
rows.append(f"training/{original.name}\t1\t0\t{evidence['checksums'][name]}")
receipt.write_text(
json.dumps(
{
"status": "ready_to_apply",
"claim_boundary": module.CLAIM_BOUNDARY,
"evidence": evidence,
}
),
encoding="utf-8",
)
storage_manifest.write_text("\n".join(rows) + "\n", encoding="utf-8")
backup_checksums = tmp_path / "CHECKSUMS.sha256"
backup_checksums.write_text(
f"{_sha(storage_manifest.read_bytes())} storage-manifest.tsv\n",
encoding="utf-8",
)
args.evidence_receipt_path = str(receipt)
args.backup_storage_manifest_path = str(storage_manifest)
args.backup_checksums_path = str(backup_checksums)
recovered = module.inspect_evidence(args)
assert recovered["checksums"] == evidence["checksums"]
assert recovered["recovery_receipt"]["missing_artifacts_not_recreated"] == list(missing_names)
storage_manifest.write_text(storage_manifest.read_text(encoding="utf-8") + "tampered\n", encoding="utf-8")
exit_code, payload = module.migrate(args)
assert exit_code == 2
assert "backup storage manifest SHA-256 mismatch" in payload["message"]
@@ -0,0 +1,325 @@
from __future__ import annotations
from hashlib import sha256
import json
from pathlib import Path
from uuid import uuid4
import pytest
from app.core.errors import AppError
from app.models import DatasetQuarantine, SourceRegistry, SourceSnapshot
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
class FakeSession:
"""Explicit database double for production-runtime provenance tests."""
def __init__(self, objects: dict[tuple[type, object], object] | None = None) -> None:
self.objects = objects or {}
def get(self, model, item_id):
return self.objects.get((model, item_id))
def _write_sidecar(
model_path: Path,
*,
model_id: str = "yolo-configured",
task_type: str = "object_detection",
framework: str = "ultralytics/pytorch",
source_version: str = "test-v1",
source_registry_id: str | None = None,
source_snapshot_id: str | None = None,
) -> Path:
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
payload = {
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
"data_contract": {
"key": "geointel.model.pytorch",
"version": "1.0.0",
},
"model": {
"model_id": model_id,
"task_type": task_type,
"sha256": model_sha256,
"model_format": "pytorch",
"framework": framework,
"class_mapping": {"0": "building"},
"source_version": source_version,
},
"source": {
"source_registry_id": source_registry_id or str(uuid4()),
"source_snapshot_id": source_snapshot_id or str(uuid4()),
"source_registry_key": "model",
"source_snapshot_checksum_sha256": model_sha256,
},
"lineage": {
"upstream_asset_ids": ["training-corpus:test-v1"],
"upstream_checksums_sha256": ["a" * 64],
"transformations": [
{
"name": "pytorch-training",
"version": "1.0.0",
"checksum_sha256": "b" * 64,
}
],
},
"metadata": {
"training_manifest_sha256": "c" * 64,
},
"imported_at": "2026-08-01T10:00:00+00:00",
}
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
sidecar_path = RuntimeModelProvenanceService.manifest_path_for_model(model_path)
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
return sidecar_path
def _governed_model_database(
*,
source_registry_id,
source_snapshot_id,
model_checksum: str,
source_version: str = "test-v1",
) -> tuple[FakeSession, SourceRegistry, SourceSnapshot]:
registry = SourceRegistry(
id=source_registry_id,
source_key="model",
display_name="Governed test model artifacts",
classification="experimental",
authority_name="GeoIntel test fixture",
freshness_status="current",
ingest_status="configured",
)
snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key=f"model-{source_version}",
source_version=source_version,
checksum_sha256=model_checksum,
freshness_status="current",
ingest_status="ingested",
)
return (
FakeSession(
{
(SourceRegistry, source_registry_id): registry,
(SourceSnapshot, source_snapshot_id): snapshot,
}
),
registry,
snapshot,
)
def test_runtime_model_provenance_accepts_byte_bound_pytorch_sidecar_for_structural_preflight(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"trusted local model bytes")
sidecar_path = _write_sidecar(model_path, source_version="v1")
evidence = RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
expected_model_version="v1",
allowed_frameworks=("ultralytics/pytorch",),
)
assert evidence.model_sha256 == sha256(model_path.read_bytes()).hexdigest()
assert evidence.manifest_path == str(sidecar_path.resolve())
assert evidence.data_contract_key == "geointel.model.pytorch"
assert evidence.data_contract_version == "1.0.0"
assert len(evidence.validation_report_sha256) == 64
def test_production_runtime_requires_db_bound_model_source_snapshot(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"governed local model bytes")
source_registry_id = uuid4()
source_snapshot_id = uuid4()
_write_sidecar(
model_path,
source_version="v1",
source_registry_id=str(source_registry_id),
source_snapshot_id=str(source_snapshot_id),
)
db, _, _ = _governed_model_database(
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
model_checksum=sha256(model_path.read_bytes()).hexdigest(),
source_version="v1",
)
evidence = RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
expected_model_version="v1",
allowed_frameworks=("ultralytics/pytorch",),
)
assert evidence.source_registry_id == str(source_registry_id)
assert evidence.source_snapshot_id == str(source_snapshot_id)
assert evidence.source_snapshot_checksum_sha256 == evidence.model_sha256
def test_production_runtime_rejects_missing_database_source_binding(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"unbound model bytes")
_write_sidecar(model_path)
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_production_runtime(
db=FakeSession(),
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
def test_production_runtime_requires_a_database_session(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"model bytes")
_write_sidecar(model_path)
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_production_runtime(
db=None,
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == "MODEL_PROVENANCE_DATABASE_REQUIRED"
@pytest.mark.parametrize(
("mutation", "expected_code"),
(
("registry_unsafe", "MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE"),
("snapshot_registry_mismatch", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH"),
("snapshot_missing", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND"),
("snapshot_quarantined", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE"),
("snapshot_checksum_mismatch", "MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH"),
("active_quarantine", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED"),
),
)
def test_production_runtime_rejects_unsafe_or_inconsistent_database_snapshot(
tmp_path: Path,
mutation: str,
expected_code: str,
) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"governed model bytes")
source_registry_id = uuid4()
source_snapshot_id = uuid4()
_write_sidecar(
model_path,
source_registry_id=str(source_registry_id),
source_snapshot_id=str(source_snapshot_id),
)
db, registry, snapshot = _governed_model_database(
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
model_checksum=sha256(model_path.read_bytes()).hexdigest(),
)
if mutation == "registry_unsafe":
registry.ingest_status = "quarantined"
elif mutation == "snapshot_registry_mismatch":
snapshot.source_registry_id = uuid4()
elif mutation == "snapshot_missing":
db.objects.pop((SourceSnapshot, source_snapshot_id))
elif mutation == "snapshot_quarantined":
snapshot.ingest_status = "quarantined"
elif mutation == "snapshot_checksum_mismatch":
snapshot.checksum_sha256 = "f" * 64
elif mutation == "active_quarantine":
snapshot.quarantines = [
DatasetQuarantine(
source_snapshot_id=source_snapshot_id,
stage="test",
reason_code="test_active_quarantine",
status="quarantined",
)
]
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == expected_code
def test_runtime_model_provenance_rejects_missing_sidecar(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"unmanifested local model bytes")
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_MISSING"
def test_runtime_model_provenance_rejects_model_bytes_tampered_after_manifest(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"original local model bytes")
_write_sidecar(model_path)
model_path.write_bytes(b"tampered local model bytes")
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == "MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH"
def test_runtime_model_provenance_rejects_tampered_manifest_contents(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"original local model bytes")
sidecar_path = _write_sidecar(model_path)
payload = json.loads(sidecar_path.read_text(encoding="utf-8"))
payload["model"]["class_mapping"]["1"] = "road"
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH"
def test_runtime_model_provenance_rejects_other_contract_even_if_structurally_valid(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local model bytes")
sidecar_path = _write_sidecar(model_path)
payload = json.loads(sidecar_path.read_text(encoding="utf-8"))
payload["data_contract"]["key"] = "geointel.vector.geojson"
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_INVALID"
@@ -0,0 +1,67 @@
from __future__ import annotations
from datetime import datetime, timezone
from unittest.mock import MagicMock
from app.models import AoiOperation, AoiOperationPartition, AnalysisRun, Job
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
def test_reconciliation_terminalizes_only_running_work() -> None:
db = MagicMock()
jobs = MagicMock()
runs = MagicMock()
resumable_partitions = MagicMock()
exhausted_partitions = MagicMock()
operations = MagicMock()
jobs.filter.return_value.update.return_value = 5
runs.filter.return_value.update.return_value = 2
resumable_partitions.filter.return_value.update.return_value = 3
exhausted_partitions.filter.return_value.update.return_value = 1
operations.filter.return_value.update.return_value = 2
db.query.side_effect = [jobs, runs, resumable_partitions, exhausted_partitions, operations]
finished_at = datetime(2026, 7, 17, 20, 0, tzinfo=timezone.utc)
result = RuntimeReconciliationService.reconcile(
db,
finished_at=finished_at,
)
assert result.interrupted_jobs == 5
assert result.interrupted_analysis_runs == 2
assert result.resumed_aoi_partitions == 3
assert result.exhausted_aoi_partitions == 1
jobs.filter.assert_called_once()
runs.filter.assert_called_once()
resumable_partitions.filter.assert_called_once()
exhausted_partitions.filter.assert_called_once()
operations.filter.assert_called_once()
job_values = jobs.filter.return_value.update.call_args.args[0]
run_values = runs.filter.return_value.update.call_args.args[0]
assert job_values[Job.status] == "failed"
assert job_values[Job.finished_at] == finished_at
assert "PROCESS_INTERRUPTED" in job_values[Job.error_message]
assert run_values[AnalysisRun.status] == "failed"
assert run_values[AnalysisRun.finished_at] == finished_at
assert "PROCESS_INTERRUPTED" in run_values[AnalysisRun.error_message]
resumed_values = resumable_partitions.filter.return_value.update.call_args.args[0]
exhausted_values = exhausted_partitions.filter.return_value.update.call_args.args[0]
operation_values = operations.filter.return_value.update.call_args.args[0]
assert resumed_values[AoiOperationPartition.status] == "queued"
assert exhausted_values[AoiOperationPartition.status] == "failed"
assert operation_values[AoiOperation.status] == "queued"
db.commit.assert_called_once_with()
def test_startup_reconciliation_is_enabled_only_in_all_in_one_runtime() -> None:
start_script = (
__import__("pathlib").Path(__file__).resolve().parents[2]
/ "deploy"
/ "unraid"
/ "all-in-one-start.sh"
).read_text(encoding="utf-8")
assert (
'GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true'
in start_script
)
@@ -0,0 +1,39 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
SCRIPT = Path(__file__).parents[2] / "scripts" / "refine_yolo_labels_with_sam.py"
SPEC = importlib.util.spec_from_file_location("sam_roof_refinement", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def test_plausible_refinement_is_fail_closed() -> None:
source = (10.0, 10.0, 30.0, 30.0)
limits = dict(
min_iou=0.15,
min_area_ratio=0.25,
max_area_ratio=4.0,
max_center_shift_ratio=0.75,
min_dimension_ratio=0.5,
max_dimension_ratio=2.0,
)
assert MODULE.plausible_refinement(source, (8.0, 9.0, 31.0, 32.0), **limits)
assert not MODULE.plausible_refinement(source, (100.0, 100.0, 120.0, 120.0), **limits)
assert not MODULE.plausible_refinement(source, (0.0, 0.0, 100.0, 100.0), **limits)
assert not MODULE.plausible_refinement(source, (10.0, 10.0, 51.0, 20.0), **limits)
assert not MODULE.plausible_refinement(source, (24.0, 24.0, 44.0, 44.0), **limits)
def test_yolo_round_trip_shape() -> None:
line = MODULE.yolo_line((10.0, 20.0, 30.0, 40.0), 100, 100)
assert line == "0 0.20000000 0.30000000 0.20000000 0.20000000"
def test_cli_exposes_explicit_fallback_policy() -> None:
source = SCRIPT.read_text(encoding="utf-8")
assert 'choices=("retain", "drop")' in source
assert '"dropped_fallback_label_count"' in source
@@ -0,0 +1,17 @@
from pydantic import BaseModel
from app.schemas import detection, segmentation
def test_model_prefixed_api_fields_are_explicitly_supported() -> None:
schemas = [
value
for module in (detection, segmentation)
for value in vars(module).values()
if isinstance(value, type)
and issubclass(value, BaseModel)
and any(field_name.startswith("model_") for field_name in value.model_fields)
]
assert schemas
assert all(schema.model_config.get("protected_namespaces") == () for schema in schemas)
@@ -0,0 +1,64 @@
from __future__ import annotations
from types import SimpleNamespace
import pytest
from app.core.config import Settings
from app.core.errors import AppError
from app.services.segmentation_adapter import YoloSegmentationAdapter
def _settings(*, require_cuda: bool, device: str) -> Settings:
return Settings(
_env_file=None,
YOLO_REQUIRE_CUDA=require_cuda,
YOLO_DEVICE=device,
)
def test_segmentation_runtime_allows_cpu_only_when_cuda_is_not_required() -> None:
adapter = YoloSegmentationAdapter(_settings(require_cuda=False, device="cpu"))
adapter.validate_runtime()
def test_segmentation_runtime_rejects_missing_cuda(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(
__import__("sys").modules,
"torch",
SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)),
)
adapter = YoloSegmentationAdapter(_settings(require_cuda=True, device="cuda:0"))
with pytest.raises(AppError) as exc_info:
adapter.validate_runtime()
assert exc_info.value.code == "SEGMENTATION_ACCELERATOR_UNAVAILABLE"
def test_segmentation_runtime_rejects_cpu_device_when_cuda_is_required(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setitem(
__import__("sys").modules,
"torch",
SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: True)),
)
adapter = YoloSegmentationAdapter(_settings(require_cuda=True, device="cpu"))
with pytest.raises(AppError) as exc_info:
adapter.validate_runtime()
assert exc_info.value.code == "SEGMENTATION_ACCELERATOR_MISCONFIGURED"
def test_segmentation_runtime_accepts_configured_cuda(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(
__import__("sys").modules,
"torch",
SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: True)),
)
adapter = YoloSegmentationAdapter(_settings(require_cuda=True, device="cuda:0"))
adapter.validate_runtime()
@@ -0,0 +1,567 @@
from __future__ import annotations
from hashlib import sha256
import json
from pathlib import Path
from uuid import uuid4
import pytest
from geoalchemy2.shape import to_shape
from app.core.config import Settings
from app.models import Dataset, DatasetVersion, Project, Segmentation, SourceRegistry, SourceSnapshot
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
from app.services.model_registry_service import ModelRegistryService
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.segmentation_service import SegmentationService
from app.services.tile_manifest_service import TileManifestService
ROOT = Path(__file__).resolve().parents[2]
class FakeSession:
def __init__(self, objects=None) -> None:
self.objects = objects or {}
self.added = []
self.commits = 0
self.refreshes = []
def get(self, model, item_id):
return self.objects.get((model, item_id))
def add(self, item) -> None:
self.added.append(item)
if getattr(item, "id", None) is not None:
self.objects[(item.__class__, item.id)] = item
def commit(self) -> None:
self.commits += 1
def refresh(self, item) -> None:
self.refreshes.append(item)
class AvailableSegAdapter:
def __init__(self, settings: Settings) -> None:
self.settings = settings
@staticmethod
def dependencies_available() -> bool:
return True
def load_model(self, model_path: Path):
return object()
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
return [
{
"class_name": "building",
"confidence": 0.91,
"points": [[10.0, 20.0], [30.0, 20.0], [30.0, 40.0], [10.0, 40.0]],
"bbox": [10.0, 20.0, 30.0, 40.0],
"properties": {"class_id": 0},
}
]
class ClassAgnosticSamAdapter(AvailableSegAdapter):
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
return [
{
"class_name": "segment",
"confidence": None,
"points": [[5.0, 5.0], [25.0, 5.0], [25.0, 25.0], [5.0, 25.0]],
"bbox": [5.0, 5.0, 25.0, 25.0],
"properties": {"class_id": -1},
}
]
class MissingDependencySegAdapter(AvailableSegAdapter):
@staticmethod
def dependencies_available() -> bool:
return False
class NeverLoadSegAdapter(AvailableSegAdapter):
load_calls = 0
def load_model(self, model_path: Path):
type(self).load_calls += 1
raise AssertionError("unmanifested weights must not reach adapter.load_model")
def _project_and_dataset(dataset_type: str = "raster"):
project_id = uuid4()
dataset_id = uuid4()
source_id = uuid4()
snapshot_id = uuid4()
checksum = "a" * 64
project = Project(id=project_id, name="Mol")
source = SourceRegistry(
id=source_id,
source_key="digitaal_vlaanderen_orthophoto",
display_name="Governed orthophoto test source",
classification="contextual",
authority_name="Digitaal Vlaanderen",
authority_scope_json={"zone": "Flanders", "role": "imagery"},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key="configured-segmentation-orthophoto",
checksum_sha256=checksum,
ingest_status="ingested",
freshness_status="current",
)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="ortho.tif",
dataset_type=dataset_type,
source="digitaal_vlaanderen_orthophoto",
source_name="digitaal_vlaanderen_orthophoto",
storage_path="storage/uploads/ortho.tif",
checksum_sha256=checksum,
crs="EPSG:4326",
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
source_registry_id=source_id,
source_snapshot_id=snapshot_id,
data_contract_key="geointel.raster.geotiff",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
status="ready",
)
dataset.source_registry = source
dataset.source_snapshot = snapshot
dataset.versions.append(
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
)
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
return db, project_id, dataset_id
def _settings(tmp_path: Path, **overrides) -> Settings:
values = {
# The runtime only consumes artifacts under the storage root, so a
# test that writes tiles into tmp_path must say that is the root.
"storage_root": str(tmp_path),
"yolo_seg_enabled": True,
"yolo_seg_model_path": str(tmp_path / "seg.pt"),
"sam_enabled": True,
"sam_model_path": str(tmp_path / "sam.pt"),
"yolo_max_tiles": 4,
}
values.update(overrides)
return Settings(**values)
def _write_model_sidecar(
model_path: Path,
*,
model_id: str,
framework: str,
source_version: str | None,
db: FakeSession | None = None,
) -> None:
"""Create explicit local test evidence; no production code creates sidecars."""
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
source_registry_id = uuid4()
source_snapshot_id = uuid4()
resolved_source_version = source_version or "test-v1"
if db is not None:
source_registry = SourceRegistry(
id=source_registry_id,
source_key="model",
display_name="Governed test model artifact",
classification="experimental",
authority_name="GeoIntel test fixture",
freshness_status="current",
ingest_status="configured",
)
source_snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key=f"model-{model_id}-{resolved_source_version}",
source_version=resolved_source_version,
checksum_sha256=model_sha256,
freshness_status="current",
ingest_status="ingested",
)
db.objects[(SourceRegistry, source_registry_id)] = source_registry
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
payload = {
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
"model": {
"model_id": model_id,
"task_type": "segmentation",
"sha256": model_sha256,
"model_format": "pytorch",
"framework": framework,
"class_mapping": {"0": "segment"},
"source_version": resolved_source_version,
},
"source": {
"source_registry_id": str(source_registry_id),
"source_snapshot_id": str(source_snapshot_id),
"source_registry_key": "model",
"source_snapshot_checksum_sha256": model_sha256,
},
"lineage": {
"upstream_asset_ids": ["test-training-corpus"],
"upstream_checksums_sha256": ["a" * 64],
"transformations": [
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
],
},
"metadata": {"training_manifest_sha256": "c" * 64},
"imported_at": "2026-08-01T10:00:00+00:00",
}
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
json.dumps(payload, sort_keys=True),
encoding="utf-8",
)
def _write_configured_model_sidecars(
tmp_path: Path,
settings: Settings,
*,
include_yolo: bool = True,
include_sam: bool = True,
db: FakeSession | None = None,
) -> None:
if include_yolo:
_write_model_sidecar(
tmp_path / "seg.pt",
model_id=settings.yolo_seg_model_id,
framework="ultralytics/pytorch",
source_version=settings.yolo_seg_model_version,
db=db,
)
if include_sam:
_write_model_sidecar(
tmp_path / "sam.pt",
model_id=settings.sam_model_id,
framework="ultralytics/sam",
source_version=settings.sam_model_version,
db=db,
)
def _manifest(
tmp_path: Path,
tile_count: int = 1,
*,
db: FakeSession | None = None,
dataset: Dataset | None = None,
) -> Path:
tiles = []
for index in range(tile_count):
tile_path = tmp_path / f"tile_{index:04d}.tif"
tile_path.write_bytes(b"fixture")
tile = {
"path": str(tile_path),
"pixel_window": [0, 0, 100, 100],
"bounds": [4.0, 51.0, 5.0, 52.0],
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
"crs": "EPSG:4326",
"index": index,
**TileManifestService.tile_integrity(tile_path),
}
tiles.append(tile)
binding = TileManifestService.dataset_binding(db or FakeSession(), dataset) if dataset is not None else {}
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text(
json.dumps(
{
**binding,
"tile_set_id": "tiles-fixture",
"source_dataset_id": binding.get("source_dataset_id", str(uuid4())),
"source_raster_id": binding.get("source_raster_id", str(uuid4())),
"crs": "EPSG:4326",
"bounds": [4.0, 51.0, 5.0, 52.0],
"tile_size": 100,
"overlap": 0,
"count": tile_count,
"tiles": tiles,
}
),
encoding="utf-8",
)
return manifest_path
def test_segmentation_models_report_not_configured_when_disabled(tmp_path: Path) -> None:
settings = _settings(tmp_path, yolo_seg_enabled=False, sam_enabled=False)
models = {
model.model_id: model
for model in ModelRegistryService.list_segmentation_model_capabilities(settings=settings)
}
assert models["yolo-seg-configured"].configured is False
assert models["yolo-seg-configured"].status == "not_configured"
assert models["sam-configured"].configured is False
assert models["sam-configured"].status == "not_configured"
def test_segmentation_models_report_dependency_unavailable(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"weights")
(tmp_path / "sam.pt").write_bytes(b"weights")
settings = _settings(tmp_path)
models = {
model.model_id: model
for model in ModelRegistryService.list_segmentation_model_capabilities(
settings=settings,
yolo_seg_adapter_class=MissingDependencySegAdapter,
sam_adapter_class=MissingDependencySegAdapter,
)
}
assert models["yolo-seg-configured"].status == "dependency_unavailable"
assert models["sam-configured"].status == "dependency_unavailable"
def test_segmentation_models_require_runtime_provenance_sidecars(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"unmanifested yolo segmentation weights")
(tmp_path / "sam.pt").write_bytes(b"unmanifested sam weights")
settings = _settings(tmp_path)
models = {
model.model_id: model
for model in ModelRegistryService.list_segmentation_model_capabilities(
settings=settings,
yolo_seg_adapter_class=AvailableSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
}
assert models["yolo-seg-configured"].configured is False
assert models["yolo-seg-configured"].status == "contract_incomplete"
assert models["sam-configured"].configured is False
assert models["sam-configured"].status == "contract_incomplete"
def test_segmentation_models_report_configured_with_local_weights(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"weights")
(tmp_path / "sam.pt").write_bytes(b"weights")
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings)
models = {
model.model_id: model
for model in ModelRegistryService.list_segmentation_model_capabilities(
settings=settings,
yolo_seg_adapter_class=AvailableSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
}
assert models["yolo-seg-configured"].configured is True
assert models["yolo-seg-configured"].status == "configured"
assert models["sam-configured"].configured is True
assert models["sam-configured"].status == "configured"
def test_segmentation_dependency_check_uses_real_imports_not_find_spec() -> None:
source = (ROOT / "backend" / "app" / "services" / "segmentation_adapter.py").read_text(encoding="utf-8")
assert 'find_spec("ultralytics")' not in source
assert "import ultralytics" in source
assert "import torch" in source
def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
with pytest.raises(Exception) as exc_info:
SegmentationService.run_segmentation(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-seg-configured",
confidence_threshold=0.5,
settings=settings,
yolo_seg_adapter_class=AvailableSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
assert getattr(exc_info.value, "code", None) == "SEGMENTATION_TILE_MANIFEST_REQUIRED"
def test_configured_segmentation_fails_before_adapter_load_without_sidecar(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"unmanifested weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
NeverLoadSegAdapter.load_calls = 0
response = SegmentationService.run_segmentation(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-seg-configured",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
settings=settings,
yolo_seg_adapter_class=NeverLoadSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
assert response.status == "failed"
assert response.error_code == "SEGMENTATION_MODEL_UNAVAILABLE"
assert NeverLoadSegAdapter.load_calls == 0
def test_configured_segmentation_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"structurally valid but unbound weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
# The sidecar passes catalog validation but its source registry/snapshot
# was never registered in this production-session fixture.
_write_configured_model_sidecars(tmp_path, settings, include_sam=False)
NeverLoadSegAdapter.load_calls = 0
response = SegmentationService.run_segmentation(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-seg-configured",
confidence_threshold=0.5,
tile_manifest_path=str(
_manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
),
settings=settings,
yolo_seg_adapter_class=NeverLoadSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
assert response.status == "failed"
assert response.error_code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
assert NeverLoadSegAdapter.load_calls == 0
def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
manifest_path = _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
response = SegmentationService.run_segmentation(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-seg-configured",
confidence_threshold=0.5,
tile_manifest_path=str(manifest_path),
settings=settings,
yolo_seg_adapter_class=AvailableSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
assert response.status == "success"
assert response.segmentation_count == 1
persisted = [item for item in db.added if isinstance(item, Segmentation)]
assert len(persisted) == 1
segmentation = persisted[0]
assert segmentation.class_name == "building"
assert segmentation.confidence == pytest.approx(0.91)
geometry = to_shape(segmentation.geometry)
assert geometry.geom_type == "MultiPolygon"
min_x, min_y, max_x, max_y = geometry.bounds
assert 4.0 <= min_x <= 5.0
assert 51.0 <= min_y <= 52.0
assert max_x <= 5.0
assert max_y <= 52.0
assert segmentation.area_m2 is not None and segmentation.area_m2 > 0
assert segmentation.provenance_json["inference"] == "local"
assert segmentation.provenance_json["model_id"] == "yolo-seg-configured"
assert segmentation.provenance_json["runtime_model_provenance"]["data_contract_key"] == "geointel.model.pytorch"
def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None:
(tmp_path / "sam.pt").write_bytes(b"weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings, include_yolo=False, db=db)
manifest_path = _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
response = SegmentationService.run_segmentation(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="sam-configured",
confidence_threshold=0.5,
tile_manifest_path=str(manifest_path),
settings=settings,
yolo_seg_adapter_class=AvailableSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
assert response.status == "success"
assert response.segmentation_count == 1
persisted = [item for item in db.added if isinstance(item, Segmentation)]
assert persisted[0].class_name == "segment"
assert persisted[0].confidence is None
def test_unconfigured_segmentation_run_fails_closed(tmp_path: Path) -> None:
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path, yolo_seg_enabled=False)
manifest_path = _manifest(tmp_path)
response = SegmentationService.run_segmentation(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-seg-configured",
confidence_threshold=0.5,
tile_manifest_path=str(manifest_path),
settings=settings,
yolo_seg_adapter_class=AvailableSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
assert response.status == "failed"
assert response.error_code == "SEGMENTATION_MODEL_UNAVAILABLE"
assert not [item for item in db.added if isinstance(item, Segmentation)]
def test_pixel_points_to_epsg4326_polygon_uses_tile_transform() -> None:
tile = {
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
"bounds": [4.0, 51.0, 5.0, 52.0],
"pixel_window": [0, 0, 100, 100],
}
polygon = pixel_points_to_epsg4326_polygon(
points=[[0.0, 0.0], [100.0, 0.0], [100.0, 100.0], [0.0, 100.0]],
tile=tile,
crs="EPSG:4326",
)
min_x, min_y, max_x, max_y = polygon.bounds
assert min_x == pytest.approx(4.0)
assert max_x == pytest.approx(5.0)
assert min_y == pytest.approx(51.0)
assert max_y == pytest.approx(52.0)
def test_pixel_points_to_epsg4326_polygon_rejects_degenerate_input() -> None:
tile = {"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01]}
with pytest.raises(Exception) as exc_info:
pixel_points_to_epsg4326_polygon(points=[[0.0, 0.0], [1.0, 1.0]], tile=tile)
assert getattr(exc_info.value, "code", None) == "SEGMENTATION_INVALID_MASK"
@@ -0,0 +1,61 @@
"""Segmentation must honour the same post-processing configuration as detection.
Segmentation reuses the detection suppressor but passed only the IoU threshold,
so it silently fell back to the hardcoded containment constant while detection
read a configured one. A deployment tuning containment for a promoted model
changed detection behaviour and left segmentation on the old value.
"""
from __future__ import annotations
import pytest
from shapely.geometry import box
from app.core.config import Settings
def test_segmentation_has_its_own_containment_setting() -> None:
settings = Settings(_env_file=None)
assert settings.segmentation_containment_nms_threshold == pytest.approx(0.85)
def test_the_setting_is_independent_of_the_detection_one() -> None:
"""Masks and boxes overlap differently; one value need not fit both."""
settings = Settings(
_env_file=None,
yolo_containment_nms_threshold=0.7,
segmentation_containment_nms_threshold=0.95,
)
assert settings.yolo_containment_nms_threshold == pytest.approx(0.7)
assert settings.segmentation_containment_nms_threshold == pytest.approx(0.95)
def test_the_configured_value_reaches_the_suppressor() -> None:
from app.services.detection_service import DetectionService
def candidate(name: str, geometry, confidence: float):
return {
"class_name": "building",
"confidence": confidence,
"geometry": geometry,
"bbox": [0.0, 0.0, 1.0, 1.0],
"source_tile_path": f"/tiles/{name}.tif",
"properties": {"name": name},
}
outer = candidate("outer", box(0, 0, 10, 10), 0.9)
# Containment 0.9, IoU 0.09: only the containment rule can act on this pair.
mostly_nested = candidate("mostly", box(8.2, 1, 10.2, 6), 0.5)
strict = DetectionService._suppress_duplicate_candidates(
[outer, mostly_nested], iou_threshold=0.5, containment_threshold=0.95
)
loose = DetectionService._suppress_duplicate_candidates(
[outer, mostly_nested], iou_threshold=0.5, containment_threshold=0.7
)
assert len(strict) == 2
assert len(loose) == 1
@@ -0,0 +1,174 @@
"""Segmentation QA must score against the area it actually inferred.
Detection QA already clips both populations to the union of the persisted
inference tiles. Segmentation QA compared candidates against every reference
feature in the dataset, so every building outside the inferred tiles counted
as a false negative and recall collapsed for no modelling reason.
"""
from __future__ import annotations
import json
from pathlib import Path
from uuid import uuid4
import pytest
from geoalchemy2.shape import from_shape
from shapely.geometry import MultiPolygon, box
from app.core.errors import AppError
from app.models import AnalysisRun, Dataset, Segmentation, VectorFeature
from app.services.segmentation_service import SegmentationService
from tests.test_sprint9_segmentation_foundation import ( # noqa: F401
FakeSession,
_authoritative_reference,
)
def _manifest(tmp_path: Path, dataset_id, bounds: list[float]) -> str:
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text(
json.dumps(
{
"source_dataset_id": str(dataset_id),
"crs": "EPSG:4326",
"tiles": [{"path": "tile_0000.tif", "bounds": bounds, "crs": "EPSG:4326"}],
}
),
encoding="utf-8",
)
return str(manifest_path)
def _segmentation(project_id, dataset_id, analysis_run_id, geom):
return Segmentation(
id=uuid4(),
project_id=project_id,
dataset_id=dataset_id,
analysis_run_id=analysis_run_id,
job_id=uuid4(),
model_name="fixture-segmenter",
model_version="fixture-v1",
class_name="building",
confidence=0.9,
geometry=from_shape(MultiPolygon([geom]), srid=4326),
)
def _reference(dataset_id, geom) -> VectorFeature:
return VectorFeature(
id=uuid4(),
dataset_id=dataset_id,
feature_class="building",
geometry=from_shape(geom, srid=4326),
)
def _session(tmp_path: Path, *, with_manifest: bool):
project_id = uuid4()
dataset_id = uuid4()
reference_dataset_id = uuid4()
analysis_run_id = uuid4()
parameters = {}
if with_manifest:
parameters = {"tile_manifest_path": _manifest(tmp_path, dataset_id, [0.0, 0.0, 1.0, 1.0])}
reference_dataset = _authoritative_reference(
Dataset(
id=reference_dataset_id,
project_id=project_id,
name="reference.geojson",
dataset_type="vector",
source="test",
dataset_role="reference",
)
)
db = FakeSession(
objects={
(AnalysisRun, analysis_run_id): AnalysisRun(
id=analysis_run_id,
project_id=project_id,
dataset_id=dataset_id,
analysis_type="segmentation",
status="success",
parameters_json=parameters,
),
(Dataset, dataset_id): Dataset(
id=dataset_id, project_id=project_id, name="fixture.tif", dataset_type="raster", source="test"
),
(Dataset, reference_dataset_id): reference_dataset,
},
query_rows={
Segmentation: [_segmentation(project_id, dataset_id, analysis_run_id, box(0.1, 0.1, 0.2, 0.2))],
VectorFeature: [
# Inside the inferred tile: a genuine match.
_reference(reference_dataset_id, box(0.1, 0.1, 0.2, 0.2)),
# Far outside it: never looked at by the model.
_reference(reference_dataset_id, box(8.0, 8.0, 8.1, 8.1)),
_reference(reference_dataset_id, box(9.0, 9.0, 9.1, 9.1)),
],
},
)
return db, analysis_run_id, reference_dataset_id
def test_segmentation_qa_scores_only_inside_persisted_tile_coverage(tmp_path: Path, monkeypatch) -> None:
# A manifest written into tmp_path is only a governed artifact if
# tmp_path is the storage root.
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
db, analysis_run_id, reference_dataset_id = _session(tmp_path, with_manifest=True)
result = SegmentationService.compare_segmentations_with_reference(
db=db,
analysis_run_id=analysis_run_id,
reference_dataset_id=reference_dataset_id,
iou_threshold=0.5,
)
assert result["matches"] == 1
assert result["false_negatives"] == 0
assert result["recall"] == 1.0
assert result["coverage"]["applied"] is True
assert result["coverage"]["reference_raw_count"] == 3
assert result["coverage"]["reference_evaluated_count"] == 1
assert result["coverage"]["reference_excluded_outside_count"] == 2
assert any("tile" in warning for warning in result["warnings"])
def test_segmentation_qa_without_manifest_reports_unbounded_coverage(tmp_path: Path) -> None:
db, analysis_run_id, reference_dataset_id = _session(tmp_path, with_manifest=False)
result = SegmentationService.compare_segmentations_with_reference(
db=db,
analysis_run_id=analysis_run_id,
reference_dataset_id=reference_dataset_id,
iou_threshold=0.5,
)
# Unchanged behaviour, but the response now says the score was not bounded
# by an inference footprint so the recall can be read correctly.
assert result["false_negatives"] == 2
assert result["coverage"]["applied"] is False
assert result["coverage"]["mode"] == "unbounded_no_manifest"
def test_segmentation_qa_rejects_reference_entirely_outside_coverage(tmp_path: Path, monkeypatch) -> None:
# A manifest written into tmp_path is only a governed artifact if
# tmp_path is the storage root.
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
db, analysis_run_id, reference_dataset_id = _session(tmp_path, with_manifest=True)
db.query_rows[VectorFeature] = [
_reference(reference_dataset_id, box(8.0, 8.0, 8.1, 8.1)),
]
with pytest.raises(AppError) as exc_info:
SegmentationService.compare_segmentations_with_reference(
db=db,
analysis_run_id=analysis_run_id,
reference_dataset_id=reference_dataset_id,
iou_threshold=0.5,
)
assert exc_info.value.code == "REFERENCE_FEATURES_OUTSIDE_COVERAGE"
@@ -0,0 +1,151 @@
"""Regression coverage for bounded, stable segmentation result listings."""
from __future__ import annotations
from datetime import UTC, datetime
from types import SimpleNamespace
from uuid import UUID
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from app.api.routes import segmentation as segmentation_routes
from app.db.session import get_db
from app.schemas.segmentation import SegmentationListResponse
from app.services.segmentation_service import SegmentationService
RUN_ID = UUID("00000000-0000-0000-0000-000000000101")
DATASET_ID = UUID("00000000-0000-0000-0000-000000000102")
PROJECT_ID = UUID("00000000-0000-0000-0000-000000000103")
def _segmentation(index: int) -> SimpleNamespace:
return SimpleNamespace(
id=UUID(int=index + 1),
project_id=PROJECT_ID,
dataset_id=DATASET_ID,
analysis_run_id=RUN_ID,
job_id=None,
model_name="segmentation-test-model",
model_version="1",
class_name="building",
confidence=0.99 - index / 100,
bbox_json=None,
area_m2=float(index + 1),
mask_path=None,
source_tile_path=None,
tile_index=index,
properties_json={},
provenance_json={},
created_at=datetime(2026, 8, 23, tzinfo=UTC),
)
class _Session:
def get(self, _model, identifier):
if identifier == RUN_ID:
return SimpleNamespace(analysis_type="segmentation")
return None
def test_service_returns_one_stable_page_with_complete_metadata(monkeypatch) -> None:
rows = [_segmentation(index) for index in range(5)]
monkeypatch.setattr(
SegmentationService,
"_query_segmentation_rows",
staticmethod(lambda _db, **_filters: rows),
)
result = SegmentationService.list_segmentations(
_Session(),
analysis_run_id=RUN_ID,
dataset_id=DATASET_ID,
limit=2,
offset=1,
)
assert [item.id for item in result.items] == [rows[1].id, rows[2].id]
assert result.total == 5
assert result.limit == 2
assert result.offset == 1
assert result.truncated is True
def test_service_pages_cover_the_stably_ordered_population_once(monkeypatch) -> None:
rows = [_segmentation(index) for index in range(5)]
monkeypatch.setattr(
SegmentationService,
"_query_segmentation_rows",
staticmethod(lambda _db, **_filters: rows),
)
seen = []
for offset in (0, 2, 4):
result = SegmentationService.list_segmentations(
_Session(),
dataset_id=DATASET_ID,
limit=2,
offset=offset,
)
seen.extend(item.id for item in result.items)
assert result.total == len(rows)
assert result.offset == offset
assert seen == [row.id for row in rows]
@pytest.mark.parametrize(
("path", "expected_run_id", "expected_dataset_id"),
[
(f"/api/v1/segmentation/runs/{RUN_ID}/segmentations", RUN_ID, None),
(f"/api/v1/segmentation/datasets/{DATASET_ID}/segmentations", None, DATASET_ID),
],
)
def test_both_listing_routes_forward_the_page_window_and_return_it(
monkeypatch,
path: str,
expected_run_id: UUID | None,
expected_dataset_id: UUID | None,
) -> None:
calls: list[dict] = []
def _list(_db, analysis_run_id=None, **parameters):
calls.append({"analysis_run_id": analysis_run_id, **parameters})
return SegmentationListResponse(
items=[],
total=9,
limit=2,
offset=4,
truncated=True,
)
monkeypatch.setattr(SegmentationService, "list_segmentations", staticmethod(_list))
app = FastAPI()
app.include_router(segmentation_routes.router, prefix="/api/v1")
app.dependency_overrides[get_db] = lambda: object()
response = TestClient(app).get(
path,
params={"limit": 2, "offset": 4, "class_name": "building", "min_confidence": 0.5},
)
assert response.status_code == 200
assert response.json()["data"] == {
"items": [],
"total": 9,
"limit": 2,
"offset": 4,
"truncated": True,
}
assert calls == [
{
"analysis_run_id": expected_run_id,
"limit": 2,
"offset": 4,
"dataset_id": expected_dataset_id,
"class_name": "building",
"min_confidence": 0.5,
}
]
@@ -0,0 +1,103 @@
from __future__ import annotations
from uuid import uuid4
import pytest
from pydantic import ValidationError
from app.api.routes.selection_partitions import select_vector_partitions
from app.core.errors import AppError
from app.models import Dataset
from app.schemas.selection_partitions import VectorPartitionSelectionRequest
from app.services.vector_feature_service import VectorFeatureService
class DatasetQuery:
def __init__(self, datasets):
self.datasets = datasets
def filter(self, *_args):
return self
def all(self):
return self.datasets
class DatasetSession:
def __init__(self, datasets):
self.datasets = datasets
def query(self, model):
assert model is Dataset
return DatasetQuery(self.datasets)
def make_dataset(project_id, dataset_id, *, source_name="grb", product_key="buildings"):
return Dataset(
id=dataset_id,
project_id=project_id,
name=f"{source_name}-{product_key}",
dataset_type="vector",
source="official",
dataset_role="reference",
source_name=source_name,
reference_layer_name=product_key,
source_metadata={"product_key": product_key, "theme": product_key},
provenance_metadata={},
metadata_json={},
status="ready",
)
def test_vector_partition_route_combines_one_governed_product(monkeypatch) -> None:
project_id = uuid4()
dataset_ids = [uuid4(), uuid4()]
db = DatasetSession([make_dataset(project_id, dataset_id) for dataset_id in dataset_ids])
captured = {}
def select_features(_db, **kwargs):
captured.update(kwargs)
return {
"selection_bbox": kwargs["bbox"],
"feature_count": 1,
"total_feature_count": 3,
"limit": kwargs["limit"],
"truncated": False,
"geojson": {"type": "FeatureCollection", "features": []},
"summary": None,
}
monkeypatch.setattr(VectorFeatureService, "select_features_by_bbox", select_features)
payload = VectorPartitionSelectionRequest(
dataset_ids=dataset_ids,
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.3, "max_y": 51.2},
)
response = select_vector_partitions(project_id, payload, db)
assert captured["dataset_ids"] == dataset_ids
assert captured["deduplicate_source_features"] is True
assert response["data"]["partition_count"] == 2
assert response["data"]["dataset_ids"] == dataset_ids
def test_vector_partition_request_has_a_bounded_fan_out() -> None:
with pytest.raises(ValidationError):
VectorPartitionSelectionRequest(
dataset_ids=[uuid4() for _ in range(4097)],
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.3, "max_y": 51.2},
)
def test_vector_partition_route_rejects_mixed_source_products() -> None:
project_id = uuid4()
datasets = [
make_dataset(project_id, uuid4(), source_name="grb", product_key="buildings"),
make_dataset(project_id, uuid4(), source_name="spw_picc", product_key="picc_buildings"),
]
payload = VectorPartitionSelectionRequest(
dataset_ids=[dataset.id for dataset in datasets],
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.3, "max_y": 51.2},
)
with pytest.raises(AppError) as exc_info:
select_vector_partitions(project_id, payload, DatasetSession(datasets))
assert getattr(exc_info.value, "code", None) == "VECTOR_PARTITION_SOURCE_MISMATCH"
@@ -0,0 +1,121 @@
"""A selection finer than the source raster must answer, not return zero.
End-to-end counterpart to ``test_raster_cell_selection``: the analysis reads a
real GeoTIFF, so it proves the fallback survives the clip/mask path the service
actually uses rather than only the helper in isolation.
"""
from __future__ import annotations
from pathlib import Path
from uuid import uuid4
import pytest
np = pytest.importorskip("numpy")
rasterio = pytest.importorskip("rasterio")
from pyproj import Transformer # noqa: E402 - optional rasterio gate precedes geospatial imports
from rasterio.transform import from_origin # noqa: E402 - optional rasterio gate precedes geospatial imports
from app.core.config import Settings # noqa: E402 - optional rasterio gate precedes app imports
from app.models import Dataset # noqa: E402 - optional rasterio gate precedes app imports
from app.schemas.flood_hazard import FloodHazardSelectionRequest # noqa: E402 - optional rasterio gate precedes app imports
from app.services.flood_hazard_acquisition_service import ( # noqa: E402 - optional rasterio gate precedes app imports
FloodHazardAcquisitionService,
)
from app.services.flood_hazard_analysis_service import ( # noqa: E402 - optional rasterio gate precedes app imports
FloodHazardAnalysisService,
)
TO_4326 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
PRODUCT_KEY = "fluviaal_current_t100"
class FakeSession:
def __init__(self, objects):
self.objects = objects
def get(self, model, item_id):
return self.objects.get((model, item_id))
def _write_raster(path: Path, *, resolution: float, depth: float) -> None:
values = np.full((4, 4), depth, dtype="float32")
with rasterio.open(
path,
"w",
driver="GTiff",
width=4,
height=4,
count=1,
dtype="float32",
crs="EPSG:31370",
transform=from_origin(200_000, 210_000, resolution, resolution),
nodata=-9999.0,
) as output:
output.write(values, 1)
def _dataset(project_id, dataset_id, path: Path) -> Dataset:
return Dataset(
id=dataset_id,
project_id=project_id,
name="vmm-flood.tif",
dataset_type="raster",
source="vmm",
source_name=FloodHazardAcquisitionService.PROVIDER,
status="ready",
storage_path=str(path),
source_metadata={"product_key": PRODUCT_KEY, "normalized_value_unit": "m"},
)
def _bbox_for(min_x: float, min_y: float, max_x: float, max_y: float) -> dict:
left, bottom = TO_4326.transform(min_x, min_y)
right, top = TO_4326.transform(max_x, max_y)
return {"min_x": left, "min_y": bottom, "max_x": right, "max_y": top, "crs": "EPSG:4326"}
def _analyze(tmp_path: Path, bbox: dict, *, resolution: float = 100.0) -> dict:
project_id = uuid4()
dataset_id = uuid4()
path = tmp_path / "flood.tif"
_write_raster(path, resolution=resolution, depth=2.0)
dataset = _dataset(project_id, dataset_id, path)
db = FakeSession({(Dataset, dataset_id): dataset})
return FloodHazardAnalysisService.analyze(
db,
project_id,
dataset_id,
FloodHazardSelectionRequest(bbox=bbox),
settings=Settings(_env_file=None),
)
def test_a_selection_smaller_than_one_cell_reports_the_cell_it_touches(tmp_path: Path) -> None:
# A 40 x 30 m rectangle wholly inside one 100 m cell: no cell centre falls
# inside it, so the centre rule alone would report an empty selection.
result = _analyze(tmp_path, _bbox_for(200_010, 209_960, 200_050, 209_990))
assert result["inundated_cell_count"] == 1
assert result["inundated_fraction"] == pytest.approx(1.0)
assert "kleiner dan één rastercel" in result["coverage_warning"]
def test_a_normal_selection_is_unaffected(tmp_path: Path) -> None:
result = _analyze(tmp_path, _bbox_for(200_000, 209_700, 200_300, 210_000))
assert result["inundated_cell_count"] >= 9
assert result["coverage_warning"] is None
def test_the_reported_area_matches_the_cells_that_were_analysed(tmp_path: Path) -> None:
result = _analyze(tmp_path, _bbox_for(200_010, 209_960, 200_050, 209_990))
metrics = {item["metric_key"]: item["metric_value"] for item in result["summary"]["metrics"]}
# One 100 x 100 m cell, not the 0.12 ha that was drawn.
assert metrics["modelled_inundated_area_ha"] == pytest.approx(1.0)
assert metrics["selection_area_ha"] == pytest.approx(1.0)

Some files were not shown because too many files have changed in this diff Show More