Initial public release
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
"""Canonical backend-test import boundary.
|
||||
|
||||
Pytest is intentionally runnable from ``backend/`` because that is the CI
|
||||
entrypoint. Some contract tests exercise repository-level deterministic
|
||||
scripts; put the canonical repository root ahead of the legacy
|
||||
``backend/scripts`` helper directory so those imports resolve to the code that
|
||||
is actually shipped by the root Docker build.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
||||
repository_root_text = str(REPOSITORY_ROOT)
|
||||
if repository_root_text not in sys.path:
|
||||
sys.path.insert(0, repository_root_text)
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Helpers for asserting frontend wiring from the backend test suite.
|
||||
|
||||
Most of this suite checks frontend behaviour by reading TSX files and
|
||||
asserting that literal substrings occur in them. That reds the suite on every
|
||||
rename and every copy change while proving nothing about behaviour: renaming a
|
||||
button label is not a regression, and ``source.count("useEffect(") == 1`` is a
|
||||
formatting rule, not a contract.
|
||||
|
||||
These helpers keep the useful half of that idea — that a documented product
|
||||
contract must remain wired somewhere in the frontend — and drop the brittle
|
||||
half. Assert on identifiers, API paths and prop names, which only change when
|
||||
the wiring genuinely changes. Do not assert on user-visible copy; put that in
|
||||
a frontend component test where the rendered output can be checked properly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
FRONTEND_SRC = ROOT / "frontend" / "src"
|
||||
|
||||
|
||||
# A feature is one behaviour spread over several modules: a container, its
|
||||
# hooks, its domain layer, its pure helpers. A contract belongs to the feature,
|
||||
# not to whichever file currently holds it, so moving code between siblings
|
||||
# must not red the suite. Missing entries are skipped, so a group survives a
|
||||
# module being split further, renamed or merged back.
|
||||
#
|
||||
# Use these for *positive* contracts ("this is wired"). A negative contract
|
||||
# ("this component performs no transport") is a statement about one file and
|
||||
# must keep reading that file, or widening it would quietly weaken the check.
|
||||
FEATURE_SOURCES: dict[str, tuple[str, ...]] = {
|
||||
"map_workspace": (
|
||||
"components/map/MapWorkspace.tsx",
|
||||
"components/map/mapWorkspaceProps.ts",
|
||||
"components/map/mapWorkspaceThemes.ts",
|
||||
"components/map/mapWorkspaceUtils.ts",
|
||||
"components/map/useMapWorkspaceViewModel.ts",
|
||||
"components/map/MapExplorerView.tsx",
|
||||
"components/map/MapAdvancedWorkbench.tsx",
|
||||
"components/map/MunicipalitySearch.tsx",
|
||||
"hooks/useMapImageOverlays.ts",
|
||||
"hooks/useMapRectangleSelection.ts",
|
||||
"hooks/useFullGisWorkflow.ts",
|
||||
"hooks/useMapThemeSelectionInsights.ts",
|
||||
"hooks/useMapSelectionExtract.ts",
|
||||
"hooks/useMapWorkspaceState.ts",
|
||||
"hooks/useMapSelectionDataset.ts",
|
||||
"hooks/useMapSelectionQa.ts",
|
||||
"hooks/useMapThemeSelectionInsights.ts",
|
||||
"hooks/useTemporalComparison.ts",
|
||||
"hooks/useCoverageResolver.ts",
|
||||
"hooks/useOfficialMapProducts.ts",
|
||||
),
|
||||
# The presentational half of the map workspace. Transport belongs to the
|
||||
# hooks, so "this performs no transport" is a contract about these modules
|
||||
# and would fail — correctly — against the whole feature.
|
||||
"map_workspace_presentation": (
|
||||
"components/map/MapWorkspace.tsx",
|
||||
"components/map/MapExplorerView.tsx",
|
||||
"components/map/MapAdvancedWorkbench.tsx",
|
||||
),
|
||||
"detection": (
|
||||
"components/detection/DetectionLab.tsx",
|
||||
"components/detection/DetectionModelManagement.tsx",
|
||||
"components/detection/detectionProfiles.ts",
|
||||
"components/models/ModelSelector.tsx",
|
||||
"components/models/modelOptions.ts",
|
||||
"hooks/useDetectionWorkflow.ts",
|
||||
),
|
||||
"segmentation": (
|
||||
"components/segmentation/SegmentationLab.tsx",
|
||||
"hooks/useSegmentationWorkflow.ts",
|
||||
),
|
||||
"quality": (
|
||||
"components/quality/QualityResultsPanel.tsx",
|
||||
"components/quality/DetectionReviewPanel.tsx",
|
||||
"hooks/useQualityWorkflow.ts",
|
||||
),
|
||||
"datasets": (
|
||||
"components/datasets/DatasetPanel.tsx",
|
||||
"components/datasets/DatasetDetailPanel.tsx",
|
||||
"components/datasets/RasterControls.tsx",
|
||||
"components/datasets/VectorControls.tsx",
|
||||
"components/datasets/SourceCatalogPanel.tsx",
|
||||
"hooks/useDatasetWorkflow.ts",
|
||||
"services/api/datasets.ts",
|
||||
),
|
||||
"exports": (
|
||||
"components/exports/ExportCenter.tsx",
|
||||
"hooks/useExportWorkflow.ts",
|
||||
),
|
||||
"shell": (
|
||||
"App.tsx",
|
||||
"WorkbenchApp.tsx",
|
||||
"components/shell/WorkbenchNavigation.tsx",
|
||||
"components/shell/SecondaryDisplay.tsx",
|
||||
"components/inspector/WorkbenchInspector.tsx",
|
||||
"components/overview/OverviewWorkspace.tsx",
|
||||
"hooks/useProjectWorkspace.ts",
|
||||
"hooks/useWorkbenchBootstrap.ts",
|
||||
),
|
||||
}
|
||||
|
||||
MAP_WORKSPACE_SOURCES = FEATURE_SOURCES["map_workspace"]
|
||||
|
||||
|
||||
def read_frontend(relative_path: str) -> str:
|
||||
"""Read one frontend source file relative to ``frontend/src``."""
|
||||
|
||||
return (FRONTEND_SRC / relative_path).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def read_frontend_area(*relative_paths: str) -> str:
|
||||
"""Read several related sources as one body of code.
|
||||
|
||||
Files that do not exist are skipped, so this survives a module being split
|
||||
further or merged back.
|
||||
"""
|
||||
|
||||
parts: list[str] = []
|
||||
for relative_path in relative_paths:
|
||||
path = FRONTEND_SRC / relative_path
|
||||
if path.is_file():
|
||||
parts.append(path.read_text(encoding="utf-8"))
|
||||
return chr(10).join(parts)
|
||||
|
||||
|
||||
def read_feature(name: str) -> str:
|
||||
"""Every module of one feature, whichever files it is currently split into."""
|
||||
|
||||
try:
|
||||
sources = FEATURE_SOURCES[name]
|
||||
except KeyError: # pragma: no cover - a typo should fail loudly
|
||||
raise AssertionError(
|
||||
f"Unknown frontend feature {name!r}; known: {sorted(FEATURE_SOURCES)}"
|
||||
) from None
|
||||
return read_frontend_area(*sources)
|
||||
|
||||
|
||||
def read_map_workspace() -> str:
|
||||
"""The whole map workspace feature, whichever modules it is split into."""
|
||||
|
||||
return read_feature("map_workspace")
|
||||
|
||||
|
||||
def assert_wired(source: str, *identifiers: str, context: str = "frontend source") -> None:
|
||||
"""Every identifier must appear in ``source``.
|
||||
|
||||
Use for symbols, hook names, prop names and API paths — things a refactor
|
||||
renames deliberately — never for sentences shown to a user.
|
||||
"""
|
||||
|
||||
missing = [identifier for identifier in identifiers if identifier not in source]
|
||||
assert not missing, f"{context} no longer wires: {missing}"
|
||||
|
||||
|
||||
def assert_calls(source: str, function_name: str, *, first_argument: str) -> None:
|
||||
"""Assert ``function_name`` is called with ``first_argument`` as argument 1.
|
||||
|
||||
Tolerates whatever the remaining arguments have been refactored into, which
|
||||
is the part that keeps changing while the wiring stays the same.
|
||||
"""
|
||||
|
||||
pattern = rf"{re.escape(function_name)}\(\s*{re.escape(first_argument)}\s*[,)]"
|
||||
assert re.search(pattern, source), f"{function_name}({first_argument}, …) is no longer called"
|
||||
|
||||
|
||||
def assert_mentions(source: str, *phrases: str, context: str = "frontend source") -> None:
|
||||
"""Case-insensitive check that a concept is still surfaced to the operator.
|
||||
|
||||
A deliberately weak assertion: it survives rewording but still fails if a
|
||||
whole explanation is deleted. Prefer ``assert_wired`` where an identifier
|
||||
exists to check instead.
|
||||
"""
|
||||
|
||||
folded = source.casefold()
|
||||
missing = [phrase for phrase in phrases if phrase.casefold() not in folded]
|
||||
assert not missing, f"{context} no longer mentions: {missing}"
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "run_accuracy_phase2_foundation_audit.py"
|
||||
SPEC = importlib.util.spec_from_file_location("accuracy_phase2_foundation_audit", SCRIPT)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def test_phase2_foundation_audit_enumerates_exact_source_and_contract_policies() -> None:
|
||||
payload = MODULE.collect()
|
||||
|
||||
assert payload["phase"] == "P2"
|
||||
assert payload["migration_revision"] == "202608010001"
|
||||
assert payload["source_registry"]["definition_count"] >= 40
|
||||
assert payload["source_registry"]["required_building_policy"] == {
|
||||
"grb_primary_building_validation": "primary",
|
||||
"buildings_register_classification": "authoritative",
|
||||
"sentinel_2_classification": "contextual",
|
||||
"dhmv_classification": "authoritative",
|
||||
"osm_ground_truth_allowed": False,
|
||||
}
|
||||
assert {(item["key"], item["version"]) for item in payload["data_contracts"]} == {
|
||||
("geointel.vector.geojson", "1.0.0"),
|
||||
("geointel.raster.geotiff", "1.0.0"),
|
||||
("geointel.label.yolo", "1.0.0"),
|
||||
("geointel.label.yolo", "1.1.0"),
|
||||
("geointel.model.pytorch", "1.0.0"),
|
||||
}
|
||||
@@ -0,0 +1,772 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
from importlib.util import module_from_spec, spec_from_file_location
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import CheckConstraint
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import (
|
||||
Dataset,
|
||||
DatasetLineageEdge,
|
||||
DatasetQuarantine,
|
||||
DatasetVersion,
|
||||
SourceRegistry,
|
||||
SourceSnapshot,
|
||||
)
|
||||
from app.services.coverage_registry_service import SOURCE_DEFINITIONS
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.source_registry_service import (
|
||||
SERVER_OWNED_SOURCE_DEFINITIONS,
|
||||
SourceRegistryService,
|
||||
)
|
||||
|
||||
|
||||
class _Query:
|
||||
def __init__(self, session: "InMemorySession", model: type) -> None:
|
||||
self.session = session
|
||||
self.model = model
|
||||
self.predicates = []
|
||||
|
||||
def filter(self, *predicates):
|
||||
self.predicates.extend(predicates)
|
||||
return self
|
||||
|
||||
def one_or_none(self):
|
||||
matches = self._matches()
|
||||
if len(matches) > 1:
|
||||
raise AssertionError(
|
||||
f"Expected one {self.model.__name__}, found {len(matches)}"
|
||||
)
|
||||
return matches[0] if matches else None
|
||||
|
||||
def all(self):
|
||||
return self._matches()
|
||||
|
||||
def _matches(self):
|
||||
matches = list(self.session.objects.get(self.model, []))
|
||||
for predicate in self.predicates:
|
||||
field_name = predicate.left.key
|
||||
expected = predicate.right.value
|
||||
operator_name = getattr(predicate.operator, "__name__", "")
|
||||
if operator_name == "in_op":
|
||||
matches = [
|
||||
item for item in matches if getattr(item, field_name) in expected
|
||||
]
|
||||
else:
|
||||
matches = [
|
||||
item for item in matches if getattr(item, field_name) == expected
|
||||
]
|
||||
return matches
|
||||
|
||||
|
||||
class InMemorySession:
|
||||
def __init__(self, *objects: object) -> None:
|
||||
self.objects: dict[type, list[object]] = {}
|
||||
self.added: list[object] = []
|
||||
self.flushes = 0
|
||||
for item in objects:
|
||||
self._store(item)
|
||||
|
||||
def query(self, model: type) -> _Query:
|
||||
return _Query(self, model)
|
||||
|
||||
def add(self, item: object) -> None:
|
||||
if getattr(item, "id", None) is None:
|
||||
setattr(item, "id", uuid4())
|
||||
self._store(item)
|
||||
self.added.append(item)
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flushes += 1
|
||||
|
||||
def _store(self, item: object) -> None:
|
||||
self.objects.setdefault(type(item), []).append(item)
|
||||
|
||||
|
||||
def _registry(source_key: str) -> SourceRegistry:
|
||||
definition = SERVER_OWNED_SOURCE_DEFINITIONS[source_key]
|
||||
return SourceRegistry(id=uuid4(), **definition.as_model_values())
|
||||
|
||||
|
||||
def _dataset() -> Dataset:
|
||||
return Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="candidate.tif",
|
||||
dataset_type="raster",
|
||||
source="governed",
|
||||
status="ready",
|
||||
validation_status="not_validated",
|
||||
provenance_status="incomplete",
|
||||
lineage_status="incomplete",
|
||||
quarantine_status="not_quarantined",
|
||||
)
|
||||
|
||||
|
||||
def test_server_owned_definitions_encode_building_authority_and_non_ground_truth_sources() -> (
|
||||
None
|
||||
):
|
||||
grb = SERVER_OWNED_SOURCE_DEFINITIONS["grb"]
|
||||
buildings_register = SERVER_OWNED_SOURCE_DEFINITIONS[
|
||||
"digitaal_vlaanderen_buildings_addresses_register"
|
||||
]
|
||||
sentinel = SERVER_OWNED_SOURCE_DEFINITIONS["sentinel_2"]
|
||||
dhmv = SERVER_OWNED_SOURCE_DEFINITIONS["digitaal_vlaanderen_dhmv"]
|
||||
osm = SERVER_OWNED_SOURCE_DEFINITIONS["osm"]
|
||||
|
||||
assert grb.classification == "authoritative"
|
||||
assert grb.usage_policy["ground_truth_allowed"] is True
|
||||
assert grb.usage_policy["validation_authority"]["building_validation"] == "primary"
|
||||
assert buildings_register.classification == "authoritative"
|
||||
assert buildings_register.usage_policy["ground_truth_allowed"] is False
|
||||
assert (
|
||||
buildings_register.usage_policy["validation_authority"]["building_validation"]
|
||||
== "corroborative"
|
||||
)
|
||||
assert (
|
||||
buildings_register.usage_policy["validation_authority"][
|
||||
"building_register_validation"
|
||||
]
|
||||
== "primary"
|
||||
)
|
||||
assert sentinel.classification == "contextual"
|
||||
assert dhmv.classification == "authoritative"
|
||||
assert dhmv.usage_policy["ground_truth_allowed"] is False
|
||||
assert (
|
||||
dhmv.usage_policy["validation_authority"]["building_validation"]
|
||||
== "corroborative"
|
||||
)
|
||||
assert (
|
||||
dhmv.usage_policy["validation_authority"]["elevation_validation"] == "primary"
|
||||
)
|
||||
assert osm.classification == "contextual"
|
||||
assert osm.usage_policy["ground_truth_allowed"] is False
|
||||
assert osm.usage_policy["automatic_ground_truth"] is False
|
||||
assert osm.usage_policy["training_allowed"] is False
|
||||
assert {
|
||||
"ngi_adminvector",
|
||||
"rbins_marine_reporting_units",
|
||||
"rbins_msp_2026",
|
||||
"grb",
|
||||
"digitaal_vlaanderen",
|
||||
"vrbg",
|
||||
"digitaal_vlaanderen_buildings_addresses_register",
|
||||
"digitaal_vlaanderen_orthophoto",
|
||||
"spw_orthophoto",
|
||||
"urbis_orthophoto",
|
||||
"digitaal_vlaanderen_dhmv",
|
||||
"spw_terrain",
|
||||
"spw_walous_land_cover",
|
||||
"spw_geoportail",
|
||||
"spw_picc",
|
||||
"urbis",
|
||||
"vmm_flood_hazard",
|
||||
"vmm_vha_bathymetry_profiles",
|
||||
"dov_soil_map",
|
||||
"statbel",
|
||||
"waterinfo",
|
||||
"agentschap_landbouw_zeevisserij_agricultural_parcels",
|
||||
"sentinel_2",
|
||||
"osm",
|
||||
"manual",
|
||||
"fixture",
|
||||
"map_selection",
|
||||
"derived",
|
||||
"training_label",
|
||||
"model",
|
||||
"experimental",
|
||||
"mdk_bcp_bathymetry",
|
||||
}.issubset(SERVER_OWNED_SOURCE_DEFINITIONS)
|
||||
|
||||
for umbrella_key in ("digitaal_vlaanderen", "spw_geoportail"):
|
||||
definition = SERVER_OWNED_SOURCE_DEFINITIONS[umbrella_key]
|
||||
assert definition.classification == "authoritative"
|
||||
assert definition.usage_policy["ground_truth_allowed"] is False
|
||||
assert definition.usage_policy["automatic_ground_truth"] is False
|
||||
|
||||
assert (
|
||||
SERVER_OWNED_SOURCE_DEFINITIONS["mdk_bcp_bathymetry"].ingest_status
|
||||
== "not_configured"
|
||||
)
|
||||
|
||||
|
||||
def test_coverage_and_direct_adapter_source_keys_are_registry_backed() -> None:
|
||||
coverage_source_keys = {
|
||||
definition.contract.source_name for definition in SOURCE_DEFINITIONS
|
||||
}
|
||||
coverage_materialization_keys = {
|
||||
source_key
|
||||
for definition in SOURCE_DEFINITIONS
|
||||
for source_key in definition.materialized_source_names
|
||||
}
|
||||
direct_adapter_source_keys = {
|
||||
"digitaal_vlaanderen",
|
||||
"spw_geoportail",
|
||||
"mdk_bcp_bathymetry",
|
||||
}
|
||||
|
||||
assert (
|
||||
coverage_source_keys
|
||||
| coverage_materialization_keys
|
||||
| direct_adapter_source_keys
|
||||
<= set(SERVER_OWNED_SOURCE_DEFINITIONS)
|
||||
)
|
||||
|
||||
|
||||
def test_new_adapter_source_seed_rows_match_server_owned_registry_semantics() -> None:
|
||||
migration_path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "alembic"
|
||||
/ "versions"
|
||||
/ "202608010001_source_registry_provenance.py"
|
||||
)
|
||||
spec = spec_from_file_location("phase2_source_registry_migration", migration_path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
migration = module_from_spec(spec)
|
||||
spec.loader.exec_module(migration)
|
||||
seed_rows = {row["source_key"]: row for row in migration._seed_rows()}
|
||||
|
||||
for source_key in ("digitaal_vlaanderen", "spw_geoportail", "mdk_bcp_bathymetry"):
|
||||
expected = SERVER_OWNED_SOURCE_DEFINITIONS[source_key].as_model_values()
|
||||
observed = seed_rows[source_key]
|
||||
for field in (
|
||||
"source_key",
|
||||
"display_name",
|
||||
"classification",
|
||||
"authority_name",
|
||||
"authority_scope_json",
|
||||
"provider_adapter_key",
|
||||
"source_url",
|
||||
"default_crs",
|
||||
"default_units",
|
||||
"geographic_coverage_json",
|
||||
"usage_policy_json",
|
||||
"freshness_status",
|
||||
"ingest_status",
|
||||
"known_limitations_json",
|
||||
):
|
||||
assert observed[field] == expected[field]
|
||||
|
||||
|
||||
def test_ensure_source_is_idempotent_and_rejects_caller_owned_unknown_sources() -> None:
|
||||
grb = _registry("grb")
|
||||
session = InMemorySession(grb)
|
||||
|
||||
assert SourceRegistryService.ensure_server_owned_source(session, "GRB") is grb
|
||||
assert session.added == []
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
SourceRegistryService.ensure_server_owned_source(session, "caller_claimed_grb")
|
||||
|
||||
assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND"
|
||||
|
||||
|
||||
def test_snapshot_is_checksum_bound_and_idempotent() -> None:
|
||||
grb = _registry("grb")
|
||||
session = InMemorySession(grb)
|
||||
checksum = "a" * 64
|
||||
|
||||
snapshot = SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key="2026-08-01-gbg",
|
||||
checksum_sha256=checksum,
|
||||
source_version="2026-08-01",
|
||||
crs="EPSG:31370",
|
||||
units="metres",
|
||||
)
|
||||
|
||||
assert snapshot.source_registry_id == grb.id
|
||||
assert snapshot.checksum_sha256 == checksum
|
||||
assert snapshot.ingest_status == "ingested"
|
||||
assert (
|
||||
SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key="2026-08-01-gbg",
|
||||
checksum_sha256=checksum,
|
||||
)
|
||||
is snapshot
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key="2026-08-01-gbg",
|
||||
checksum_sha256="b" * 64,
|
||||
)
|
||||
assert exc_info.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
|
||||
|
||||
with pytest.raises(AppError) as version_conflict:
|
||||
SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key="2026-08-01-gbg",
|
||||
checksum_sha256=checksum,
|
||||
source_version="2026-08-02",
|
||||
)
|
||||
assert version_conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
|
||||
|
||||
with pytest.raises(AppError) as invalid_checksum:
|
||||
SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key="bad-checksum",
|
||||
checksum_sha256="not-a-checksum",
|
||||
)
|
||||
assert invalid_checksum.value.code == "SOURCE_SNAPSHOT_CHECKSUM_INVALID"
|
||||
|
||||
|
||||
def test_governed_import_reuses_an_identical_snapshot_without_rewriting_fetched_at() -> None:
|
||||
"""A second project may bind the same immutable source snapshot safely."""
|
||||
|
||||
grb = _registry("grb")
|
||||
session = InMemorySession(grb)
|
||||
checksum = "a" * 64
|
||||
observed_at = None
|
||||
metadata = {
|
||||
"dataset_type": "vector",
|
||||
"bounds_json": {"min_x": 4.0, "min_y": 50.0, "max_x": 4.1, "max_y": 50.1},
|
||||
}
|
||||
source_metadata = {"source_url": "https://example.test/grb", "units": "metres"}
|
||||
|
||||
# Dataset ingest keys are project-scoped, while a source snapshot is
|
||||
# globally keyed by immutable source evidence. This represents the same
|
||||
# source file arriving through two independently resumable imports.
|
||||
project_one, project_two = uuid4(), uuid4()
|
||||
assert (
|
||||
DatasetService._ingest_key(
|
||||
project_id=project_one,
|
||||
source_key="grb",
|
||||
checksum_sha256=checksum,
|
||||
dataset_type="vector",
|
||||
dataset_role="source",
|
||||
area_id=None,
|
||||
reference_layer_name=None,
|
||||
source_version="2026-08-01",
|
||||
)
|
||||
!= DatasetService._ingest_key(
|
||||
project_id=project_two,
|
||||
source_key="grb",
|
||||
checksum_sha256=checksum,
|
||||
dataset_type="vector",
|
||||
dataset_role="source",
|
||||
area_id=None,
|
||||
reference_layer_name=None,
|
||||
source_version="2026-08-01",
|
||||
)
|
||||
)
|
||||
|
||||
first_source, first_snapshot = DatasetService._record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
checksum_sha256=checksum,
|
||||
source_version="2026-08-01",
|
||||
observed_at=observed_at,
|
||||
valid_from=None,
|
||||
valid_to=None,
|
||||
source_crs="EPSG:31370",
|
||||
source_metadata=source_metadata,
|
||||
metadata=metadata,
|
||||
)
|
||||
original_fetched_at = first_snapshot.fetched_at
|
||||
|
||||
replay_source, replay_snapshot = DatasetService._record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
checksum_sha256=checksum,
|
||||
source_version="2026-08-01",
|
||||
observed_at=observed_at,
|
||||
valid_from=None,
|
||||
valid_to=None,
|
||||
source_crs="EPSG:31370",
|
||||
source_metadata=source_metadata,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
assert replay_source is first_source
|
||||
assert replay_snapshot is first_snapshot
|
||||
assert replay_snapshot.fetched_at == original_fetched_at
|
||||
assert session.objects[SourceSnapshot] == [first_snapshot]
|
||||
|
||||
# Outside the governed replay path, a contradictory acquisition timestamp
|
||||
# remains immutable evidence and is still rejected.
|
||||
with pytest.raises(AppError) as fetched_at_conflict:
|
||||
SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key=first_snapshot.snapshot_key,
|
||||
checksum_sha256=checksum,
|
||||
fetched_at=original_fetched_at + timedelta(seconds=1),
|
||||
)
|
||||
assert fetched_at_conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
|
||||
|
||||
# Replay mode is narrow: a changed immutable evidence field still fails.
|
||||
with pytest.raises(AppError) as conflict:
|
||||
SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key=first_snapshot.snapshot_key,
|
||||
checksum_sha256=checksum,
|
||||
crs="EPSG:4326",
|
||||
reuse_existing_snapshot=True,
|
||||
)
|
||||
assert conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
|
||||
|
||||
|
||||
def test_snapshot_schema_requires_a_canonical_sha256() -> None:
|
||||
constraints = {
|
||||
constraint.name: str(constraint.sqltext)
|
||||
for constraint in SourceSnapshot.__table__.constraints
|
||||
if isinstance(constraint, CheckConstraint)
|
||||
}
|
||||
|
||||
assert SourceSnapshot.__table__.c.checksum_sha256.nullable is False
|
||||
assert "ck_source_snapshots_checksum_sha256" in constraints
|
||||
assert (
|
||||
"lower(checksum_sha256)" in constraints["ck_source_snapshots_checksum_sha256"]
|
||||
)
|
||||
|
||||
|
||||
def test_complete_provenance_binding_is_required_before_authoritative_validation() -> (
|
||||
None
|
||||
):
|
||||
grb = _registry("grb")
|
||||
snapshot = SourceSnapshot(
|
||||
id=uuid4(),
|
||||
source_registry_id=grb.id,
|
||||
snapshot_key="governed-grb",
|
||||
checksum_sha256="c" * 64,
|
||||
ingest_status="ingested",
|
||||
)
|
||||
dataset = _dataset()
|
||||
|
||||
SourceRegistryService.bind_dataset_provenance(
|
||||
dataset,
|
||||
source=grb,
|
||||
snapshot=snapshot,
|
||||
data_contract_key="vector.grb.buildings",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="complete",
|
||||
)
|
||||
|
||||
assert SourceRegistryService.is_dataset_eligible_for_authoritative_validation(
|
||||
dataset,
|
||||
source=grb,
|
||||
snapshot=snapshot,
|
||||
task="building_validation",
|
||||
)
|
||||
|
||||
osm = _registry("osm")
|
||||
dataset.source_registry_id = osm.id
|
||||
snapshot.source_registry_id = osm.id
|
||||
assert not SourceRegistryService.is_dataset_eligible_for_authoritative_validation(
|
||||
dataset,
|
||||
source=osm,
|
||||
snapshot=snapshot,
|
||||
task="building_validation",
|
||||
)
|
||||
|
||||
|
||||
def test_lineage_and_quarantine_are_fail_closed_and_observable() -> None:
|
||||
session = InMemorySession()
|
||||
parent_id = uuid4()
|
||||
child_id = uuid4()
|
||||
edge = SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=parent_id,
|
||||
child_dataset_id=child_id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="vector_clip",
|
||||
input_checksum_sha256="d" * 64,
|
||||
output_checksum_sha256="e" * 64,
|
||||
)
|
||||
|
||||
assert isinstance(edge, DatasetLineageEdge)
|
||||
assert (
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=parent_id,
|
||||
child_dataset_id=child_id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="vector_clip",
|
||||
input_checksum_sha256="d" * 64,
|
||||
output_checksum_sha256="e" * 64,
|
||||
)
|
||||
is edge
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as self_reference:
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=parent_id,
|
||||
child_dataset_id=parent_id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="vector_clip",
|
||||
)
|
||||
assert self_reference.value.code == "DATASET_LINEAGE_SELF_REFERENCE"
|
||||
|
||||
grandchild_id = uuid4()
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=child_id,
|
||||
child_dataset_id=grandchild_id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="vector_buffer",
|
||||
)
|
||||
with pytest.raises(AppError) as cycle:
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=grandchild_id,
|
||||
child_dataset_id=parent_id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="vector_union",
|
||||
)
|
||||
assert cycle.value.code == "DATASET_LINEAGE_CYCLE_DETECTED"
|
||||
|
||||
dataset = _dataset()
|
||||
record = SourceRegistryService.quarantine_dataset(
|
||||
session,
|
||||
dataset=dataset,
|
||||
stage="vector_ingest",
|
||||
reason_code="CRS_UNVERIFIED",
|
||||
details={"observed_crs": None},
|
||||
)
|
||||
assert isinstance(record, DatasetQuarantine)
|
||||
assert dataset.status == "quarantined"
|
||||
assert dataset.quarantine_status == "quarantined"
|
||||
assert dataset.validation_status == "failed"
|
||||
|
||||
version_parent = _dataset()
|
||||
version = DatasetVersion(id=uuid4(), dataset_id=version_parent.id, version=1)
|
||||
version_session = InMemorySession(version_parent, version)
|
||||
version_record = SourceRegistryService.quarantine_dataset(
|
||||
version_session,
|
||||
dataset_version=version,
|
||||
stage="dataset_version_validation",
|
||||
reason_code="CHECKSUM_MISMATCH",
|
||||
)
|
||||
assert version_record.dataset_id == version_parent.id
|
||||
assert version_record.dataset_version_id == version.id
|
||||
assert version_parent.status == "quarantined"
|
||||
assert version_parent.quarantine_status == "quarantined"
|
||||
assert version_parent.validation_status == "failed"
|
||||
assert version_parent.provenance_status == "incomplete"
|
||||
assert version_parent.lineage_status == "incomplete"
|
||||
assert version.validation_status == "failed"
|
||||
assert version.provenance_status == "incomplete"
|
||||
|
||||
snapshot = SourceSnapshot(
|
||||
id=uuid4(),
|
||||
source_registry_id=uuid4(),
|
||||
snapshot_key="quarantined-source",
|
||||
checksum_sha256="f" * 64,
|
||||
ingest_status="ingested",
|
||||
)
|
||||
SourceRegistryService.quarantine_dataset(
|
||||
session,
|
||||
source_snapshot=snapshot,
|
||||
stage="source_snapshot_validation",
|
||||
reason_code="CHECKSUM_MISMATCH",
|
||||
)
|
||||
assert snapshot.ingest_status == "quarantined"
|
||||
|
||||
|
||||
def test_quarantine_propagates_transitively_to_descendant_dataset_and_version_consumption_gates() -> (
|
||||
None
|
||||
):
|
||||
"""A -> B -> C must fail closed when the governing A artifact is rejected."""
|
||||
|
||||
source = _registry("grb")
|
||||
snapshot = SourceSnapshot(
|
||||
id=uuid4(),
|
||||
source_registry_id=source.id,
|
||||
snapshot_key="transitive-quarantine-source",
|
||||
checksum_sha256="a" * 64,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
|
||||
def governed_dataset(name: str) -> Dataset:
|
||||
dataset = _dataset()
|
||||
dataset.name = name
|
||||
dataset.source = "grb"
|
||||
dataset.source_name = "grb"
|
||||
dataset.dataset_role = "source"
|
||||
dataset.checksum_sha256 = snapshot.checksum_sha256
|
||||
dataset.source_registry_id = source.id
|
||||
dataset.source_snapshot_id = snapshot.id
|
||||
dataset.data_contract_key = "geointel.raster.geotiff"
|
||||
dataset.data_contract_version = "1.0.0"
|
||||
dataset.validation_status = "passed"
|
||||
dataset.provenance_status = "complete"
|
||||
dataset.lineage_status = "complete"
|
||||
dataset.quarantine_status = "not_quarantined"
|
||||
dataset.status = "ready"
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
return dataset
|
||||
|
||||
parent = governed_dataset("parent.tif")
|
||||
child = governed_dataset("child.tif")
|
||||
grandchild = governed_dataset("grandchild.tif")
|
||||
parent_version = DatasetVersion(
|
||||
id=uuid4(),
|
||||
dataset_id=parent.id,
|
||||
version=1,
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="complete",
|
||||
)
|
||||
child_version = DatasetVersion(
|
||||
id=uuid4(),
|
||||
dataset_id=child.id,
|
||||
version=1,
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="complete",
|
||||
)
|
||||
grandchild_version = DatasetVersion(
|
||||
id=uuid4(),
|
||||
dataset_id=grandchild.id,
|
||||
version=1,
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="complete",
|
||||
)
|
||||
session = InMemorySession(
|
||||
parent,
|
||||
child,
|
||||
grandchild,
|
||||
parent_version,
|
||||
child_version,
|
||||
grandchild_version,
|
||||
)
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=parent.id,
|
||||
child_dataset_id=child.id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="clip",
|
||||
)
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=child.id,
|
||||
child_dataset_id=grandchild.id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="buffer",
|
||||
)
|
||||
|
||||
assert (
|
||||
DatasetConsumptionGate.evaluate(child, purpose="production_inference").eligible
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
DatasetConsumptionGate.evaluate(
|
||||
grandchild, purpose="production_inference"
|
||||
).eligible
|
||||
is True
|
||||
)
|
||||
|
||||
SourceRegistryService.quarantine_dataset(
|
||||
session,
|
||||
dataset=parent,
|
||||
stage="contract_validation",
|
||||
reason_code="CHECKSUM_MISMATCH",
|
||||
)
|
||||
|
||||
for dataset in (parent, child, grandchild):
|
||||
decision = DatasetConsumptionGate.evaluate(
|
||||
dataset, purpose="production_inference"
|
||||
)
|
||||
assert dataset.status == "quarantined"
|
||||
assert dataset.quarantine_status == "quarantined"
|
||||
assert dataset.validation_status == "failed"
|
||||
assert dataset.provenance_status == "incomplete"
|
||||
assert dataset.lineage_status == "incomplete"
|
||||
assert decision.eligible is False
|
||||
assert "dataset_quarantined" in decision.reasons
|
||||
for dataset_version in (parent_version, child_version, grandchild_version):
|
||||
assert dataset_version.validation_status == "failed"
|
||||
assert dataset_version.provenance_status == "incomplete"
|
||||
assert dataset_version.lineage_status == "incomplete"
|
||||
|
||||
|
||||
def test_ingest_keys_are_scoped_and_migration_keeps_unknown_legacy_unbound() -> None:
|
||||
project_id = uuid4()
|
||||
dataset = _dataset()
|
||||
dataset.project_id = project_id
|
||||
dataset.ingest_key = "grb:2026-08-01:gbg:area-sha"
|
||||
version = DatasetVersion(
|
||||
id=uuid4(),
|
||||
dataset_id=dataset.id,
|
||||
ingest_key=dataset.ingest_key,
|
||||
validation_status="not_validated",
|
||||
provenance_status="incomplete",
|
||||
lineage_status="incomplete",
|
||||
)
|
||||
session = InMemorySession(dataset, version)
|
||||
|
||||
assert (
|
||||
SourceRegistryService.find_dataset_by_ingest_key(
|
||||
session, project_id, dataset.ingest_key
|
||||
)
|
||||
is dataset
|
||||
)
|
||||
assert (
|
||||
SourceRegistryService.find_dataset_version_by_ingest_key(
|
||||
session, dataset.id, dataset.ingest_key
|
||||
)
|
||||
is version
|
||||
)
|
||||
with pytest.raises(AppError) as invalid_key:
|
||||
SourceRegistryService.find_dataset_by_ingest_key(session, project_id, " ")
|
||||
assert invalid_key.value.code == "INGEST_KEY_INVALID"
|
||||
|
||||
migration = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "alembic"
|
||||
/ "versions"
|
||||
/ "202608010001_source_registry_provenance.py"
|
||||
).read_text(encoding="utf-8")
|
||||
assert "uuid_generate_v5" not in migration
|
||||
assert "__unregistered_legacy_source__" in migration
|
||||
assert "uq_datasets_project_ingest_key" in migration
|
||||
assert "uq_dataset_versions_dataset_ingest_key" in migration
|
||||
|
||||
|
||||
def test_migration_contains_database_guards_for_snapshot_pairing_contract_lineage_and_quarantine() -> (
|
||||
None
|
||||
):
|
||||
migration = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "alembic"
|
||||
/ "versions"
|
||||
/ "202608010001_source_registry_provenance.py"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "trg_datasets_snapshot_registry_guard" in migration
|
||||
assert "trg_dataset_versions_snapshot_registry_guard" in migration
|
||||
assert "trg_source_registry_write_guard" in migration
|
||||
assert "trg_source_snapshots_evidence_immutable" in migration
|
||||
assert "geointel_phase2_contract_report_guard" in migration
|
||||
assert "trg_datasets_contract_report_guard" in migration
|
||||
assert "trg_dataset_versions_contract_report_guard" in migration
|
||||
assert "matching complete validation report" in migration
|
||||
assert "geointel_phase2_lineage_cycle_guard" in migration
|
||||
assert "trg_dataset_lineage_edges_cycle_guard" in migration
|
||||
assert "geointel_phase2_lineage_edge_immutable_guard" in migration
|
||||
assert "trg_dataset_lineage_edges_immutable" in migration
|
||||
assert "WITH RECURSIVE descendants" in migration
|
||||
assert "geointel_phase2_quarantine_lineage_descendants" in migration
|
||||
assert "geointel_phase2_quarantine_state_guard" in migration
|
||||
assert "trg_dataset_quarantines_state_guard" in migration
|
||||
assert "accepted dataset artifact and provenance evidence is immutable" in migration
|
||||
@@ -0,0 +1,131 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "run_accuracy_phase3_full_data_scan.py"
|
||||
|
||||
|
||||
def run_scan(
|
||||
repo: Path,
|
||||
output: Path,
|
||||
*,
|
||||
resume: bool = False,
|
||||
unreachable_scopes: tuple[str, ...] = (),
|
||||
) -> dict:
|
||||
command = [
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--repo-root",
|
||||
str(repo),
|
||||
"--output-dir",
|
||||
str(output),
|
||||
"--roots",
|
||||
"data",
|
||||
"--batch-size",
|
||||
"2",
|
||||
]
|
||||
if resume:
|
||||
command.append("--resume")
|
||||
for scope in unreachable_scopes:
|
||||
command.extend(("--unreachable-scope", scope))
|
||||
completed = subprocess.run(command, check=True, capture_output=True, text=True)
|
||||
return json.loads(completed.stdout)
|
||||
|
||||
|
||||
def test_phase3_scan_reconciles_and_resumes_deterministically(tmp_path: Path) -> None:
|
||||
data = tmp_path / "data"
|
||||
data.mkdir()
|
||||
(data / "valid.geojson").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {"source": "GRB"},
|
||||
"geometry": {"type": "Point", "coordinates": [4.4, 50.8]},
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
invalid = data / "invalid.geojson"
|
||||
invalid.write_text(
|
||||
'{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0,0],[1,1],[1,0],[0,1],[0,0]]]}}]}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
duplicate_payload = '{"schema_version":1,"value":"same"}'
|
||||
(data / "one.json").write_text(duplicate_payload, encoding="utf-8")
|
||||
(data / "two.json").write_text(duplicate_payload, encoding="utf-8")
|
||||
(data / "broken.tif").write_bytes(b"not a geotiff")
|
||||
|
||||
output = tmp_path / "evidence"
|
||||
unreachable_scopes = (
|
||||
"external://tower-corpora=Tower corpora are not mounted in this fixture",
|
||||
"external://mounted-model-volumes=Model volumes are not mounted in this fixture",
|
||||
"external://production-postgis-or-api=Production database is not configured in this fixture",
|
||||
)
|
||||
first = run_scan(tmp_path, output, unreachable_scopes=unreachable_scopes)
|
||||
second = run_scan(
|
||||
tmp_path,
|
||||
output,
|
||||
resume=True,
|
||||
unreachable_scopes=unreachable_scopes,
|
||||
)
|
||||
manifest = json.loads((output / "full-scan-manifest.json").read_text(encoding="utf-8"))
|
||||
quarantine = json.loads((output / "quarantine-manifest.json").read_text(encoding="utf-8"))
|
||||
|
||||
assert first["reconciliation"] == {"examined": 5, "skipped": 0, "unreachable": 3, "inventory_total": 8, "reconciles": True}
|
||||
assert second["content_hash"] == first["content_hash"]
|
||||
assert manifest["determinism"]["content_hash"] == first["content_hash"]
|
||||
assert any(item["path"] == "data/broken.tif" for item in quarantine["items"])
|
||||
assert any(item["path"] == "data/invalid.geojson" for item in quarantine["items"])
|
||||
assert len(manifest["duplicates"]["exact_duplicate_groups"]) == 1
|
||||
|
||||
|
||||
def test_phase3_scan_does_not_invent_unreachable_production_boundaries(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
data = tmp_path / "data"
|
||||
data.mkdir()
|
||||
(data / "present.json").write_text("{}", encoding="utf-8")
|
||||
output = tmp_path / "evidence"
|
||||
|
||||
result = run_scan(tmp_path, output)
|
||||
|
||||
assert result["reconciliation"] == {
|
||||
"examined": 1,
|
||||
"skipped": 0,
|
||||
"unreachable": 0,
|
||||
"inventory_total": 1,
|
||||
"reconciles": True,
|
||||
}
|
||||
|
||||
|
||||
def test_phase3_scan_records_a_missing_requested_root(tmp_path: Path) -> None:
|
||||
output = tmp_path / "evidence"
|
||||
completed = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--repo-root",
|
||||
str(tmp_path),
|
||||
"--output-dir",
|
||||
str(output),
|
||||
"--roots",
|
||||
"missing-data",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
result = json.loads(completed.stdout)
|
||||
manifest = json.loads((output / "full-scan-manifest.json").read_text(encoding="utf-8"))
|
||||
|
||||
assert result["reconciliation"]["unreachable"] == 1
|
||||
assert manifest["items"][0]["path"] == "root://missing-data"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,718 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
from accuracy_phase4_evaluator import ( # noqa: E402
|
||||
TASKS,
|
||||
EXPECTED_PROTECTED_POLICY,
|
||||
canonical_hash,
|
||||
count_metrics,
|
||||
detection_ap,
|
||||
evaluate_cases,
|
||||
evaluate_object_detection,
|
||||
evaluate_footprint_segmentation,
|
||||
evaluate_raster_classification,
|
||||
evaluate_terrain,
|
||||
evaluate_validation,
|
||||
evaluate_vector_comparison,
|
||||
subgroup_report,
|
||||
task_inventory,
|
||||
)
|
||||
|
||||
|
||||
METADATA = {
|
||||
"region": "flanders",
|
||||
"municipality": "Mol",
|
||||
"urbanity": "urban",
|
||||
"object_size": "medium",
|
||||
"source": "synthetic-source",
|
||||
"sensor": "synthetic-sensor",
|
||||
"resolution_m": 0.25,
|
||||
"context": "dense_urban",
|
||||
"season": "summer",
|
||||
"date": "2026-01-01",
|
||||
"vegetation": "partial",
|
||||
"occlusion": "none",
|
||||
"difficulty": "normal",
|
||||
}
|
||||
|
||||
|
||||
def lineage(sample_id: str) -> dict:
|
||||
return {
|
||||
"reference": {
|
||||
"source_id": f"synthetic:{sample_id}:reference",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_contract_fixture",
|
||||
},
|
||||
"prediction": {
|
||||
"source_id": f"synthetic:{sample_id}:prediction",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_fixed_output",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def detection_case(sample_id: str = "det-1") -> dict:
|
||||
return {
|
||||
"sample_id": sample_id,
|
||||
"task": "object_detection",
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": {"confidence_threshold": 0.5, "match_iou": 0.5},
|
||||
"lineage": lineage(sample_id),
|
||||
"classes": ["building", "tank"],
|
||||
"references": [{"id": "r-building", "class": "building", "bbox": [0, 0, 4, 4]}],
|
||||
"predictions": [
|
||||
{
|
||||
"id": "p-building",
|
||||
"class": "building",
|
||||
"bbox": [0, 0, 4, 4],
|
||||
"confidence": 0.8,
|
||||
},
|
||||
{
|
||||
"id": "p-filtered",
|
||||
"class": "building",
|
||||
"bbox": [10, 10, 12, 12],
|
||||
"confidence": 0.2,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def raster_case(sample_id: str = "raster-1") -> dict:
|
||||
reference_side = {
|
||||
"crs": "EPSG:31370",
|
||||
"transform": [1, 0, 100000, 0, -1, 200000],
|
||||
"shape": [2, 2],
|
||||
"nodata": -9999,
|
||||
"mask": [[True, True], [True, True]],
|
||||
}
|
||||
return {
|
||||
"sample_id": sample_id,
|
||||
"task": "raster_classification",
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": {},
|
||||
"lineage": lineage(sample_id),
|
||||
"classes": [0, 1],
|
||||
"references": [[0, 1], [1, 0]],
|
||||
"predictions": [[0, 1], [1, 0]],
|
||||
"raster_context": {
|
||||
"reference": reference_side,
|
||||
"prediction": copy.deepcopy(reference_side),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def polygon_case(task: str = "vector_comparison") -> dict:
|
||||
sample_id = f"{task}-1"
|
||||
config = {"match_iou": 0.5}
|
||||
if task == "footprint_segmentation":
|
||||
config["boundary_tolerance_m"] = 1.0
|
||||
polygon = [
|
||||
[100000, 200000],
|
||||
[100010, 200000],
|
||||
[100010, 200010],
|
||||
[100000, 200010],
|
||||
[100000, 200000],
|
||||
]
|
||||
return {
|
||||
"sample_id": sample_id,
|
||||
"task": task,
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": config,
|
||||
"lineage": lineage(sample_id),
|
||||
"classes": ["building"],
|
||||
"spatial_context": {
|
||||
"crs": "EPSG:31370",
|
||||
"coordinate_units": "m",
|
||||
"metric": True,
|
||||
},
|
||||
"references": [{"id": "reference", "class": "building", "polygon": polygon}],
|
||||
"predictions": [{"id": "prediction", "class": "building", "polygon": polygon}],
|
||||
}
|
||||
|
||||
|
||||
def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> None:
|
||||
case = detection_case()
|
||||
portfolio = {
|
||||
"schema_version": 2,
|
||||
"portfolio_kind": "synthetic_contract",
|
||||
"portfolio_id": "synthetic-hardening-test",
|
||||
"portfolio_lineage": {
|
||||
"origin": "repository_fixture",
|
||||
"source_path": "synthetic.json",
|
||||
"version": "1",
|
||||
},
|
||||
"split_roles": ["test"],
|
||||
"selection_policy": "Fixed before evaluation; no selection.",
|
||||
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
|
||||
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
|
||||
"cases": [case],
|
||||
}
|
||||
path = tmp_path / "portfolio.json"
|
||||
path.write_text(
|
||||
json.dumps(portfolio, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
report = evaluate_cases(path, {case["sample_id"]})
|
||||
raw = report["results"][0]["raw"]
|
||||
|
||||
assert raw["references"] == case["references"]
|
||||
assert raw["predictions_pre_filter"] == case["predictions"]
|
||||
assert raw["predictions_post_filter"] == case["predictions"][:1]
|
||||
assert raw["config"] == case["config"]
|
||||
assert raw["split"] == "test"
|
||||
assert raw["input_lineage"] == case["lineage"]
|
||||
assert raw["portfolio_lineage"]["declared"] == portfolio["portfolio_lineage"]
|
||||
assert raw["hashes"]["case_input_canonical_json_sha256"] == canonical_hash(case)
|
||||
assert raw["hashes"]["references_canonical_json_sha256"] == canonical_hash(
|
||||
case["references"]
|
||||
)
|
||||
assert (
|
||||
report["portfolio_file_sha256"] == hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
)
|
||||
assert report["portfolio_canonical_json_sha256"] == canonical_hash(portfolio)
|
||||
assert report["results_canonical_json_sha256"] == canonical_hash(report["results"])
|
||||
high_threshold = next(
|
||||
row
|
||||
for row in report["results"][0]["metrics"]["coverage_risk"]
|
||||
if row["threshold"] == 0.9
|
||||
)
|
||||
assert high_threshold["retained_prediction_coverage"] == 0.0
|
||||
assert high_threshold["reference_coverage"] == 0.0
|
||||
assert high_threshold["false_negative_count"] == 1
|
||||
assert high_threshold["risk"] == 1.0
|
||||
|
||||
challenge_exposed = copy.deepcopy(portfolio)
|
||||
challenge_exposed["challenge_labels"] = []
|
||||
path.write_text(json.dumps(challenge_exposed, ensure_ascii=False), encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="Challenge cases and labels"):
|
||||
evaluate_cases(path, {case["sample_id"]})
|
||||
|
||||
|
||||
def test_portfolio_schema_policy_metadata_and_lineage_are_strict(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
case = detection_case("strict-contract")
|
||||
portfolio = {
|
||||
"schema_version": 2,
|
||||
"portfolio_kind": "synthetic_contract",
|
||||
"portfolio_id": "synthetic-strict-contract",
|
||||
"portfolio_lineage": {
|
||||
"origin": "repository_fixture",
|
||||
"source_path": "synthetic.json",
|
||||
"version": "1",
|
||||
},
|
||||
"split_roles": ["test"],
|
||||
"selection_policy": "Fixed before evaluation; no selection.",
|
||||
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
|
||||
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
|
||||
"cases": [case],
|
||||
}
|
||||
path = tmp_path / "strict.json"
|
||||
|
||||
def evaluate(value: dict) -> dict:
|
||||
path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
|
||||
return evaluate_cases(path, {case["sample_id"]})
|
||||
|
||||
assert evaluate(portfolio)["case_count"] == 1
|
||||
|
||||
for invalid_version in (1, True, "2"):
|
||||
invalid = copy.deepcopy(portfolio)
|
||||
invalid["schema_version"] = invalid_version
|
||||
with pytest.raises(
|
||||
ValueError, match="schema_version must be exactly integer 2"
|
||||
):
|
||||
evaluate(invalid)
|
||||
|
||||
invalid_policy = copy.deepcopy(portfolio)
|
||||
invalid_policy["protected_policy"]["test_feedback_allowed"] = True
|
||||
with pytest.raises(ValueError, match="protected_policy must exactly equal"):
|
||||
evaluate(invalid_policy)
|
||||
|
||||
invalid_metadata = copy.deepcopy(portfolio)
|
||||
invalid_metadata["cases"][0]["metadata"]["source"] = "unknown"
|
||||
with pytest.raises(ValueError, match="metadata.source must be a meaningful"):
|
||||
evaluate(invalid_metadata)
|
||||
|
||||
invalid_resolution = copy.deepcopy(portfolio)
|
||||
invalid_resolution["cases"][0]["metadata"]["resolution_m"] = 0
|
||||
with pytest.raises(ValueError, match="metadata.resolution_m must be positive"):
|
||||
evaluate(invalid_resolution)
|
||||
|
||||
invalid_lineage = copy.deepcopy(portfolio)
|
||||
del invalid_lineage["cases"][0]["lineage"]["prediction"]["derivation"]
|
||||
with pytest.raises(ValueError, match="lineage.prediction missing"):
|
||||
evaluate(invalid_lineage)
|
||||
|
||||
|
||||
def test_portfolio_kind_separates_synthetic_and_governed_product_claims(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
fixture_path = (
|
||||
ROOT / "fixtures" / "accuracy" / "p4" / "protected-baseline-cases.json"
|
||||
)
|
||||
synthetic = json.loads(fixture_path.read_text(encoding="utf-8"))
|
||||
allowed = {item["sample_id"] for item in synthetic["cases"]}
|
||||
path = tmp_path / "portfolio.json"
|
||||
|
||||
missing_kind = copy.deepcopy(synthetic)
|
||||
del missing_kind["portfolio_kind"]
|
||||
path.write_text(json.dumps(missing_kind), encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="portfolio_kind"):
|
||||
evaluate_cases(path, allowed)
|
||||
|
||||
confused = copy.deepcopy(synthetic)
|
||||
confused["claim_boundary"] = "Governed product baseline accuracy evidence."
|
||||
path.write_text(json.dumps(confused), encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="synthetic_contract"):
|
||||
evaluate_cases(path, allowed)
|
||||
|
||||
governed = json.loads(
|
||||
json.dumps(synthetic)
|
||||
.replace("Synthetic", "Governed")
|
||||
.replace("synthetic", "governed")
|
||||
.replace("repository_fixture", "governed_product_evaluation")
|
||||
)
|
||||
governed["portfolio_kind"] = "governed_product_baseline"
|
||||
governed["portfolio_id"] = "governed-product-baseline-test"
|
||||
governed["claim_boundary"] = (
|
||||
"Governed product baseline metrics recomputed from protected raw cases; "
|
||||
"inference provenance is validated separately."
|
||||
)
|
||||
path.write_text(json.dumps(governed), encoding="utf-8")
|
||||
report = evaluate_cases(path, allowed)
|
||||
assert report["portfolio_kind"] == "governed_product_baseline"
|
||||
assert set(report["evaluated_task_families"]) == TASKS
|
||||
|
||||
governed["cases"] = governed["cases"][:-1]
|
||||
path.write_text(json.dumps(governed), encoding="utf-8")
|
||||
|
||||
|
||||
def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
|
||||
references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}]
|
||||
predictions = [
|
||||
{
|
||||
"id": "z-true",
|
||||
"class": "building",
|
||||
"bbox": [0, 0, 4, 4],
|
||||
"confidence": 0.8,
|
||||
},
|
||||
{
|
||||
"id": "a-false",
|
||||
"class": "building",
|
||||
"bbox": [10, 10, 12, 12],
|
||||
"confidence": 0.8,
|
||||
},
|
||||
]
|
||||
forward = detection_ap(predictions, references, 0.5)
|
||||
reverse = detection_ap(list(reversed(predictions)), references, 0.5)
|
||||
assert forward == reverse == pytest.approx(0.5)
|
||||
|
||||
wrong_class = copy.deepcopy(predictions)
|
||||
wrong_class[1] = {
|
||||
"id": "a-tank",
|
||||
"class": "tank",
|
||||
"bbox": [0, 0, 4, 4],
|
||||
"confidence": 0.95,
|
||||
}
|
||||
assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5)
|
||||
|
||||
|
||||
def test_detection_ap_and_calibration_are_pooled_globally_and_per_subgroup(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
first = detection_case("a-case")
|
||||
first["predictions"] = [
|
||||
{
|
||||
"id": "p-true",
|
||||
"class": "building",
|
||||
"bbox": [0, 0, 4, 4],
|
||||
"confidence": 0.9,
|
||||
}
|
||||
]
|
||||
second = detection_case("b-case")
|
||||
second["predictions"] = [
|
||||
{
|
||||
"id": "p-false",
|
||||
"class": "building",
|
||||
"bbox": [10, 10, 12, 12],
|
||||
"confidence": 0.9,
|
||||
},
|
||||
{
|
||||
"id": "p-true",
|
||||
"class": "building",
|
||||
"bbox": [0, 0, 4, 4],
|
||||
"confidence": 0.8,
|
||||
},
|
||||
]
|
||||
portfolio = {
|
||||
"schema_version": 2,
|
||||
"portfolio_kind": "synthetic_contract",
|
||||
"portfolio_id": "synthetic-pooled-detection",
|
||||
"portfolio_lineage": {
|
||||
"origin": "repository_fixture",
|
||||
"source_path": "pooled.json",
|
||||
"version": "1",
|
||||
},
|
||||
"split_roles": ["test"],
|
||||
"selection_policy": "Fixed before evaluation; no selection.",
|
||||
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
|
||||
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
|
||||
"cases": [first, second],
|
||||
}
|
||||
path = tmp_path / "pooled.json"
|
||||
path.write_text(json.dumps(portfolio, ensure_ascii=False), encoding="utf-8")
|
||||
report = evaluate_cases(path, {"a-case", "b-case"})
|
||||
|
||||
case_ap = [item["metrics"]["ap50"] for item in report["results"]]
|
||||
pooled = report["portfolio_metrics"]["object_detection"]["micro"]
|
||||
expected_pooled = detection_ap(
|
||||
[
|
||||
{**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
|
||||
for item in first["predictions"]
|
||||
]
|
||||
+ [
|
||||
{**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
|
||||
for item in second["predictions"]
|
||||
],
|
||||
[
|
||||
{**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
|
||||
for item in first["references"]
|
||||
]
|
||||
+ [
|
||||
{**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
|
||||
for item in second["references"]
|
||||
],
|
||||
0.5,
|
||||
)
|
||||
assert pooled["ap50"] == expected_pooled
|
||||
assert pooled["ap50"] != pytest.approx(sum(case_ap) / len(case_ap))
|
||||
assert sum(item["count"] for item in pooled["calibration"]["bins"]) == 3
|
||||
subgroup = report["subgroups"]["dimensions"]["region"]["strata"]["flanders"]
|
||||
subgroup_calibration = subgroup["task_metrics"]["object_detection"]["micro"][
|
||||
"calibration"
|
||||
]
|
||||
assert sum(item["count"] for item in subgroup_calibration["bins"]) == 3
|
||||
|
||||
|
||||
def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None:
|
||||
invalid_case = detection_case("invalid-class")
|
||||
invalid_case["predictions"][0]["class"] = "road"
|
||||
with pytest.raises(ValueError, match="outside the declared ontology"):
|
||||
evaluate_object_detection(invalid_case)
|
||||
|
||||
valid = raster_case()
|
||||
valid["predictions"][0][1] = -9999
|
||||
valid["raster_context"]["prediction"]["mask"][0][1] = False
|
||||
result = evaluate_raster_classification(valid)
|
||||
assert result["metrics"]["prediction_coverage"] == pytest.approx(0.75)
|
||||
assert result["metrics"]["per_class"]["1"]["false_negative"] == 1
|
||||
|
||||
jagged = raster_case("jagged")
|
||||
jagged["predictions"][1].pop()
|
||||
with pytest.raises(ValueError, match="exactly rectangular"):
|
||||
evaluate_raster_classification(jagged)
|
||||
|
||||
missing_metadata = raster_case("missing-metadata")
|
||||
del missing_metadata["raster_context"]["prediction"]["crs"]
|
||||
with pytest.raises(ValueError, match="missing"):
|
||||
evaluate_raster_classification(missing_metadata)
|
||||
|
||||
shifted = raster_case("shifted")
|
||||
shifted["raster_context"]["prediction"]["transform"][2] += 1
|
||||
with pytest.raises(ValueError, match="affine alignment differs"):
|
||||
evaluate_raster_classification(shifted)
|
||||
|
||||
invalid_class = raster_case("invalid-class")
|
||||
invalid_class["predictions"][0][0] = 3
|
||||
with pytest.raises(ValueError, match="prediction class outside ontology"):
|
||||
evaluate_raster_classification(invalid_class)
|
||||
|
||||
invalid_nodata = raster_case("invalid-nodata")
|
||||
invalid_nodata["predictions"][0][0] = -9999
|
||||
with pytest.raises(ValueError, match="marks nodata as valid"):
|
||||
evaluate_raster_classification(invalid_nodata)
|
||||
singular = raster_case("singular")
|
||||
for side in ("reference", "prediction"):
|
||||
singular["raster_context"][side]["transform"] = [1, 2, 0, 2, 4, 0]
|
||||
with pytest.raises(ValueError, match="affine transform is singular"):
|
||||
evaluate_raster_classification(singular)
|
||||
|
||||
|
||||
def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None:
|
||||
assert evaluate_vector_comparison(polygon_case())["metrics"]["f1"] == 1.0
|
||||
assert (
|
||||
evaluate_footprint_segmentation(polygon_case("footprint_segmentation"))[
|
||||
"metrics"
|
||||
]["mean_iou"]
|
||||
== 1.0
|
||||
)
|
||||
outer = [
|
||||
[100000, 200000],
|
||||
[100020, 200000],
|
||||
[100020, 200020],
|
||||
[100000, 200020],
|
||||
[100000, 200000],
|
||||
]
|
||||
hole = [
|
||||
[100005, 200005],
|
||||
[100010, 200005],
|
||||
[100010, 200010],
|
||||
[100005, 200010],
|
||||
[100005, 200005],
|
||||
]
|
||||
polygon_geometry = {"type": "Polygon", "coordinates": [outer, hole]}
|
||||
geojson_polygon = polygon_case()
|
||||
for side in ("references", "predictions"):
|
||||
del geojson_polygon[side][0]["polygon"]
|
||||
geojson_polygon[side][0]["geometry"] = copy.deepcopy(polygon_geometry)
|
||||
polygon_result = evaluate_vector_comparison(geojson_polygon)
|
||||
assert polygon_result["metrics"]["mean_iou"] == 1.0
|
||||
assert polygon_result["raw"]["references"][0]["geometry"] == polygon_geometry
|
||||
|
||||
second = [
|
||||
[100030, 200000],
|
||||
[100040, 200000],
|
||||
[100040, 200010],
|
||||
[100030, 200010],
|
||||
[100030, 200000],
|
||||
]
|
||||
multipolygon_geometry = {
|
||||
"type": "MultiPolygon",
|
||||
"coordinates": [[outer, hole], [second]],
|
||||
}
|
||||
geojson_multi = polygon_case()
|
||||
for side in ("references", "predictions"):
|
||||
del geojson_multi[side][0]["polygon"]
|
||||
geojson_multi[side][0]["geometry"] = copy.deepcopy(multipolygon_geometry)
|
||||
assert evaluate_vector_comparison(geojson_multi)["metrics"]["mean_iou"] == 1.0
|
||||
|
||||
geographic = polygon_case()
|
||||
geographic["spatial_context"]["crs"] = "EPSG:4326"
|
||||
with pytest.raises(ValueError, match="projected CRS"):
|
||||
evaluate_vector_comparison(geographic)
|
||||
mercator = polygon_case()
|
||||
mercator["spatial_context"]["crs"] = "EPSG:3857"
|
||||
with pytest.raises(ValueError, match="Mercator is unsuitable"):
|
||||
evaluate_vector_comparison(mercator)
|
||||
|
||||
wrong_geography = polygon_case()
|
||||
wrong_geography["spatial_context"]["crs"] = "EPSG:32660"
|
||||
with pytest.raises(ValueError, match="does not overlap"):
|
||||
evaluate_vector_comparison(wrong_geography)
|
||||
|
||||
wrong_units = polygon_case()
|
||||
wrong_units["spatial_context"]["coordinate_units"] = "degree"
|
||||
with pytest.raises(ValueError, match="must be 'm'"):
|
||||
evaluate_vector_comparison(wrong_units)
|
||||
|
||||
bowtie = polygon_case()
|
||||
bowtie["predictions"][0]["polygon"] = [
|
||||
[100000, 200000],
|
||||
[100010, 200010],
|
||||
[100010, 200000],
|
||||
[100000, 200010],
|
||||
[100000, 200000],
|
||||
]
|
||||
with pytest.raises(ValueError, match="positive-area and valid"):
|
||||
evaluate_vector_comparison(bowtie)
|
||||
|
||||
|
||||
def test_failure_gallery_covers_geometry_raster_calibration_and_contexts() -> None:
|
||||
segmentation = polygon_case("footprint_segmentation")
|
||||
segmentation["predictions"][0]["polygon"] = [
|
||||
[100000, 200000],
|
||||
[100012, 200000],
|
||||
[100012, 200010],
|
||||
[100000, 200010],
|
||||
[100000, 200000],
|
||||
]
|
||||
segmentation_result = evaluate_footprint_segmentation(segmentation)
|
||||
segmentation_codes = {
|
||||
item["error_code"] for item in segmentation_result["failures"]
|
||||
}
|
||||
assert {"M-BOUNDARY", "M-AREA-BIAS"} <= segmentation_codes
|
||||
|
||||
raster = raster_case("raster-taxonomy")
|
||||
raster["metadata"]["tile_edge"] = True
|
||||
raster["predictions"][0][0] = 1
|
||||
raster_result = evaluate_raster_classification(raster)
|
||||
raster_failure = next(
|
||||
item
|
||||
for item in raster_result["failures"]
|
||||
if item["kind"] == "raster_misclassification"
|
||||
)
|
||||
assert raster_failure["error_code"] == "M-CLASS"
|
||||
assert "tile_edge" in raster_failure["contexts"]
|
||||
|
||||
detection = detection_case("context-taxonomy")
|
||||
detection["references"] = []
|
||||
detection["predictions"] = [
|
||||
{
|
||||
"id": "high-confidence-fp",
|
||||
"class": "building",
|
||||
"bbox": [10, 10, 12, 12],
|
||||
"confidence": 0.95,
|
||||
}
|
||||
]
|
||||
detection["config"]["fixed_diagnostic_risk_thresholds"] = [0.5, 0.9]
|
||||
detection["metadata"]["tile_edge"] = True
|
||||
detection["metadata"]["ood"] = True
|
||||
detection_result = evaluate_object_detection(detection)
|
||||
false_positive = next(
|
||||
item
|
||||
for item in detection_result["failures"]
|
||||
if item["kind"] == "false_positive"
|
||||
)
|
||||
assert {"tile_edge", "high_confidence", "out_of_distribution"} <= set(
|
||||
false_positive["contexts"]
|
||||
)
|
||||
assert {"M-MISCALIBRATED", "M-OOD"} <= set(false_positive["secondary_error_codes"])
|
||||
assert any(
|
||||
item["error_code"] == "M-MISCALIBRATED" for item in detection_result["failures"]
|
||||
)
|
||||
|
||||
|
||||
def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> (
|
||||
None
|
||||
):
|
||||
terrain = {
|
||||
"sample_id": "terrain",
|
||||
"task": "terrain_interpretation",
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": {},
|
||||
"lineage": lineage("terrain"),
|
||||
"units": "m_TAW",
|
||||
"references": [1.0, 2.0],
|
||||
"predictions": [1.1, None],
|
||||
}
|
||||
assert evaluate_terrain(terrain)["metrics"]["coverage"] == 0.5
|
||||
for field, value in (("references", math.nan), ("predictions", math.inf)):
|
||||
invalid = copy.deepcopy(terrain)
|
||||
invalid[field][0] = value
|
||||
with pytest.raises(ValueError, match="finite number"):
|
||||
evaluate_terrain(invalid)
|
||||
|
||||
validation = {
|
||||
"sample_id": "validation",
|
||||
"task": "geospatial_data_validation",
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": {},
|
||||
"lineage": lineage("validation"),
|
||||
"expected_anomalies": [{"code": "D-MAJOR", "severity": "major"}],
|
||||
"observed_anomalies": [],
|
||||
}
|
||||
assert (
|
||||
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
|
||||
== 0
|
||||
)
|
||||
validation["expected_anomalies"].append(
|
||||
{"code": "D-CRITICAL", "severity": "critical"}
|
||||
)
|
||||
assert (
|
||||
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
|
||||
== 1
|
||||
)
|
||||
validation["expected_anomalies"] = [{"code": "D-SEVERITY", "severity": "critical"}]
|
||||
validation["observed_anomalies"] = [{"code": "D-SEVERITY", "severity": "minor"}]
|
||||
severity_result = evaluate_validation(validation)
|
||||
assert severity_result["metrics"]["true_positive"] == 0
|
||||
assert severity_result["metrics"]["false_positive"] == 1
|
||||
assert severity_result["metrics"]["false_negative"] == 1
|
||||
assert severity_result["metrics"]["severity_mismatch_count"] == 1
|
||||
assert severity_result["metrics"]["blocker_or_critical_miss_count"] == 1
|
||||
|
||||
validation["expected_anomalies"] = ["D-NO-SEVERITY"]
|
||||
with pytest.raises(ValueError, match="include code and severity"):
|
||||
evaluate_validation(validation)
|
||||
|
||||
|
||||
def _subgroup_result(region: str, index: int, tp: int, fp: int, fn: int) -> dict:
|
||||
metadata = copy.deepcopy(METADATA)
|
||||
metadata["region"] = region
|
||||
sample_id = f"{region}-{index}"
|
||||
reference = {"id": "r", "class": "building", "bbox": [0, 0, 1, 1]}
|
||||
prediction = {
|
||||
"id": "p",
|
||||
"class": "building",
|
||||
"bbox": [0, 0, 1, 1],
|
||||
"confidence": 0.8,
|
||||
}
|
||||
return {
|
||||
"sample_id": sample_id,
|
||||
"task": "object_detection",
|
||||
"metadata": metadata,
|
||||
"metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4},
|
||||
"raw": {
|
||||
"sample_id": sample_id,
|
||||
"classes": ["building"],
|
||||
"references": [reference],
|
||||
"predictions_pre_filter": [prediction],
|
||||
"predictions_post_filter": [prediction],
|
||||
"matches": [
|
||||
{
|
||||
"prediction_id": "p",
|
||||
"reference_id": "r",
|
||||
"overlap": 1.0,
|
||||
"confidence": 0.8,
|
||||
"class": "building",
|
||||
}
|
||||
],
|
||||
},
|
||||
"failures": [],
|
||||
}
|
||||
|
||||
|
||||
def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
|
||||
results = [
|
||||
*[_subgroup_result("strong", index, 10, 0, 0) for index in range(5)],
|
||||
*[_subgroup_result("weak", index, 1, 4, 4) for index in range(5)],
|
||||
]
|
||||
report = subgroup_report(results)
|
||||
region = report["dimensions"]["region"]
|
||||
weak = region["strata"]["weak"]["task_metrics"]["object_detection"]
|
||||
|
||||
assert weak["status"] == "evaluable"
|
||||
assert weak["case_support"] == 5
|
||||
assert weak["micro"]["precision_ci95_wilson"]["status"] == "computed"
|
||||
assert weak["macro"]["f1_case_support"] == 5
|
||||
assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak"
|
||||
|
||||
insufficient = subgroup_report([_subgroup_result("thin", 0, 1, 0, 0)])
|
||||
thin = insufficient["dimensions"]["region"]["strata"]["thin"]
|
||||
assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support"
|
||||
assert thin["release_gate_status"] == "not_evaluable"
|
||||
assert insufficient["overall_status"] == "not_evaluable"
|
||||
|
||||
|
||||
def test_capability_inventory_is_comprehensive_and_honest() -> None:
|
||||
inventory = task_inventory()
|
||||
assert {item["task"] for item in inventory} == TASKS
|
||||
assert len(inventory) >= 15
|
||||
assert all(item["implementation_paths"] for item in inventory)
|
||||
assert all(item["suitable_metrics"] for item in inventory)
|
||||
assistant = next(
|
||||
item
|
||||
for item in inventory
|
||||
if item["capability_id"] == "geo_assistant_orchestration"
|
||||
)
|
||||
assert assistant["evaluation_status"].startswith("no_independent_accuracy_score")
|
||||
@@ -0,0 +1,538 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
from generate_accuracy_phase4_splits import ( # noqa: E402
|
||||
LeakageError,
|
||||
assert_training_inputs_safe,
|
||||
build_manifests,
|
||||
generate,
|
||||
)
|
||||
|
||||
|
||||
SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json"
|
||||
|
||||
|
||||
def load_source() -> dict:
|
||||
return json.loads(SOURCE.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def build_fixture_manifests(source: dict) -> tuple[dict, dict, dict]:
|
||||
return build_manifests(source, trusted_fixture_mode=True)
|
||||
|
||||
|
||||
def assert_fixture_training_inputs_safe(
|
||||
input_paths: list[Path], input_records: list[dict], protected: dict
|
||||
) -> None:
|
||||
assert_training_inputs_safe(
|
||||
input_paths, input_records, protected, trusted_fixture_mode=True
|
||||
)
|
||||
|
||||
|
||||
def test_normative_roles_hashes_and_source_order_are_enforced() -> None:
|
||||
source = load_source()
|
||||
development, protected, leakage = build_fixture_manifests(source)
|
||||
reversed_source = copy.deepcopy(source)
|
||||
reversed_source["samples"].reverse()
|
||||
reversed_development, reversed_protected, reversed_leakage = (
|
||||
build_fixture_manifests(reversed_source)
|
||||
)
|
||||
|
||||
assert leakage["status"] == "pass"
|
||||
assert leakage["finding_count"] == 0
|
||||
assert leakage["split_counts"] == {
|
||||
"background-test": 2,
|
||||
"calibration": 2,
|
||||
"challenge": 4,
|
||||
"test": 7,
|
||||
"train": 3,
|
||||
"val": 3,
|
||||
}
|
||||
assert leakage["crs_validation"] == {
|
||||
"status": "pass",
|
||||
"crs": "EPSG:31370",
|
||||
"distance_units": "m",
|
||||
}
|
||||
assert development["training_access_allowed_by_split"] == {
|
||||
"train": True,
|
||||
"val": False,
|
||||
"calibration": False,
|
||||
}
|
||||
assert protected["labels_available_by_split"]["challenge"] == "sealed_external"
|
||||
assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
|
||||
assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
|
||||
assert reversed_leakage == leakage
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "expected_code"),
|
||||
[
|
||||
("group_id", "S-SPATIAL-GROUP"),
|
||||
("source_family", "S-SOURCE-FAMILY"),
|
||||
("temporal_family", "S-TEMPORAL-FAMILY"),
|
||||
("raw_image_sha256", "S-RAW-IMAGE-DUPLICATE"),
|
||||
("processed_image_sha256", "S-PROCESSED-IMAGE-DUPLICATE"),
|
||||
("label_sha256", "S-LABEL-DUPLICATE"),
|
||||
("label_geometry_hash", "S-LABEL-GEOMETRY-DUPLICATE"),
|
||||
("parent_raster_id", "S-PARENT-RASTER"),
|
||||
("acquisition_id", "S-ACQUISITION"),
|
||||
],
|
||||
)
|
||||
def test_cross_split_lineage_and_content_collisions_fail(
|
||||
field: str, expected_code: str
|
||||
) -> None:
|
||||
source = load_source()
|
||||
source["samples"][8][field] = source["samples"][0][field]
|
||||
_development, _protected, leakage = build_fixture_manifests(source)
|
||||
|
||||
assert leakage["status"] == "fail"
|
||||
assert expected_code in {item["code"] for item in leakage["findings"]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "expected_code"),
|
||||
[
|
||||
("perceptual_image_hash", "S-PERCEPTUAL-IMAGE-NEAR-DUPLICATE"),
|
||||
("label_geometry_fingerprint", "S-LABEL-GEOMETRY-NEAR-DUPLICATE"),
|
||||
],
|
||||
)
|
||||
def test_near_duplicate_fingerprints_fail(field: str, expected_code: str) -> None:
|
||||
source = load_source()
|
||||
source["samples"][8][field] = source["samples"][0][field]
|
||||
_development, _protected, leakage = build_fixture_manifests(source)
|
||||
|
||||
assert leakage["status"] == "fail"
|
||||
assert expected_code in {item["code"] for item in leakage["findings"]}
|
||||
|
||||
|
||||
def test_object_native_feature_and_spatial_collisions_fail() -> None:
|
||||
source = load_source()
|
||||
source["samples"][8]["object_ids"] = source["samples"][0]["object_ids"]
|
||||
source["samples"][9]["native_feature_ids"] = source["samples"][1][
|
||||
"native_feature_ids"
|
||||
]
|
||||
source["samples"][10]["bbox"] = source["samples"][2]["bbox"]
|
||||
_development, _protected, leakage = build_fixture_manifests(source)
|
||||
codes = {item["code"] for item in leakage["findings"]}
|
||||
|
||||
assert {"S-OBJECT-INSTANCE", "S-NATIVE-FEATURE", "S-SPATIAL-OVERLAP"} <= codes
|
||||
|
||||
|
||||
def test_non_metric_crs_and_missing_normative_role_fail_closed() -> None:
|
||||
geographic = load_source()
|
||||
geographic["crs"] = "EPSG:4326"
|
||||
with pytest.raises(LeakageError, match="projected in metres"):
|
||||
build_fixture_manifests(geographic)
|
||||
|
||||
missing = load_source()
|
||||
missing["samples"] = [
|
||||
item for item in missing["samples"] if item["split"] != "calibration"
|
||||
]
|
||||
with pytest.raises(LeakageError, match="Required splits are absent"):
|
||||
build_fixture_manifests(missing)
|
||||
|
||||
|
||||
def test_training_firewall_only_allows_train_and_binds_protected_lineage() -> None:
|
||||
development, protected, leakage = build_fixture_manifests(load_source())
|
||||
assert leakage["status"] == "pass"
|
||||
train = [item for item in development["samples"] if item["split"] == "train"]
|
||||
validation = next(item for item in development["samples"] if item["split"] == "val")
|
||||
protected_item = protected["samples"][0]
|
||||
|
||||
assert_fixture_training_inputs_safe([], train, protected)
|
||||
with pytest.raises(LeakageError, match="non_train_role"):
|
||||
assert_fixture_training_inputs_safe([], [validation], protected)
|
||||
with pytest.raises(LeakageError, match="protected_identity"):
|
||||
disguised = copy.deepcopy(train[0])
|
||||
disguised["source_family"] = protected_item["source_family"]
|
||||
assert_fixture_training_inputs_safe([], [disguised], protected)
|
||||
with pytest.raises(LeakageError, match="protected_path"):
|
||||
assert_fixture_training_inputs_safe(
|
||||
[Path("vault/protected/test.json")], [], protected
|
||||
)
|
||||
|
||||
|
||||
def test_failed_generation_writes_status_but_no_consumable_manifests(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
source = load_source()
|
||||
source["samples"][8]["group_id"] = source["samples"][0]["group_id"]
|
||||
source_path = tmp_path / "source.json"
|
||||
source_path.write_text(json.dumps(source), encoding="utf-8")
|
||||
output = tmp_path / "out"
|
||||
|
||||
with pytest.raises(LeakageError, match="Leakage gate failed"):
|
||||
generate(source_path, output, trusted_fixture_mode=True)
|
||||
|
||||
status = json.loads((output / "generation-status.json").read_text(encoding="utf-8"))
|
||||
assert status["status"] == "fail"
|
||||
assert status["consumable_manifests_valid"] is False
|
||||
for name in (
|
||||
"development-split-manifest.json",
|
||||
"protected-split-manifest.json",
|
||||
):
|
||||
tombstone = json.loads((output / name).read_text(encoding="utf-8"))
|
||||
assert tombstone["status"] == "invalidated"
|
||||
assert tombstone["consumable"] is False
|
||||
|
||||
|
||||
def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together() -> (
|
||||
None
|
||||
):
|
||||
source = load_source()
|
||||
source["assignment_mode"] = "deterministic_grouped"
|
||||
source["split_assignment"] = {
|
||||
"seed": "fixed-phase4-test-seed",
|
||||
"roles": [
|
||||
"train",
|
||||
"val",
|
||||
"calibration",
|
||||
"test",
|
||||
"background-test",
|
||||
"challenge",
|
||||
],
|
||||
"weights": {
|
||||
"train": 6,
|
||||
"val": 2,
|
||||
"calibration": 1,
|
||||
"test": 2,
|
||||
"background-test": 1,
|
||||
"challenge": 1,
|
||||
},
|
||||
"stratify_by": ["task"],
|
||||
}
|
||||
for item in source["samples"]:
|
||||
item.pop("split")
|
||||
source["samples"][1]["group_id"] = source["samples"][0]["group_id"]
|
||||
|
||||
development, protected, leakage = build_fixture_manifests(source)
|
||||
reversed_source = copy.deepcopy(source)
|
||||
reversed_source["samples"].reverse()
|
||||
reversed_development, reversed_protected, reversed_leakage = (
|
||||
build_fixture_manifests(reversed_source)
|
||||
)
|
||||
|
||||
assigned = {
|
||||
item["sample_id"]: item["split"]
|
||||
for item in development["samples"] + protected["samples"]
|
||||
}
|
||||
assert assigned["det-train-a"] == assigned["seg-train-a"]
|
||||
assert set(leakage["split_counts"]) == {
|
||||
"train",
|
||||
"val",
|
||||
"calibration",
|
||||
"test",
|
||||
"background-test",
|
||||
"challenge",
|
||||
}
|
||||
assert leakage["status"] == "pass"
|
||||
assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
|
||||
assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
|
||||
assert reversed_leakage == leakage
|
||||
|
||||
|
||||
def test_source_cannot_weaken_mandatory_roles_or_policy_floors() -> None:
|
||||
source = load_source()
|
||||
source["required_splits"] = ["train", "val", "test"]
|
||||
with pytest.raises(LeakageError, match="mandatory role order"):
|
||||
build_fixture_manifests(source)
|
||||
|
||||
grouped = load_source()
|
||||
grouped["assignment_mode"] = "deterministic_grouped"
|
||||
grouped["split_assignment"] = {"roles": ["train", "val"]}
|
||||
for item in grouped["samples"]:
|
||||
item.pop("split")
|
||||
with pytest.raises(LeakageError, match="mandatory role order"):
|
||||
build_fixture_manifests(grouped)
|
||||
|
||||
for field, value in (
|
||||
("independence_buffer_m", 1999),
|
||||
("perceptual_hamming_threshold", 3),
|
||||
("label_geometry_hamming_threshold", 1),
|
||||
):
|
||||
weakened = load_source()
|
||||
weakened[field] = value
|
||||
with pytest.raises(LeakageError, match="code-owned minimum"):
|
||||
build_fixture_manifests(weakened)
|
||||
|
||||
|
||||
def test_task_coverage_gap_and_even_justified_exemption_fail_honestly() -> None:
|
||||
source = load_source()
|
||||
source["samples"] = [
|
||||
item
|
||||
for item in source["samples"]
|
||||
if item["sample_id"] != "validation-test-national"
|
||||
]
|
||||
_development, _protected, leakage = build_fixture_manifests(source)
|
||||
assert leakage["status"] == "fail"
|
||||
assert "S-PROTECTED-TASK-COVERAGE-MISSING" in {
|
||||
finding["code"] for finding in leakage["findings"]
|
||||
}
|
||||
|
||||
source["protected_task_exemptions"] = {
|
||||
"geospatial_data_validation": (
|
||||
"No evaluator-visible reference exists; challenge data remains sealed."
|
||||
)
|
||||
}
|
||||
_development, _protected, leakage = build_fixture_manifests(source)
|
||||
assert leakage["status"] == "fail"
|
||||
assert "S-PROTECTED-TASK-COVERAGE-EXEMPTED" in {
|
||||
finding["code"] for finding in leakage["findings"]
|
||||
}
|
||||
|
||||
|
||||
def test_identifiers_and_acquisition_dates_are_canonical_leakage_keys() -> None:
|
||||
source = load_source()
|
||||
source["samples"][8]["group_id"] = " G01 "
|
||||
_development, _protected, leakage = build_fixture_manifests(source)
|
||||
assert "S-SPATIAL-GROUP" in {item["code"] for item in leakage["findings"]}
|
||||
|
||||
temporal = load_source()
|
||||
temporal["samples"][8]["acquisition_date"] = temporal["samples"][0][
|
||||
"acquisition_date"
|
||||
]
|
||||
_development, _protected, leakage = build_fixture_manifests(temporal)
|
||||
assert "S-ACQUISITION-DATE" in {item["code"] for item in leakage["findings"]}
|
||||
|
||||
ambiguous = load_source()
|
||||
ambiguous["samples"][8]["sample_id"] = " DET-TRAIN-A "
|
||||
with pytest.raises(LeakageError, match="ambiguous canonical sample_id"):
|
||||
build_fixture_manifests(ambiguous)
|
||||
|
||||
|
||||
def test_challenge_is_sealed_in_standard_manifest() -> None:
|
||||
_development, protected, leakage = build_fixture_manifests(load_source())
|
||||
assert leakage["status"] == "pass"
|
||||
challenge = [item for item in protected["samples"] if item["split"] == "challenge"]
|
||||
forbidden = {
|
||||
"label_sha256",
|
||||
"label_geometry_hash",
|
||||
"label_geometry_fingerprint",
|
||||
"object_ids",
|
||||
"native_feature_ids",
|
||||
"record_sha256",
|
||||
"label_path",
|
||||
"label_geometry_path",
|
||||
}
|
||||
assert challenge
|
||||
assert all(item["sealed"] is True for item in challenge)
|
||||
assert all(not (forbidden & set(item)) for item in challenge)
|
||||
|
||||
|
||||
def test_firewall_rejects_empty_tampered_wrong_and_renamed_manifests(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
development, protected, leakage = build_fixture_manifests(load_source())
|
||||
assert leakage["status"] == "pass"
|
||||
train = [item for item in development["samples"] if item["split"] == "train"]
|
||||
|
||||
with pytest.raises(LeakageError, match="empty manifest"):
|
||||
assert_fixture_training_inputs_safe([], train, {})
|
||||
with pytest.raises(LeakageError, match="missing fields"):
|
||||
assert_fixture_training_inputs_safe([], train, development)
|
||||
tampered = copy.deepcopy(protected)
|
||||
tampered["samples"].pop()
|
||||
with pytest.raises(LeakageError, match="checksum mismatch"):
|
||||
assert_fixture_training_inputs_safe([], train, tampered)
|
||||
|
||||
renamed = tmp_path / "ordinary-training-input.json"
|
||||
renamed.write_text(json.dumps(protected), encoding="utf-8")
|
||||
with pytest.raises(LeakageError, match="protected_manifest_content"):
|
||||
assert_fixture_training_inputs_safe([renamed], train, protected)
|
||||
|
||||
|
||||
def _canonical_json_sha256(value: object) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def make_governed_source(tmp_path: Path) -> dict:
|
||||
source = load_source()
|
||||
source["dataset_version"] = "governed-production-v1"
|
||||
source["claim_boundary"] = "Governed production split source."
|
||||
p3_items: list[dict] = []
|
||||
provenance_records: list[dict] = []
|
||||
asset_fields = {
|
||||
"raw_image": ("raw_image_path", "raw_image_sha256"),
|
||||
"processed_image": ("processed_image_path", "processed_image_sha256"),
|
||||
"label": ("label_path", "label_sha256"),
|
||||
"label_geometry": ("label_geometry_path", "label_geometry_hash"),
|
||||
}
|
||||
for sample in source["samples"]:
|
||||
assets: dict[str, dict] = {}
|
||||
p3_ids: dict[str, str] = {}
|
||||
for role, (path_field, hash_field) in asset_fields.items():
|
||||
relative = Path("assets") / sample["sample_id"] / f"{role}.bin"
|
||||
path = tmp_path / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(f"{sample['sample_id']}:{role}:governed".encode())
|
||||
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
p3_id = hashlib.sha256(
|
||||
f"{sample['sample_id']}:{role}".encode()
|
||||
).hexdigest()[:20]
|
||||
relative_posix = relative.as_posix()
|
||||
sample[path_field] = relative_posix
|
||||
sample[hash_field] = digest
|
||||
p3_ids[role] = p3_id
|
||||
assets[role] = {
|
||||
"path": relative_posix,
|
||||
"sha256": digest,
|
||||
"size_bytes": path.stat().st_size,
|
||||
"p3_item_id": p3_id,
|
||||
}
|
||||
p3_items.append(
|
||||
{
|
||||
"item_id": p3_id,
|
||||
"path": relative_posix,
|
||||
"sha256": digest,
|
||||
"size_bytes": path.stat().st_size,
|
||||
"status": "examined",
|
||||
"read_status": "readable",
|
||||
"recommended_action": "accept",
|
||||
"empty_content": False,
|
||||
"schema_conformity": "conformant",
|
||||
"anomalies": [],
|
||||
}
|
||||
)
|
||||
provenance_id = (
|
||||
"prov-" + hashlib.sha256(sample["sample_id"].encode()).hexdigest()[:24]
|
||||
)
|
||||
sample["governance_binding"] = {
|
||||
"source_provenance_record_id": provenance_id,
|
||||
"p3_item_ids": p3_ids,
|
||||
}
|
||||
provenance_records.append(
|
||||
{
|
||||
"record_id": provenance_id,
|
||||
"sample_id": sample["sample_id"],
|
||||
"status": "accepted",
|
||||
"lineage_status": "complete",
|
||||
"training_allowed": True,
|
||||
"perceptual_image_hash": sample["perceptual_image_hash"],
|
||||
"label_geometry_fingerprint": sample["label_geometry_fingerprint"],
|
||||
"assets": assets,
|
||||
}
|
||||
)
|
||||
p3 = {
|
||||
"schema_version": 1,
|
||||
"scan_id": "p3-test-governed",
|
||||
"scanner_version": "3.0.3",
|
||||
"completed_at": "2026-08-02T12:00:00+02:00",
|
||||
"items": p3_items,
|
||||
"reconciliation": {
|
||||
"examined": len(p3_items),
|
||||
"skipped": 0,
|
||||
"unreachable": 0,
|
||||
"inventory_total": len(p3_items),
|
||||
"reconciles": True,
|
||||
},
|
||||
}
|
||||
provenance = {
|
||||
"schema_version": 1,
|
||||
"manifest_type": "geointel_phase4_source_provenance",
|
||||
"status": "pass",
|
||||
"records": provenance_records,
|
||||
"records_canonical_json_sha256": _canonical_json_sha256(provenance_records),
|
||||
}
|
||||
p3_path = tmp_path / "p3.json"
|
||||
provenance_path = tmp_path / "provenance.json"
|
||||
p3_path.write_text(json.dumps(p3), encoding="utf-8")
|
||||
provenance_path.write_text(json.dumps(provenance), encoding="utf-8")
|
||||
source["governance_evidence"] = {
|
||||
"p3_scan_manifest": {
|
||||
"path": p3_path.name,
|
||||
"sha256": hashlib.sha256(p3_path.read_bytes()).hexdigest(),
|
||||
},
|
||||
"source_provenance_manifest": {
|
||||
"path": provenance_path.name,
|
||||
"sha256": hashlib.sha256(provenance_path.read_bytes()).hexdigest(),
|
||||
},
|
||||
}
|
||||
return source
|
||||
|
||||
|
||||
def test_fixture_mode_is_explicit_and_source_metadata_cannot_enable_it() -> None:
|
||||
with pytest.raises(LeakageError, match="explicit trusted_fixture_mode"):
|
||||
build_manifests(load_source())
|
||||
|
||||
source = load_source()
|
||||
source["trusted_fixture_mode"] = True
|
||||
with pytest.raises(LeakageError, match="cannot be enabled by source metadata"):
|
||||
build_manifests(source, trusted_fixture_mode=True)
|
||||
|
||||
|
||||
def test_empty_or_arbitrary_governance_json_is_rejected(tmp_path: Path) -> None:
|
||||
source = load_source()
|
||||
source["dataset_version"] = "governed-production-v1"
|
||||
source["claim_boundary"] = "Governed production split source."
|
||||
p3 = tmp_path / "p3.json"
|
||||
provenance = tmp_path / "provenance.json"
|
||||
p3.write_text("{}", encoding="utf-8")
|
||||
provenance.write_text("{}", encoding="utf-8")
|
||||
source["governance_evidence"] = {
|
||||
"p3_scan_manifest": {
|
||||
"path": p3.name,
|
||||
"sha256": hashlib.sha256(p3.read_bytes()).hexdigest(),
|
||||
},
|
||||
"source_provenance_manifest": {
|
||||
"path": provenance.name,
|
||||
"sha256": hashlib.sha256(provenance.read_bytes()).hexdigest(),
|
||||
},
|
||||
}
|
||||
with pytest.raises(LeakageError, match="non-empty JSON object"):
|
||||
build_manifests(source, source_root=tmp_path)
|
||||
|
||||
|
||||
def test_governed_records_require_exact_provenance_paths_and_live_bytes(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
source = make_governed_source(tmp_path)
|
||||
development, protected, leakage = build_manifests(source, source_root=tmp_path)
|
||||
assert leakage["status"] == "pass"
|
||||
assert protected["source_trust"]["production_accuracy_use_allowed"] is True
|
||||
train = [item for item in development["samples"] if item["split"] == "train"]
|
||||
assert_training_inputs_safe([], train, protected)
|
||||
|
||||
no_paths = copy.deepcopy(train[0])
|
||||
no_paths.pop("content_path_bindings")
|
||||
with pytest.raises(LeakageError, match="missing_accessible_content_paths"):
|
||||
assert_training_inputs_safe([], [no_paths], protected)
|
||||
|
||||
relabeled = copy.deepcopy(
|
||||
next(item for item in protected["samples"] if item["split"] == "test")
|
||||
)
|
||||
relabeled["split"] = "train"
|
||||
relabeled.pop("content_path_bindings")
|
||||
for field in ("sample_id", "group_id", "source_family", "temporal_family"):
|
||||
relabeled[field] = f"spoofed-{field}"
|
||||
with pytest.raises(
|
||||
LeakageError, match="unavailable_provenance_record|protected_identity"
|
||||
):
|
||||
assert_training_inputs_safe([], [relabeled], protected)
|
||||
|
||||
broken_binding = copy.deepcopy(source)
|
||||
broken_binding["samples"][0]["governance_binding"]["p3_item_ids"]["raw_image"] = (
|
||||
"0" * 20
|
||||
)
|
||||
with pytest.raises(LeakageError, match="provenance binding mismatch|P3 record"):
|
||||
build_manifests(broken_binding, source_root=tmp_path)
|
||||
|
||||
raw_path = Path(train[0]["content_path_bindings"]["raw_image"]["resolved_path"])
|
||||
raw_path.write_bytes(b"mutated after manifest creation")
|
||||
with pytest.raises(LeakageError, match="record_path_hash_binding_mismatch"):
|
||||
assert_training_inputs_safe([], train, protected)
|
||||
@@ -0,0 +1,10 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_alembic_logging_formatter_uses_runtime_interpolation_tokens() -> None:
|
||||
config = Path(__file__).resolve().parents[1] / "alembic.ini"
|
||||
content = config.read_text(encoding="utf-8")
|
||||
|
||||
assert "format = %(levelname)-5.5s [%(name)s] %(message)s" in content
|
||||
assert "%%(levelname)" not in content
|
||||
assert "%%(message)" not in content
|
||||
@@ -0,0 +1,135 @@
|
||||
"""A queued run must be claimed once, even if two workers look at it.
|
||||
|
||||
The worker selected queued jobs and then set them to running in a second
|
||||
statement. Two workers — an API restart overlapping the previous process, or a
|
||||
second replica — could both select the same row and both start tiled GPU
|
||||
inference on it, producing duplicate analysis runs and doubling the GPU load.
|
||||
|
||||
The AOI worker beside it already claims with ``FOR UPDATE SKIP LOCKED``. This
|
||||
uses a conditional update, which is the same guarantee expressed in one
|
||||
statement: exactly one caller sees a row count of 1.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from app.models import Job
|
||||
from app.services.analysis_job_worker import AnalysisJobWorker
|
||||
|
||||
|
||||
class _Update:
|
||||
"""Mimics a conditional UPDATE: the first caller wins, the rest see zero."""
|
||||
|
||||
def __init__(self, store: dict, job_id):
|
||||
self.store = store
|
||||
self.job_id = job_id
|
||||
|
||||
def update(self, values, **_kwargs) -> int:
|
||||
if self.store.get(self.job_id) != "queued":
|
||||
return 0
|
||||
self.store[self.job_id] = "running"
|
||||
return 1
|
||||
|
||||
|
||||
class _Query:
|
||||
def __init__(self, session, model):
|
||||
self.session = session
|
||||
self.model = model
|
||||
self.job_id = None
|
||||
|
||||
def filter(self, *criteria):
|
||||
for criterion in criteria:
|
||||
right = getattr(criterion, "right", None)
|
||||
value = getattr(right, "value", None)
|
||||
if isinstance(value, type(uuid4())):
|
||||
self.job_id = value
|
||||
return self
|
||||
|
||||
def update(self, values, **kwargs) -> int:
|
||||
return _Update(self.session.statuses, self.job_id).update(values, **kwargs)
|
||||
|
||||
def order_by(self, *_args):
|
||||
return self
|
||||
|
||||
def limit(self, _count):
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return list(self.session.rows)
|
||||
|
||||
|
||||
class _Session:
|
||||
def __init__(self, rows: list[Job]):
|
||||
self.rows = rows
|
||||
self.statuses = {row.id: row.status for row in rows}
|
||||
self.committed = 0
|
||||
|
||||
def query(self, model):
|
||||
return _Query(self, model)
|
||||
|
||||
def get(self, _model, item_id):
|
||||
return next((row for row in self.rows if row.id == item_id), None)
|
||||
|
||||
def add(self, _item):
|
||||
return None
|
||||
|
||||
def commit(self):
|
||||
self.committed += 1
|
||||
|
||||
def rollback(self):
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
|
||||
def _job() -> Job:
|
||||
return Job(
|
||||
id=uuid4(),
|
||||
job_type="detection.run",
|
||||
status="queued",
|
||||
project_id=uuid4(),
|
||||
parameters_json={},
|
||||
)
|
||||
|
||||
|
||||
def test_the_first_claim_wins() -> None:
|
||||
job = _job()
|
||||
session = _Session([job])
|
||||
|
||||
assert AnalysisJobWorker.claim(session, job) is True
|
||||
assert job.status == "running"
|
||||
|
||||
|
||||
def test_a_second_claim_on_the_same_job_is_refused() -> None:
|
||||
job = _job()
|
||||
session = _Session([job])
|
||||
|
||||
assert AnalysisJobWorker.claim(session, job) is True
|
||||
assert AnalysisJobWorker.claim(session, job) is False
|
||||
|
||||
|
||||
def test_a_job_that_is_no_longer_queued_cannot_be_claimed() -> None:
|
||||
job = _job()
|
||||
session = _Session([job])
|
||||
session.statuses[job.id] = "success"
|
||||
|
||||
assert AnalysisJobWorker.claim(session, job) is False
|
||||
|
||||
|
||||
def test_an_unclaimable_job_is_skipped_rather_than_run(monkeypatch) -> None:
|
||||
job = _job()
|
||||
session = _Session([job])
|
||||
session.statuses[job.id] = "running"
|
||||
dispatched: list[Job] = []
|
||||
monkeypatch.setattr(
|
||||
AnalysisJobWorker,
|
||||
"_dispatch",
|
||||
staticmethod(lambda _db, item: dispatched.append(item)),
|
||||
)
|
||||
|
||||
processed = AnalysisJobWorker.run_once(db=session)
|
||||
|
||||
assert processed == 0
|
||||
assert dispatched == []
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Tiled GPU inference must not run inside an HTTP request.
|
||||
|
||||
A configured YOLO run walks up to ``YOLO_MAX_TILES`` tiles through the GPU.
|
||||
Doing that in the request handler holds a worker thread for minutes, gives the
|
||||
operator no progress, and times the client out before the result exists. The
|
||||
run is queued as a Job instead and executed by a background worker, which is
|
||||
the same pattern the AOI operations already use.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Job
|
||||
from app.services.analysis_job_worker import AnalysisJobWorker
|
||||
from app.services.detection_service import DetectionService
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def __init__(self, rows):
|
||||
self.rows = list(rows)
|
||||
|
||||
def filter(self, *criteria):
|
||||
return self
|
||||
|
||||
def update(self, values, **_kwargs) -> int:
|
||||
"""Stand in for the conditional claim: succeeds while still queued."""
|
||||
|
||||
claimed = 0
|
||||
for row in self.rows:
|
||||
if getattr(row, "status", None) == "queued":
|
||||
row.status = "running"
|
||||
claimed += 1
|
||||
return claimed
|
||||
|
||||
def order_by(self, *_args):
|
||||
return self
|
||||
|
||||
def limit(self, count):
|
||||
self.rows = self.rows[:count]
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return list(self.rows)
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, objects=None, query_rows=None):
|
||||
self.objects = dict(objects or {})
|
||||
self.query_rows = query_rows or {}
|
||||
self.added = []
|
||||
|
||||
def get(self, model, item_id):
|
||||
return self.objects.get((model, item_id))
|
||||
|
||||
def query(self, model):
|
||||
return FakeQuery(self.query_rows.get(model, []))
|
||||
|
||||
def add(self, item):
|
||||
self.added.append(item)
|
||||
if getattr(item, "id", None) is not None:
|
||||
self.objects[(item.__class__, item.id)] = item
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
def rollback(self):
|
||||
return None
|
||||
|
||||
def refresh(self, _item):
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
|
||||
def _queued_job(**parameters) -> Job:
|
||||
payload = {
|
||||
"project_id": str(uuid4()),
|
||||
"dataset_id": str(uuid4()),
|
||||
"model_id": "yolo-configured",
|
||||
"confidence_threshold": 0.4,
|
||||
"class_filter": ["building"],
|
||||
"tile_manifest_path": "/tiles/manifest.json",
|
||||
"parameters_json": {},
|
||||
}
|
||||
payload.update(parameters)
|
||||
return Job(
|
||||
id=uuid4(),
|
||||
job_type="detection.run",
|
||||
status="queued",
|
||||
project_id=uuid4(),
|
||||
parameters_json=payload,
|
||||
)
|
||||
|
||||
|
||||
def test_queued_detection_job_is_dispatched_to_the_detection_service(monkeypatch) -> None:
|
||||
job = _queued_job()
|
||||
db = FakeSession(objects={(Job, job.id): job}, query_rows={Job: [job]})
|
||||
calls: list[dict] = []
|
||||
|
||||
def fake_run(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return type(
|
||||
"Result",
|
||||
(),
|
||||
{
|
||||
"status": "success",
|
||||
"detection_count": 3,
|
||||
"analysis_run_id": uuid4(),
|
||||
"job_id": kwargs["existing_job"].id,
|
||||
"model_dump": lambda self, **_: {"status": "success", "detection_count": 3},
|
||||
},
|
||||
)()
|
||||
|
||||
monkeypatch.setattr(DetectionService, "run_detection", staticmethod(fake_run))
|
||||
|
||||
processed = AnalysisJobWorker.run_once(db=db)
|
||||
|
||||
assert processed == 1
|
||||
assert calls[0]["model_id"] == "yolo-configured"
|
||||
assert calls[0]["confidence_threshold"] == 0.4
|
||||
assert calls[0]["tile_manifest_path"] == "/tiles/manifest.json"
|
||||
assert calls[0]["existing_job"] is job
|
||||
assert job.status == "success"
|
||||
|
||||
|
||||
def test_a_failing_run_marks_the_job_failed_instead_of_leaving_it_running(monkeypatch) -> None:
|
||||
job = _queued_job()
|
||||
db = FakeSession(objects={(Job, job.id): job}, query_rows={Job: [job]})
|
||||
|
||||
def exploding(**_kwargs):
|
||||
raise AppError(code="DETECTION_TILE_NOT_FOUND", message="missing tile", status_code=422)
|
||||
|
||||
monkeypatch.setattr(DetectionService, "run_detection", staticmethod(exploding))
|
||||
|
||||
processed = AnalysisJobWorker.run_once(db=db)
|
||||
|
||||
assert processed == 1
|
||||
assert job.status == "failed"
|
||||
assert job.error_message == "missing tile"
|
||||
assert job.result_json["error_code"] == "DETECTION_TILE_NOT_FOUND"
|
||||
|
||||
|
||||
def test_an_unexpected_error_still_closes_the_job(monkeypatch) -> None:
|
||||
job = _queued_job()
|
||||
db = FakeSession(objects={(Job, job.id): job}, query_rows={Job: [job]})
|
||||
|
||||
def exploding(**_kwargs):
|
||||
raise RuntimeError("CUDA out of memory")
|
||||
|
||||
monkeypatch.setattr(DetectionService, "run_detection", staticmethod(exploding))
|
||||
|
||||
AnalysisJobWorker.run_once(db=db)
|
||||
|
||||
assert job.status == "failed"
|
||||
assert job.result_json["error_code"] == "ANALYSIS_JOB_INTERNAL_ERROR"
|
||||
|
||||
|
||||
def test_job_types_the_worker_does_not_own_are_left_alone() -> None:
|
||||
job = _queued_job()
|
||||
job.job_type = "raster.clip"
|
||||
db = FakeSession(objects={(Job, job.id): job}, query_rows={Job: [job]})
|
||||
|
||||
assert AnalysisJobWorker.run_once(db=db) == 0
|
||||
assert job.status == "queued"
|
||||
|
||||
|
||||
def test_enqueue_validates_before_accepting_the_job() -> None:
|
||||
"""A bad request is rejected up front, not minutes later in the worker."""
|
||||
|
||||
db = FakeSession()
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DetectionService.enqueue_detection(
|
||||
db=db,
|
||||
project_id=uuid4(),
|
||||
dataset_id=uuid4(),
|
||||
model_id="yolo-configured",
|
||||
confidence_threshold=0.4,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "PROJECT_NOT_FOUND"
|
||||
@@ -0,0 +1,159 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from geoalchemy2.shape import from_shape, to_shape
|
||||
from pyproj import Transformer
|
||||
import pytest
|
||||
from shapely.geometry import Polygon, mapping
|
||||
from shapely.ops import transform
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Project
|
||||
from app.schemas.area import AreaCreate, AreaUpdate
|
||||
from app.services.area_service import AreaService
|
||||
from app.utils.geometry import area_m2, normalize_area_to_epsg4326
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, objects=None) -> None:
|
||||
self.objects = objects or {}
|
||||
self.added = []
|
||||
self.commits = 0
|
||||
self.refreshes = []
|
||||
|
||||
def get(self, model, item_id):
|
||||
return self.objects.get((model, item_id))
|
||||
|
||||
def add(self, item) -> None:
|
||||
self.added.append(item)
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def refresh(self, item) -> None:
|
||||
self.refreshes.append(item)
|
||||
|
||||
|
||||
def _wgs84_polygon(offset: float = 0.0) -> Polygon:
|
||||
return Polygon(
|
||||
[
|
||||
(5.00 + offset, 51.00),
|
||||
(5.01 + offset, 51.00),
|
||||
(5.01 + offset, 51.01),
|
||||
(5.00 + offset, 51.01),
|
||||
(5.00 + offset, 51.00),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _to_lambert(geometry: Polygon) -> Polygon:
|
||||
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
return transform(transformer.transform, geometry)
|
||||
|
||||
|
||||
def test_create_area_transforms_declared_lambert_geometry_before_storage() -> None:
|
||||
project_id = uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
|
||||
source = _wgs84_polygon()
|
||||
|
||||
area = AreaService.create_area(
|
||||
db,
|
||||
project_id,
|
||||
AreaCreate(name="Lambert AOI", geometry=mapping(_to_lambert(source)), crs="EPSG:31370"),
|
||||
)
|
||||
|
||||
stored = to_shape(area.geometry)
|
||||
assert stored.bounds == pytest.approx(source.bounds, abs=1e-7)
|
||||
assert area.original_crs == "EPSG:31370"
|
||||
assert area.area_m2 == pytest.approx(area_m2(normalize_area_to_epsg4326(mapping(source), "EPSG:4326")[0]))
|
||||
assert area.area_m2 and area.area_m2 > 0
|
||||
assert to_shape(area.bbox).bounds == pytest.approx(source.bounds, abs=1e-7)
|
||||
|
||||
|
||||
def test_patch_area_replaces_geometry_and_recomputes_all_spatial_fields() -> None:
|
||||
area_id = uuid4()
|
||||
project_id = uuid4()
|
||||
original = _wgs84_polygon()
|
||||
normalized, _ = normalize_area_to_epsg4326(mapping(original), "EPSG:4326")
|
||||
area = Area(
|
||||
id=area_id,
|
||||
project_id=project_id,
|
||||
name="Original",
|
||||
geometry=from_shape(normalized, srid=4326),
|
||||
bbox=from_shape(normalized.envelope, srid=4326),
|
||||
original_crs="EPSG:4326",
|
||||
area_m2=area_m2(normalized),
|
||||
)
|
||||
db = FakeSession({(Area, area_id): area})
|
||||
replacement = _wgs84_polygon(offset=0.05)
|
||||
|
||||
updated = AreaService.update_area(
|
||||
db,
|
||||
area_id,
|
||||
AreaUpdate(
|
||||
name="Replacement",
|
||||
geometry=mapping(_to_lambert(replacement)),
|
||||
crs="EPSG:31370",
|
||||
),
|
||||
)
|
||||
|
||||
assert updated.name == "Replacement"
|
||||
assert updated.original_crs == "EPSG:31370"
|
||||
assert to_shape(updated.geometry).bounds == pytest.approx(replacement.bounds, abs=1e-7)
|
||||
assert to_shape(updated.bbox).bounds == pytest.approx(replacement.bounds, abs=1e-7)
|
||||
assert updated.area_m2 and updated.area_m2 > 0
|
||||
assert db.commits == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("geometry", "crs", "message_fragment"),
|
||||
[
|
||||
(mapping(_wgs84_polygon()), "EPSG:not-real", "unknown or invalid"),
|
||||
(mapping(_wgs84_polygon()), "EPSG:4979", "exactly two spatial axes"),
|
||||
(
|
||||
{
|
||||
"type": "Polygon",
|
||||
"coordinates": [[[5.0, 51.0], [float("nan"), 51.0], [5.1, 51.1], [5.0, 51.0]]],
|
||||
},
|
||||
"EPSG:4326",
|
||||
"finite",
|
||||
),
|
||||
(mapping(Polygon([(10.0, 51.0), (10.1, 51.0), (10.1, 51.1), (10.0, 51.0)])), "EPSG:4326", "workbench domain"),
|
||||
(
|
||||
{
|
||||
"type": "Polygon",
|
||||
"coordinates": [[[5.0, 51.0], [5.1, 51.1], [5.1, 51.0], [5.0, 51.1], [5.0, 51.0]]],
|
||||
},
|
||||
"EPSG:4326",
|
||||
"invalid",
|
||||
),
|
||||
({"type": "Point", "coordinates": [5.0, 51.0]}, "EPSG:4326", "Polygon or MultiPolygon"),
|
||||
],
|
||||
)
|
||||
def test_create_area_rejects_invalid_crs_nonfinite_and_out_of_domain_geometry(
|
||||
geometry: dict,
|
||||
crs: str,
|
||||
message_fragment: str,
|
||||
) -> None:
|
||||
project_id = uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
AreaService.create_area(db, project_id, AreaCreate(name="Invalid", geometry=geometry, crs=crs))
|
||||
|
||||
assert exc_info.value.code == "INVALID_GEOMETRY"
|
||||
assert message_fragment in exc_info.value.message
|
||||
assert db.commits == 0
|
||||
|
||||
|
||||
def test_patch_area_rejects_crs_without_replacement_geometry() -> None:
|
||||
area_id = uuid4()
|
||||
area = Area(id=area_id, project_id=uuid4(), name="AOI", original_crs="EPSG:4326")
|
||||
db = FakeSession({(Area, area_id): area})
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
AreaService.update_area(db, area_id, AreaUpdate(crs="EPSG:31370"))
|
||||
|
||||
assert exc_info.value.code == "INVALID_AREA_CRS_UPDATE"
|
||||
assert db.commits == 0
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Estimate disclosure must come from the data, not from patching prose.
|
||||
|
||||
``ensure_estimate_disclosure`` rewrites the model's sentences with regular
|
||||
expressions to insert the word "schatting". That only fires when the generated
|
||||
text happens to contain one of the phrasings it knows, so whether a number is
|
||||
labelled an estimate depends on how the language model worded it. The
|
||||
disclosure is derived from the metric metadata instead, so the honesty of the
|
||||
answer no longer depends on string matching.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from app.schemas.assistant import AssistantContextMetric
|
||||
from app.services.geo_assistant_service import GeoAssistantService
|
||||
|
||||
|
||||
def _metric(theme: str, label: str, *, is_estimate: bool) -> AssistantContextMetric:
|
||||
return AssistantContextMetric(
|
||||
theme=theme,
|
||||
label=label,
|
||||
value=36783.0,
|
||||
unit="inwoners",
|
||||
source="Statbel",
|
||||
dataset_id=uuid4(),
|
||||
observed_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
is_estimate=is_estimate,
|
||||
)
|
||||
|
||||
|
||||
def test_every_estimated_metric_produces_a_disclosure() -> None:
|
||||
metrics = [
|
||||
_metric("population", "Inwoners", is_estimate=True),
|
||||
_metric("buildings", "Gebouwen", is_estimate=False),
|
||||
]
|
||||
|
||||
disclosures = GeoAssistantService.estimate_disclosures(metrics)
|
||||
|
||||
assert len(disclosures) == 1
|
||||
assert disclosures[0].theme == "population"
|
||||
assert disclosures[0].label == "Inwoners"
|
||||
assert disclosures[0].source == "Statbel"
|
||||
assert disclosures[0].dataset_id == metrics[0].dataset_id
|
||||
assert "schatting" in disclosures[0].reason.casefold()
|
||||
|
||||
|
||||
def test_disclosure_does_not_depend_on_the_generated_wording() -> None:
|
||||
"""The regex path only fires on phrasings it recognises; this does not."""
|
||||
|
||||
metrics = [_metric("population", "Inwoners", is_estimate=True)]
|
||||
|
||||
patched = GeoAssistantService.ensure_estimate_disclosure(
|
||||
"Er wonen daar 36.783 mensen.", metrics
|
||||
)
|
||||
disclosures = GeoAssistantService.estimate_disclosures(metrics)
|
||||
|
||||
# The prose was left untouched because no known phrase matched...
|
||||
assert "Datakwaliteit" not in patched
|
||||
# ...but the structured disclosure is present regardless.
|
||||
assert len(disclosures) == 1
|
||||
|
||||
|
||||
def test_no_estimates_means_no_disclosures() -> None:
|
||||
metrics = [_metric("buildings", "Gebouwen", is_estimate=False)]
|
||||
|
||||
assert GeoAssistantService.estimate_disclosures(metrics) == []
|
||||
|
||||
|
||||
def test_disclosures_are_deduplicated_per_theme_and_dataset() -> None:
|
||||
shared = _metric("population", "Inwoners", is_estimate=True)
|
||||
duplicate = AssistantContextMetric(**{**shared.model_dump(), "label": "Inwoners (2024)"})
|
||||
|
||||
disclosures = GeoAssistantService.estimate_disclosures([shared, duplicate])
|
||||
|
||||
assert len(disclosures) == 1
|
||||
|
||||
|
||||
def test_disclosures_are_ordered_deterministically() -> None:
|
||||
metrics = [
|
||||
_metric("space_occupation", "Ruimtebeslag", is_estimate=True),
|
||||
_metric("population", "Inwoners", is_estimate=True),
|
||||
]
|
||||
|
||||
themes = [item.theme for item in GeoAssistantService.estimate_disclosures(metrics)]
|
||||
|
||||
assert themes == ["population", "space_occupation"]
|
||||
@@ -0,0 +1,506 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.public_demo import PUBLIC_DEMO_PROJECT_ID
|
||||
from app.db.session import get_db
|
||||
from app.main import create_app
|
||||
from app.schemas.demo import DemoWorkflowResponse
|
||||
from app.services.auth_service import AuthService
|
||||
from app.services.change_detection_service import ChangeDetectionService
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.demo_workflow_service import DemoWorkflowService
|
||||
from app.services.raster_operations_service import RasterOperationsService
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
from app.services.job_service import JobService
|
||||
|
||||
|
||||
def auth_client(
|
||||
monkeypatch,
|
||||
*,
|
||||
guest_access: bool = False,
|
||||
require_https: bool = False,
|
||||
base_url: str = "http://testserver",
|
||||
) -> TestClient:
|
||||
password_hash = AuthService.hash_password(
|
||||
"correct horse battery staple",
|
||||
salt=b"geointel-test-salt",
|
||||
iterations=100_000,
|
||||
)
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_ENABLED", "true")
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_REQUIRE_HTTPS", "true" if require_https else "false")
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_USERNAME", "operator")
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_PASSWORD_HASH", password_hash)
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_SESSION_SECRET", "test-session-secret-that-is-long-enough")
|
||||
monkeypatch.setenv("GEOINTEL_GUEST_ACCESS_ENABLED", "true" if guest_access else "false")
|
||||
monkeypatch.setenv("GEOINTEL_GUEST_DISPLAY_NAME", "Gast")
|
||||
monkeypatch.setenv("GEOINTEL_GUEST_SESSION_TTL_SECONDS", "7200")
|
||||
return TestClient(create_app(), base_url=base_url)
|
||||
|
||||
|
||||
def test_guest_access_defaults_off_when_operator_authentication_is_enabled(monkeypatch) -> None:
|
||||
password_hash = AuthService.hash_password(
|
||||
"correct horse battery staple",
|
||||
salt=b"geointel-test-salt",
|
||||
iterations=100_000,
|
||||
)
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_ENABLED", "true")
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_USERNAME", "operator")
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_PASSWORD_HASH", password_hash)
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_SESSION_SECRET", "test-session-secret-that-is-long-enough")
|
||||
monkeypatch.delenv("GEOINTEL_GUEST_ACCESS_ENABLED", raising=False)
|
||||
|
||||
client = TestClient(create_app())
|
||||
session = client.get("/api/v1/auth/session")
|
||||
|
||||
assert session.status_code == 200
|
||||
assert session.json()["data"]["authentication_required"] is True
|
||||
assert session.json()["data"]["guest_access_enabled"] is False
|
||||
|
||||
|
||||
def test_guest_default_is_inactive_but_valid_when_operator_authentication_is_disabled(monkeypatch) -> None:
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_ENABLED", "false")
|
||||
monkeypatch.delenv("GEOINTEL_AUTH_USERNAME", raising=False)
|
||||
monkeypatch.delenv("GEOINTEL_AUTH_PASSWORD_HASH", raising=False)
|
||||
monkeypatch.delenv("GEOINTEL_AUTH_SESSION_SECRET", raising=False)
|
||||
monkeypatch.delenv("GEOINTEL_GUEST_ACCESS_ENABLED", raising=False)
|
||||
|
||||
client = TestClient(create_app())
|
||||
session = client.get("/api/v1/auth/session")
|
||||
|
||||
assert session.status_code == 200
|
||||
assert session.json()["data"]["authentication_required"] is False
|
||||
assert session.json()["data"]["authenticated"] is True
|
||||
assert session.json()["data"]["guest_access_enabled"] is False
|
||||
|
||||
|
||||
def test_auth_session_and_health_are_public_but_api_is_protected(monkeypatch) -> None:
|
||||
client = auth_client(monkeypatch)
|
||||
|
||||
session = client.get("/api/v1/auth/session")
|
||||
protected = client.get("/api/v1/protected-probe")
|
||||
health = client.get("/health/live")
|
||||
|
||||
assert session.status_code == 200
|
||||
assert session.json()["data"] == {
|
||||
"authentication_required": True,
|
||||
"authenticated": False,
|
||||
"username": None,
|
||||
"expires_at": None,
|
||||
"role": None,
|
||||
"guest_access_enabled": False,
|
||||
"authentik_enabled": False,
|
||||
"guest_project_id": None,
|
||||
}
|
||||
assert protected.status_code == 401
|
||||
assert protected.json()["error"] == "AUTHENTICATION_REQUIRED"
|
||||
assert health.status_code == 200
|
||||
|
||||
|
||||
def test_login_uses_http_only_session_cookie_and_logout_revokes_browser_access(monkeypatch) -> None:
|
||||
client = auth_client(monkeypatch, guest_access=True)
|
||||
|
||||
invalid = client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"username": "operator", "password": "wrong"},
|
||||
)
|
||||
login = client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"username": "operator", "password": "correct horse battery staple"},
|
||||
)
|
||||
authenticated = client.get("/api/v1/auth/session")
|
||||
protected_after_login = client.get("/api/v1/protected-probe")
|
||||
logout = client.post("/api/v1/auth/logout")
|
||||
protected_after_logout = client.get("/api/v1/protected-probe")
|
||||
|
||||
assert invalid.status_code == 401
|
||||
assert invalid.json()["error"] == "INVALID_CREDENTIALS"
|
||||
assert login.status_code == 200
|
||||
assert login.json()["data"] == {
|
||||
"authentication_required": True,
|
||||
"authenticated": True,
|
||||
"username": "operator",
|
||||
"expires_at": login.json()["data"]["expires_at"],
|
||||
"role": "operator",
|
||||
"guest_access_enabled": True,
|
||||
"authentik_enabled": False,
|
||||
"guest_project_id": None,
|
||||
}
|
||||
cookie = login.headers["set-cookie"].lower()
|
||||
assert "httponly" in cookie
|
||||
assert "samesite=strict" in cookie
|
||||
assert authenticated.json()["data"]["authenticated"] is True
|
||||
assert authenticated.json()["data"]["role"] == "operator"
|
||||
assert protected_after_login.status_code == 404
|
||||
assert logout.status_code == 200
|
||||
assert logout.json()["data"]["guest_access_enabled"] is True
|
||||
assert protected_after_logout.status_code == 401
|
||||
|
||||
|
||||
def test_operator_login_can_require_https(monkeypatch) -> None:
|
||||
insecure_client = auth_client(monkeypatch, require_https=True)
|
||||
rejected = insecure_client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"username": "operator", "password": "correct horse battery staple"},
|
||||
)
|
||||
spoofed = insecure_client.post(
|
||||
"/api/v1/auth/login",
|
||||
headers={"x-forwarded-proto": "https", "x-real-ip": "203.0.113.9"},
|
||||
json={"username": "operator", "password": "correct horse battery staple"},
|
||||
)
|
||||
|
||||
secure_client = auth_client(
|
||||
monkeypatch,
|
||||
require_https=True,
|
||||
base_url="https://testserver",
|
||||
)
|
||||
accepted = secure_client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"username": "operator", "password": "correct horse battery staple"},
|
||||
)
|
||||
|
||||
assert rejected.status_code == 426
|
||||
assert rejected.json()["error"] == "AUTH_HTTPS_REQUIRED"
|
||||
assert spoofed.status_code == 426
|
||||
assert accepted.status_code == 200
|
||||
assert "secure" in accepted.headers["set-cookie"].lower()
|
||||
|
||||
|
||||
def test_guest_login_exposes_models_but_rejects_management_and_cross_project_requests(monkeypatch) -> None:
|
||||
project_id = PUBLIC_DEMO_PROJECT_ID
|
||||
demo = DemoWorkflowResponse(
|
||||
project_id=project_id,
|
||||
area_id=UUID("00000000-0000-0000-0000-000000000124"),
|
||||
reference_dataset_id=UUID("00000000-0000-0000-0000-000000000125"),
|
||||
candidate_dataset_id=UUID("00000000-0000-0000-0000-000000000126"),
|
||||
raster_dataset_id=UUID("00000000-0000-0000-0000-000000000127"),
|
||||
quality_check_id=UUID("00000000-0000-0000-0000-000000000128"),
|
||||
metric_count=6,
|
||||
status="ok",
|
||||
message="Demo ready",
|
||||
created=False,
|
||||
)
|
||||
monkeypatch.setattr(DemoWorkflowService, "seed", staticmethod(lambda _db: demo))
|
||||
client = auth_client(monkeypatch, guest_access=True)
|
||||
|
||||
def fake_db():
|
||||
yield object()
|
||||
|
||||
client.app.dependency_overrides[get_db] = fake_db
|
||||
|
||||
guest_login = client.post("/api/v1/auth/guest")
|
||||
guest_session = client.get("/api/v1/auth/session")
|
||||
mutation = client.post("/api/v1/projects", json={"name": "Not allowed"})
|
||||
other_project = client.get("/api/v1/projects/00000000-0000-0000-0000-000000000999")
|
||||
detection_models = client.get("/api/v1/detection/models")
|
||||
segmentation_models = client.get("/api/v1/segmentation/models")
|
||||
global_source_registry = client.get("/api/v1/source-registry/grb")
|
||||
cross_project_runs = client.get(
|
||||
"/api/v1/detection/runs?project_id=00000000-0000-0000-0000-000000000999"
|
||||
)
|
||||
cross_project_coverage = client.post(
|
||||
"/api/v1/external/coverage/resolve",
|
||||
json={
|
||||
"project_id": "00000000-0000-0000-0000-000000000999",
|
||||
"bbox": {"minx": 4.9, "miny": 51.0, "maxx": 5.0, "maxy": 51.1},
|
||||
"themes": [],
|
||||
},
|
||||
)
|
||||
bounded_acquisition = client.post(
|
||||
f"/api/v1/projects/{project_id}/datasets/orthophoto/acquire",
|
||||
json={},
|
||||
)
|
||||
cross_project_acquisition = client.post(
|
||||
"/api/v1/projects/00000000-0000-0000-0000-000000000999/datasets/orthophoto/acquire",
|
||||
json={},
|
||||
)
|
||||
bounded_derived_selection = client.post(
|
||||
f"/api/v1/projects/{project_id}/datasets/{demo.candidate_dataset_id}/vector/select/derive",
|
||||
json={},
|
||||
)
|
||||
|
||||
assert guest_login.status_code == 200
|
||||
assert guest_login.json()["data"]["role"] == "guest"
|
||||
assert guest_login.json()["data"]["username"] == "Gast"
|
||||
assert guest_login.json()["data"]["guest_project_id"] == str(project_id)
|
||||
assert "httponly" in guest_login.headers["set-cookie"].lower()
|
||||
assert guest_session.json()["data"]["role"] == "guest"
|
||||
assert mutation.status_code == 403
|
||||
assert mutation.json()["error"] == "GUEST_READ_ONLY"
|
||||
assert other_project.status_code == 403
|
||||
assert other_project.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
|
||||
assert detection_models.status_code == 200
|
||||
assert detection_models.json()["data"]["models"]
|
||||
assert segmentation_models.status_code == 200
|
||||
assert global_source_registry.status_code == 403
|
||||
assert global_source_registry.json()["error"] == "GUEST_ROUTE_NOT_AVAILABLE"
|
||||
assert cross_project_runs.status_code == 403
|
||||
assert cross_project_runs.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
|
||||
assert cross_project_coverage.status_code == 403
|
||||
assert cross_project_coverage.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
|
||||
assert bounded_acquisition.status_code == 422
|
||||
assert bounded_acquisition.json()["error"] != "GUEST_READ_ONLY"
|
||||
assert cross_project_acquisition.status_code == 403
|
||||
assert cross_project_acquisition.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
|
||||
assert bounded_derived_selection.status_code == 422
|
||||
assert bounded_derived_selection.json()["error"] != "GUEST_READ_ONLY"
|
||||
|
||||
|
||||
def test_guest_change_detection_binds_both_datasets_to_signed_demo_project(monkeypatch) -> None:
|
||||
project_id = PUBLIC_DEMO_PROJECT_ID
|
||||
other_project_id = UUID("00000000-0000-0000-0000-000000000999")
|
||||
source_dataset_id = UUID("00000000-0000-0000-0000-000000000125")
|
||||
target_dataset_id = UUID("00000000-0000-0000-0000-000000000126")
|
||||
cross_project_dataset_id = UUID("00000000-0000-0000-0000-000000000998")
|
||||
demo = DemoWorkflowResponse(
|
||||
project_id=project_id,
|
||||
area_id=UUID("00000000-0000-0000-0000-000000000124"),
|
||||
reference_dataset_id=source_dataset_id,
|
||||
candidate_dataset_id=target_dataset_id,
|
||||
raster_dataset_id=UUID("00000000-0000-0000-0000-000000000127"),
|
||||
quality_check_id=UUID("00000000-0000-0000-0000-000000000128"),
|
||||
metric_count=6,
|
||||
status="ok",
|
||||
message="Demo ready",
|
||||
created=False,
|
||||
)
|
||||
monkeypatch.setattr(DemoWorkflowService, "seed", staticmethod(lambda _db: demo))
|
||||
|
||||
class FakeDb:
|
||||
def get(self, _model, dataset_id):
|
||||
bound_project_id = other_project_id if dataset_id == cross_project_dataset_id else project_id
|
||||
return SimpleNamespace(id=dataset_id, project_id=bound_project_id, dataset_type="vector")
|
||||
|
||||
validated_datasets: list[tuple[UUID, UUID, str]] = []
|
||||
|
||||
def validate_dataset(_db, dataset_id, requested_project_id, label):
|
||||
validated_datasets.append((dataset_id, requested_project_id, label))
|
||||
return SimpleNamespace(id=dataset_id, project_id=requested_project_id, dataset_type="vector")
|
||||
|
||||
monkeypatch.setattr(
|
||||
ChangeDetectionService,
|
||||
"_get_project_vector_dataset",
|
||||
staticmethod(validate_dataset),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
JobService,
|
||||
"run_sync_job",
|
||||
staticmethod(
|
||||
lambda **kwargs: SimpleNamespace(
|
||||
id=uuid4(),
|
||||
job_type=kwargs["job_type"],
|
||||
status="success",
|
||||
project_id=kwargs["project_id"],
|
||||
dataset_id=source_dataset_id,
|
||||
input_dataset_id=source_dataset_id,
|
||||
output_dataset_id=None,
|
||||
parameters_json=kwargs["parameters"],
|
||||
result_json={},
|
||||
error_message=None,
|
||||
created_at=None,
|
||||
started_at=None,
|
||||
finished_at=None,
|
||||
)
|
||||
),
|
||||
)
|
||||
client = auth_client(monkeypatch, guest_access=True)
|
||||
|
||||
def fake_db():
|
||||
yield FakeDb()
|
||||
|
||||
client.app.dependency_overrides[get_db] = fake_db
|
||||
assert client.post("/api/v1/auth/guest").status_code == 200
|
||||
|
||||
accepted = client.post(
|
||||
"/api/v1/analysis/change-detection",
|
||||
json={
|
||||
"source_dataset_id": str(source_dataset_id),
|
||||
"target_dataset_id": str(target_dataset_id),
|
||||
},
|
||||
)
|
||||
rejected = client.post(
|
||||
"/api/v1/analysis/change-detection",
|
||||
json={
|
||||
"source_dataset_id": str(cross_project_dataset_id),
|
||||
"target_dataset_id": str(target_dataset_id),
|
||||
},
|
||||
)
|
||||
|
||||
assert accepted.status_code == 200
|
||||
assert accepted.json()["data"]["project_id"] == str(project_id)
|
||||
assert validated_datasets == [
|
||||
(source_dataset_id, project_id, "Source"),
|
||||
(target_dataset_id, project_id, "Target"),
|
||||
]
|
||||
assert rejected.status_code == 403
|
||||
assert rejected.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
|
||||
|
||||
|
||||
def test_guest_can_prepare_tiles_and_queue_project_scoped_detection(monkeypatch) -> None:
|
||||
project_id = PUBLIC_DEMO_PROJECT_ID
|
||||
raster_dataset_id = UUID("00000000-0000-0000-0000-000000000127")
|
||||
manifest_path = "/app/storage/tiles/demo/manifest.json"
|
||||
demo = DemoWorkflowResponse(
|
||||
project_id=project_id,
|
||||
area_id=UUID("00000000-0000-0000-0000-000000000124"),
|
||||
reference_dataset_id=UUID("00000000-0000-0000-0000-000000000125"),
|
||||
candidate_dataset_id=UUID("00000000-0000-0000-0000-000000000126"),
|
||||
raster_dataset_id=raster_dataset_id,
|
||||
quality_check_id=UUID("00000000-0000-0000-0000-000000000128"),
|
||||
metric_count=6,
|
||||
status="ok",
|
||||
message="Demo ready",
|
||||
created=False,
|
||||
)
|
||||
monkeypatch.setattr(DemoWorkflowService, "seed", staticmethod(lambda _db: demo))
|
||||
monkeypatch.setattr(
|
||||
DatasetService,
|
||||
"get_dataset",
|
||||
staticmethod(lambda _db, _dataset_id: SimpleNamespace(project_id=project_id)),
|
||||
)
|
||||
|
||||
def job(*, job_type: str, result_json: dict | None = None):
|
||||
return SimpleNamespace(
|
||||
id=uuid4(),
|
||||
job_type=job_type,
|
||||
status="success" if result_json else "queued",
|
||||
project_id=project_id,
|
||||
dataset_id=raster_dataset_id,
|
||||
input_dataset_id=raster_dataset_id,
|
||||
output_dataset_id=None,
|
||||
parameters_json={},
|
||||
result_json=result_json,
|
||||
error_message=None,
|
||||
created_at=None,
|
||||
started_at=None,
|
||||
finished_at=None,
|
||||
)
|
||||
|
||||
tile_parameters: dict = {}
|
||||
|
||||
def tile(_db, _dataset_id, **kwargs):
|
||||
tile_parameters.update(kwargs)
|
||||
return {"manifest_path": manifest_path}
|
||||
|
||||
monkeypatch.setattr(RasterOperationsService, "tile", staticmethod(tile))
|
||||
monkeypatch.setattr(
|
||||
"app.api.routes.datasets._run_job_sync",
|
||||
lambda **kwargs: job(job_type="raster.tile", result_json=kwargs["operation"]()),
|
||||
)
|
||||
queued_parameters: dict = {}
|
||||
|
||||
def enqueue_detection(**kwargs):
|
||||
queued_parameters.update(kwargs)
|
||||
return job(job_type="detection.run")
|
||||
|
||||
monkeypatch.setattr(DetectionService, "enqueue_detection", staticmethod(enqueue_detection))
|
||||
queued_segmentation_parameters: dict = {}
|
||||
|
||||
def enqueue_segmentation(**kwargs):
|
||||
queued_segmentation_parameters.update(kwargs)
|
||||
return job(job_type="segmentation.run")
|
||||
|
||||
monkeypatch.setattr(SegmentationService, "enqueue_segmentation", staticmethod(enqueue_segmentation))
|
||||
client = auth_client(monkeypatch, guest_access=True)
|
||||
|
||||
def fake_db():
|
||||
yield object()
|
||||
|
||||
client.app.dependency_overrides[get_db] = fake_db
|
||||
assert client.post("/api/v1/auth/guest").status_code == 200
|
||||
|
||||
tile_response = client.post(
|
||||
f"/api/v1/projects/{project_id}/datasets/{raster_dataset_id}/raster/tile",
|
||||
json={"tile_size": 512, "overlap": 64},
|
||||
)
|
||||
detection_response = client.post(
|
||||
f"/api/v1/detection/run-async?project_id={project_id}",
|
||||
json={
|
||||
"project_id": str(project_id),
|
||||
"dataset_id": str(raster_dataset_id),
|
||||
"model_id": "yolo-configured",
|
||||
"model_asset_id": "active-model",
|
||||
"confidence_threshold": 0.15,
|
||||
"tile_manifest_path": manifest_path,
|
||||
"parameters_json": {},
|
||||
},
|
||||
)
|
||||
segmentation_response = client.post(
|
||||
f"/api/v1/segmentation/run-async?project_id={project_id}",
|
||||
json={
|
||||
"project_id": str(project_id),
|
||||
"dataset_id": str(raster_dataset_id),
|
||||
"model_id": "sam-configured",
|
||||
"confidence_threshold": 0.5,
|
||||
"tile_manifest_path": manifest_path,
|
||||
"parameters_json": {},
|
||||
},
|
||||
)
|
||||
|
||||
assert tile_response.status_code == 201
|
||||
assert tile_response.json()["data"]["result_json"]["manifest_path"] == manifest_path
|
||||
assert detection_response.status_code == 200
|
||||
assert detection_response.json()["data"]["status"] == "queued"
|
||||
assert segmentation_response.status_code == 200
|
||||
assert segmentation_response.json()["data"]["status"] == "queued"
|
||||
assert queued_parameters["project_id"] == project_id
|
||||
assert queued_parameters["dataset_id"] == raster_dataset_id
|
||||
assert queued_parameters["tile_manifest_path"] == manifest_path
|
||||
assert queued_segmentation_parameters["project_id"] == project_id
|
||||
assert queued_segmentation_parameters["dataset_id"] == raster_dataset_id
|
||||
assert queued_segmentation_parameters["tile_manifest_path"] == manifest_path
|
||||
assert tile_parameters["max_tiles"] == get_settings().yolo_max_tiles
|
||||
|
||||
|
||||
def test_guest_session_tokens_fail_closed_without_a_project_scope(monkeypatch) -> None:
|
||||
auth_client(monkeypatch, guest_access=True)
|
||||
settings = get_settings()
|
||||
|
||||
try:
|
||||
AuthService.create_session_token("Gast", settings, role="guest")
|
||||
except ValueError as error:
|
||||
assert "demo project" in str(error)
|
||||
else: # pragma: no cover - defensive assertion
|
||||
raise AssertionError("An unscoped guest token should not be created")
|
||||
|
||||
|
||||
def test_password_hash_and_session_signatures_fail_closed(monkeypatch) -> None:
|
||||
client = auth_client(monkeypatch)
|
||||
login = client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"username": "operator", "password": "correct horse battery staple"},
|
||||
)
|
||||
token = login.cookies.get("geointel_session")
|
||||
|
||||
assert token
|
||||
client.cookies.set("geointel_session", f"{token}tampered")
|
||||
session = client.get("/api/v1/auth/session")
|
||||
|
||||
assert session.status_code == 200
|
||||
assert session.json()["data"]["authenticated"] is False
|
||||
|
||||
|
||||
def test_unraid_runtime_carries_only_hashed_operator_credentials() -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
runner = (root / "deploy/unraid/run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
example = (root / "deploy/unraid/geointel.env.example").read_text(encoding="utf-8")
|
||||
browser_smoke = (root / "scripts/verify_browser_runtime.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert '-e GEOINTEL_AUTH_PASSWORD_HASH="$GEOINTEL_AUTH_PASSWORD_HASH"' in runner
|
||||
assert '-e GEOINTEL_AUTHENTIK_CLIENT_SECRET="$GEOINTEL_AUTHENTIK_CLIENT_SECRET"' in runner
|
||||
assert "GEOINTEL_AUTH_PASSWORD_HASH=" in example
|
||||
assert "GEOINTEL_AUTHENTIK_CLIENT_SECRET=" in example
|
||||
assert "GEOINTEL_AUTH_PASSWORD=" not in runner
|
||||
assert "GEOINTEL_AUTH_ENABLED=true" in example
|
||||
assert "GEOINTEL_AUTH_REQUIRE_HTTPS=true" in example
|
||||
assert "GEOINTEL_GUEST_ACCESS_ENABLED=false" in example
|
||||
assert 'GEOINTEL_AUTH_ENABLED="${GEOINTEL_AUTH_ENABLED:-true}"' in runner
|
||||
assert 'GEOINTEL_AUTH_REQUIRE_HTTPS="${GEOINTEL_AUTH_REQUIRE_HTTPS:-true}"' in runner
|
||||
assert 'GEOINTEL_GUEST_ACCESS_ENABLED="${GEOINTEL_GUEST_ACCESS_ENABLED:-false}"' in runner
|
||||
assert "/api/v1/auth/session" in browser_smoke
|
||||
@@ -0,0 +1,183 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.services.authentik_oidc_service import (
|
||||
MAX_OIDC_JSON_BYTES,
|
||||
AuthentikOidcService,
|
||||
)
|
||||
|
||||
|
||||
ISSUER = "https://auth.example.test/application/o/geointel"
|
||||
|
||||
|
||||
def configured_settings(**overrides: object) -> Settings:
|
||||
values: dict[str, object] = {
|
||||
"auth_enabled": True,
|
||||
"auth_username": "ITWorx",
|
||||
"auth_password_hash": "pbkdf2_sha256$1$salt$digest",
|
||||
"auth_session_secret": "s" * 48,
|
||||
"authentik_issuer": ISSUER,
|
||||
"authentik_client_id": "geointel-client",
|
||||
"authentik_client_secret": "client-secret",
|
||||
"authentik_allowed_email": "operator@example.test",
|
||||
"public_base_url": "https://geointel.example.test",
|
||||
}
|
||||
values.update(overrides)
|
||||
return Settings(_env_file=None, **values)
|
||||
|
||||
|
||||
def discovery_document() -> dict[str, str]:
|
||||
return {
|
||||
"issuer": ISSUER,
|
||||
"authorization_endpoint": f"{ISSUER}/authorize",
|
||||
"token_endpoint": f"{ISSUER}/token",
|
||||
"jwks_uri": f"{ISSUER}/jwks",
|
||||
}
|
||||
|
||||
|
||||
def test_authentik_configuration_is_all_or_nothing_and_https_only() -> None:
|
||||
with pytest.raises(ValidationError, match="configured together"):
|
||||
configured_settings(authentik_client_secret=None)
|
||||
with pytest.raises(ValidationError, match="absolute HTTPS URL"):
|
||||
configured_settings(authentik_issuer="http://auth.example.test/issuer")
|
||||
with pytest.raises(ValidationError, match="must not contain a path"):
|
||||
configured_settings(public_base_url="https://geointel.example.test/app")
|
||||
|
||||
|
||||
def test_start_uses_same_origin_discovery_and_pkce(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
service = AuthentikOidcService(configured_settings())
|
||||
monkeypatch.setattr(service, "_fetch_json", lambda *_args, **_kwargs: discovery_document())
|
||||
|
||||
location, flow_cookie = service.start()
|
||||
|
||||
parsed = urlsplit(location)
|
||||
query = parse_qs(parsed.query)
|
||||
flow = service.serializer.loads(flow_cookie, max_age=600)
|
||||
assert f"{parsed.scheme}://{parsed.netloc}{parsed.path}" == f"{ISSUER}/authorize"
|
||||
assert query["redirect_uri"] == [
|
||||
"https://geointel.example.test/api/v1/auth/authentik/callback"
|
||||
]
|
||||
assert query["code_challenge_method"] == ["S256"]
|
||||
assert query["state"] == [flow["state"]]
|
||||
assert query["nonce"] == [flow["nonce"]]
|
||||
assert query["code_challenge"][0]
|
||||
|
||||
|
||||
def test_discovery_rejects_cross_origin_endpoints(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
service = AuthentikOidcService(configured_settings())
|
||||
document = discovery_document()
|
||||
document["jwks_uri"] = "https://attacker.example.test/jwks"
|
||||
monkeypatch.setattr(service, "_fetch_json", lambda *_args, **_kwargs: document)
|
||||
|
||||
with pytest.raises(ValueError, match="outside the configured issuer origin"):
|
||||
service._discovery()
|
||||
|
||||
|
||||
def test_finish_verifies_signature_nonce_and_exact_allowed_email(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
service = AuthentikOidcService(configured_settings())
|
||||
state, nonce, verifier = "state-value", "nonce-value", "verifier-value"
|
||||
flow_cookie = service.serializer.dumps(
|
||||
{"state": state, "nonce": nonce, "verifier": verifier}
|
||||
)
|
||||
private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
public_jwk = jwt.algorithms.RSAAlgorithm.to_jwk(
|
||||
private_key.public_key(), as_dict=True
|
||||
)
|
||||
public_jwk["kid"] = "operator-key"
|
||||
now = int(time.time())
|
||||
token = jwt.encode(
|
||||
{
|
||||
"iss": ISSUER,
|
||||
"aud": "geointel-client",
|
||||
"sub": "authentik-user-id",
|
||||
"iat": now,
|
||||
"exp": now + 300,
|
||||
"nonce": nonce,
|
||||
"email": "Operator@Example.Test",
|
||||
"email_verified": True,
|
||||
},
|
||||
private_key,
|
||||
algorithm="RS256",
|
||||
headers={"kid": "operator-key"},
|
||||
)
|
||||
token_holder = {"value": token}
|
||||
|
||||
def fetch(url: str, data: dict[str, str] | None = None) -> dict:
|
||||
if url.endswith("openid-configuration"):
|
||||
return discovery_document()
|
||||
if url.endswith("/token"):
|
||||
assert data is not None
|
||||
assert data["code_verifier"] == verifier
|
||||
return {"id_token": token_holder["value"]}
|
||||
if url.endswith("/jwks"):
|
||||
return {"keys": [public_jwk]}
|
||||
raise AssertionError(url)
|
||||
|
||||
monkeypatch.setattr(service, "_fetch_json", fetch)
|
||||
|
||||
claims = service.finish(code="authorization-code", state=state, flow_cookie=flow_cookie)
|
||||
|
||||
assert claims["sub"] == "authentik-user-id"
|
||||
token_holder["value"] = jwt.encode(
|
||||
{
|
||||
"iss": ISSUER,
|
||||
"aud": "geointel-client",
|
||||
"sub": "different-user",
|
||||
"iat": now,
|
||||
"exp": now + 300,
|
||||
"nonce": nonce,
|
||||
"email": "other@example.test",
|
||||
"email_verified": True,
|
||||
},
|
||||
private_key,
|
||||
algorithm="RS256",
|
||||
headers={"kid": "operator-key"},
|
||||
)
|
||||
with pytest.raises(ValueError, match="not authorized"):
|
||||
service.finish(code="authorization-code", state=state, flow_cookie=flow_cookie)
|
||||
with pytest.raises(ValueError, match="state mismatch"):
|
||||
service.finish(
|
||||
code="authorization-code",
|
||||
state="different-state",
|
||||
flow_cookie=flow_cookie,
|
||||
)
|
||||
|
||||
|
||||
def test_fetch_json_rejects_declared_oversize_response(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
service = AuthentikOidcService(configured_settings())
|
||||
|
||||
class OversizeResponse:
|
||||
headers = {"Content-Length": str(MAX_OIDC_JSON_BYTES + 1)}
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
return None
|
||||
|
||||
def read(self, _size: int) -> bytes:
|
||||
raise AssertionError("oversized responses must not be read")
|
||||
|
||||
class Opener:
|
||||
def open(self, *_args: object, **_kwargs: object) -> OversizeResponse:
|
||||
return OversizeResponse()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.authentik_oidc_service.build_opener",
|
||||
lambda *_args: Opener(),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="size limit"):
|
||||
service._fetch_json(f"{ISSUER}/oversized")
|
||||
@@ -0,0 +1,128 @@
|
||||
"""A provider that ignores ``resultOffset`` must not produce duplicated data.
|
||||
|
||||
ArcGIS layers without ``supportsPagination`` accept ``resultOffset`` and ignore
|
||||
it, answering every page with the first one. The profile reader advanced its
|
||||
offset by the page length and stopped when it reached the announced count, so
|
||||
for a count that is a multiple of the page size it collected N copies of page
|
||||
one, matched the expected total exactly, and stored that as an official
|
||||
dataset. The watercourse-name reader had no bound at all: it looped for as long
|
||||
as the provider kept setting ``exceededTransferLimit``.
|
||||
|
||||
The sibling reader for official vector products already refuses a repeated page
|
||||
and deduplicates on feature identity. These tests hold this reader to the same
|
||||
rule.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.services.bathymetry_profile_acquisition_service import (
|
||||
BathymetryProfileAcquisitionService,
|
||||
)
|
||||
|
||||
BBOX = (4.30, 51.20, 4.32, 51.22)
|
||||
|
||||
|
||||
def _settings(**overrides: Any) -> Settings:
|
||||
base = get_settings()
|
||||
return base.model_copy(update={"bathymetry_profiles_page_size": 2, **overrides})
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, payload: dict[str, Any]) -> None:
|
||||
self._body = json.dumps(payload).encode("utf-8")
|
||||
|
||||
def read(self, _limit: int | None = None) -> bytes:
|
||||
return self._body
|
||||
|
||||
def __enter__(self) -> "_Response":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc: object) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _profile(object_id: int) -> dict[str, Any]:
|
||||
return {
|
||||
"attributes": {"OBJECTID": object_id, "vhag": 7, "opg_diepte": 1.5},
|
||||
"geometry": {"x": 4.31, "y": 51.21},
|
||||
}
|
||||
|
||||
|
||||
class StuckProvider:
|
||||
"""Answers every page with the same records, as an unpaged layer does."""
|
||||
|
||||
def __init__(self, *, count: int, page: list[dict[str, Any]]) -> None:
|
||||
self.count = count
|
||||
self.page = page
|
||||
self.requests: list[str] = []
|
||||
|
||||
def __call__(self, request: Any, **_kwargs: Any) -> _Response:
|
||||
url = request.full_url if hasattr(request, "full_url") else str(request)
|
||||
self.requests.append(url)
|
||||
query = parse_qs(urlparse(url).query)
|
||||
if query.get("returnCountOnly") == ["true"]:
|
||||
return _Response({"count": self.count})
|
||||
return _Response({"features": list(self.page), "exceededTransferLimit": True})
|
||||
|
||||
|
||||
def test_a_provider_that_ignores_the_offset_is_refused_not_duplicated() -> None:
|
||||
"""Four announced records, two per page, the same two every time.
|
||||
|
||||
Advancing by page length reaches the announced total after two pages, so the
|
||||
completeness check passed while every record was stored twice.
|
||||
"""
|
||||
|
||||
provider = StuckProvider(count=4, page=[_profile(1), _profile(2)])
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
BathymetryProfileAcquisitionService._fetch_profiles(BBOX, _settings(), provider)
|
||||
|
||||
assert exc_info.value.code == "BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION"
|
||||
assert exc_info.value.status_code == 502
|
||||
|
||||
|
||||
def test_honest_pagination_still_returns_every_record() -> None:
|
||||
"""The guard must not reject a provider that pages correctly."""
|
||||
|
||||
pages = [[_profile(1), _profile(2)], [_profile(3), _profile(4)]]
|
||||
|
||||
def opener(request: Any, **_kwargs: Any) -> _Response:
|
||||
url = request.full_url
|
||||
query = parse_qs(urlparse(url).query)
|
||||
if query.get("returnCountOnly") == ["true"]:
|
||||
return _Response({"count": 4})
|
||||
offset = int(query.get("resultOffset", ["0"])[0])
|
||||
index = offset // 2
|
||||
page = pages[index] if index < len(pages) else []
|
||||
return _Response({"features": page})
|
||||
|
||||
features, provenance = BathymetryProfileAcquisitionService._fetch_profiles(
|
||||
BBOX, _settings(), opener
|
||||
)
|
||||
|
||||
assert [item["attributes"]["OBJECTID"] for item in features] == [1, 2, 3, 4]
|
||||
assert provenance["candidate_count"] == 4
|
||||
|
||||
|
||||
def test_watercourse_names_stop_instead_of_looping_forever() -> None:
|
||||
"""``exceededTransferLimit`` forever is not a reason to request forever."""
|
||||
|
||||
provider = StuckProvider(
|
||||
count=0,
|
||||
page=[{"attributes": {"wlasvl.vhag": 7, "VHAG_TABEL.naam": "Schelde"}}],
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
BathymetryProfileAcquisitionService._fetch_watercourse_names({7}, _settings(), provider)
|
||||
|
||||
assert exc_info.value.code == "BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION"
|
||||
# Bounded, and bounded early: it must not have hammered the provider first.
|
||||
assert len(provider.requests) <= 3
|
||||
@@ -0,0 +1,40 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "evaluate_belgium_building_candidate.py"
|
||||
SPEC = importlib.util.spec_from_file_location("candidate_evaluation", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def test_iou_and_one_to_one_matching() -> None:
|
||||
reference = [(0.0, 0.0, 10.0, 10.0)]
|
||||
predictions = [((0.0, 0.0, 10.0, 10.0), 0.9), ((0.0, 0.0, 10.0, 10.0), 0.8)]
|
||||
assert MODULE.iou(reference[0], reference[0]) == 1.0
|
||||
assert MODULE.match_boxes(predictions, reference, confidence=0.25, match_iou=0.5) == (1, 1, 0)
|
||||
|
||||
|
||||
def test_empty_reference_counts_false_positives() -> None:
|
||||
predictions = [((0.0, 0.0, 10.0, 10.0), 0.4)]
|
||||
assert MODULE.match_boxes(predictions, [], confidence=0.25, match_iou=0.5) == (0, 1, 0)
|
||||
assert MODULE.match_boxes(predictions, [], confidence=0.5, match_iou=0.5) == (0, 0, 0)
|
||||
|
||||
|
||||
def test_box_scaling_preserves_center() -> None:
|
||||
assert MODULE.scale_box((10.0, 20.0, 30.0, 40.0), 1.5) == (5.0, 15.0, 35.0, 45.0)
|
||||
|
||||
|
||||
def test_containment_suppression_removes_nested_lower_score_box() -> None:
|
||||
predictions = [
|
||||
((0.0, 0.0, 20.0, 20.0), 0.9),
|
||||
((5.0, 5.0, 15.0, 15.0), 0.8),
|
||||
((25.0, 0.0, 35.0, 10.0), 0.7),
|
||||
]
|
||||
assert MODULE.suppress_contained_predictions(predictions, 0.8) == [
|
||||
predictions[0],
|
||||
predictions[2],
|
||||
]
|
||||
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "assess_belgium_building_training_iteration.py"
|
||||
SPEC = importlib.util.spec_from_file_location("iteration_assessment", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def test_calibration_selection_prefers_worst_region_then_aggregate() -> None:
|
||||
report = {
|
||||
"sweeps": [
|
||||
{"threshold": 0.1, "pure_empty_false_positives": 0, "aggregate": {"f1": 0.8}, "regions": {"a": {"f1": 0.2}}},
|
||||
{"threshold": 0.2, "pure_empty_false_positives": 0, "aggregate": {"f1": 0.6}, "regions": {"a": {"f1": 0.5}}},
|
||||
]
|
||||
}
|
||||
assert MODULE.select_calibration_threshold(report)["threshold"] == 0.2
|
||||
|
||||
|
||||
def test_threshold_lookup_is_exact() -> None:
|
||||
report = {"sweeps": [{"threshold": 0.25, "aggregate": {}}]}
|
||||
assert MODULE.find_threshold(report, 0.25)["threshold"] == 0.25
|
||||
|
||||
|
||||
def test_release_assessment_rejects_changed_inference_configuration() -> None:
|
||||
calibration = {field: None for field in MODULE.INFERENCE_CONFIG_FIELDS}
|
||||
calibration.update({"model": "/models/candidate.pt", "nms_iou": 0.3, "containment_nms": 0.95})
|
||||
test = dict(calibration)
|
||||
test["nms_iou"] = 0.4
|
||||
with pytest.raises(ValueError, match="nms_iou"):
|
||||
MODULE.assert_same_inference_config(calibration, test, "test")
|
||||
@@ -0,0 +1,510 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "run_belgium_building_training_loop.py"
|
||||
SPEC = importlib.util.spec_from_file_location("training_loop", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
from training_release_manifest import create_training_release_manifest # noqa: E402
|
||||
|
||||
|
||||
def write_fixture_manifest(path: Path) -> None:
|
||||
policy = "geointel-training-source-eligibility/v1"
|
||||
def eligible(sample_slug: str) -> dict[str, object]:
|
||||
return {
|
||||
"policy_version": policy,
|
||||
"eligible": True,
|
||||
"fixture_mode": True,
|
||||
"raster": {
|
||||
"eligible": True,
|
||||
"reasons": [],
|
||||
"evidence": {
|
||||
"dataset_id": f"raster:{sample_slug}",
|
||||
"checksum_sha256": "a" * 64,
|
||||
"source_registry_id": "fixture-raster",
|
||||
"source_snapshot_id": "fixture-raster-snapshot",
|
||||
},
|
||||
},
|
||||
"reference": {
|
||||
"eligible": True,
|
||||
"reasons": [],
|
||||
"evidence": {
|
||||
"dataset_id": f"reference:{sample_slug}",
|
||||
"checksum_sha256": "b" * 64,
|
||||
"source_registry_id": "fixture-reference",
|
||||
"source_snapshot_id": "fixture-reference-snapshot",
|
||||
},
|
||||
},
|
||||
}
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"training_eligibility": {
|
||||
"policy_version": policy,
|
||||
"status": "eligible",
|
||||
"fixture_mode": True,
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"sample_slug": sample_slug,
|
||||
"split": split,
|
||||
"raster_dataset_id": f"raster:{sample_slug}",
|
||||
"reference_dataset_id": f"reference:{sample_slug}",
|
||||
"training_eligibility": eligible(sample_slug),
|
||||
}
|
||||
for sample_slug, split in (("fixture-train", "train"), ("fixture-val", "val"))
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(path.parent / "corpus-freeze.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 2,
|
||||
"manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
"immutable": True,
|
||||
"training_eligibility_policy": policy,
|
||||
"fixture_mode": True,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def write_fixture_training_release(tmp_path: Path, manifest: Path) -> Path:
|
||||
dataset_dir = tmp_path / "fixture-dataset"
|
||||
for split, sample_slug in (("train", "fixture-train"), ("val", "fixture-val")):
|
||||
image = dataset_dir / "images" / split / f"{sample_slug}.png"
|
||||
label = dataset_dir / "labels" / split / f"{sample_slug}.txt"
|
||||
image.parent.mkdir(parents=True, exist_ok=True)
|
||||
label.parent.mkdir(parents=True, exist_ok=True)
|
||||
image.write_bytes(split.encode("utf-8"))
|
||||
label.write_text("0 0.5 0.5 0.2 0.2\n", encoding="utf-8")
|
||||
yaml_path = dataset_dir / "dataset.yaml"
|
||||
yaml_path.write_text(
|
||||
f"path: {dataset_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
create_training_release_manifest(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=manifest,
|
||||
fixture_mode=True,
|
||||
)
|
||||
return yaml_path
|
||||
|
||||
|
||||
def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_path: Path) -> None:
|
||||
command = MODULE.training_command(
|
||||
"yolo",
|
||||
model=tmp_path / "base.pt",
|
||||
data=tmp_path / "dataset.yaml",
|
||||
project=tmp_path / "runs",
|
||||
name="iteration-001",
|
||||
epochs=160,
|
||||
seed=42,
|
||||
batch=2,
|
||||
workers=4,
|
||||
)
|
||||
assert command[:2] == ["yolo", "train"]
|
||||
assert "device=0" in command
|
||||
assert "deterministic=True" in command
|
||||
assert "seed=42" in command
|
||||
assert "epochs=160" in command
|
||||
assert "patience=18" in command
|
||||
assert "max_det=1000" in command
|
||||
assert "imgsz=640" in command
|
||||
assert "optimizer=auto" in command
|
||||
assert "mosaic=1.0" in command
|
||||
|
||||
|
||||
def test_training_command_supports_conservative_aerial_finetuning(tmp_path: Path) -> None:
|
||||
command = MODULE.training_command(
|
||||
"yolo",
|
||||
model=tmp_path / "base.pt",
|
||||
data=tmp_path / "dataset.yaml",
|
||||
project=tmp_path / "runs",
|
||||
name="aerial",
|
||||
epochs=50,
|
||||
seed=42,
|
||||
batch=2,
|
||||
workers=0,
|
||||
optimizer="AdamW",
|
||||
lr0=0.0001,
|
||||
mosaic=0.0,
|
||||
scale=0.2,
|
||||
translate=0.05,
|
||||
)
|
||||
assert "optimizer=AdamW" in command
|
||||
assert "lr0=0.0001" in command
|
||||
assert "mosaic=0.0" in command
|
||||
assert "scale=0.2" in command
|
||||
assert "translate=0.05" in command
|
||||
assert "degrees=0.0" in command
|
||||
assert "flipud=0.0" in command
|
||||
assert "fliplr=0.5" in command
|
||||
assert "warmup_epochs=1.0" in command
|
||||
assert "warmup_bias_lr=0.01" in command
|
||||
assert "hsv_h=0.01" in command
|
||||
assert "hsv_s=0.2" in command
|
||||
assert "hsv_v=0.15" in command
|
||||
assert f"data={tmp_path / 'dataset.yaml'}" in command
|
||||
|
||||
|
||||
def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_path: Path) -> None:
|
||||
command = MODULE.failure_sampling_command(
|
||||
scripts_dir=tmp_path / "scripts",
|
||||
train_summary=tmp_path / "train-summary.json",
|
||||
corpus_manifest=tmp_path / "manifest.json",
|
||||
assessment=tmp_path / "assessment.json",
|
||||
output_dir=tmp_path / "iteration-001" / "failure-driven-training",
|
||||
review_audit=tmp_path / "review-audit.json",
|
||||
)
|
||||
assert command[1].endswith("build_failure_driven_yolo_sampling.py")
|
||||
assert command[command.index("--summary") + 1].endswith("train-summary.json")
|
||||
assert command[command.index("--assessment") + 1].endswith("assessment.json")
|
||||
assert command[command.index("--output-dir") + 1].endswith("failure-driven-training")
|
||||
|
||||
|
||||
def test_protected_assessment_uses_frozen_threshold_and_configured_gates(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
command = MODULE.protected_assessment_command(
|
||||
scripts_dir=tmp_path / "scripts",
|
||||
calibration=tmp_path / "calibration.json",
|
||||
test=tmp_path / "test.json",
|
||||
background=tmp_path / "background.json",
|
||||
output=tmp_path / "assessment.json",
|
||||
selected_threshold=0.275,
|
||||
min_aggregate_f1=0.71,
|
||||
min_region_f1=0.62,
|
||||
min_region_precision=0.73,
|
||||
min_region_recall=0.58,
|
||||
max_pure_empty_fp=1,
|
||||
)
|
||||
|
||||
assert command[command.index("--selected-threshold") + 1] == "0.275"
|
||||
assert command[command.index("--min-aggregate-f1") + 1] == "0.71"
|
||||
assert command[command.index("--min-region-f1") + 1] == "0.62"
|
||||
assert command[command.index("--min-region-precision") + 1] == "0.73"
|
||||
assert command[command.index("--min-region-recall") + 1] == "0.58"
|
||||
assert command[command.index("--max-pure-empty-fp") + 1] == "1"
|
||||
|
||||
|
||||
def test_partial_iteration_resume_uses_exact_checkpoint_and_cuda(tmp_path: Path) -> None:
|
||||
checkpoint = tmp_path / "runs" / "iteration-002" / "weights" / "last.pt"
|
||||
assert MODULE.resumable_training_command("yolo", checkpoint) == [
|
||||
"yolo", "train", f"resume={checkpoint}", "device=0"
|
||||
]
|
||||
|
||||
|
||||
def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -> None:
|
||||
audit = tmp_path / "audit.json"
|
||||
audit.write_text(json.dumps({
|
||||
"status": "needs_human_review", "failures": [],
|
||||
"manifest_immutable": True, "spatial_leakage_status": "ok",
|
||||
}))
|
||||
quality = tmp_path / "quality.json"
|
||||
quality.write_text(json.dumps({
|
||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||
}))
|
||||
manifest = tmp_path / "manifest.json"
|
||||
write_fixture_manifest(manifest)
|
||||
train_yaml = write_fixture_training_release(tmp_path, manifest)
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable, str(SCRIPT),
|
||||
"--initial-model", str(tmp_path / "candidate.pt"),
|
||||
"--train-yaml", str(train_yaml),
|
||||
"--train-summary", str(tmp_path / "train-summary.json"),
|
||||
"--dataset-audit", str(audit),
|
||||
"--train-quality-audit", str(quality),
|
||||
"--calibration-summary", str(tmp_path / "cal.json"),
|
||||
"--test-summary", str(tmp_path / "test.json"),
|
||||
"--background-summary", str(tmp_path / "background.json"),
|
||||
"--corpus-manifest", str(manifest),
|
||||
"--output-dir", str(tmp_path / "output"),
|
||||
"--evaluate-initial-model", "--fixture-mode", "--dry-run",
|
||||
], capture_output=True, text=True, check=False,
|
||||
)
|
||||
assert result.returncode == 0
|
||||
assert json.loads(result.stdout) == {
|
||||
"training_command": None, "evaluate_existing": True, "resume_partial": False
|
||||
}
|
||||
|
||||
|
||||
def test_existing_checkpoint_iteration_directory_can_be_created_without_yolo(tmp_path: Path) -> None:
|
||||
iteration_dir = tmp_path / "closed-loop" / "iteration-001"
|
||||
iteration_dir.mkdir(parents=True, exist_ok=True)
|
||||
assert iteration_dir.is_dir()
|
||||
|
||||
|
||||
def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
||||
audit = tmp_path / "audit.json"
|
||||
audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4}))
|
||||
quality = tmp_path / "quality.json"
|
||||
quality.write_text(json.dumps({
|
||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||
}))
|
||||
manifest = tmp_path / "manifest.json"
|
||||
write_fixture_manifest(manifest)
|
||||
train_yaml = write_fixture_training_release(tmp_path, manifest)
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--initial-model",
|
||||
str(tmp_path / "base.pt"),
|
||||
"--train-yaml",
|
||||
str(train_yaml),
|
||||
"--train-summary",
|
||||
str(tmp_path / "train-summary.json"),
|
||||
"--dataset-audit",
|
||||
str(audit),
|
||||
"--train-quality-audit",
|
||||
str(quality),
|
||||
"--calibration-summary",
|
||||
str(tmp_path / "cal.json"),
|
||||
"--test-summary",
|
||||
str(tmp_path / "test.json"),
|
||||
"--background-summary",
|
||||
str(tmp_path / "background.json"),
|
||||
"--corpus-manifest",
|
||||
str(manifest),
|
||||
"--output-dir",
|
||||
str(tmp_path / "output"),
|
||||
"--fixture-mode",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode != 0
|
||||
assert "Dataset audit is not eligible for training" in result.stderr
|
||||
|
||||
|
||||
def test_loop_rejects_manifest_without_source_eligibility_before_cuda_training(tmp_path: Path) -> None:
|
||||
audit = tmp_path / "audit.json"
|
||||
audit.write_text(json.dumps({
|
||||
"status": "needs_human_review", "failures": [],
|
||||
"manifest_immutable": True, "spatial_leakage_status": "ok",
|
||||
}))
|
||||
quality = tmp_path / "quality.json"
|
||||
quality.write_text(json.dumps({
|
||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||
}))
|
||||
manifest = tmp_path / "manifest.json"
|
||||
manifest.write_text(json.dumps({"samples": [{"sample_slug": "unproven"}]}), encoding="utf-8")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable, str(SCRIPT),
|
||||
"--initial-model", str(tmp_path / "candidate.pt"),
|
||||
"--train-yaml", str(tmp_path / "dataset.yaml"),
|
||||
"--train-summary", str(tmp_path / "train-summary.json"),
|
||||
"--dataset-audit", str(audit),
|
||||
"--train-quality-audit", str(quality),
|
||||
"--calibration-summary", str(tmp_path / "cal.json"),
|
||||
"--test-summary", str(tmp_path / "test.json"),
|
||||
"--background-summary", str(tmp_path / "background.json"),
|
||||
"--corpus-manifest", str(manifest),
|
||||
"--output-dir", str(tmp_path / "output"),
|
||||
"--evaluate-initial-model", "--dry-run",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "manifest_training_eligibility_missing" in result.stderr
|
||||
|
||||
|
||||
def test_pending_human_review_blocks_operational_training() -> None:
|
||||
audit = {
|
||||
"status": "needs_human_review",
|
||||
"failures": [],
|
||||
"manifest_immutable": True,
|
||||
"spatial_leakage_status": "ok",
|
||||
"low_variance_positive_tile_count": 0,
|
||||
"review_complete": False,
|
||||
}
|
||||
quality = {
|
||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||
}
|
||||
failures = MODULE.dataset_audit_failures(audit, quality)
|
||||
assert "unsupported audit status: needs_human_review" in failures
|
||||
assert "review_complete_not_true" in failures
|
||||
assert "accepted_human_review_evidence_missing" in failures
|
||||
|
||||
|
||||
def test_fixture_mode_can_relax_review_only_after_fixture_manifest_gate() -> None:
|
||||
audit = {
|
||||
"status": "needs_human_review",
|
||||
"failures": [],
|
||||
"manifest_immutable": True,
|
||||
"spatial_leakage_status": "ok",
|
||||
"review_complete": False,
|
||||
}
|
||||
quality = {
|
||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||
}
|
||||
assert MODULE.dataset_audit_failures(audit, quality, fixture_mode=True) == []
|
||||
|
||||
|
||||
def test_operational_dataset_audit_must_be_the_one_bound_into_the_release(tmp_path: Path) -> None:
|
||||
bound = tmp_path / "bound-audit.json"
|
||||
other = tmp_path / "other-audit.json"
|
||||
bound.write_text("{}", encoding="utf-8")
|
||||
other.write_text("{}", encoding="utf-8")
|
||||
release = {"human_review": {"audit_path": str(bound.resolve())}}
|
||||
|
||||
MODULE.assert_dataset_audit_bound_to_release(
|
||||
release=release,
|
||||
dataset_audit=bound,
|
||||
fixture_mode=False,
|
||||
)
|
||||
try:
|
||||
MODULE.assert_dataset_audit_bound_to_release(
|
||||
release=release,
|
||||
dataset_audit=other,
|
||||
fixture_mode=False,
|
||||
)
|
||||
except MODULE.TrainingReleaseError as exc:
|
||||
assert "does not match" in str(exc)
|
||||
else:
|
||||
raise AssertionError("unbound dataset audit was accepted")
|
||||
|
||||
|
||||
def test_protected_assessment_feedback_is_terminal_and_cannot_seed_another_yaml() -> None:
|
||||
assert MODULE.protected_feedback_roles(
|
||||
{"status": "continue_training_loop", "test": {"aggregate": {}}, "background": None}
|
||||
) == ["test"]
|
||||
assert MODULE.protected_feedback_roles(
|
||||
{"status": "continue_training_loop", "test": None, "background": {"aggregate": {}}}
|
||||
) == ["background"]
|
||||
|
||||
|
||||
def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None:
|
||||
audit = {
|
||||
"status": "needs_human_review",
|
||||
"failures": ["wallonia/test below minimum"],
|
||||
"manifest_immutable": False,
|
||||
"spatial_leakage_status": "failed",
|
||||
"low_variance_positive_tile_count": 2,
|
||||
}
|
||||
quality = {
|
||||
"status": "failed", "low_variance_positive_tile_count": 2,
|
||||
"label_stats": {"invalid_label_count": 1, "missing_label_file_count": 1},
|
||||
}
|
||||
failures = MODULE.dataset_audit_failures(audit, quality)
|
||||
assert "wallonia/test below minimum" in failures
|
||||
assert "corpus manifest is not immutable" in failures
|
||||
assert "spatial leakage audit is not ok" in failures
|
||||
assert "dataset contains blank/low-variance positive tiles" in failures
|
||||
assert "train tile quality audit is not ok" in failures
|
||||
assert "train tile quality audit contains invalid labels" in failures
|
||||
assert "train tile quality audit contains missing label files" in failures
|
||||
|
||||
|
||||
def test_missing_tile_quality_evidence_fails_closed() -> None:
|
||||
audit = {
|
||||
"status": "needs_human_review", "failures": [],
|
||||
"manifest_immutable": True, "spatial_leakage_status": "ok",
|
||||
}
|
||||
failures = MODULE.dataset_audit_failures(audit, {})
|
||||
assert "train tile quality audit is not ok" in failures
|
||||
assert "train tile quality audit contains invalid labels" in failures
|
||||
assert "train tile quality audit contains missing label files" in failures
|
||||
assert "dataset contains blank/low-variance positive tiles" in failures
|
||||
|
||||
|
||||
def test_calibration_failure_blocks_protected_evaluation() -> None:
|
||||
chosen = {
|
||||
"threshold": 0.1,
|
||||
"aggregate": {"f1": 0.54},
|
||||
"regions": {
|
||||
"flanders": {"f1": 0.44, "precision": 0.49, "recall": 0.39},
|
||||
"wallonia": {"f1": 0.6, "precision": 0.6, "recall": 0.6},
|
||||
},
|
||||
"pure_empty_false_positives": 0,
|
||||
}
|
||||
failures = MODULE.calibration_failures(
|
||||
chosen,
|
||||
min_aggregate_f1=0.55,
|
||||
min_region_f1=0.45,
|
||||
min_region_precision=0.5,
|
||||
min_region_recall=0.4,
|
||||
max_pure_empty_fp=0,
|
||||
)
|
||||
assert failures == [
|
||||
"calibration_aggregate_f1_below_gate",
|
||||
"calibration_flanders_f1_below_gate",
|
||||
"calibration_flanders_precision_below_gate",
|
||||
"calibration_flanders_recall_below_gate",
|
||||
]
|
||||
|
||||
|
||||
def test_threshold_selection_uses_worst_region_then_aggregate() -> None:
|
||||
report = {
|
||||
"sweeps": [
|
||||
{
|
||||
"threshold": 0.1,
|
||||
"aggregate": {"f1": 0.8},
|
||||
"regions": {"a": {"f1": 0.4}, "b": {"f1": 0.7}},
|
||||
"pure_empty_false_positives": 0,
|
||||
},
|
||||
{
|
||||
"threshold": 0.2,
|
||||
"aggregate": {"f1": 0.6},
|
||||
"regions": {"a": {"f1": 0.5}, "b": {"f1": 0.5}},
|
||||
"pure_empty_false_positives": 0,
|
||||
},
|
||||
]
|
||||
}
|
||||
assert MODULE.select_calibration_threshold(report)["threshold"] == 0.2
|
||||
|
||||
|
||||
def test_rejected_candidate_score_prioritizes_weakest_release_gate() -> None:
|
||||
gates = {
|
||||
"min_aggregate_f1": 0.55,
|
||||
"min_region_f1": 0.45,
|
||||
"min_region_precision": 0.5,
|
||||
"min_region_recall": 0.4,
|
||||
}
|
||||
incumbent = {
|
||||
"gates": gates,
|
||||
"calibration": {
|
||||
"aggregate": {"f1": 0.58},
|
||||
"regions": {
|
||||
"flanders": {"f1": 0.34, "precision": 0.38, "recall": 0.31},
|
||||
"wallonia": {"f1": 0.60, "precision": 0.50, "recall": 0.75},
|
||||
},
|
||||
},
|
||||
}
|
||||
regressed = {
|
||||
"gates": gates,
|
||||
"calibration": {
|
||||
"aggregate": {"f1": 0.60},
|
||||
"regions": {
|
||||
"flanders": {"f1": 0.31, "precision": 0.45, "recall": 0.24},
|
||||
"wallonia": {"f1": 0.62, "precision": 0.52, "recall": 0.77},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
assert MODULE.rejected_candidate_score(incumbent) > MODULE.rejected_candidate_score(regressed)
|
||||
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SPEC = importlib.util.spec_from_file_location(
|
||||
"building_portfolio", ROOT / "scripts" / "provision_belgium_building_training_portfolio.py"
|
||||
)
|
||||
assert SPEC and SPEC.loader
|
||||
module = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = module
|
||||
SPEC.loader.exec_module(module)
|
||||
|
||||
|
||||
def test_portfolio_covers_every_region_split_and_context_family() -> None:
|
||||
assert len({aoi.slug for aoi in module.AOIS}) == len(module.AOIS)
|
||||
counts = Counter((aoi.region, aoi.split) for aoi in module.AOIS)
|
||||
for region in module.REGION_CONTRACT:
|
||||
assert counts[(region, "train")] >= 15
|
||||
assert counts[(region, "val")] >= 2
|
||||
assert counts[(region, "calibration")] >= 3
|
||||
assert counts[(region, "test")] >= 3
|
||||
assert counts[(region, "background-test")] >= 2
|
||||
|
||||
|
||||
def test_portfolio_bbox_is_metric_sized() -> None:
|
||||
bbox = module.bbox_for_center(4.35, 50.85, 256.0)
|
||||
to_metric = module.Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
bounds = to_metric.transform_bounds(bbox["min_x"], bbox["min_y"], bbox["max_x"], bbox["max_y"])
|
||||
assert 255 <= bounds[2] - bounds[0] <= 258
|
||||
assert 255 <= bounds[3] - bounds[1] <= 258
|
||||
|
||||
|
||||
def test_failure_driven_expansion_is_train_only_and_context_complete() -> None:
|
||||
additions = [aoi for aoi in module.AOIS if aoi.slug.endswith("-v31") or "-v31-bg" in aoi.slug]
|
||||
assert len(additions) == 18
|
||||
assert {aoi.split for aoi in additions} == {"train"}
|
||||
assert {aoi.region for aoi in additions} == {"flanders", "wallonia"}
|
||||
contexts = {(aoi.region, aoi.context) for aoi in additions}
|
||||
assert {
|
||||
("flanders", "industrial"),
|
||||
("flanders", "ribbon-development"),
|
||||
("flanders", "coastal-urban"),
|
||||
("wallonia", "dense-urban"),
|
||||
("wallonia", "rural-town"),
|
||||
("wallonia", "regional-architecture"),
|
||||
} <= contexts
|
||||
@@ -0,0 +1,261 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from rasterio.transform import from_origin
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "normalize_belgium_building_labels.py"
|
||||
SPEC = importlib.util.spec_from_file_location("normalize_buildings", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
module = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(module)
|
||||
|
||||
ASSEMBLER_SPEC = importlib.util.spec_from_file_location(
|
||||
"assemble_building_corpus", ROOT / "scripts" / "assemble_belgium_building_corpus.py"
|
||||
)
|
||||
assert ASSEMBLER_SPEC and ASSEMBLER_SPEC.loader
|
||||
assembler = importlib.util.module_from_spec(ASSEMBLER_SPEC)
|
||||
ASSEMBLER_SPEC.loader.exec_module(assembler)
|
||||
|
||||
|
||||
def test_normalizer_retains_native_identity_and_records_rejections(tmp_path: Path) -> None:
|
||||
raster_path = tmp_path / "image.tif"
|
||||
with rasterio.open(
|
||||
raster_path,
|
||||
"w",
|
||||
driver="GTiff",
|
||||
width=100,
|
||||
height=100,
|
||||
count=3,
|
||||
dtype="uint8",
|
||||
crs="EPSG:4326",
|
||||
transform=from_origin(4.0, 51.0, 0.001, 0.001),
|
||||
) as dataset:
|
||||
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
|
||||
valid = {
|
||||
"type": "Feature",
|
||||
"id": "native-1",
|
||||
"properties": {"TYPE": "main building"},
|
||||
"geometry": {"type": "Polygon", "coordinates": [[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]]},
|
||||
}
|
||||
duplicate = json.loads(json.dumps(valid))
|
||||
duplicate["id"] = "native-2"
|
||||
canopy = json.loads(json.dumps(valid))
|
||||
canopy["id"] = "native-3"
|
||||
canopy["properties"]["TYPE"] = "canopy"
|
||||
tiny = json.loads(json.dumps(valid))
|
||||
tiny["id"] = "native-4"
|
||||
tiny["geometry"] = {"type": "Polygon", "coordinates": [[[4.03, 50.97], [4.031, 50.97], [4.031, 50.969], [4.03, 50.969], [4.03, 50.97]]]}
|
||||
reference_path = tmp_path / "reference.geojson"
|
||||
reference_path.write_text(json.dumps({"type": "FeatureCollection", "features": [valid, duplicate, canopy, tiny]}), encoding="utf-8")
|
||||
|
||||
normalized, audit = module.normalize(
|
||||
reference_path=reference_path,
|
||||
raster_path=raster_path,
|
||||
source_name="urbis",
|
||||
min_label_px=3,
|
||||
imagery_observed_at="2026-01-01T00:00:00Z",
|
||||
reference_observed_at="2025-12-01T00:00:00Z",
|
||||
)
|
||||
|
||||
assert len(normalized["features"]) == 1
|
||||
properties = normalized["features"][0]["properties"]
|
||||
assert properties["canonical_class"] == "building"
|
||||
assert properties["source_name"] == "urbis"
|
||||
assert properties["source_feature_id"] == "native-1"
|
||||
assert properties["source_class"] == "main building"
|
||||
assert audit["decision_counts"] == {
|
||||
"accepted": 1,
|
||||
"below_resolvable_pixel_size": 1,
|
||||
"duplicate_geometry": 1,
|
||||
"excluded_canopy": 1,
|
||||
}
|
||||
assert audit["temporal_mismatch_days"] == 31
|
||||
|
||||
|
||||
def test_spatial_leakage_audit_fails_cross_split_neighbors() -> None:
|
||||
samples = [
|
||||
{"sample_slug": "train-a", "split": "train", "bbox_epsg4326": [4.0, 50.0, 4.01, 50.01]},
|
||||
{"sample_slug": "val-a", "split": "val", "bbox_epsg4326": [4.005, 50.005, 4.02, 50.02]},
|
||||
{"sample_slug": "test-far", "split": "test", "bbox_epsg4326": [5.0, 51.0, 5.01, 51.01]},
|
||||
]
|
||||
audit = assembler.audit_spatial_leakage(samples)
|
||||
assert audit["status"] == "failed"
|
||||
assert audit["findings"][0]["left"] == "train-a"
|
||||
assert audit["findings"][0]["right"] == "val-a"
|
||||
|
||||
|
||||
def test_normalizer_rejects_features_created_after_dated_imagery(tmp_path: Path) -> None:
|
||||
raster_path = tmp_path / "image.tif"
|
||||
with rasterio.open(
|
||||
raster_path,
|
||||
"w",
|
||||
driver="GTiff",
|
||||
width=100,
|
||||
height=100,
|
||||
count=3,
|
||||
dtype="uint8",
|
||||
crs="EPSG:4326",
|
||||
transform=from_origin(4.0, 51.0, 0.001, 0.001),
|
||||
) as dataset:
|
||||
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
|
||||
geometry = {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]],
|
||||
}
|
||||
reference_path = tmp_path / "reference.geojson"
|
||||
reference_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{"type": "Feature", "id": "old", "properties": {"BEGINDATUM": "2024-01-01"}, "geometry": geometry},
|
||||
{"type": "Feature", "id": "new", "properties": {"BEGINDATUM": "2026-01-01"}, "geometry": geometry},
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
normalized, audit = module.normalize(
|
||||
reference_path=reference_path,
|
||||
raster_path=raster_path,
|
||||
source_name="grb",
|
||||
min_label_px=3,
|
||||
imagery_observed_at="2025-01-01T00:00:00Z",
|
||||
imagery_valid_to="2025-12-31T23:59:59Z",
|
||||
reference_observed_at="2026-07-01T00:00:00Z",
|
||||
)
|
||||
assert len(normalized["features"]) == 1
|
||||
assert audit["decision_counts"] == {"accepted": 1, "created_after_imagery_period": 1}
|
||||
|
||||
|
||||
def test_normalizer_rejects_same_year_feature_after_annual_mosaic_start(tmp_path: Path) -> None:
|
||||
raster_path = tmp_path / "image.tif"
|
||||
with rasterio.open(
|
||||
raster_path, "w", driver="GTiff", width=100, height=100, count=3,
|
||||
dtype="uint8", crs="EPSG:4326", transform=from_origin(4.0, 51.0, 0.001, 0.001),
|
||||
) as dataset:
|
||||
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
|
||||
geometry = {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]],
|
||||
}
|
||||
reference_path = tmp_path / "reference.geojson"
|
||||
reference_path.write_text(json.dumps({
|
||||
"type": "FeatureCollection",
|
||||
"features": [{"type": "Feature", "properties": {"BEGINDATUM": "2025-08-18"}, "geometry": geometry}],
|
||||
}), encoding="utf-8")
|
||||
|
||||
normalized, audit = module.normalize(
|
||||
reference_path=reference_path,
|
||||
raster_path=raster_path,
|
||||
source_name="grb",
|
||||
min_label_px=3,
|
||||
imagery_observed_at="2025-01-01T00:00:00Z",
|
||||
imagery_valid_to="2025-12-31T23:59:59Z",
|
||||
reference_observed_at="2026-07-01T00:00:00Z",
|
||||
)
|
||||
|
||||
assert normalized["features"] == []
|
||||
assert audit["decision_counts"] == {"created_after_imagery_period": 1}
|
||||
assert audit["imagery_feature_creation_cutoff"] == "2025-01-01T00:00:00+00:00"
|
||||
|
||||
|
||||
def test_normalizer_applies_provider_native_source_class_allowlist(tmp_path: Path) -> None:
|
||||
raster_path = tmp_path / "image.tif"
|
||||
with rasterio.open(
|
||||
raster_path, "w", driver="GTiff", width=100, height=100, count=3,
|
||||
dtype="uint8", crs="EPSG:4326", transform=from_origin(4.0, 51.0, 0.001, 0.001),
|
||||
) as dataset:
|
||||
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
|
||||
def feature(feature_id: str, source_type: int, left: float) -> dict:
|
||||
return {
|
||||
"type": "Feature", "id": feature_id, "properties": {"TYPE": source_type},
|
||||
"geometry": {"type": "Polygon", "coordinates": [[[left, 50.99], [left + .01, 50.99], [left + .01, 50.98], [left, 50.98], [left, 50.99]]]},
|
||||
}
|
||||
reference_path = tmp_path / "reference.geojson"
|
||||
reference_path.write_text(json.dumps({"type": "FeatureCollection", "features": [feature("main", 1, 4.01), feature("annex", 2, 4.03)]}), encoding="utf-8")
|
||||
|
||||
normalized, audit = module.normalize(
|
||||
reference_path=reference_path, raster_path=raster_path, source_name="grb",
|
||||
min_label_px=3, imagery_observed_at=None, reference_observed_at=None,
|
||||
allowed_source_classes={"1"},
|
||||
)
|
||||
|
||||
assert [item["id"] for item in normalized["features"]] == ["grb:main"]
|
||||
assert audit["allowed_source_classes"] == ["1"]
|
||||
assert audit["decision_counts"] == {"accepted": 1, "source_class_not_allowed": 1}
|
||||
|
||||
|
||||
def test_normalizer_merges_only_touching_visible_roof_instances(tmp_path: Path) -> None:
|
||||
raster_path = tmp_path / "image.tif"
|
||||
with rasterio.open(
|
||||
raster_path,
|
||||
"w",
|
||||
driver="GTiff",
|
||||
width=100,
|
||||
height=100,
|
||||
count=3,
|
||||
dtype="uint8",
|
||||
crs="EPSG:4326",
|
||||
transform=from_origin(4.0, 51.0, 0.001, 0.001),
|
||||
) as dataset:
|
||||
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
|
||||
polygons = [
|
||||
[[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]],
|
||||
[[[4.02, 50.99], [4.03, 50.99], [4.03, 50.98], [4.02, 50.98], [4.02, 50.99]]],
|
||||
[[[4.04, 50.99], [4.05, 50.99], [4.05, 50.98], [4.04, 50.98], [4.04, 50.99]]],
|
||||
]
|
||||
reference_path = tmp_path / "reference.geojson"
|
||||
reference_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{"type": "Feature", "id": str(index), "properties": {}, "geometry": {"type": "Polygon", "coordinates": coordinates}}
|
||||
for index, coordinates in enumerate(polygons)
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
normalized, audit = module.normalize(
|
||||
reference_path=reference_path,
|
||||
raster_path=raster_path,
|
||||
source_name="urbis",
|
||||
min_label_px=3,
|
||||
imagery_observed_at=None,
|
||||
reference_observed_at=None,
|
||||
merge_touching_roofs=True,
|
||||
)
|
||||
assert len(normalized["features"]) == 2
|
||||
assert sorted(item["properties"]["source_feature_count"] for item in normalized["features"]) == [1, 2]
|
||||
assert audit["accepted_source_feature_count"] == 3
|
||||
assert audit["accepted_feature_count"] == 2
|
||||
|
||||
|
||||
def test_visible_roof_merge_retains_large_touching_chains() -> None:
|
||||
features = []
|
||||
for index in range(13):
|
||||
left = float(index)
|
||||
features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": str(index),
|
||||
"properties": {"source_feature_id": str(index)},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[[left, 0], [left + 1, 0], [left + 1, 1], [left, 1], [left, 0]]],
|
||||
},
|
||||
}
|
||||
)
|
||||
merged = module.merge_touching_roof_instances(features, "grb")
|
||||
assert len(merged) == 13
|
||||
assert {item["properties"]["label_semantics"] for item in merged} == {
|
||||
"native_instance_complex_touch_group"
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "train_building_proposal_filter.py"
|
||||
SPEC = importlib.util.spec_from_file_location("building_proposal_filter", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def test_evenly_limited_is_deterministic() -> None:
|
||||
assert MODULE.evenly_limited(list(range(10)), 3) == [0, 3, 6]
|
||||
|
||||
|
||||
def test_validation_threshold_is_selected_without_test_data() -> None:
|
||||
threshold, metrics = MODULE.choose_threshold([0.9, 0.8, 0.2, 0.1], [1, 1, 0, 0])
|
||||
assert 0.2 < threshold <= 0.8
|
||||
assert metrics["f1"] == 1.0
|
||||
|
||||
|
||||
def test_negative_match_iou() -> None:
|
||||
assert MODULE.iou((0, 0, 10, 10), (0, 0, 10, 10)) == 1.0
|
||||
assert MODULE.iou((0, 0, 10, 10), (20, 20, 30, 30)) == 0.0
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Change detection must answer the question the operator actually asked.
|
||||
|
||||
Every other analysis in the workbench is bounded by the drawn selection.
|
||||
Change detection was not: it compared two datasets in full, loaded every
|
||||
feature of both into Python, and — with ``include_unchanged`` defaulting to
|
||||
true — returned a FeatureCollection containing both datasets entire. For a
|
||||
regional building layer that is the wrong answer to "what changed here" and a
|
||||
response no browser should be asked to hold.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.services.change_detection_service import ChangeDetectionService
|
||||
|
||||
|
||||
def _feature(feature_id: str, geometry):
|
||||
return {"feature_id": feature_id, "properties": {}, "geometry": geometry}
|
||||
|
||||
|
||||
INSIDE = box(0.0, 0.0, 1.0, 1.0)
|
||||
OUTSIDE = box(50.0, 50.0, 51.0, 51.0)
|
||||
SELECTION = box(-1.0, -1.0, 2.0, 2.0)
|
||||
|
||||
|
||||
def test_features_outside_the_selection_are_not_compared() -> None:
|
||||
kept = ChangeDetectionService.restrict_to_selection(
|
||||
[_feature("in", INSIDE), _feature("out", OUTSIDE)],
|
||||
SELECTION,
|
||||
)
|
||||
|
||||
assert [item["feature_id"] for item in kept] == ["in"]
|
||||
|
||||
|
||||
def test_a_feature_crossing_the_selection_edge_is_kept_and_flagged() -> None:
|
||||
crossing = box(1.5, 1.5, 3.0, 3.0)
|
||||
|
||||
kept = ChangeDetectionService.restrict_to_selection(
|
||||
[_feature("crossing", crossing)],
|
||||
SELECTION,
|
||||
)
|
||||
|
||||
assert len(kept) == 1
|
||||
assert kept[0]["partially_covered"] is True
|
||||
# The geometry is not clipped: a change class describes a whole object, and
|
||||
# comparing a clipped 2020 footprint with an unclipped 2024 one would
|
||||
# invent change at the selection edge.
|
||||
assert kept[0]["geometry"].equals(crossing)
|
||||
|
||||
|
||||
def test_a_feature_wholly_inside_is_not_flagged() -> None:
|
||||
kept = ChangeDetectionService.restrict_to_selection([_feature("in", INSIDE)], SELECTION)
|
||||
|
||||
assert kept[0]["partially_covered"] is False
|
||||
|
||||
|
||||
def test_no_selection_leaves_the_population_untouched() -> None:
|
||||
features = [_feature("in", INSIDE), _feature("out", OUTSIDE)]
|
||||
|
||||
assert ChangeDetectionService.restrict_to_selection(features, None) == features
|
||||
|
||||
|
||||
def test_an_empty_intersection_is_an_explicit_error_not_a_silent_zero() -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
ChangeDetectionService.restrict_to_selection([_feature("out", OUTSIDE)], SELECTION, label="Source")
|
||||
|
||||
assert exc_info.value.code == "CHANGE_DETECTION_SELECTION_EMPTY"
|
||||
assert "Source" in exc_info.value.message
|
||||
|
||||
|
||||
def test_the_preview_is_capped_while_the_counts_stay_complete() -> None:
|
||||
features = [
|
||||
{
|
||||
"change_type": "added" if index % 2 else "removed",
|
||||
"geometry": box(index, 0, index + 1, 1),
|
||||
"source_feature_id": None,
|
||||
"target_feature_id": f"t{index}",
|
||||
"iou": None,
|
||||
"properties": {},
|
||||
}
|
||||
for index in range(250)
|
||||
]
|
||||
|
||||
preview, truncated = ChangeDetectionService.limit_preview(features, limit=100)
|
||||
|
||||
assert len(preview) == 100
|
||||
assert truncated is True
|
||||
|
||||
|
||||
def test_a_short_result_is_not_reported_as_truncated() -> None:
|
||||
features = [
|
||||
{
|
||||
"change_type": "added",
|
||||
"geometry": box(0, 0, 1, 1),
|
||||
"source_feature_id": None,
|
||||
"target_feature_id": "t",
|
||||
"iou": None,
|
||||
"properties": {},
|
||||
}
|
||||
]
|
||||
|
||||
preview, truncated = ChangeDetectionService.limit_preview(features, limit=100)
|
||||
|
||||
assert len(preview) == 1
|
||||
assert truncated is False
|
||||
|
||||
|
||||
def test_the_preview_prefers_changes_over_unchanged_features() -> None:
|
||||
"""A cap must not spend its budget on the least interesting class."""
|
||||
|
||||
features = [
|
||||
{"change_type": "unchanged", "geometry": box(index, 0, index + 1, 1), "source_feature_id": f"s{index}",
|
||||
"target_feature_id": f"t{index}", "iou": 1.0, "properties": {}}
|
||||
for index in range(100)
|
||||
] + [
|
||||
{"change_type": "added", "geometry": box(0, 5, 1, 6), "source_feature_id": None,
|
||||
"target_feature_id": "new", "iou": None, "properties": {}},
|
||||
{"change_type": "modified", "geometry": box(0, 7, 1, 8), "source_feature_id": "s",
|
||||
"target_feature_id": "t", "iou": 0.6, "properties": {}},
|
||||
]
|
||||
|
||||
preview, truncated = ChangeDetectionService.limit_preview(features, limit=3)
|
||||
|
||||
assert truncated is True
|
||||
assert sorted(item["change_type"] for item in preview[:2]) == ["added", "modified"]
|
||||
@@ -0,0 +1,114 @@
|
||||
"""An extended building is a change, not a deletion plus a new building.
|
||||
|
||||
With only added/removed/unchanged, a footprint that grew by an annexe drops
|
||||
below the IoU threshold and is reported twice: once as removed and once as
|
||||
added. That hides exactly the category a change-detection product exists to
|
||||
show, and inflates both counts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.services.change_detection_service import ChangeDetectionService
|
||||
|
||||
|
||||
def _feature(feature_id: str, geometry):
|
||||
return {"feature_id": feature_id, "properties": {}, "geometry": geometry}
|
||||
|
||||
|
||||
def _classify(source, target, *, iou_threshold=0.8, modified_threshold=0.3):
|
||||
return ChangeDetectionService._classify_features(
|
||||
source,
|
||||
target,
|
||||
iou_threshold=iou_threshold,
|
||||
modified_threshold=modified_threshold,
|
||||
)
|
||||
|
||||
|
||||
def test_an_extended_footprint_is_reported_as_modified() -> None:
|
||||
source = [_feature("b1", box(0, 0, 10, 10))]
|
||||
target = [_feature("b1-new", box(0, 0, 10, 14))] # IoU 100/140 = 0.71
|
||||
|
||||
result = _classify(source, target)
|
||||
|
||||
assert [item["change_type"] for item in result] == ["modified"]
|
||||
assert result[0]["source_feature_id"] == "b1"
|
||||
assert result[0]["target_feature_id"] == "b1-new"
|
||||
assert result[0]["iou"] == pytest.approx(100 / 140)
|
||||
|
||||
|
||||
def test_a_nearly_identical_footprint_is_unchanged() -> None:
|
||||
source = [_feature("b1", box(0, 0, 10, 10))]
|
||||
target = [_feature("b1", box(0, 0, 10, 10.2))]
|
||||
|
||||
result = _classify(source, target)
|
||||
|
||||
assert [item["change_type"] for item in result] == ["unchanged"]
|
||||
|
||||
|
||||
def test_a_genuinely_new_building_stays_added() -> None:
|
||||
source = [_feature("b1", box(0, 0, 10, 10))]
|
||||
target = [_feature("b1", box(0, 0, 10, 10)), _feature("b2", box(50, 50, 60, 60))]
|
||||
|
||||
result = _classify(source, target)
|
||||
|
||||
assert sorted(item["change_type"] for item in result) == ["added", "unchanged"]
|
||||
|
||||
|
||||
def test_a_demolished_building_stays_removed() -> None:
|
||||
source = [_feature("b1", box(0, 0, 10, 10)), _feature("b2", box(50, 50, 60, 60))]
|
||||
target = [_feature("b1", box(0, 0, 10, 10))]
|
||||
|
||||
result = _classify(source, target)
|
||||
|
||||
assert sorted(item["change_type"] for item in result) == ["removed", "unchanged"]
|
||||
|
||||
|
||||
def test_barely_overlapping_footprints_are_not_called_modified() -> None:
|
||||
"""Below the modified floor the two are separate objects, not one changed."""
|
||||
|
||||
source = [_feature("b1", box(0, 0, 10, 10))]
|
||||
target = [_feature("b2", box(9, 9, 19, 19))] # IoU ~0.005
|
||||
|
||||
result = _classify(source, target)
|
||||
|
||||
assert sorted(item["change_type"] for item in result) == ["added", "removed"]
|
||||
|
||||
|
||||
def test_each_target_is_claimed_at_most_once() -> None:
|
||||
source = [_feature("a", box(0, 0, 10, 10)), _feature("b", box(0, 0, 10, 12))]
|
||||
target = [_feature("t", box(0, 0, 10, 10))]
|
||||
|
||||
result = _classify(source, target)
|
||||
|
||||
claimed = [item for item in result if item["target_feature_id"] == "t"]
|
||||
assert len(claimed) == 1
|
||||
|
||||
|
||||
def test_classification_is_independent_of_input_order() -> None:
|
||||
source = [_feature("a", box(0, 0, 10, 10)), _feature("b", box(30, 30, 40, 40))]
|
||||
target = [_feature("a2", box(0, 0, 10, 14)), _feature("c", box(70, 70, 80, 80))]
|
||||
|
||||
forward = _classify(source, target)
|
||||
reverse = _classify(list(reversed(source)), list(reversed(target)))
|
||||
|
||||
def signature(items):
|
||||
return sorted(
|
||||
(item["change_type"], item["source_feature_id"], item["target_feature_id"]) for item in items
|
||||
)
|
||||
|
||||
assert signature(forward) == signature(reverse)
|
||||
|
||||
|
||||
def test_large_populations_do_not_use_a_full_cross_product() -> None:
|
||||
"""A spatial index keeps a city-sized comparison tractable."""
|
||||
|
||||
source = [_feature(f"s{i}", box(i * 10, 0, i * 10 + 8, 8)) for i in range(400)]
|
||||
target = [_feature(f"t{i}", box(i * 10, 0, i * 10 + 8, 8)) for i in range(400)]
|
||||
|
||||
result = _classify(source, target)
|
||||
|
||||
assert all(item["change_type"] == "unchanged" for item in result)
|
||||
assert len(result) == 400
|
||||
@@ -0,0 +1,449 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.services.data_contract_validation import (
|
||||
AttributeRule,
|
||||
BoundingBox,
|
||||
ContractKind,
|
||||
DataAssetValidationInput,
|
||||
DataContract,
|
||||
DataContractRegistry,
|
||||
DataContractValidator,
|
||||
FreshnessRules,
|
||||
GeometryRecord,
|
||||
GeometryRules,
|
||||
LineageEvidence,
|
||||
LineageRules,
|
||||
RasterRules,
|
||||
RequirementLevel,
|
||||
Resolution,
|
||||
ResolutionRules,
|
||||
TransformationEvidence,
|
||||
ValidationStatus,
|
||||
build_default_data_contract_registry,
|
||||
build_label_validation_input,
|
||||
build_model_validation_input,
|
||||
build_raster_ingest_input,
|
||||
build_vector_ingest_input,
|
||||
validate_registered_asset,
|
||||
)
|
||||
from app.services.data_quarantine_service import AssetUse, DataQuarantineService
|
||||
|
||||
|
||||
FIXTURE_ROOT = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "data-contracts"
|
||||
NOW = datetime(2026, 8, 1, 12, 0, tzinfo=timezone.utc)
|
||||
CHECKSUM_A = "a" * 64
|
||||
|
||||
|
||||
def _fixture_json(name: str) -> tuple[bytes, object]:
|
||||
raw = (FIXTURE_ROOT / name).read_bytes()
|
||||
return raw, json.loads(raw)
|
||||
|
||||
|
||||
def _checksum(raw: bytes) -> str:
|
||||
return sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def _lineage_with_transform() -> LineageEvidence:
|
||||
return LineageEvidence(
|
||||
transformations=(
|
||||
TransformationEvidence(
|
||||
name="epsg31370-to-epsg4326",
|
||||
version="1.0.0",
|
||||
checksum_sha256=CHECKSUM_A,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _vector_input_from_fixture(name: str, *, source_crs: str = "EPSG:31370", storage_crs: str = "EPSG:4326") -> DataAssetValidationInput:
|
||||
raw, payload = _fixture_json(name)
|
||||
assert isinstance(payload, dict)
|
||||
return build_vector_ingest_input(
|
||||
asset_id=f"fixture:{name}",
|
||||
source_crs=source_crs,
|
||||
storage_crs=storage_crs,
|
||||
feature_collection=payload,
|
||||
checksum_sha256=_checksum(raw),
|
||||
computed_checksum_sha256=_checksum(raw),
|
||||
content=raw,
|
||||
source_registry_id="source:digitaal-vlaanderen:grb",
|
||||
source_snapshot_id="snapshot:grb:2026-07-31",
|
||||
imported_at=NOW,
|
||||
metadata={"license": "Open Data Licence", "provider": "Digitaal Vlaanderen"},
|
||||
observed_at=NOW - timedelta(days=1),
|
||||
source_version="2026.07.31",
|
||||
lineage=_lineage_with_transform() if source_crs != storage_crs else LineageEvidence(),
|
||||
)
|
||||
|
||||
|
||||
def _issue_codes(report) -> set[str]:
|
||||
return {issue.code for issue in report.issues}
|
||||
|
||||
|
||||
def test_default_vector_contract_accepts_transformed_geojson_with_complete_provenance() -> None:
|
||||
report = validate_registered_asset(_vector_input_from_fixture("vector-building-valid.geojson"), now=NOW)
|
||||
|
||||
assert report.validation_status == ValidationStatus.PASSED
|
||||
assert report.quarantine_status == "not_quarantined"
|
||||
assert report.provenance_status == "complete"
|
||||
assert report.lineage_status == "complete"
|
||||
persisted = report.persistence_fields()
|
||||
assert persisted["data_contract_key"] == "geointel.vector.geojson"
|
||||
assert persisted["data_contract_version"] == "1.0.0"
|
||||
assert persisted["validation_report_json"]["report_sha256"] == report.report_sha256
|
||||
|
||||
|
||||
def test_default_vector_contract_quarantines_lambert_coordinates_mislabelled_as_epsg4326() -> None:
|
||||
report = validate_registered_asset(
|
||||
_vector_input_from_fixture(
|
||||
"vector-lambert-mislabelled-as-4326.geojson",
|
||||
source_crs="EPSG:4326",
|
||||
storage_crs="EPSG:4326",
|
||||
),
|
||||
now=NOW,
|
||||
)
|
||||
|
||||
assert report.validation_status == ValidationStatus.FAILED
|
||||
assert report.quarantine_status == "quarantined"
|
||||
assert "CRS_COORDINATE_DOMAIN_VIOLATION" in _issue_codes(report)
|
||||
|
||||
|
||||
def test_vector_contract_checks_geometry_attributes_bounds_and_topology_fail_closed() -> None:
|
||||
contract = DataContract(
|
||||
key="test.vector.buildings",
|
||||
version="1.0.0",
|
||||
kind=ContractKind.VECTOR,
|
||||
accepted_source_crs=frozenset({"EPSG:4326"}),
|
||||
canonical_storage_crs="EPSG:4326",
|
||||
spatial_domain=BoundingBox(2.0, 49.0, 7.0, 52.0),
|
||||
require_bounds=True,
|
||||
geometry_rules=GeometryRules(
|
||||
allowed_geometry_types=frozenset({"Polygon"}),
|
||||
attribute_rules=(AttributeRule("native_id", accepted_types=("integer",)),),
|
||||
forbid_shared_area=True,
|
||||
),
|
||||
)
|
||||
raw = b"overlapping-vector"
|
||||
asset = DataAssetValidationInput(
|
||||
asset_id="vector:bad",
|
||||
data_contract_key=contract.key,
|
||||
data_contract_version=contract.version,
|
||||
kind=ContractKind.VECTOR,
|
||||
source_crs="EPSG:4326",
|
||||
storage_crs="EPSG:4326",
|
||||
bounds=BoundingBox(4.0, 51.0, 4.1, 51.1),
|
||||
checksum_sha256=_checksum(raw),
|
||||
computed_checksum_sha256=_checksum(raw),
|
||||
content=raw,
|
||||
geometry_records=(
|
||||
GeometryRecord(box(4.0, 51.0, 4.05, 51.05), {"native_id": "wrong-type"}),
|
||||
GeometryRecord(box(4.025, 51.025, 4.075, 51.075), {}),
|
||||
),
|
||||
source_registry_id="source:test",
|
||||
source_snapshot_id="snapshot:test",
|
||||
imported_at=NOW,
|
||||
)
|
||||
|
||||
report = DataContractValidator.validate(contract, asset, now=NOW)
|
||||
|
||||
assert report.validation_status == ValidationStatus.FAILED
|
||||
assert {"ATTRIBUTE_TYPE_INVALID", "ATTRIBUTE_REQUIRED", "TOPOLOGY_SHARED_AREA"} <= _issue_codes(report)
|
||||
assert "BOUNDS_GEOMETRY_MISMATCH" in _issue_codes(report)
|
||||
|
||||
|
||||
def test_default_vector_contract_validates_replayable_large_partition_stream_without_materialising_geometry_list() -> None:
|
||||
"""Regional imports may be large but remain fully schema/domain checked.
|
||||
|
||||
The default contract has no source-specific shared-area rule, so the
|
||||
validator must make its bounds/schema passes over a replayable stream
|
||||
without accumulating every Shapely geometry in memory. A stricter
|
||||
source-specific contract can still opt into a bounded topology batch.
|
||||
"""
|
||||
|
||||
class ReplayableRecords:
|
||||
def __init__(self, count: int) -> None:
|
||||
self.count = count
|
||||
self.iterations = 0
|
||||
|
||||
def __iter__(self):
|
||||
self.iterations += 1
|
||||
for index in range(self.count):
|
||||
yield GeometryRecord(
|
||||
box(4.69, 51.09, 4.70, 51.10),
|
||||
{"partition_feature": index},
|
||||
)
|
||||
|
||||
raw = b"partitioned-vector-stream"
|
||||
records = ReplayableRecords(12_000)
|
||||
asset = DataAssetValidationInput(
|
||||
asset_id="vector:partitioned-stream",
|
||||
data_contract_key="geointel.vector.geojson",
|
||||
data_contract_version="1.0.0",
|
||||
kind=ContractKind.VECTOR,
|
||||
source_crs="EPSG:4326",
|
||||
storage_crs="EPSG:4326",
|
||||
bounds=BoundingBox(4.69, 51.09, 4.70, 51.10),
|
||||
checksum_sha256=_checksum(raw),
|
||||
computed_checksum_sha256=_checksum(raw),
|
||||
content=raw,
|
||||
metadata={"license": "Open Data"},
|
||||
geometry_records=records,
|
||||
source_registry_id="source:grb",
|
||||
source_snapshot_id="snapshot:grb:partitioned",
|
||||
imported_at=NOW,
|
||||
observed_at=NOW,
|
||||
source_version="2026-08-01",
|
||||
)
|
||||
|
||||
report = validate_registered_asset(asset, now=NOW)
|
||||
|
||||
assert report.validation_status == ValidationStatus.PASSED
|
||||
assert records.iterations >= 2
|
||||
|
||||
|
||||
def test_raster_contract_accepts_explicit_units_and_quarantines_stale_bad_profile() -> None:
|
||||
raw = b"raster-stage"
|
||||
valid = build_raster_ingest_input(
|
||||
asset_id="raster:valid",
|
||||
source_crs="EPSG:31370",
|
||||
storage_crs="EPSG:31370",
|
||||
raster_profile={"width": 512, "height": 512, "band_count": 3, "dtype": ["uint8"]},
|
||||
bounds=BoundingBox(193_277.5, 205_708.3, 193_777.5, 206_208.3),
|
||||
resolution=Resolution(0.9765625, 0.9765625, "m"),
|
||||
checksum_sha256=_checksum(raw),
|
||||
computed_checksum_sha256=_checksum(raw),
|
||||
content=raw,
|
||||
source_registry_id="source:orthophoto",
|
||||
source_snapshot_id="snapshot:orthophoto:2026.01",
|
||||
imported_at=NOW,
|
||||
metadata={"license": "Open Data"},
|
||||
observed_at=None,
|
||||
temporal_unknown_reason="latest mosaic has no per-pixel observation date",
|
||||
source_version=None,
|
||||
source_version_unknown_reason="provider did not publish an edition",
|
||||
)
|
||||
assert validate_registered_asset(valid, now=NOW).validation_status == ValidationStatus.PASSED
|
||||
|
||||
strict = DataContract(
|
||||
key="test.raster.strict",
|
||||
version="1.0.0",
|
||||
kind=ContractKind.RASTER,
|
||||
accepted_source_crs=frozenset({"EPSG:31370"}),
|
||||
require_bounds=True,
|
||||
raster_rules=RasterRules(allowed_band_counts=frozenset({3}), allowed_dtypes=frozenset({"uint8"})),
|
||||
resolution_rules=ResolutionRules(allowed_units=frozenset({"m"}), min_x=0.2, max_x=1.0, min_y=0.2, max_y=1.0),
|
||||
freshness_rules=FreshnessRules(observed_at=RequirementLevel.REQUIRED, max_age=timedelta(days=30)),
|
||||
lineage_rules=LineageRules(require_transformation_when_crs_changes=False),
|
||||
)
|
||||
invalid = DataAssetValidationInput(
|
||||
asset_id="raster:bad",
|
||||
data_contract_key=strict.key,
|
||||
data_contract_version=strict.version,
|
||||
kind=ContractKind.RASTER,
|
||||
source_crs="EPSG:31370",
|
||||
storage_crs="EPSG:31370",
|
||||
bounds=BoundingBox(100.0, 100.0, 200.0, 200.0),
|
||||
checksum_sha256=_checksum(raw),
|
||||
computed_checksum_sha256=_checksum(raw),
|
||||
content=raw,
|
||||
raster_profile={"width": 0, "height": 10, "band_count": 2, "dtype": ["float32"]},
|
||||
resolution=Resolution(2.0, 0.1, "degree"),
|
||||
source_registry_id="source:raster",
|
||||
source_snapshot_id="snapshot:raster",
|
||||
imported_at=NOW,
|
||||
observed_at=NOW - timedelta(days=31),
|
||||
)
|
||||
report = DataContractValidator.validate(strict, invalid, now=NOW)
|
||||
|
||||
assert report.validation_status == ValidationStatus.FAILED
|
||||
assert {
|
||||
"RASTER_PROFILE_VALUE_INVALID",
|
||||
"RASTER_BAND_COUNT_NOT_ALLOWED",
|
||||
"RASTER_DTYPE_NOT_ALLOWED",
|
||||
"RESOLUTION_UNIT_NOT_ALLOWED",
|
||||
"RESOLUTION_OUT_OF_RANGE",
|
||||
"FRESHNESS_EXCEEDED",
|
||||
} <= _issue_codes(report)
|
||||
|
||||
|
||||
def test_default_label_and_model_contracts_validate_good_and_bad_fixtures() -> None:
|
||||
valid_raw, valid_labels = _fixture_json("labels-valid.json")
|
||||
invalid_raw, invalid_labels = _fixture_json("labels-invalid.json")
|
||||
assert isinstance(valid_labels, list)
|
||||
assert isinstance(invalid_labels, list)
|
||||
lineage = LineageEvidence(upstream_asset_ids=("image:1",), upstream_checksums_sha256=(CHECKSUM_A,))
|
||||
valid_label = build_label_validation_input(
|
||||
asset_id="label:valid",
|
||||
label_records=valid_labels,
|
||||
checksum_sha256=_checksum(valid_raw),
|
||||
computed_checksum_sha256=_checksum(valid_raw),
|
||||
content=valid_raw,
|
||||
source_registry_id="source:labels",
|
||||
source_snapshot_id="snapshot:labels:1",
|
||||
imported_at=NOW,
|
||||
metadata={
|
||||
"image_checksum_sha256": CHECKSUM_A,
|
||||
"class_ontology_version": "buildings-v1",
|
||||
"source_corpus_manifest_sha256": CHECKSUM_A,
|
||||
},
|
||||
temporal_unknown_reason="labels inherit image observation handling",
|
||||
source_version_unknown_reason="label release is represented by its snapshot",
|
||||
lineage=lineage,
|
||||
)
|
||||
valid_report = validate_registered_asset(valid_label, now=NOW)
|
||||
assert valid_report.validation_status == ValidationStatus.PASSED
|
||||
|
||||
invalid_label = build_label_validation_input(
|
||||
asset_id="label:invalid",
|
||||
label_records=invalid_labels,
|
||||
checksum_sha256=_checksum(invalid_raw),
|
||||
computed_checksum_sha256=_checksum(invalid_raw),
|
||||
content=invalid_raw,
|
||||
source_registry_id="source:labels",
|
||||
source_snapshot_id="snapshot:labels:1",
|
||||
imported_at=NOW,
|
||||
metadata={
|
||||
"image_checksum_sha256": "not-a-sha256",
|
||||
"class_ontology_version": "buildings-v1",
|
||||
"source_corpus_manifest_sha256": CHECKSUM_A,
|
||||
},
|
||||
temporal_unknown_reason="labels inherit image observation handling",
|
||||
source_version_unknown_reason="label release is represented by its snapshot",
|
||||
lineage=lineage,
|
||||
)
|
||||
invalid_report = validate_registered_asset(invalid_label, now=NOW)
|
||||
assert invalid_report.validation_status == ValidationStatus.FAILED
|
||||
assert {
|
||||
"LABEL_CLASS_ID_NOT_ALLOWED",
|
||||
"LABEL_NORMALIZED_COORDINATE_INVALID",
|
||||
"METADATA_CHECKSUM_INVALID",
|
||||
} <= _issue_codes(invalid_report)
|
||||
|
||||
pure_background_raw = b""
|
||||
pure_background_metadata = {
|
||||
"image_checksum_sha256": CHECKSUM_A,
|
||||
"class_ontology_version": "buildings-v1",
|
||||
"source_corpus_manifest_sha256": CHECKSUM_A,
|
||||
"label_mode": "pure_background",
|
||||
"sample_slug": "forest-background-aoi",
|
||||
"split": "train",
|
||||
"raster_dataset_id": "dataset:raster:1",
|
||||
"reference_dataset_id": "dataset:reference:1",
|
||||
"review_decision": "accepted",
|
||||
"reviewer_id": "reviewer@example.test",
|
||||
"reviewed_at": "2026-08-01T11:00:00+00:00",
|
||||
"review_artifact_sha256": CHECKSUM_A,
|
||||
}
|
||||
pure_background = build_label_validation_input(
|
||||
asset_id="label:pure-background",
|
||||
label_records=(),
|
||||
label_mode="pure_background",
|
||||
checksum_sha256=_checksum(pure_background_raw),
|
||||
computed_checksum_sha256=_checksum(pure_background_raw),
|
||||
content=pure_background_raw,
|
||||
source_registry_id="source:labels",
|
||||
source_snapshot_id="snapshot:labels:1",
|
||||
imported_at=NOW,
|
||||
metadata=pure_background_metadata,
|
||||
temporal_unknown_reason="labels inherit image observation handling",
|
||||
source_version_unknown_reason="label release is represented by its snapshot",
|
||||
lineage=LineageEvidence(
|
||||
upstream_asset_ids=("dataset:raster:1", "dataset:reference:1"),
|
||||
upstream_checksums_sha256=(CHECKSUM_A, CHECKSUM_A),
|
||||
),
|
||||
)
|
||||
assert validate_registered_asset(pure_background, now=NOW).validation_status == ValidationStatus.PASSED
|
||||
|
||||
unmarked_empty = build_label_validation_input(
|
||||
asset_id="label:unmarked-empty",
|
||||
label_records=(),
|
||||
checksum_sha256=_checksum(pure_background_raw),
|
||||
computed_checksum_sha256=_checksum(pure_background_raw),
|
||||
content=pure_background_raw,
|
||||
source_registry_id="source:labels",
|
||||
source_snapshot_id="snapshot:labels:1",
|
||||
imported_at=NOW,
|
||||
metadata={
|
||||
"image_checksum_sha256": CHECKSUM_A,
|
||||
"class_ontology_version": "buildings-v1",
|
||||
"source_corpus_manifest_sha256": CHECKSUM_A,
|
||||
},
|
||||
temporal_unknown_reason="labels inherit image observation handling",
|
||||
source_version_unknown_reason="label release is represented by its snapshot",
|
||||
lineage=LineageEvidence(
|
||||
upstream_asset_ids=("dataset:raster:1", "dataset:reference:1"),
|
||||
upstream_checksums_sha256=(CHECKSUM_A, CHECKSUM_A),
|
||||
),
|
||||
)
|
||||
assert "PURE_BACKGROUND_MODE_REQUIRED" in _issue_codes(validate_registered_asset(unmarked_empty, now=NOW))
|
||||
|
||||
model_raw = b"model-asset"
|
||||
model = build_model_validation_input(
|
||||
asset_id="model:valid",
|
||||
model_metadata={"model_format": "pytorch", "framework": "torch", "class_mapping": {"0": "building"}},
|
||||
checksum_sha256=_checksum(model_raw),
|
||||
computed_checksum_sha256=_checksum(model_raw),
|
||||
content=model_raw,
|
||||
source_registry_id="source:model-registry",
|
||||
source_snapshot_id="snapshot:model:1",
|
||||
imported_at=NOW,
|
||||
source_version="candidate-1",
|
||||
metadata={"training_manifest_sha256": CHECKSUM_A, "runtime_manifest_sha256": CHECKSUM_A},
|
||||
lineage=lineage,
|
||||
)
|
||||
assert validate_registered_asset(model, now=NOW).validation_status == ValidationStatus.PASSED
|
||||
|
||||
|
||||
def test_unknown_contract_and_quarantine_gate_are_deterministic_and_fail_closed() -> None:
|
||||
unknown = DataAssetValidationInput(
|
||||
asset_id="asset:unknown",
|
||||
data_contract_key="does.not.exist",
|
||||
data_contract_version="9.9.9",
|
||||
kind=ContractKind.VECTOR,
|
||||
)
|
||||
report = DataContractRegistry().validate(unknown, now=NOW)
|
||||
assert report.validation_status == ValidationStatus.FAILED
|
||||
assert report.quarantine_status == "quarantined"
|
||||
assert _issue_codes(report) == {"DATA_CONTRACT_UNKNOWN"}
|
||||
|
||||
first = DataQuarantineService.decide(report)
|
||||
second = DataQuarantineService.decide(report)
|
||||
assert first.idempotency_key == second.idempotency_key
|
||||
assert first.reason_codes == ("DATA_CONTRACT_UNKNOWN",)
|
||||
with pytest.raises(AppError, match="cannot enter this pipeline") as exc_info:
|
||||
DataQuarantineService.require_eligible(first, use=AssetUse.PRODUCTION_INFERENCE)
|
||||
assert exc_info.value.code == "DATASET_QUARANTINED"
|
||||
assert exc_info.value.details["use"] == "production_inference"
|
||||
|
||||
clean_report = validate_registered_asset(_vector_input_from_fixture("vector-building-valid.geojson"), now=NOW)
|
||||
release_request = DataQuarantineService.decide(clean_report, previous=first)
|
||||
assert release_request.quarantine_status == "quarantined"
|
||||
assert release_request.requires_explicit_release is True
|
||||
assert release_request.reason_codes == ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",)
|
||||
|
||||
|
||||
def test_registry_requires_exact_contract_version_and_fingerprints_schema() -> None:
|
||||
registry = build_default_data_contract_registry()
|
||||
version_mismatch = _vector_input_from_fixture("vector-building-valid.geojson")
|
||||
mismatched = DataAssetValidationInput(
|
||||
**{**version_mismatch.__dict__, "data_contract_version": "2.0.0"},
|
||||
)
|
||||
|
||||
report = registry.validate(mismatched, now=NOW)
|
||||
assert report.validation_status == ValidationStatus.FAILED
|
||||
assert "DATA_CONTRACT_UNKNOWN" in _issue_codes(report)
|
||||
|
||||
contract = registry.resolve("geointel.vector.geojson", "1.0.0")
|
||||
assert contract is not None
|
||||
direct_report = DataContractValidator.validate(contract, mismatched, now=NOW)
|
||||
assert direct_report.validation_status == ValidationStatus.FAILED
|
||||
assert "DATA_CONTRACT_IDENTITY_MISMATCH" in _issue_codes(direct_report)
|
||||
@@ -0,0 +1,327 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, SourceRegistry, SourceSnapshot
|
||||
from app.services.coverage_registry_service import CoverageRegistryService, SOURCE_DEFINITIONS
|
||||
import app.services.dataset_consumption_gate_service as gate_module
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
from app.services.export_service import ExportService
|
||||
|
||||
|
||||
def _governed_dataset(
|
||||
*,
|
||||
source_key: str = "grb",
|
||||
classification: str = "authoritative",
|
||||
snapshot_freshness_status: str = "current",
|
||||
) -> Dataset:
|
||||
source_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
checksum = "a" * 64
|
||||
source = SourceRegistry(
|
||||
id=source_id,
|
||||
source_key=source_key,
|
||||
display_name=f"{source_key} test source",
|
||||
classification=classification,
|
||||
authority_name="GeoIntel test authority",
|
||||
authority_scope_json={"scope": "test"},
|
||||
usage_policy_json={
|
||||
"ground_truth_allowed": classification == "authoritative",
|
||||
"validation_authority": {"building_validation": "primary"}
|
||||
if classification == "authoritative"
|
||||
else {},
|
||||
},
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=source_id,
|
||||
snapshot_key="test-snapshot",
|
||||
checksum_sha256=checksum,
|
||||
freshness_status=snapshot_freshness_status,
|
||||
ingest_status="ingested",
|
||||
)
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="governed.tif",
|
||||
dataset_type="raster",
|
||||
source=source_key,
|
||||
source_name=source_key,
|
||||
dataset_role="source",
|
||||
checksum_sha256=checksum,
|
||||
source_registry_id=source_id,
|
||||
source_snapshot_id=snapshot_id,
|
||||
data_contract_key="geointel.raster.geotiff",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
status="ready",
|
||||
)
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
return dataset
|
||||
|
||||
|
||||
def test_governed_dataset_passes_production_inference_and_authoritative_coverage() -> None:
|
||||
dataset = _governed_dataset()
|
||||
|
||||
inference = DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
|
||||
coverage = DatasetConsumptionGate.assert_eligible(dataset, purpose="authoritative_coverage")
|
||||
|
||||
assert inference.eligible is True
|
||||
assert coverage.eligible is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value", "error_code"),
|
||||
(
|
||||
("provenance_status", "incomplete", "DATASET_PROVENANCE_INCOMPLETE"),
|
||||
("validation_status", "failed", "DATASET_QUARANTINED"),
|
||||
("quarantine_status", "quarantined", "DATASET_QUARANTINED"),
|
||||
),
|
||||
)
|
||||
def test_explicit_unsafe_states_can_never_be_relaxed(field: str, value: str, error_code: str) -> None:
|
||||
dataset = _governed_dataset()
|
||||
setattr(dataset, field, value)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
dataset,
|
||||
purpose="production_inference",
|
||||
fixture_mode=True,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == error_code
|
||||
assert field.replace("_status", "") in " ".join(exc_info.value.details["reasons"])
|
||||
|
||||
|
||||
def test_legacy_fixture_can_support_fixture_qa_but_never_authoritative_coverage() -> None:
|
||||
fixture = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="fixture.tif",
|
||||
dataset_type="raster",
|
||||
source="fixture",
|
||||
)
|
||||
|
||||
qa = DatasetConsumptionGate.assert_eligible(fixture, purpose="quality_assessment")
|
||||
with pytest.raises(AppError) as inference_error:
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
fixture,
|
||||
purpose="production_inference",
|
||||
fixture_mode=True,
|
||||
)
|
||||
with pytest.raises(AppError) as export_error:
|
||||
DatasetConsumptionGate.assert_eligible(fixture, purpose="export")
|
||||
with pytest.raises(AppError) as fixture_export_error:
|
||||
DatasetConsumptionGate.assert_eligible(fixture, purpose="export", fixture_mode=True)
|
||||
coverage = DatasetConsumptionGate.evaluate(fixture, purpose="authoritative_coverage")
|
||||
|
||||
assert qa.fixture_legacy_exception is True
|
||||
assert inference_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "fixture_qa_only" in inference_error.value.details["reasons"]
|
||||
assert export_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert fixture_export_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "fixture_qa_only" in fixture_export_error.value.details["reasons"]
|
||||
assert coverage.eligible is False
|
||||
assert "fixture_not_authoritative_coverage" in coverage.reasons
|
||||
|
||||
|
||||
def test_unprovenanced_persistent_dataset_is_blocked(monkeypatch) -> None:
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="manual.tif",
|
||||
dataset_type="raster",
|
||||
source="manual_upload",
|
||||
)
|
||||
monkeypatch.setattr(gate_module, "sa_inspect", lambda _dataset: SimpleNamespace(transient=False))
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
dataset,
|
||||
purpose="production_inference",
|
||||
fixture_mode=True,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "phase2_provenance_missing" in exc_info.value.details["reasons"]
|
||||
assert "fixture_source_required" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_transient_orm_test_double_can_only_bypass_missing_legacy_fields_for_qa() -> None:
|
||||
transient = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="transient-test.tif",
|
||||
dataset_type="raster",
|
||||
source="manual_upload",
|
||||
)
|
||||
|
||||
decision = DatasetConsumptionGate.assert_eligible(transient, purpose="quality_assessment")
|
||||
coverage = DatasetConsumptionGate.evaluate(transient, purpose="authoritative_coverage")
|
||||
with pytest.raises(AppError) as production_error:
|
||||
DatasetConsumptionGate.assert_eligible(transient, purpose="production_inference")
|
||||
|
||||
assert decision.fixture_legacy_exception is True
|
||||
assert production_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert coverage.eligible is False
|
||||
assert "phase2_provenance_missing" in coverage.reasons
|
||||
|
||||
|
||||
@pytest.mark.parametrize("purpose", ("production_inference", "derived_processing", "export"))
|
||||
def test_passed_manual_or_experimental_dataset_cannot_cross_production_boundary(purpose: str) -> None:
|
||||
"""A syntactically valid manual upload remains experimental, never production-ready."""
|
||||
|
||||
manual = _governed_dataset(source_key="manual", classification="experimental")
|
||||
manual.source = "manual_upload"
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(manual, purpose=purpose) # type: ignore[arg-type]
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_fully_governed_demo_fixture_still_cannot_enter_production_inference() -> None:
|
||||
fixture = _governed_dataset(source_key="fixture", classification="experimental")
|
||||
fixture.source_metadata = {"fixture": True, "usage": "offline demo raster workflow only"}
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(fixture, purpose="production_inference")
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_reference_validation_requires_authoritative_ground_truth_reference() -> None:
|
||||
reference = _governed_dataset()
|
||||
reference.dataset_type = "vector"
|
||||
reference.dataset_role = "reference"
|
||||
|
||||
decision = DatasetConsumptionGate.assert_eligible(
|
||||
reference,
|
||||
purpose="reference_validation",
|
||||
reference_task="building_validation",
|
||||
)
|
||||
assert decision.eligible is True
|
||||
|
||||
reference.source_registry.classification = "corroborative"
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
reference,
|
||||
purpose="reference_validation",
|
||||
reference_task="building_validation",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "reference_source_not_authoritative" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_pending_regional_building_authority_cannot_become_truth_without_approval() -> None:
|
||||
reference = _governed_dataset(source_key="spw_picc", classification="authoritative")
|
||||
reference.dataset_type = "vector"
|
||||
reference.dataset_role = "reference"
|
||||
reference.source_registry.authority_scope_json = {"zone": "Wallonia"}
|
||||
reference.source_registry.usage_policy_json = {
|
||||
"ground_truth_allowed": True,
|
||||
"validation_authority": {"building_validation": "regional_primary_pending_contract"},
|
||||
}
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
reference,
|
||||
purpose="reference_validation",
|
||||
reference_task="building_validation",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "reference_task_authority_not_approved" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_source_snapshot_must_belong_to_the_dataset_source_registry() -> None:
|
||||
dataset = _governed_dataset()
|
||||
dataset.source_snapshot.source_registry_id = uuid4()
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "source_snapshot_registry_mismatch" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("freshness_status", ("unknown", "review_required", "due", "stale"))
|
||||
def test_non_consumable_source_snapshot_freshness_is_blocked_at_production_boundaries(
|
||||
freshness_status: str,
|
||||
) -> None:
|
||||
dataset = _governed_dataset(snapshot_freshness_status=freshness_status)
|
||||
|
||||
for purpose in ("production_inference", "authoritative_coverage"):
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(dataset, purpose=purpose) # type: ignore[arg-type]
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "source_snapshot_freshness_not_eligible" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_coverage_registry_ignores_explicitly_incomplete_materialization() -> None:
|
||||
definition = next(item for item in SOURCE_DEFINITIONS if item.contract.source_name == "digitaal_vlaanderen")
|
||||
unsafe_materialization = SimpleNamespace(
|
||||
id=uuid4(),
|
||||
status="ready",
|
||||
source_name="grb",
|
||||
validation_status="passed",
|
||||
provenance_status="incomplete",
|
||||
lineage_status="complete",
|
||||
quarantine_status="not_quarantined",
|
||||
)
|
||||
|
||||
matches, fully_covered = CoverageRegistryService._matching_datasets(
|
||||
[unsafe_materialization],
|
||||
definition,
|
||||
"buildings",
|
||||
"flanders",
|
||||
box(4.0, 50.8, 4.1, 50.9),
|
||||
)
|
||||
|
||||
assert matches == []
|
||||
assert fully_covered is False
|
||||
|
||||
|
||||
def test_vector_export_is_fail_closed_before_selection(monkeypatch) -> None:
|
||||
dataset = _governed_dataset()
|
||||
dataset.dataset_type = "vector"
|
||||
dataset.status = "quarantined"
|
||||
queried = False
|
||||
|
||||
class _Session:
|
||||
@staticmethod
|
||||
def get(model, item_id):
|
||||
return dataset if model is Dataset and item_id == dataset.id else None
|
||||
|
||||
def _unexpected_selection(*_args, **_kwargs):
|
||||
nonlocal queried
|
||||
queried = True
|
||||
raise AssertionError("unsafe dataset must be rejected before querying vector features")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.export_service.VectorFeatureService.select_features_by_bbox",
|
||||
_unexpected_selection,
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
ExportService.export_vector_selection_geojson(
|
||||
_Session(),
|
||||
dataset.id,
|
||||
{"min_x": 4.0, "min_y": 50.0, "max_x": 4.1, "max_y": 50.1, "crs": "EPSG:4326"},
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "DATASET_QUARANTINED"
|
||||
assert queried is False
|
||||
@@ -0,0 +1,125 @@
|
||||
"""A precise failure must not be reported as a generic one.
|
||||
|
||||
``get_dataset_geojson`` wrapped the JSON parse, the metadata read, the CRS
|
||||
resolution and the canonicalisation in one ``try``, and reported everything as
|
||||
"Stored dataset is not valid JSON" with a 500. An operator whose dataset has an
|
||||
unusable CRS was sent to inspect a file that parses perfectly well, and the
|
||||
specific AppError the canonicaliser raised — with its own code and status —
|
||||
never reached them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, dataset: Dataset) -> None:
|
||||
self.dataset = dataset
|
||||
|
||||
def get(self, _model, item_id):
|
||||
return self.dataset if item_id == self.dataset.id else None
|
||||
|
||||
|
||||
def _dataset(tmp_path: Path, payload: str) -> Dataset:
|
||||
path = tmp_path / "features.geojson"
|
||||
path.write_text(payload, encoding="utf-8")
|
||||
return Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="features.geojson",
|
||||
dataset_type="geojson",
|
||||
source="test",
|
||||
status="ready",
|
||||
storage_path=str(path),
|
||||
crs="EPSG:4326",
|
||||
)
|
||||
|
||||
|
||||
def test_a_valid_dataset_is_returned(tmp_path: Path) -> None:
|
||||
dataset = _dataset(
|
||||
tmp_path,
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
result = DatasetService.get_dataset_geojson(FakeSession(dataset), dataset.id)
|
||||
|
||||
assert result["type"] == "FeatureCollection"
|
||||
|
||||
|
||||
def test_unparseable_bytes_are_reported_as_invalid_json(tmp_path: Path) -> None:
|
||||
dataset = _dataset(tmp_path, "{ not json")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetService.get_dataset_geojson(FakeSession(dataset), dataset.id)
|
||||
|
||||
assert exc_info.value.code == "INVALID_GEOJSON"
|
||||
assert "JSON" in exc_info.value.message
|
||||
|
||||
|
||||
def test_a_parseable_file_that_is_not_a_feature_collection_says_so(tmp_path: Path) -> None:
|
||||
"""Valid JSON, wrong shape. Telling the operator it is not JSON sends them
|
||||
to inspect a file that parses perfectly well."""
|
||||
|
||||
dataset = _dataset(tmp_path, json.dumps({"type": "Feature", "geometry": None}))
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetService.get_dataset_geojson(FakeSession(dataset), dataset.id)
|
||||
|
||||
assert exc_info.value.code == "INVALID_GEOJSON"
|
||||
assert "FeatureCollection" in exc_info.value.message
|
||||
# The canonicaliser's own status survives; it is a bad request, not a
|
||||
# server fault.
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
def test_a_canonicalisation_failure_keeps_its_own_code(tmp_path: Path, monkeypatch) -> None:
|
||||
dataset = _dataset(tmp_path, json.dumps({"type": "FeatureCollection", "features": []}))
|
||||
|
||||
def failing(*_args, **_kwargs):
|
||||
raise AppError(code="INVALID_CRS", message="Unusable source CRS", status_code=409)
|
||||
|
||||
monkeypatch.setattr(VectorFeatureService, "canonicalize_geojson_payload", staticmethod(failing))
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetService.get_dataset_geojson(FakeSession(dataset), dataset.id)
|
||||
|
||||
assert exc_info.value.code == "INVALID_CRS"
|
||||
assert exc_info.value.status_code == 409
|
||||
|
||||
|
||||
def test_an_unexpected_failure_still_becomes_a_server_error(tmp_path: Path, monkeypatch) -> None:
|
||||
"""A genuine bug must not masquerade as a client mistake."""
|
||||
|
||||
dataset = _dataset(tmp_path, json.dumps({"type": "FeatureCollection", "features": []}))
|
||||
|
||||
def exploding(*_args, **_kwargs):
|
||||
raise RuntimeError("pyproj blew up")
|
||||
|
||||
monkeypatch.setattr(VectorFeatureService, "canonicalize_geojson_payload", staticmethod(exploding))
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetService.get_dataset_geojson(FakeSession(dataset), dataset.id)
|
||||
|
||||
assert exc_info.value.status_code == 500
|
||||
assert exc_info.value.code == "DATASET_GEOJSON_UNREADABLE"
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Calibrating a confidence threshold does not need one inference run per value.
|
||||
|
||||
The workbench ran the model over every tile once per threshold — three GPU
|
||||
passes to compare 0.50, 0.25 and 0.15. The answer is already in a single run at
|
||||
the lowest value: detections above a higher cut are a subset of it, and
|
||||
suppression walks candidates in descending confidence, so a lower-confidence
|
||||
box can never displace a higher-confidence one. The kept set above any cut is
|
||||
therefore identical whichever threshold the run used.
|
||||
|
||||
One matching pass produces every operating point exactly, so the sweep is free
|
||||
rather than N times the cost.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.services.detection_metrics_service import DetectionMetricsService
|
||||
|
||||
|
||||
def _candidate(name: str, geometry, confidence: float):
|
||||
return ({"id": name, "confidence": confidence}, geometry)
|
||||
|
||||
|
||||
def _reference(name: str, geometry):
|
||||
return ({"id": name}, geometry)
|
||||
|
||||
|
||||
REFERENCES = [
|
||||
_reference("r1", box(0, 0, 1, 1)),
|
||||
_reference("r2", box(5, 5, 6, 6)),
|
||||
_reference("r3", box(10, 10, 11, 11)),
|
||||
]
|
||||
CANDIDATES = [
|
||||
_candidate("hit-high", box(0, 0, 1, 1), 0.90),
|
||||
_candidate("hit-mid", box(5, 5, 6, 6), 0.40),
|
||||
_candidate("junk-low", box(30, 30, 31, 31), 0.20),
|
||||
]
|
||||
|
||||
|
||||
def _curve():
|
||||
return DetectionMetricsService.precision_recall_curve(CANDIDATES, REFERENCES, iou_threshold=0.5)
|
||||
|
||||
|
||||
class TestOperatingPoints:
|
||||
def test_a_strict_cut_keeps_only_the_confident_detection(self) -> None:
|
||||
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.5)
|
||||
|
||||
assert point["true_positives"] == 1
|
||||
assert point["false_positives"] == 0
|
||||
assert point["false_negatives"] == 2
|
||||
assert point["precision"] == pytest.approx(1.0)
|
||||
assert point["recall"] == pytest.approx(1 / 3)
|
||||
|
||||
def test_a_looser_cut_finds_more_and_stays_exact(self) -> None:
|
||||
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.3)
|
||||
|
||||
assert point["true_positives"] == 2
|
||||
assert point["false_positives"] == 0
|
||||
assert point["recall"] == pytest.approx(2 / 3)
|
||||
|
||||
def test_the_loosest_cut_admits_the_false_positive(self) -> None:
|
||||
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.1)
|
||||
|
||||
assert point["true_positives"] == 2
|
||||
assert point["false_positives"] == 1
|
||||
assert point["precision"] == pytest.approx(2 / 3)
|
||||
|
||||
def test_a_cut_above_every_detection_finds_nothing_but_still_reports(self) -> None:
|
||||
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.99)
|
||||
|
||||
assert point["true_positives"] == 0
|
||||
assert point["false_negatives"] == 3
|
||||
assert point["recall"] == pytest.approx(0.0)
|
||||
assert point["precision"] is None
|
||||
|
||||
def test_the_requested_threshold_is_echoed_back(self) -> None:
|
||||
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.42)
|
||||
|
||||
assert point["min_confidence"] == pytest.approx(0.42)
|
||||
# The nearest actual operating point sits at the detection's own
|
||||
# confidence, which is what the numbers describe.
|
||||
assert point["confidence_threshold"] == pytest.approx(0.9)
|
||||
|
||||
|
||||
class TestSweep:
|
||||
def test_a_sweep_returns_one_row_per_requested_threshold(self) -> None:
|
||||
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.5, 0.3, 0.1])
|
||||
|
||||
assert [row["min_confidence"] for row in rows] == [0.5, 0.3, 0.1]
|
||||
|
||||
def test_the_sweep_is_ordered_from_strict_to_loose(self) -> None:
|
||||
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.1, 0.5, 0.3])
|
||||
|
||||
assert [row["min_confidence"] for row in rows] == [0.5, 0.3, 0.1]
|
||||
|
||||
def test_a_repeated_threshold_is_asked_once(self) -> None:
|
||||
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.3, 0.3])
|
||||
|
||||
assert len(rows) == 1
|
||||
|
||||
def test_recall_never_falls_as_the_cut_loosens(self) -> None:
|
||||
"""The monotonicity that makes one run sufficient."""
|
||||
|
||||
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.9, 0.5, 0.3, 0.1])
|
||||
recalls = [row["recall"] for row in rows]
|
||||
|
||||
assert recalls == sorted(recalls)
|
||||
|
||||
def test_an_empty_sweep_is_not_an_error(self) -> None:
|
||||
assert DetectionMetricsService.calibration_sweep(_curve(), thresholds=[]) == []
|
||||
|
||||
def test_the_sweep_marks_the_f1_optimal_row(self) -> None:
|
||||
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.5, 0.3, 0.1])
|
||||
best = [row for row in rows if row["best_f1_in_sweep"]]
|
||||
|
||||
assert len(best) == 1
|
||||
assert best[0]["f1_score"] == max(row["f1_score"] for row in rows)
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Inference must refuse to guess a CRS.
|
||||
|
||||
Detection QA rejects a tile without explicit CRS metadata, but the inference
|
||||
side silently assumed EPSG:4326. That produced geometry that renders as a
|
||||
plausible polygon in the wrong place, which is worse than a clear failure:
|
||||
"fail closed" is the stated rule for the runtime.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.services.detection_georeferencing import (
|
||||
pixel_bbox_to_epsg4326_polygon,
|
||||
pixel_points_to_epsg4326_polygon,
|
||||
)
|
||||
from app.services.detection_service import DetectionService
|
||||
|
||||
|
||||
TILE_WITHOUT_CRS = {
|
||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||
"pixel_window": [0, 0, 100, 100],
|
||||
}
|
||||
|
||||
|
||||
def test_manifest_without_crs_is_rejected() -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DetectionService._require_manifest_crs({"tiles": [{"bounds": [0, 0, 1, 1]}]})
|
||||
|
||||
assert exc_info.value.code == "DETECTION_TILE_MANIFEST_INVALID"
|
||||
|
||||
|
||||
def test_manifest_crs_is_read_from_any_of_the_documented_keys() -> None:
|
||||
assert DetectionService._require_manifest_crs({"crs": "EPSG:31370"}) == "EPSG:31370"
|
||||
assert DetectionService._require_manifest_crs({"source_crs": "EPSG:31370"}) == "EPSG:31370"
|
||||
assert DetectionService._require_manifest_crs({"dataset_crs": "EPSG:3812"}) == "EPSG:3812"
|
||||
|
||||
|
||||
def test_bbox_georeferencing_requires_an_explicit_crs() -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
pixel_bbox_to_epsg4326_polygon(bbox=[0.0, 0.0, 10.0, 10.0], tile=TILE_WITHOUT_CRS)
|
||||
|
||||
assert exc_info.value.code == "DETECTION_TILE_CRS_REQUIRED"
|
||||
|
||||
|
||||
def test_mask_georeferencing_requires_an_explicit_crs() -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
pixel_points_to_epsg4326_polygon(
|
||||
points=[[0.0, 0.0], [10.0, 0.0], [10.0, 10.0]], tile=TILE_WITHOUT_CRS
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "DETECTION_TILE_CRS_REQUIRED"
|
||||
|
||||
|
||||
def test_explicit_crs_on_the_tile_is_used() -> None:
|
||||
tile = {**TILE_WITHOUT_CRS, "crs": "EPSG:4326"}
|
||||
|
||||
polygon = pixel_bbox_to_epsg4326_polygon(bbox=[0.0, 0.0, 50.0, 50.0], tile=tile)
|
||||
|
||||
assert polygon.bounds == pytest.approx((4.0, 51.5, 4.5, 52.0))
|
||||
|
||||
|
||||
def test_projected_bounds_are_reprojected_as_a_whole_rectangle() -> None:
|
||||
# Lambert 72 around Mol. All four corners must be transformed, otherwise a
|
||||
# rotated footprint is understated.
|
||||
bounds = DetectionService._bounds_to_epsg4326([200000.0, 200000.0, 201000.0, 201000.0], "EPSG:31370")
|
||||
|
||||
assert bounds is not None
|
||||
min_x, min_y, max_x, max_y = bounds
|
||||
assert 4.0 < min_x < 6.0
|
||||
assert 50.0 < min_y < 52.0
|
||||
assert max_x > min_x and max_y > min_y
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Comparing two models must not compare two different questions.
|
||||
|
||||
The workbench ranks model variants by a stored F1, each measured at that
|
||||
model's own confidence threshold. A conservatively calibrated model then looks
|
||||
worse than a liberal one without detecting anything differently — the number
|
||||
says as much about the threshold as about the model.
|
||||
|
||||
It also says nothing about whether the two runs are comparable at all. Two runs
|
||||
over different rasters, or with different inference coverage, produce numbers
|
||||
that cannot be placed side by side however they were measured.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.services.detection_comparison_service import DetectionComparisonService
|
||||
|
||||
|
||||
def _entry(
|
||||
*,
|
||||
dataset_id,
|
||||
model_asset_id: str = "asset-a",
|
||||
coverage_mode: str = "persisted_tile_manifest_union",
|
||||
reference_dataset_id=None,
|
||||
reference_evaluated: int = 100,
|
||||
):
|
||||
return {
|
||||
"analysis_run_id": uuid4(),
|
||||
"dataset_id": dataset_id,
|
||||
"model_id": "yolo-configured",
|
||||
"model_asset_id": model_asset_id,
|
||||
"reference_dataset_id": reference_dataset_id or uuid4(),
|
||||
"coverage_mode": coverage_mode,
|
||||
"reference_evaluated_count": reference_evaluated,
|
||||
}
|
||||
|
||||
|
||||
class TestComparability:
|
||||
def test_runs_over_the_same_raster_and_reference_are_comparable(self) -> None:
|
||||
dataset_id, reference_id = uuid4(), uuid4()
|
||||
entries = [
|
||||
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a"),
|
||||
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="b"),
|
||||
]
|
||||
|
||||
report = DetectionComparisonService.assess_comparability(entries)
|
||||
|
||||
assert report["comparable"] is True
|
||||
assert report["blocking_reasons"] == []
|
||||
|
||||
def test_runs_over_different_rasters_are_not_comparable(self) -> None:
|
||||
reference_id = uuid4()
|
||||
entries = [
|
||||
_entry(dataset_id=uuid4(), reference_dataset_id=reference_id),
|
||||
_entry(dataset_id=uuid4(), reference_dataset_id=reference_id),
|
||||
]
|
||||
|
||||
report = DetectionComparisonService.assess_comparability(entries)
|
||||
|
||||
assert report["comparable"] is False
|
||||
assert "different_source_raster" in report["blocking_reasons"]
|
||||
|
||||
def test_runs_scored_against_different_references_are_not_comparable(self) -> None:
|
||||
dataset_id = uuid4()
|
||||
entries = [_entry(dataset_id=dataset_id), _entry(dataset_id=dataset_id)]
|
||||
|
||||
report = DetectionComparisonService.assess_comparability(entries)
|
||||
|
||||
assert report["comparable"] is False
|
||||
assert "different_reference_dataset" in report["blocking_reasons"]
|
||||
|
||||
def test_a_run_without_proven_coverage_is_flagged(self) -> None:
|
||||
dataset_id, reference_id = uuid4(), uuid4()
|
||||
entries = [
|
||||
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a"),
|
||||
_entry(
|
||||
dataset_id=dataset_id,
|
||||
reference_dataset_id=reference_id,
|
||||
model_asset_id="b",
|
||||
coverage_mode="unbounded_no_manifest",
|
||||
),
|
||||
]
|
||||
|
||||
report = DetectionComparisonService.assess_comparability(entries)
|
||||
|
||||
assert report["comparable"] is False
|
||||
assert "coverage_not_proven" in report["blocking_reasons"]
|
||||
|
||||
def test_a_differing_evaluated_population_is_flagged(self) -> None:
|
||||
"""Same raster and reference, but the runs did not see the same ground."""
|
||||
|
||||
dataset_id, reference_id = uuid4(), uuid4()
|
||||
entries = [
|
||||
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a", reference_evaluated=100),
|
||||
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="b", reference_evaluated=60),
|
||||
]
|
||||
|
||||
report = DetectionComparisonService.assess_comparability(entries)
|
||||
|
||||
assert report["comparable"] is False
|
||||
assert "different_evaluated_population" in report["blocking_reasons"]
|
||||
|
||||
def test_one_run_is_never_a_comparison(self) -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DetectionComparisonService.assess_comparability([_entry(dataset_id=uuid4())])
|
||||
|
||||
assert exc_info.value.code == "DETECTION_COMPARISON_NEEDS_TWO_RUNS"
|
||||
|
||||
|
||||
class TestRanking:
|
||||
def _row(self, name: str, *, ap: float, best_f1: float, threshold_f1: float):
|
||||
return {
|
||||
"model_asset_id": name,
|
||||
"average_precision": ap,
|
||||
"best_f1": best_f1,
|
||||
"best_f1_threshold": 0.3,
|
||||
"f1_at_run_threshold": threshold_f1,
|
||||
}
|
||||
|
||||
def test_ranking_uses_average_precision_not_the_run_threshold_f1(self) -> None:
|
||||
rows = [
|
||||
self._row("liberal", ap=0.55, best_f1=0.60, threshold_f1=0.61),
|
||||
self._row("conservative", ap=0.72, best_f1=0.71, threshold_f1=0.44),
|
||||
]
|
||||
|
||||
ranked = DetectionComparisonService.rank(rows)
|
||||
|
||||
# The conservative model detects better; its stored F1 only looked worse
|
||||
# because it was measured at a stricter cut.
|
||||
assert [row["model_asset_id"] for row in ranked] == ["conservative", "liberal"]
|
||||
assert ranked[0]["rank"] == 1
|
||||
|
||||
def test_every_row_states_how_far_behind_the_leader_it_is(self) -> None:
|
||||
rows = [
|
||||
self._row("a", ap=0.72, best_f1=0.71, threshold_f1=0.44),
|
||||
self._row("b", ap=0.55, best_f1=0.60, threshold_f1=0.61),
|
||||
]
|
||||
|
||||
ranked = DetectionComparisonService.rank(rows)
|
||||
|
||||
assert ranked[0]["average_precision_gap"] == pytest.approx(0.0)
|
||||
assert ranked[1]["average_precision_gap"] == pytest.approx(0.17)
|
||||
|
||||
def test_only_the_leader_carries_a_lead(self) -> None:
|
||||
"""A follower's gap must never be readable as an advantage."""
|
||||
|
||||
rows = [
|
||||
self._row("a", ap=0.72, best_f1=0.71, threshold_f1=0.44),
|
||||
self._row("b", ap=0.55, best_f1=0.60, threshold_f1=0.61),
|
||||
]
|
||||
|
||||
ranked = DetectionComparisonService.rank(rows)
|
||||
|
||||
assert ranked[0]["lead_over_next"] == pytest.approx(0.17)
|
||||
assert ranked[1]["lead_over_next"] is None
|
||||
|
||||
def test_a_tied_leader_claims_no_lead(self) -> None:
|
||||
rows = [
|
||||
self._row("a", ap=0.6, best_f1=0.6, threshold_f1=0.6),
|
||||
self._row("b", ap=0.6, best_f1=0.5, threshold_f1=0.5),
|
||||
]
|
||||
|
||||
assert all(row["lead_over_next"] is None for row in DetectionComparisonService.rank(rows))
|
||||
|
||||
def test_a_tie_is_reported_as_a_tie_rather_than_an_arbitrary_winner(self) -> None:
|
||||
rows = [
|
||||
self._row("a", ap=0.6, best_f1=0.6, threshold_f1=0.6),
|
||||
self._row("b", ap=0.6, best_f1=0.5, threshold_f1=0.5),
|
||||
]
|
||||
|
||||
ranked = DetectionComparisonService.rank(rows)
|
||||
|
||||
assert [row["rank"] for row in ranked] == [1, 1]
|
||||
assert all(row["tied"] for row in ranked)
|
||||
|
||||
|
||||
class TestComparingRealRuns:
|
||||
"""Through the same QA path the workbench uses, so the two cannot drift."""
|
||||
|
||||
def test_two_runs_are_scored_ranked_and_judged_comparable(self, monkeypatch) -> None:
|
||||
from app.services.detection_comparison_service import DetectionComparisonService as Service
|
||||
from app.services.detection_service import DetectionService
|
||||
|
||||
dataset_id, reference_id = uuid4(), uuid4()
|
||||
run_a, run_b = uuid4(), uuid4()
|
||||
|
||||
class _Run:
|
||||
def __init__(self, asset: str, threshold: float) -> None:
|
||||
self.dataset_id = dataset_id
|
||||
self.model_name = "yolo-configured"
|
||||
self.parameters_json = {"model_asset_id": asset, "confidence_threshold": threshold}
|
||||
# Both runs post-processed identically, so they stay comparable.
|
||||
self.result_json = {
|
||||
"containment_suppression_threshold": 0.85,
|
||||
"duplicate_iou_threshold": 0.5,
|
||||
}
|
||||
|
||||
runs = {run_a: _Run("liberal", 0.15), run_b: _Run("conservative", 0.45)}
|
||||
scores = {
|
||||
run_a: {"average_precision": 0.55, "best_f1": 0.60, "f1": 0.61},
|
||||
run_b: {"average_precision": 0.72, "best_f1": 0.71, "f1": 0.44},
|
||||
}
|
||||
|
||||
class _Session:
|
||||
def get(self, _model, item_id):
|
||||
return runs.get(item_id)
|
||||
|
||||
def fake_qa(_db, *, analysis_run_id, reference_dataset_id, iou_threshold, **_kwargs):
|
||||
score = scores[analysis_run_id]
|
||||
return {
|
||||
"quality_check_id": str(uuid4()),
|
||||
"f1_score": score["f1"],
|
||||
"precision": 0.6,
|
||||
"recall": 0.6,
|
||||
"coverage": {"mode": "persisted_tile_manifest_union", "reference_evaluated_count": 100},
|
||||
"precision_recall_curve": {
|
||||
"average_precision": score["average_precision"],
|
||||
"best_f1": score["best_f1"],
|
||||
"best_f1_threshold": 0.3,
|
||||
},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(DetectionService, "compare_detections_with_reference", staticmethod(fake_qa))
|
||||
|
||||
report = Service.compare_runs(
|
||||
_Session(),
|
||||
analysis_run_ids=[run_a, run_b],
|
||||
reference_dataset_id=reference_id,
|
||||
iou_threshold=0.5,
|
||||
)
|
||||
|
||||
assert report["comparability"]["comparable"] is True
|
||||
assert report["ranking_metric"] == "average_precision"
|
||||
# Ranked on AP, so the conservative model leads despite the lower F1 at
|
||||
# its own threshold — which stays visible next to it.
|
||||
assert [row["model_asset_id"] for row in report["rows"]] == ["conservative", "liberal"]
|
||||
assert report["rows"][0]["f1_at_run_threshold"] == pytest.approx(0.44)
|
||||
assert report["rows"][1]["f1_at_run_threshold"] == pytest.approx(0.61)
|
||||
|
||||
def test_one_run_twice_is_refused(self) -> None:
|
||||
run_id = uuid4()
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DetectionComparisonService.compare_runs(
|
||||
object(),
|
||||
analysis_run_ids=[run_id, run_id],
|
||||
reference_dataset_id=uuid4(),
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "DETECTION_COMPARISON_NEEDS_TWO_RUNS"
|
||||
@@ -0,0 +1,116 @@
|
||||
"""A single F1 at one arbitrary confidence cut cannot compare two models.
|
||||
|
||||
Reporting F1 at whichever threshold the operator happened to type makes two
|
||||
models look better or worse depending on their calibration rather than their
|
||||
detection quality. The sweep produces the standard curve instead: precision
|
||||
and recall at every operating point, average precision, and the threshold
|
||||
where F1 actually peaks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.services.detection_metrics_service import DetectionMetricsService
|
||||
|
||||
|
||||
def _candidate(name: str, geometry, confidence: float):
|
||||
return ({"id": name, "confidence": confidence}, geometry)
|
||||
|
||||
|
||||
def _reference(name: str, geometry):
|
||||
return ({"id": name}, geometry)
|
||||
|
||||
|
||||
def test_perfect_detector_reaches_average_precision_one() -> None:
|
||||
references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(5, 5, 6, 6))]
|
||||
candidates = [
|
||||
_candidate("c1", box(0, 0, 1, 1), 0.9),
|
||||
_candidate("c2", box(5, 5, 6, 6), 0.8),
|
||||
]
|
||||
|
||||
curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
|
||||
|
||||
assert curve["average_precision"] == pytest.approx(1.0)
|
||||
assert curve["best_f1"] == pytest.approx(1.0)
|
||||
assert curve["reference_count"] == 2
|
||||
|
||||
|
||||
def test_low_confidence_false_positive_is_only_penalised_below_its_threshold() -> None:
|
||||
references = [_reference("r1", box(0, 0, 1, 1))]
|
||||
candidates = [
|
||||
_candidate("hit", box(0, 0, 1, 1), 0.9),
|
||||
_candidate("junk", box(20, 20, 21, 21), 0.2),
|
||||
]
|
||||
|
||||
curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
|
||||
|
||||
# Cutting at 0.2 admits the junk box, so precision there is 0.5.
|
||||
low = next(point for point in curve["points"] if point["confidence_threshold"] == pytest.approx(0.2))
|
||||
assert low["precision"] == pytest.approx(0.5)
|
||||
assert low["recall"] == pytest.approx(1.0)
|
||||
|
||||
# The optimum simply drops it.
|
||||
assert curve["best_f1"] == pytest.approx(1.0)
|
||||
assert curve["best_f1_threshold"] == pytest.approx(0.9)
|
||||
# AP is computed over the ranking, so one trailing false positive after
|
||||
# full recall does not reduce it.
|
||||
assert curve["average_precision"] == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_ranking_quality_is_visible_in_average_precision() -> None:
|
||||
"""A detector that ranks its mistake above its hit scores worse."""
|
||||
|
||||
references = [_reference("r1", box(0, 0, 1, 1))]
|
||||
good = DetectionMetricsService.precision_recall_curve(
|
||||
[_candidate("hit", box(0, 0, 1, 1), 0.9), _candidate("junk", box(20, 20, 21, 21), 0.1)],
|
||||
references,
|
||||
iou_threshold=0.5,
|
||||
)
|
||||
bad = DetectionMetricsService.precision_recall_curve(
|
||||
[_candidate("hit", box(0, 0, 1, 1), 0.1), _candidate("junk", box(20, 20, 21, 21), 0.9)],
|
||||
references,
|
||||
iou_threshold=0.5,
|
||||
)
|
||||
|
||||
assert good["average_precision"] > bad["average_precision"]
|
||||
assert bad["average_precision"] == pytest.approx(0.5)
|
||||
|
||||
|
||||
def test_missed_reference_caps_recall_and_average_precision() -> None:
|
||||
references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(9, 9, 10, 10))]
|
||||
candidates = [_candidate("hit", box(0, 0, 1, 1), 0.9)]
|
||||
|
||||
curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
|
||||
|
||||
assert curve["points"][0]["recall"] == pytest.approx(0.5)
|
||||
assert curve["average_precision"] == pytest.approx(0.5)
|
||||
assert curve["best_f1"] == pytest.approx(2 / 3)
|
||||
|
||||
|
||||
def test_curve_is_independent_of_input_order() -> None:
|
||||
references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(5, 5, 6, 6))]
|
||||
candidates = [
|
||||
_candidate("c1", box(0, 0, 1, 1), 0.9),
|
||||
_candidate("c2", box(5, 5, 6, 6), 0.4),
|
||||
_candidate("c3", box(30, 30, 31, 31), 0.6),
|
||||
]
|
||||
|
||||
forward = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
|
||||
reverse = DetectionMetricsService.precision_recall_curve(
|
||||
list(reversed(candidates)), list(reversed(references)), iou_threshold=0.5
|
||||
)
|
||||
|
||||
assert forward == reverse
|
||||
|
||||
|
||||
def test_empty_candidate_population_is_reported_not_crashed() -> None:
|
||||
curve = DetectionMetricsService.precision_recall_curve(
|
||||
[], [_reference("r1", box(0, 0, 1, 1))], iou_threshold=0.5
|
||||
)
|
||||
|
||||
assert curve["average_precision"] == 0.0
|
||||
assert curve["best_f1"] == 0.0
|
||||
assert curve["best_f1_threshold"] is None
|
||||
assert curve["points"] == []
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Every accuracy figure shown to an operator must exist in the evidence record.
|
||||
|
||||
The recommended profile published precision 0.6140895327792112, recall
|
||||
0.6062221049337548 and F1 0.6068607646002744. Those three numbers appear
|
||||
nowhere in this repository except the file that publishes them and the test
|
||||
that pinned them as literal strings. The only recorded evaluation of that
|
||||
model at that operating point — tile 512, overlap 64, threshold 0.15 — reported
|
||||
0.5898197518, 0.5769921004 and 0.5824578632, so the published figures were
|
||||
about two and a half points more flattering than anything that was measured,
|
||||
and a test guaranteed nobody would correct them.
|
||||
|
||||
An operator cannot check a number that has no source. This test refuses one.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
PROFILES = ROOT / "frontend" / "src" / "components" / "detection" / "detectionProfiles.ts"
|
||||
EVIDENCE = ROOT / "docs" / "CODEX_EXECUTION_LOG.md"
|
||||
|
||||
METRIC_FIELDS = ("precision", "recall", "f1")
|
||||
# The log rounds; the source file may carry more digits of the same value.
|
||||
TOLERANCE = 1e-9
|
||||
|
||||
|
||||
def _published_metrics() -> list[tuple[str, str, float]]:
|
||||
source = PROFILES.read_text(encoding="utf-8")
|
||||
profiles = re.findall(r"id: '([^']+)',(.*?)\n \},", source, re.S)
|
||||
assert profiles, "no operator profiles found; the file shape changed"
|
||||
|
||||
published: list[tuple[str, str, float]] = []
|
||||
for profile_id, body in profiles:
|
||||
for field in METRIC_FIELDS:
|
||||
match = re.search(rf"^\s*{field}: ([0-9.]+),", body, re.M)
|
||||
assert match, f"{profile_id} publishes no {field}"
|
||||
published.append((profile_id, field, float(match.group(1))))
|
||||
return published
|
||||
|
||||
|
||||
def _recorded_values() -> list[float]:
|
||||
text = EVIDENCE.read_text(encoding="utf-8")
|
||||
return [float(value) for value in re.findall(r"\b0\.\d{4,}\b", text)]
|
||||
|
||||
|
||||
def test_every_published_accuracy_figure_appears_in_the_evidence_record() -> None:
|
||||
recorded = _recorded_values()
|
||||
untraceable = [
|
||||
f"{profile_id}.{field} = {value}"
|
||||
for profile_id, field, value in _published_metrics()
|
||||
if not any(abs(value - candidate) <= TOLERANCE for candidate in recorded)
|
||||
]
|
||||
|
||||
assert not untraceable, (
|
||||
"These figures are shown to operators but were never recorded in "
|
||||
f"docs/CODEX_EXECUTION_LOG.md: {untraceable}. Publish the measurement "
|
||||
"that was taken, or record the evaluation that produced these."
|
||||
)
|
||||
|
||||
|
||||
def test_each_profile_names_the_measurement_behind_its_numbers() -> None:
|
||||
source = PROFILES.read_text(encoding="utf-8")
|
||||
profile_count = source.count("modelAssetId:")
|
||||
|
||||
assert source.count("evidenceReference:") == profile_count
|
||||
assert source.count("backgroundGate:") == profile_count
|
||||
assert source.count("backgroundSampleCount:") == profile_count
|
||||
|
||||
|
||||
def test_the_check_would_notice_an_invented_figure() -> None:
|
||||
"""Without this the test could pass because nothing ever matches."""
|
||||
|
||||
recorded = _recorded_values()
|
||||
|
||||
assert any(abs(0.5898197518 - value) <= TOLERANCE for value in recorded)
|
||||
assert not any(abs(0.6140895327792112 - value) <= TOLERANCE for value in recorded)
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Loading a run's results must not depend on the run being small.
|
||||
|
||||
``/detection/runs/{id}/detections`` and its GeoJSON sibling returned every
|
||||
persisted detection. A regional run holds tens of thousands, so the endpoints
|
||||
the map and the results table call after every run grew without bound. The
|
||||
counts stay complete; what is transferred does not.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.detection_service import DetectionService
|
||||
|
||||
|
||||
class _Detection:
|
||||
def __init__(self, index: int) -> None:
|
||||
self.id = uuid.uuid4()
|
||||
self.index = index
|
||||
|
||||
|
||||
def _rows(count: int) -> list[_Detection]:
|
||||
return [_Detection(index) for index in range(count)]
|
||||
|
||||
|
||||
def test_a_page_is_returned_with_the_complete_total() -> None:
|
||||
page, total, truncated = DetectionService.paginate(_rows(1_000), limit=100, offset=0)
|
||||
|
||||
assert len(page) == 100
|
||||
assert total == 1_000
|
||||
assert truncated is True
|
||||
|
||||
|
||||
def test_the_offset_walks_the_population() -> None:
|
||||
page, total, _ = DetectionService.paginate(_rows(10), limit=3, offset=6)
|
||||
|
||||
assert [row.index for row in page] == [6, 7, 8]
|
||||
assert total == 10
|
||||
|
||||
|
||||
def test_an_offset_past_the_end_yields_an_empty_page_not_an_error() -> None:
|
||||
page, total, truncated = DetectionService.paginate(_rows(5), limit=10, offset=50)
|
||||
|
||||
assert page == []
|
||||
assert total == 5
|
||||
assert truncated is True
|
||||
|
||||
|
||||
def test_a_population_inside_one_page_is_not_reported_as_truncated() -> None:
|
||||
page, total, truncated = DetectionService.paginate(_rows(7), limit=100, offset=0)
|
||||
|
||||
assert len(page) == 7
|
||||
assert total == 7
|
||||
assert truncated is False
|
||||
|
||||
|
||||
def test_a_zero_limit_returns_everything_for_callers_that_need_it() -> None:
|
||||
page, total, truncated = DetectionService.paginate(_rows(2_500), limit=0, offset=0)
|
||||
|
||||
assert len(page) == 2_500
|
||||
assert total == 2_500
|
||||
assert truncated is False
|
||||
|
||||
|
||||
def test_a_negative_offset_is_treated_as_the_start() -> None:
|
||||
page, _, _ = DetectionService.paginate(_rows(4), limit=2, offset=-5)
|
||||
|
||||
assert [row.index for row in page] == [0, 1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("limit", [1, 2, 3])
|
||||
def test_paging_covers_the_population_exactly_once(limit: int) -> None:
|
||||
rows = _rows(7)
|
||||
seen: list[int] = []
|
||||
offset = 0
|
||||
while True:
|
||||
page, total, _ = DetectionService.paginate(rows, limit=limit, offset=offset)
|
||||
if not page:
|
||||
break
|
||||
seen.extend(row.index for row in page)
|
||||
offset += limit
|
||||
|
||||
assert seen == list(range(7))
|
||||
assert total == 7
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Detections that straddle a tile seam must not become two half buildings.
|
||||
|
||||
Tiling uses a fixed overlap. An object wider than that overlap is truncated by
|
||||
both tiles, so the two boxes barely intersect and plain IoU suppression keeps
|
||||
them both: two false positives plus one missed footprint for every seam
|
||||
building. The suppressor therefore also compares overlap against the smaller
|
||||
box, and truncated boxes that sit against an interior tile edge are dropped in
|
||||
favour of the neighbouring tile's complete view.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.services.detection_service import DetectionService
|
||||
|
||||
|
||||
def _candidate(name: str, geometry, confidence: float, *, tile_index: int = 0, tile_bounds=None):
|
||||
return {
|
||||
"class_name": "building",
|
||||
"confidence": confidence,
|
||||
"geometry": geometry,
|
||||
"bbox": [0.0, 0.0, 1.0, 1.0],
|
||||
"source_tile_path": f"/tiles/tile_{tile_index:04d}.tif",
|
||||
"properties": {"tile_index": tile_index, "name": name},
|
||||
"tile_bounds": tile_bounds,
|
||||
}
|
||||
|
||||
|
||||
def test_identical_overlapping_predictions_are_still_suppressed() -> None:
|
||||
kept = DetectionService._suppress_duplicate_candidates(
|
||||
[
|
||||
_candidate("a", box(0.0, 0.0, 1.0, 1.0), 0.7),
|
||||
_candidate("b", box(0.02, 0.02, 1.02, 1.02), 0.9),
|
||||
],
|
||||
iou_threshold=0.5,
|
||||
)
|
||||
|
||||
assert [item["properties"]["name"] for item in kept] == ["b"]
|
||||
|
||||
|
||||
def test_a_box_contained_in_a_larger_one_is_suppressed() -> None:
|
||||
"""A truncated seam half sits inside the complete box from the next tile."""
|
||||
|
||||
complete = box(0.0, 0.0, 10.0, 10.0)
|
||||
truncated_half = box(0.0, 0.0, 4.0, 10.0) # IoU with ``complete`` is 0.4
|
||||
|
||||
kept = DetectionService._suppress_duplicate_candidates(
|
||||
[
|
||||
_candidate("complete", complete, 0.88),
|
||||
_candidate("truncated", truncated_half, 0.61),
|
||||
],
|
||||
iou_threshold=0.5,
|
||||
)
|
||||
|
||||
assert [item["properties"]["name"] for item in kept] == ["complete"]
|
||||
|
||||
|
||||
def test_genuinely_adjacent_buildings_are_both_kept() -> None:
|
||||
"""Terraced houses touch but do not contain one another."""
|
||||
|
||||
kept = DetectionService._suppress_duplicate_candidates(
|
||||
[
|
||||
_candidate("left", box(0.0, 0.0, 10.0, 10.0), 0.9),
|
||||
_candidate("right", box(10.0, 0.0, 20.0, 10.0), 0.85),
|
||||
],
|
||||
iou_threshold=0.5,
|
||||
)
|
||||
|
||||
assert sorted(item["properties"]["name"] for item in kept) == ["left", "right"]
|
||||
|
||||
|
||||
def test_different_classes_are_never_merged() -> None:
|
||||
first = _candidate("a", box(0.0, 0.0, 10.0, 10.0), 0.9)
|
||||
second = _candidate("b", box(0.0, 0.0, 10.0, 10.0), 0.8)
|
||||
second["class_name"] = "solar_panel"
|
||||
|
||||
kept = DetectionService._suppress_duplicate_candidates([first, second], iou_threshold=0.5)
|
||||
|
||||
assert len(kept) == 2
|
||||
|
||||
|
||||
def test_boxes_clipped_by_an_interior_tile_edge_are_dropped() -> None:
|
||||
"""The overlapping neighbour tile still sees the whole object."""
|
||||
|
||||
tile = box(0.0, 0.0, 10.0, 10.0)
|
||||
raster = box(0.0, 0.0, 30.0, 10.0)
|
||||
|
||||
candidates = [
|
||||
# Sits against the tile's right edge: truncated by the tile, not real.
|
||||
_candidate("edge", box(9.0, 2.0, 10.0, 4.0), 0.8, tile_bounds=tile.bounds),
|
||||
# Comfortably inside the tile.
|
||||
_candidate("interior", box(2.0, 2.0, 4.0, 4.0), 0.8, tile_bounds=tile.bounds),
|
||||
]
|
||||
|
||||
kept = DetectionService._drop_tile_edge_truncations(
|
||||
candidates, raster_bounds=raster.bounds, tolerance=0.001
|
||||
)
|
||||
|
||||
assert [item["properties"]["name"] for item in kept] == ["interior"]
|
||||
|
||||
|
||||
def test_boxes_against_the_raster_edge_are_kept() -> None:
|
||||
"""No neighbouring tile exists there, so the box is all the evidence there is."""
|
||||
|
||||
tile = box(0.0, 0.0, 10.0, 10.0)
|
||||
raster = box(0.0, 0.0, 10.0, 10.0)
|
||||
|
||||
candidates = [_candidate("edge", box(9.0, 2.0, 10.0, 4.0), 0.8, tile_bounds=tile.bounds)]
|
||||
|
||||
kept = DetectionService._drop_tile_edge_truncations(
|
||||
candidates, raster_bounds=raster.bounds, tolerance=0.001
|
||||
)
|
||||
|
||||
assert [item["properties"]["name"] for item in kept] == ["edge"]
|
||||
|
||||
|
||||
def test_edge_filter_keeps_candidates_without_tile_bounds() -> None:
|
||||
candidates = [_candidate("unknown", box(2.0, 2.0, 4.0, 4.0), 0.8, tile_bounds=None)]
|
||||
|
||||
kept = DetectionService._drop_tile_edge_truncations(
|
||||
candidates, raster_bounds=(0.0, 0.0, 30.0, 10.0), tolerance=0.001
|
||||
)
|
||||
|
||||
assert len(kept) == 1
|
||||
@@ -0,0 +1,512 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_backend_dockerfile_copies_package_sources_before_pip_install() -> None:
|
||||
dockerfile = ROOT / "backend" / "Dockerfile"
|
||||
lines = dockerfile.read_text(encoding="utf-8").splitlines()
|
||||
|
||||
pip_install_index = lines.index('RUN extras=".[gis]" \\')
|
||||
preceding = "\n".join(lines[:pip_install_index])
|
||||
|
||||
assert "COPY pyproject.toml README.md /app/" in preceding
|
||||
assert "COPY app /app/app" in preceding
|
||||
|
||||
|
||||
def test_backend_dockerfile_installs_approved_gis_runtime_stack() -> None:
|
||||
dockerfile = (ROOT / "backend" / "Dockerfile").read_text(encoding="utf-8")
|
||||
|
||||
assert "ARG GEOINTEL_INSTALL_AI=false" in dockerfile
|
||||
assert 'extras=".[gis]"' in dockerfile
|
||||
assert 'extras=".[gis,ai]"' in dockerfile
|
||||
assert "RUN python scripts/gis_import_smoke.py" in dockerfile
|
||||
assert "gdal-bin" in dockerfile
|
||||
assert "libgdal-dev" in dockerfile
|
||||
assert "libgeos-dev" in dockerfile
|
||||
assert "libproj-dev" in dockerfile
|
||||
assert "proj-bin" in dockerfile
|
||||
assert "libxcb1" in dockerfile
|
||||
assert "libgl1" in dockerfile
|
||||
assert "libglib2.0-0" in dockerfile
|
||||
|
||||
|
||||
def test_backend_pyproject_exposes_gis_optional_dependency_group() -> None:
|
||||
pyproject = (ROOT / "backend" / "pyproject.toml").read_text(encoding="utf-8")
|
||||
|
||||
assert "gis = [" in pyproject
|
||||
assert '"rasterio>=1.4.3"' in pyproject
|
||||
assert '"geopandas>=1.0.1"' in pyproject
|
||||
assert '"pyogrio>=0.10.0"' in pyproject
|
||||
assert '"ultralytics>=8.3,<9"' not in pyproject.split("gis = [", 1)[1].split("]", 1)[0]
|
||||
|
||||
|
||||
def test_all_in_one_dockerfile_can_opt_into_ai_dependencies_without_base_install() -> None:
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
|
||||
assert "ARG GEOINTEL_INSTALL_AI=false" in dockerfile
|
||||
assert "COPY backend/pyproject.toml /app/" in dockerfile
|
||||
assert "COPY backend/requirements-runtime.lock /app/" in dockerfile
|
||||
assert "COPY backend/requirements-ai-linux.lock /app/" in dockerfile
|
||||
assert "COPY backend/requirements-build-tools.lock /app/" in dockerfile
|
||||
assert "COPY backend/pyproject.toml backend/README.md /app/" not in dockerfile
|
||||
assert "GeoIntel backend package metadata" in dockerfile
|
||||
assert "--require-hashes -r requirements-runtime.lock" in dockerfile
|
||||
assert "--require-hashes" in dockerfile
|
||||
assert "-r requirements-ai-linux.lock" in dockerfile
|
||||
assert "-r requirements-build-tools.lock" in dockerfile
|
||||
assert "python scripts/gis_import_smoke.py" in dockerfile
|
||||
assert "yolo_preflight.py" in dockerfile
|
||||
assert "libxcb1" in dockerfile
|
||||
assert "libgl1" in dockerfile
|
||||
assert "libglib2.0-0" in dockerfile
|
||||
|
||||
|
||||
def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None:
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
|
||||
for line in dockerfile.splitlines():
|
||||
if line.startswith("COPY scripts/"):
|
||||
source_path = line.split()[1]
|
||||
assert (ROOT / source_path).is_file()
|
||||
|
||||
required_runtime_scripts = {
|
||||
"prepare_operator_real_data_samples.py",
|
||||
"export_operator_yolo_tile_dataset.py",
|
||||
"audit_operator_yolo_dataset_quality.py",
|
||||
"render_operator_yolo_label_qa_contact_sheets.py",
|
||||
"train_operator_yolo_detector.sh",
|
||||
"training_dataset_eligibility.py",
|
||||
"training_release_manifest.py",
|
||||
"verify_real_data_detection_qa_workflow.sh",
|
||||
"run_detection_quality_matrix.sh",
|
||||
"run_multi_sample_detection_quality_matrix.sh",
|
||||
"run_mol_operational_validation.sh",
|
||||
"export_detection_calibration_evidence.sh",
|
||||
"assemble_detection_calibration_evidence_portfolio.sh",
|
||||
"build_fixed_threshold_evidence_portfolio_inputs.py",
|
||||
"audit_detection_false_negative_evidence.py",
|
||||
"audit_detection_false_positive_evidence.py",
|
||||
"render_detection_false_positive_review_contact_sheets.py",
|
||||
"render_detection_false_negative_review_contact_sheets.py",
|
||||
"validate_detection_false_positive_review_decisions.py",
|
||||
"validate_detection_false_negative_review_decisions.py",
|
||||
"run_operator_hard_negative_detection_matrix.sh",
|
||||
"run_background_corpus_split_matrix.sh",
|
||||
"build_background_corpus_split_report.py",
|
||||
"build_detection_model_promotion_report.py",
|
||||
"run_split_background_promotion_workflow.sh",
|
||||
"activate_promoted_yolo_candidate.py",
|
||||
"migrate_runtime_model_provenance.py",
|
||||
"manage_grb_refresh.py",
|
||||
"orthophoto_release_preflight.py",
|
||||
"provision_walous_sources.py",
|
||||
"provision_spw_terrain_source.py",
|
||||
}
|
||||
for script_name in required_runtime_scripts:
|
||||
assert f"COPY scripts/{script_name} /app/scripts/{script_name}" in dockerfile
|
||||
|
||||
|
||||
def test_all_in_one_dockerfile_copies_operator_scripts_after_dependency_install() -> None:
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
|
||||
dependency_install_index = dockerfile.index('/usr/bin/python3.11 -m venv /opt/geointel/venv \\')
|
||||
operator_copy_index = dockerfile.index(
|
||||
"COPY scripts/render_operator_yolo_label_qa_contact_sheets.py "
|
||||
"/app/scripts/render_operator_yolo_label_qa_contact_sheets.py"
|
||||
)
|
||||
|
||||
assert operator_copy_index > dependency_install_index
|
||||
|
||||
|
||||
def test_compose_does_not_require_missing_root_env_file() -> None:
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
|
||||
assert "env_file:" not in compose
|
||||
assert "DATABASE_URL: postgresql+psycopg://${GEOINTEL_POSTGRES_USER:-geointel}" in compose
|
||||
|
||||
|
||||
def test_compose_exposes_frontend_on_configurable_host_port_with_cors_origin() -> None:
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
env_example = (ROOT / ".env.example").read_text(encoding="utf-8")
|
||||
|
||||
assert '"${GEOINTEL_BIND_ADDRESS:-127.0.0.1}:${GEOINTEL_FRONTEND_PORT:-1202}:80"' in compose
|
||||
assert '"${GEOINTEL_BIND_ADDRESS:-127.0.0.1}:${GEOINTEL_BACKEND_PORT:-8000}:8000"' in compose
|
||||
assert "CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202}" in compose
|
||||
assert "GEOINTEL_FRONTEND_PORT=1202" in env_example
|
||||
assert "GEOINTEL_BACKEND_PORT=8000" in env_example
|
||||
assert "http://localhost:1202" in env_example
|
||||
assert "http://127.0.0.1:1202" in env_example
|
||||
|
||||
|
||||
def test_packaged_runtime_uses_fail_closed_authentication_defaults() -> None:
|
||||
compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
|
||||
env_example = (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(encoding="utf-8")
|
||||
|
||||
assert "GEOINTEL_AUTH_ENABLED: ${GEOINTEL_AUTH_ENABLED:-true}" in compose
|
||||
assert "GEOINTEL_AUTH_REQUIRE_HTTPS: ${GEOINTEL_AUTH_REQUIRE_HTTPS:-true}" in compose
|
||||
assert "GEOINTEL_GUEST_ACCESS_ENABLED: ${GEOINTEL_GUEST_ACCESS_ENABLED:-false}" in compose
|
||||
assert "GEOINTEL_AUTH_ENABLED=true" in env_example
|
||||
assert "GEOINTEL_AUTH_REQUIRE_HTTPS=true" in env_example
|
||||
assert "GEOINTEL_GUEST_ACCESS_ENABLED=false" in env_example
|
||||
|
||||
|
||||
def test_env_example_uses_runtime_env_names_read_by_backend_and_frontend() -> None:
|
||||
env_example = (ROOT / ".env.example").read_text(encoding="utf-8")
|
||||
|
||||
assert "GEOINTEL_INSTALL_AI=false" in env_example
|
||||
assert "YOLO_ENABLED=false" in env_example
|
||||
assert "YOLO_MODELS_DIR=/app/models" in env_example
|
||||
assert "YOLO_MODEL_PATH=" in env_example
|
||||
assert "YOLO_CONFIG_DIR=./storage/ultralytics" in env_example
|
||||
assert "YOLO_MAX_TILES=100" in env_example
|
||||
assert "YOLO_MAX_DETECTIONS=1000" in env_example
|
||||
assert "YOLO_DUPLICATE_IOU_THRESHOLD=0.5" in env_example
|
||||
assert "ENABLE_YOLO" not in env_example
|
||||
assert "ENABLE_SAM" not in env_example
|
||||
assert "VITE_API_BASE_URL=" in env_example
|
||||
assert "VITE_API_PROXY_TARGET=http://localhost:8000" in env_example
|
||||
|
||||
|
||||
def test_walloon_runtime_settings_are_editable_in_compose_and_unraid() -> None:
|
||||
files = [
|
||||
(ROOT / "docker-compose.yml").read_text(encoding="utf-8"),
|
||||
(ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8"),
|
||||
(ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(encoding="utf-8"),
|
||||
(ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8"),
|
||||
(ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(encoding="utf-8"),
|
||||
]
|
||||
for content in files:
|
||||
assert "SPW_FLOOD_HAZARD_ENABLED" in content
|
||||
assert "SPW_FLOOD_HAZARD_MAPSERVER_URL" in content
|
||||
assert "WALOUS_ENABLED" in content
|
||||
assert "WALOUS_SOURCE_DIR" in content
|
||||
assert "WALOUS_ANALYSIS_RESOLUTION_M" in content
|
||||
assert "WALOUS_MAX_SIDE_M" in content
|
||||
assert "WALOUS_MAX_PIXELS" in content
|
||||
|
||||
|
||||
def test_in_memory_vector_limit_is_propagated_and_validated_in_every_runtime() -> None:
|
||||
expected = "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB"
|
||||
for path in (
|
||||
ROOT / ".env.example",
|
||||
ROOT / "docker-compose.yml",
|
||||
ROOT / "docker-compose.unraid.yml",
|
||||
ROOT / "deploy" / "unraid" / "geointel.env.example",
|
||||
ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml",
|
||||
):
|
||||
assert expected in path.read_text(encoding="utf-8"), path
|
||||
|
||||
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
assert 'GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}"' in run_script
|
||||
assert "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB must be between 1 and 256." in run_script
|
||||
assert '-e GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB"' in run_script
|
||||
|
||||
|
||||
def test_frontend_uses_same_origin_api_proxy_by_default() -> None:
|
||||
api_client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8")
|
||||
nginx_config = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "frontend" / "Dockerfile").read_text(encoding="utf-8")
|
||||
|
||||
assert '?? ""' in api_client
|
||||
assert "http://localhost:8000" not in api_client
|
||||
assert "FROM nginx:" in dockerfile
|
||||
assert "COPY --from=build /app/dist /usr/share/nginx/html" in dockerfile
|
||||
assert "location /api/" in nginx_config
|
||||
assert "proxy_pass http://backend:8000/api/" in nginx_config
|
||||
assert "location = /health" in nginx_config
|
||||
assert 'add_header Cache-Control "no-cache"' in nginx_config
|
||||
assert "location /assets/" in nginx_config
|
||||
assert "try_files $uri $uri/ /index.html" in nginx_config
|
||||
|
||||
|
||||
def test_nginx_runtime_allows_real_gis_upload_payloads() -> None:
|
||||
frontend_nginx = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8")
|
||||
all_in_one_nginx = (ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(encoding="utf-8")
|
||||
start_script = (ROOT / "deploy" / "unraid" / "all-in-one-start.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "client_max_body_size 250m;" in frontend_nginx
|
||||
assert "client_max_body_size __GEOINTEL_MAX_UPLOAD_MB__m;" in all_in_one_nginx
|
||||
assert 'sed -i "s/__GEOINTEL_MAX_UPLOAD_MB__/${MAX_UPLOAD_MB}/g"' in start_script
|
||||
|
||||
|
||||
def test_nginx_runtime_allows_long_ai_and_qa_requests() -> None:
|
||||
frontend_nginx = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8")
|
||||
all_in_one_nginx = (ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(encoding="utf-8")
|
||||
|
||||
for config in (frontend_nginx, all_in_one_nginx):
|
||||
assert "proxy_read_timeout 600s;" in config
|
||||
assert "proxy_send_timeout 600s;" in config
|
||||
|
||||
|
||||
def test_nginx_preserves_outer_https_scheme_for_secure_session_cookies() -> None:
|
||||
configs = (
|
||||
(ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8"),
|
||||
(ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(encoding="utf-8"),
|
||||
)
|
||||
for config in configs:
|
||||
assert "geo $geointel_trusted_forwarder" in config
|
||||
assert "default 0;" in config
|
||||
assert "172.16.0.0/12 1;" in config
|
||||
assert 'map "$geointel_trusted_forwarder:$http_x_forwarded_proto"' in config
|
||||
assert '"1:https" https;' in config
|
||||
assert "proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;" in config
|
||||
assert "proxy_set_header X-Forwarded-Proto $scheme;" not in config
|
||||
|
||||
|
||||
def test_nginx_runtime_sets_security_headers_on_all_cached_locations() -> None:
|
||||
configs = (
|
||||
(ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8"),
|
||||
(ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(
|
||||
encoding="utf-8"
|
||||
),
|
||||
)
|
||||
required = (
|
||||
'Content-Security-Policy "frame-ancestors \'none\'" always;',
|
||||
'X-Frame-Options "DENY" always;',
|
||||
'X-Content-Type-Options "nosniff" always;',
|
||||
'Referrer-Policy "strict-origin-when-cross-origin" always;',
|
||||
'Permissions-Policy "camera=(), microphone=(), geolocation=()" always;',
|
||||
)
|
||||
|
||||
for config in configs:
|
||||
cached_locations = config.count("add_header Cache-Control")
|
||||
assert cached_locations >= 2
|
||||
for header in required:
|
||||
# Nginx 1.27 locations with Cache-Control do not inherit server-level
|
||||
# add_header directives, so every cached location repeats the policy.
|
||||
assert config.count(f"add_header {header}") == cached_locations + 1
|
||||
|
||||
|
||||
def test_compose_does_not_publish_postgis_on_default_host_port() -> None:
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
|
||||
assert '"5432:5432"' not in compose
|
||||
|
||||
|
||||
def test_compose_waits_for_healthy_database_and_applies_migrations() -> None:
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
|
||||
assert "pg_isready -U ${GEOINTEL_POSTGRES_USER:-geointel} -d ${GEOINTEL_POSTGRES_DB:-geointel}" in compose
|
||||
assert "condition: service_healthy" in compose
|
||||
assert "sh /app/docker_start.sh" in compose
|
||||
|
||||
|
||||
def test_compose_mounts_demo_fixtures_for_backend_runtime() -> None:
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
|
||||
assert "./fixtures:/app/fixtures:ro" in compose
|
||||
|
||||
|
||||
def test_compose_has_backend_and_frontend_healthchecks() -> None:
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
|
||||
assert "http://127.0.0.1:8000/health/ready" in compose
|
||||
assert "urllib.request.urlopen" in compose
|
||||
assert "http://127.0.0.1/health/ready" in compose
|
||||
assert "wget -q -O -" in compose
|
||||
assert "start_period: 30s" in compose
|
||||
assert "start_period: 10s" in compose
|
||||
|
||||
|
||||
def test_frontend_waits_for_healthy_backend_in_compose() -> None:
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
frontend_section = compose.split(" frontend:", 1)[1]
|
||||
|
||||
assert "backend:" in frontend_section
|
||||
assert "condition: service_healthy" in frontend_section
|
||||
|
||||
|
||||
def test_backend_docker_start_script_waits_for_sql_connection_before_migrations() -> None:
|
||||
script = (ROOT / "backend" / "docker_start.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "Waiting for database connection" in script
|
||||
assert "create_engine(settings.database_url" in script
|
||||
assert "SELECT 1" in script
|
||||
assert "python -m alembic upgrade head" in script
|
||||
assert "uvicorn app.main:app --host 0.0.0.0 --port 8000" in script
|
||||
|
||||
|
||||
def test_runtime_sets_writable_ultralytics_config_directory() -> None:
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
unraid_compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
|
||||
start_script = (ROOT / "deploy" / "unraid" / "all-in-one-start.sh").read_text(encoding="utf-8")
|
||||
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
unraid_env = (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(encoding="utf-8")
|
||||
|
||||
assert "YOLO_CONFIG_DIR: ${YOLO_CONFIG_DIR:-/app/storage/ultralytics}" in compose
|
||||
assert "YOLO_CONFIG_DIR: ${YOLO_CONFIG_DIR:-/app/storage/ultralytics}" in unraid_compose
|
||||
assert 'export YOLO_CONFIG_DIR="${YOLO_CONFIG_DIR:-$STORAGE_ROOT/ultralytics}"' in start_script
|
||||
assert 'mkdir -p "$PGDATA" "$STORAGE_ROOT" "$YOLO_CONFIG_DIR"' in start_script
|
||||
assert 'YOLO_CONFIG_DIR="${YOLO_CONFIG_DIR:-/app/storage/ultralytics}"' in run_script
|
||||
assert '-e YOLO_CONFIG_DIR="$YOLO_CONFIG_DIR"' in run_script
|
||||
assert "YOLO_CONFIG_DIR=/app/storage/ultralytics" in unraid_env
|
||||
|
||||
|
||||
def test_regional_official_vector_sources_are_configurable_in_every_runtime() -> None:
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
unraid_compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
|
||||
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
env_example = (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
template = (ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
for key in (
|
||||
"SPW_PICC_ENABLED",
|
||||
"SPW_PICC_MAPSERVER_URL",
|
||||
"URBIS_ENABLED",
|
||||
"URBIS_WFS_URL",
|
||||
):
|
||||
assert key in compose
|
||||
assert key in unraid_compose
|
||||
assert f'{key}="${{{key}:-' in run_script
|
||||
assert f'-e {key}="${key}"' in run_script
|
||||
assert f"{key}=" in env_example
|
||||
assert f'Target="{key}"' in template
|
||||
|
||||
|
||||
def test_segmentation_and_mdk_acquisition_are_configurable_in_every_runtime() -> None:
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
unraid_compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
|
||||
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
env_example = (ROOT / ".env.example").read_text(encoding="utf-8")
|
||||
unraid_env = (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(encoding="utf-8")
|
||||
template = (ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(encoding="utf-8")
|
||||
|
||||
for key in (
|
||||
"YOLO_SEG_ENABLED",
|
||||
"YOLO_SEG_MODEL_PATH",
|
||||
"SAM_ENABLED",
|
||||
"SAM_MODEL_PATH",
|
||||
"SEGMENTATION_MAX_MASKS_PER_TILE",
|
||||
"SEGMENTATION_DUPLICATE_IOU_THRESHOLD",
|
||||
"MDK_BATHYMETRY_ACQUISITION_ENABLED",
|
||||
"MDK_BATHYMETRY_COVERAGE_ID",
|
||||
"MDK_BATHYMETRY_MAX_BBOX_DEG2",
|
||||
):
|
||||
assert key in compose, key
|
||||
assert key in unraid_compose, key
|
||||
assert f'{key}="${{{key}:-' in run_script, key
|
||||
assert f'-e {key}="${key}"' in run_script, key
|
||||
assert f"{key}=" in env_example, key
|
||||
assert f"{key}=" in unraid_env, key
|
||||
assert f'Target="{key}"' in template, key
|
||||
|
||||
|
||||
def test_compose_reconciles_interrupted_runs_after_restart_like_unraid_runtime() -> None:
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
start_script = (ROOT / "deploy" / "unraid" / "all-in-one-start.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert (
|
||||
"GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP: "
|
||||
"${GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true}"
|
||||
) in compose
|
||||
assert (
|
||||
'GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP='
|
||||
'"${GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true}"'
|
||||
) in start_script
|
||||
|
||||
|
||||
def test_docker_build_contexts_exclude_vendor_build_and_cache_outputs() -> None:
|
||||
required_patterns = {
|
||||
"node_modules",
|
||||
"dist",
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
".pytest_cache",
|
||||
}
|
||||
|
||||
for relative_path in ("backend/.dockerignore", "frontend/.dockerignore"):
|
||||
content = (ROOT / relative_path).read_text(encoding="utf-8")
|
||||
for pattern in required_patterns:
|
||||
assert pattern in content
|
||||
|
||||
|
||||
def test_browser_runtime_verification_script_detects_proxy_contract() -> None:
|
||||
script = (ROOT / "scripts" / "verify_browser_runtime.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "/api/v1/projects" in script
|
||||
assert "<!doctype html" in script
|
||||
assert "canonical GeoIntel envelope" in script
|
||||
assert '"status":"ok"' in script
|
||||
|
||||
|
||||
def test_gis_runtime_verification_script_detects_required_capabilities() -> None:
|
||||
script = (ROOT / "scripts" / "verify_gis_runtime.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "/api/v1/system/capabilities" in script
|
||||
assert '"postgis":true' in script
|
||||
assert '"rasterio":true' in script
|
||||
assert '"geopandas":true' in script
|
||||
assert "<!doctype html" in script
|
||||
|
||||
|
||||
def test_gis_import_smoke_script_checks_runtime_imports() -> None:
|
||||
docker_script = (ROOT / "backend" / "scripts" / "gis_import_smoke.py").read_text(encoding="utf-8")
|
||||
root_wrapper = (ROOT / "scripts" / "gis_import_smoke.py").read_text(encoding="utf-8")
|
||||
|
||||
assert 'REQUIRED_MODULES = ("rasterio", "geopandas", "pyogrio")' in docker_script
|
||||
assert "importlib.import_module" in docker_script
|
||||
assert '"gis_imports"' in docker_script
|
||||
assert 'ROOT / "backend" / "scripts"' in root_wrapper
|
||||
assert "from gis_import_smoke import main" in root_wrapper
|
||||
|
||||
|
||||
def test_backend_docker_context_contains_gis_import_smoke_script() -> None:
|
||||
assert (ROOT / "backend" / "scripts" / "gis_import_smoke.py").exists()
|
||||
|
||||
|
||||
def test_all_in_one_dockerfile_caches_dependencies_and_pins_driver_compatible_cuda_torch() -> None:
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
|
||||
metadata_copy_index = dockerfile.index("COPY backend/pyproject.toml /app/")
|
||||
placeholder_readme_index = dockerfile.index("GeoIntel backend package metadata")
|
||||
dependency_install_index = dockerfile.index('/usr/bin/python3.11 -m venv /opt/geointel/venv \\')
|
||||
backend_copy_index = dockerfile.index("COPY backend/ /app/")
|
||||
smoke_index = dockerfile.index("RUN python scripts/gis_import_smoke.py")
|
||||
|
||||
assert metadata_copy_index < placeholder_readme_index < dependency_install_index < backend_copy_index < smoke_index
|
||||
assert "GEOINTEL_TORCH_INDEX_URL=https://download.pytorch.org/whl/cu128" in dockerfile
|
||||
assert '--extra-index-url "$GEOINTEL_TORCH_INDEX_URL"' in dockerfile
|
||||
ai_lock = (ROOT / "backend" / "requirements-ai-linux.lock").read_text(encoding="utf-8")
|
||||
assert "torch==2.11.0+cu128 --hash=sha256:" in ai_lock
|
||||
assert "torchvision==0.26.0+cu128 --hash=sha256:" in ai_lock
|
||||
|
||||
|
||||
def test_unraid_deploy_passes_ai_build_arg_and_yolo_runtime_env() -> None:
|
||||
deploy_ps1 = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8")
|
||||
deploy_sh = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8")
|
||||
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert 'DEPLOY_GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-}"' in deploy_sh
|
||||
assert "--build-arg GEOINTEL_INSTALL_AI=" in release_script
|
||||
assert "DEPLOY_GEOINTEL_INSTALL_AI" in deploy_ps1
|
||||
assert "[string]$InstallAi" in deploy_ps1
|
||||
|
||||
assert 'YOLO_ENABLED="${YOLO_ENABLED:-false}"' in run_script
|
||||
assert '-e YOLO_ENABLED="$YOLO_ENABLED"' in run_script
|
||||
assert 'YOLO_MODELS_DIR="${YOLO_MODELS_DIR:-/app/models}"' in run_script
|
||||
assert '-e YOLO_MODELS_DIR="$YOLO_MODELS_DIR"' in run_script
|
||||
assert '-e YOLO_MODEL_PATH="$YOLO_MODEL_PATH"' in run_script
|
||||
assert '-e YOLO_MAX_TILES="$YOLO_MAX_TILES"' in run_script
|
||||
assert '-e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS"' in run_script
|
||||
assert '-e YOLO_DUPLICATE_IOU_THRESHOLD="$YOLO_DUPLICATE_IOU_THRESHOLD"' in run_script
|
||||
assert "-v \"${GEOINTEL_MODELS_PATH}:/app/models\"" in run_script
|
||||
def test_unraid_ai_runtime_requests_nvidia_and_fails_closed() -> None:
|
||||
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
|
||||
assert "--gpus all" in run_script
|
||||
assert 'YOLO_DEVICE="${YOLO_DEVICE:-cuda:0}"' in run_script
|
||||
assert 'YOLO_REQUIRE_CUDA="${YOLO_REQUIRE_CUDA:-true}"' in run_script
|
||||
assert '-e YOLO_REQUIRE_CUDA="$YOLO_REQUIRE_CUDA"' in run_script
|
||||
assert "https://download.pytorch.org/whl/cu128" in dockerfile
|
||||
@@ -0,0 +1,88 @@
|
||||
"""A broad handler must not overwrite a precise diagnosis with a generic one.
|
||||
|
||||
``except Exception: raise AppError(...)`` also catches ``AppError``, so a
|
||||
service that raised INVALID_CRS with a 409 comes out as whatever generic code
|
||||
the outer handler chose. For a product whose whole claim is that a result can
|
||||
be traced back to its cause, that is the wrong direction in which to lose
|
||||
information.
|
||||
|
||||
Wrapping a narrow call whose failure the outer message describes better is
|
||||
legitimate — coercing a CRS string, parsing one geometry, calling one's own
|
||||
private helper. Relabelling what *another* component reported is not.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
APP = Path(__file__).resolve().parents[1] / "app"
|
||||
|
||||
GENERIC_HANDLER = re.compile(r"except Exception as \w+:\s*\n\s*raise AppError\(", re.M)
|
||||
OTHER_SERVICE = re.compile(r"\b([A-Z]\w*Service)\.")
|
||||
# Cross-component entry points that carry their own considered diagnosis.
|
||||
GOVERNED_CALLS = re.compile(r"assert_eligible|assert_within_storage_root|canonicalize_|_load_dataset_payload")
|
||||
|
||||
|
||||
def _enclosing_service(text: str, handler_line: int) -> str | None:
|
||||
"""The class the handler sits in, not merely the first one in the file."""
|
||||
|
||||
enclosing = None
|
||||
for match in re.finditer(r"^class (\w+)[:(]", text, re.M):
|
||||
if text[: match.start()].count(chr(10)) > handler_line:
|
||||
break
|
||||
enclosing = match.group(1)
|
||||
return enclosing
|
||||
|
||||
|
||||
def _try_body(lines: list[str], handler_line: int) -> str:
|
||||
index = handler_line
|
||||
while index > 0 and lines[index].strip() != "try:":
|
||||
index -= 1
|
||||
return "\n".join(lines[index:handler_line])
|
||||
|
||||
|
||||
def calls_another_component(body: str, own_service: str | None) -> bool:
|
||||
if GOVERNED_CALLS.search(body):
|
||||
return True
|
||||
return any(name != own_service for name in OTHER_SERVICE.findall(body))
|
||||
|
||||
|
||||
def _offenders() -> list[str]:
|
||||
found: list[str] = []
|
||||
for path in sorted(APP.rglob("*.py")):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
lines = text.splitlines()
|
||||
for match in GENERIC_HANDLER.finditer(text):
|
||||
preceding = text[max(0, match.start() - 240):match.start() + 40]
|
||||
if "except AppError:" in preceding:
|
||||
continue
|
||||
handler_line = text[:match.start()].count("\n")
|
||||
own = _enclosing_service(text, handler_line)
|
||||
if calls_another_component(_try_body(lines, handler_line), own):
|
||||
found.append(f"{path.relative_to(APP)}:{handler_line + 1}")
|
||||
return found
|
||||
|
||||
|
||||
def test_a_call_into_another_component_keeps_the_error_it_raised() -> None:
|
||||
offenders = _offenders()
|
||||
|
||||
assert not offenders, (
|
||||
"These catch Exception around a call into another component and relabel "
|
||||
"whatever it raised, including its AppError. Add `except AppError: "
|
||||
f"raise` before the generic handler: {offenders}"
|
||||
)
|
||||
|
||||
|
||||
def test_the_check_recognises_the_shape_it_guards_against() -> None:
|
||||
"""Without this the guard could pass because its pattern never matches."""
|
||||
|
||||
other = "try:\n DatasetService.get_dataset(db, dataset_id)"
|
||||
own = "try:\n GrbAcquisitionService._extract_dimension(geometry)"
|
||||
governed = "try:\n DatasetConsumptionGate.assert_eligible(dataset)"
|
||||
|
||||
assert calls_another_component(other, "GrbAcquisitionService")
|
||||
assert calls_another_component(governed, "GrbAcquisitionService")
|
||||
# Wrapping one's own private helper is the legitimate case.
|
||||
assert not calls_another_component(own, "GrbAcquisitionService")
|
||||
assert GENERIC_HANDLER.search("except Exception as exc:\n raise AppError(")
|
||||
@@ -0,0 +1,49 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.main import app, create_app
|
||||
|
||||
|
||||
def assert_contract_error(payload: dict, code: str) -> None:
|
||||
assert payload["error"] == code
|
||||
assert isinstance(payload["message"], str)
|
||||
assert "details" in payload
|
||||
assert "request_id" in payload
|
||||
|
||||
|
||||
def test_app_error_uses_top_level_api_error_contract() -> None:
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/api/v1/external/providers/unknown")
|
||||
|
||||
assert response.status_code == 404
|
||||
assert_contract_error(response.json(), "PROVIDER_NOT_FOUND")
|
||||
assert response.json()["message"] == "Provider not found"
|
||||
|
||||
|
||||
def test_http_exception_uses_top_level_api_error_contract() -> None:
|
||||
test_app = create_app()
|
||||
|
||||
@test_app.get("/__test__/http-error")
|
||||
def raise_http_error() -> None:
|
||||
raise HTTPException(status_code=404, detail="Project not found")
|
||||
|
||||
client = TestClient(test_app)
|
||||
response = client.get("/__test__/http-error")
|
||||
|
||||
assert response.status_code == 404
|
||||
assert_contract_error(response.json(), "HTTP_ERROR")
|
||||
assert response.json()["message"] == "Project not found"
|
||||
|
||||
|
||||
def test_validation_error_uses_top_level_api_error_contract() -> None:
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/api/v1/projects/not-a-uuid")
|
||||
|
||||
assert response.status_code == 422
|
||||
assert_contract_error(response.json(), "VALIDATION_ERROR")
|
||||
assert response.json()["message"] == "Validation failed"
|
||||
assert isinstance(response.json()["details"], list)
|
||||
@@ -0,0 +1,125 @@
|
||||
"""A downloaded export must carry its own provenance and its own limits.
|
||||
|
||||
Counterpart to ``test_export_provenance_member``: this reads the file the
|
||||
service actually writes, so it proves the member survives serialisation rather
|
||||
than only that the helper builds it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from app.models import Dataset, Export
|
||||
from app.services.export_service import ExportService
|
||||
from app.services.storage_service import StorageService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
from tests.test_sprint107_map_selection_export import FakeSession, _govern_fixture_dataset
|
||||
|
||||
|
||||
BBOX = {"min_x": 4.9, "min_y": 50.9, "max_x": 5.2, "max_y": 51.2, "crs": "EPSG:4326"}
|
||||
|
||||
|
||||
def _dataset(dataset_id):
|
||||
dataset = _govern_fixture_dataset(
|
||||
Dataset(
|
||||
id=dataset_id,
|
||||
project_id=uuid4(),
|
||||
name="grb-buildings.geojson",
|
||||
dataset_type="vector",
|
||||
source="grb",
|
||||
source_name="grb",
|
||||
source_version="2024-06",
|
||||
observed_at=datetime(2024, 6, 1, tzinfo=timezone.utc),
|
||||
status="ready",
|
||||
)
|
||||
)
|
||||
return dataset
|
||||
|
||||
|
||||
def _selection(*, feature_count: int, total: int, truncated: bool, warning: str | None = None):
|
||||
return {
|
||||
"selection_bbox": BBOX,
|
||||
"feature_count": feature_count,
|
||||
"total_feature_count": total,
|
||||
"limit": 250,
|
||||
"truncated": truncated,
|
||||
"summary": {"selection_edge_warning": warning} if warning else {},
|
||||
"geojson": {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {"vector_feature_id": f"vf-{index}"},
|
||||
}
|
||||
for index in range(feature_count)
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _export(monkeypatch, tmp_path: Path, selection) -> dict:
|
||||
dataset_id = uuid4()
|
||||
dataset = _dataset(dataset_id)
|
||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||
export_path = tmp_path / "selection.geojson"
|
||||
monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path))
|
||||
monkeypatch.setattr(VectorFeatureService, "select_features_by_bbox", lambda *_args, **_kwargs: selection)
|
||||
|
||||
ExportService.export_vector_selection_geojson(db, dataset_id, BBOX, limit=250, name="selection")
|
||||
|
||||
assert [item for item in db.added if isinstance(item, Export)]
|
||||
return json.loads(export_path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_a_truncated_export_file_states_that_it_is_partial(monkeypatch, tmp_path: Path) -> None:
|
||||
written = _export(monkeypatch, tmp_path, _selection(feature_count=2, total=1_400, truncated=True))
|
||||
|
||||
provenance = written["geointel_provenance"]
|
||||
assert provenance["complete"] is False
|
||||
assert "1400" in provenance["completeness_note"].replace(".", "")
|
||||
# The features are still there; the file simply no longer implies it holds
|
||||
# everything the selection contains.
|
||||
assert len(written["features"]) == 2
|
||||
|
||||
|
||||
def test_a_complete_export_file_says_so(monkeypatch, tmp_path: Path) -> None:
|
||||
written = _export(monkeypatch, tmp_path, _selection(feature_count=2, total=2, truncated=False))
|
||||
|
||||
provenance = written["geointel_provenance"]
|
||||
assert provenance["complete"] is True
|
||||
assert provenance["completeness_note"] is None
|
||||
|
||||
|
||||
def test_the_file_identifies_its_source_edition(monkeypatch, tmp_path: Path) -> None:
|
||||
written = _export(monkeypatch, tmp_path, _selection(feature_count=1, total=1, truncated=False))
|
||||
|
||||
provenance = written["geointel_provenance"]
|
||||
assert provenance["source_name"] == "grb"
|
||||
assert provenance["source_version"] == "2024-06"
|
||||
assert provenance["observed_at"].startswith("2024-06-01")
|
||||
assert provenance["selection_bbox"] == BBOX
|
||||
|
||||
|
||||
def test_selection_caveats_travel_into_the_file(monkeypatch, tmp_path: Path) -> None:
|
||||
written = _export(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
_selection(feature_count=1, total=1, truncated=False, warning="22 objecten liggen deels buiten de selectie."),
|
||||
)
|
||||
|
||||
assert written["geointel_provenance"]["warnings"] == ["22 objecten liggen deels buiten de selectie."]
|
||||
|
||||
|
||||
def test_the_file_remains_a_valid_feature_collection(monkeypatch, tmp_path: Path) -> None:
|
||||
"""The provenance is a foreign member, not a change to the GeoJSON shape."""
|
||||
|
||||
written = _export(monkeypatch, tmp_path, _selection(feature_count=1, total=1, truncated=False))
|
||||
|
||||
assert written["type"] == "FeatureCollection"
|
||||
assert isinstance(written["features"], list)
|
||||
assert written["features"][0]["type"] == "Feature"
|
||||
@@ -0,0 +1,129 @@
|
||||
"""An exported GeoJSON must say what it is and what it leaves out.
|
||||
|
||||
The vector selection export caps its features and records ``truncated`` in the
|
||||
export *record*. The file itself said nothing: an operator downloads
|
||||
``mol-selection.geojson``, opens it in QGIS and sees 250 buildings where the
|
||||
workbench said 1.400, with nothing in the file to indicate the difference.
|
||||
|
||||
For a product whose promise is that an export is a reproducible result, a file
|
||||
that looks complete and is not is the sharpest possible violation. RFC 7946
|
||||
allows foreign members on a FeatureCollection, and the detection export already
|
||||
uses one; this makes that convention uniform.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from app.services.export_service import ExportService
|
||||
|
||||
|
||||
def test_provenance_names_the_source_and_the_moment() -> None:
|
||||
dataset_id = uuid4()
|
||||
project_id = uuid4()
|
||||
|
||||
member = ExportService.provenance_member(
|
||||
source="vector_selection",
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
source_name="grb",
|
||||
source_version="2024-06",
|
||||
observed_at=datetime(2024, 6, 1, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
assert member["source"] == "vector_selection"
|
||||
assert member["dataset_id"] == str(dataset_id)
|
||||
assert member["project_id"] == str(project_id)
|
||||
assert member["source_name"] == "grb"
|
||||
assert member["source_version"] == "2024-06"
|
||||
assert member["observed_at"].startswith("2024-06-01")
|
||||
assert member["exported_at"]
|
||||
|
||||
|
||||
def test_a_truncated_export_says_so_in_words() -> None:
|
||||
member = ExportService.provenance_member(
|
||||
source="vector_selection",
|
||||
project_id=uuid4(),
|
||||
dataset_id=uuid4(),
|
||||
feature_count=250,
|
||||
total_feature_count=1_400,
|
||||
truncated=True,
|
||||
)
|
||||
|
||||
assert member["complete"] is False
|
||||
assert member["feature_count"] == 250
|
||||
assert member["total_feature_count"] == 1_400
|
||||
assert "1400" in member["completeness_note"].replace(".", "").replace(",", "")
|
||||
assert "250" in member["completeness_note"]
|
||||
|
||||
|
||||
def test_a_complete_export_is_stated_as_complete() -> None:
|
||||
member = ExportService.provenance_member(
|
||||
source="detection_run",
|
||||
project_id=uuid4(),
|
||||
dataset_id=uuid4(),
|
||||
feature_count=12,
|
||||
total_feature_count=12,
|
||||
truncated=False,
|
||||
)
|
||||
|
||||
assert member["complete"] is True
|
||||
assert member["completeness_note"] is None
|
||||
|
||||
|
||||
def test_counts_that_disagree_are_treated_as_incomplete() -> None:
|
||||
"""A caller that forgets the flag must not produce a file claiming completeness."""
|
||||
|
||||
member = ExportService.provenance_member(
|
||||
source="vector_selection",
|
||||
project_id=uuid4(),
|
||||
dataset_id=uuid4(),
|
||||
feature_count=100,
|
||||
total_feature_count=140,
|
||||
truncated=False,
|
||||
)
|
||||
|
||||
assert member["complete"] is False
|
||||
|
||||
|
||||
def test_selection_context_travels_with_the_file() -> None:
|
||||
area_id = uuid4()
|
||||
bbox = {"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3, "crs": "EPSG:4326"}
|
||||
|
||||
member = ExportService.provenance_member(
|
||||
source="vector_selection",
|
||||
project_id=uuid4(),
|
||||
dataset_id=uuid4(),
|
||||
selection_bbox=bbox,
|
||||
selection_area_id=area_id,
|
||||
warnings=["22 van de 100 objecten liggen deels buiten de selectie."],
|
||||
)
|
||||
|
||||
assert member["selection_bbox"] == bbox
|
||||
assert member["selection_area_id"] == str(area_id)
|
||||
assert member["warnings"] == ["22 van de 100 objecten liggen deels buiten de selectie."]
|
||||
|
||||
|
||||
def test_empty_context_is_omitted_rather_than_written_as_null_noise() -> None:
|
||||
member = ExportService.provenance_member(
|
||||
source="dataset",
|
||||
project_id=uuid4(),
|
||||
dataset_id=uuid4(),
|
||||
)
|
||||
|
||||
assert "selection_bbox" not in member
|
||||
assert "warnings" not in member
|
||||
assert "source_version" not in member
|
||||
|
||||
|
||||
def test_the_member_attaches_under_a_reserved_key() -> None:
|
||||
collection = {"type": "FeatureCollection", "features": []}
|
||||
|
||||
ExportService.attach_provenance(
|
||||
collection,
|
||||
ExportService.provenance_member(source="dataset", project_id=uuid4(), dataset_id=uuid4()),
|
||||
)
|
||||
|
||||
assert collection["type"] == "FeatureCollection"
|
||||
assert collection["geointel_provenance"]["source"] == "dataset"
|
||||
@@ -0,0 +1,369 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "build_failure_driven_yolo_sampling.py"
|
||||
SPEC = importlib.util.spec_from_file_location("failure_sampling", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def test_dataset_validation_source_preserves_manifest_path(tmp_path: Path):
|
||||
source = tmp_path / "dataset.yaml"
|
||||
source.write_text(
|
||||
"path: /data/source\ntrain: /data/source/train.txt\n"
|
||||
"val: /data/source/internal-val.txt\nnames:\n 0: building\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert MODULE.dataset_validation_source(source) == "/data/source/internal-val.txt"
|
||||
|
||||
|
||||
def test_sampling_rejects_protected_test_and_background_feedback() -> None:
|
||||
manifest = {
|
||||
"samples": [
|
||||
{"sample_slug": "train-fl", "split": "train", "region": "flanders"},
|
||||
{"sample_slug": "train-wa", "split": "train", "region": "wallonia"},
|
||||
{"sample_slug": "test-fl", "split": "test", "region": "flanders"},
|
||||
]
|
||||
}
|
||||
summary = {
|
||||
"tiles": [
|
||||
{"sample_slug": "train-fl", "split": "train", "label_count": 2, "image_path": "/tmp/fl-pos.png"},
|
||||
{"sample_slug": "train-fl", "split": "train", "label_count": 0, "image_path": "/tmp/fl-neg.png"},
|
||||
{"sample_slug": "train-wa", "split": "train", "label_count": 1, "image_path": "/tmp/wa-pos.png"},
|
||||
{"sample_slug": "test-fl", "split": "val", "label_count": 1, "image_path": "/tmp/protected.png"},
|
||||
]
|
||||
}
|
||||
assessment = {
|
||||
"status": "continue_training_loop",
|
||||
"gates": {
|
||||
"min_region_f1": 0.45,
|
||||
"min_region_precision": 0.5,
|
||||
"min_region_recall": 0.4,
|
||||
"max_pure_empty_false_positives": 0,
|
||||
},
|
||||
"test": {
|
||||
"regions": {
|
||||
"flanders": {"f1": 0.2, "precision": 0.3, "recall": 0.2},
|
||||
"wallonia": {"f1": 0.6, "precision": 0.6, "recall": 0.6},
|
||||
}
|
||||
},
|
||||
"background": {"pure_empty_false_positives": 2},
|
||||
}
|
||||
with pytest.raises(ValueError, match="protected test/background evidence"):
|
||||
MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
|
||||
)
|
||||
|
||||
|
||||
def test_sampling_can_use_calibration_before_test_is_opened() -> None:
|
||||
manifest = {"samples": [{"sample_slug": "train-fl", "split": "train", "region": "flanders"}]}
|
||||
summary = {
|
||||
"tiles": [
|
||||
{"sample_slug": "train-fl", "split": "train", "label_count": 1, "image_path": "/tmp/fl.png"}
|
||||
]
|
||||
}
|
||||
assessment = {
|
||||
"status": "continue_training_loop",
|
||||
"gates": {
|
||||
"min_region_f1": 0.45,
|
||||
"min_region_precision": 0.5,
|
||||
"min_region_recall": 0.4,
|
||||
"max_pure_empty_false_positives": 0,
|
||||
},
|
||||
"calibration": {
|
||||
"regions": {"flanders": {"f1": 0.4, "precision": 0.6, "recall": 0.35}}
|
||||
},
|
||||
"test": None,
|
||||
"background": None,
|
||||
}
|
||||
paths, metadata = MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
|
||||
)
|
||||
assert len(paths) == 3
|
||||
assert metadata["failure_evidence_source"] == "calibration"
|
||||
|
||||
|
||||
def test_precision_correction_can_balance_positive_and_negative_tiles() -> None:
|
||||
manifest = {"samples": [{"sample_slug": "train-fl", "split": "train", "region": "flanders"}]}
|
||||
summary = {
|
||||
"tiles": [
|
||||
{"sample_slug": "train-fl", "split": "train", "label_count": 2, "image_path": "/tmp/fl-pos.png"},
|
||||
{"sample_slug": "train-fl", "split": "train", "label_count": 0, "image_path": "/tmp/fl-neg.png"},
|
||||
]
|
||||
}
|
||||
assessment = {
|
||||
"status": "continue_training_loop",
|
||||
"gates": {
|
||||
"min_region_f1": 0.45,
|
||||
"min_region_precision": 0.5,
|
||||
"min_region_recall": 0.4,
|
||||
"max_pure_empty_false_positives": 0,
|
||||
},
|
||||
"calibration": {
|
||||
"regions": {"flanders": {"f1": 0.46, "precision": 0.45, "recall": 0.46}}
|
||||
},
|
||||
}
|
||||
|
||||
paths, metadata = MODULE.build_sampling(
|
||||
summary=summary,
|
||||
manifest=manifest,
|
||||
assessment=assessment,
|
||||
precision_positive_repeat=2,
|
||||
negative_repeat=3,
|
||||
max_region_share=1.0,
|
||||
)
|
||||
|
||||
assert paths.count(str(Path("/tmp/fl-pos.png").resolve())) == 2
|
||||
assert paths.count(str(Path("/tmp/fl-neg.png").resolve())) == 3
|
||||
assert metadata["precision_positive_repeat"] == 2
|
||||
|
||||
|
||||
def test_sampling_targets_failed_calibration_contexts_without_using_protected_tiles() -> None:
|
||||
manifest = {
|
||||
"samples": [
|
||||
{"sample_slug": "train-industry", "split": "train", "region": "flanders", "context": "industrial"},
|
||||
{"sample_slug": "train-suburban", "split": "train", "region": "flanders", "context": "suburban"},
|
||||
{"sample_slug": "cal-industry", "split": "calibration", "region": "flanders", "context": "industrial"},
|
||||
]
|
||||
}
|
||||
summary = {
|
||||
"tiles": [
|
||||
{"sample_slug": "train-industry", "split": "train", "label_count": 2, "image_path": "/tmp/industry-pos.png"},
|
||||
{"sample_slug": "train-industry", "split": "train", "label_count": 0, "image_path": "/tmp/industry-neg.png"},
|
||||
{"sample_slug": "train-suburban", "split": "train", "label_count": 2, "image_path": "/tmp/suburban-pos.png"},
|
||||
{"sample_slug": "cal-industry", "split": "val", "label_count": 2, "image_path": "/tmp/protected.png"},
|
||||
]
|
||||
}
|
||||
assessment = {
|
||||
"status": "continue_training_loop",
|
||||
"gates": {
|
||||
"min_region_f1": 0.45,
|
||||
"min_region_precision": 0.5,
|
||||
"min_region_recall": 0.4,
|
||||
"max_pure_empty_false_positives": 0,
|
||||
},
|
||||
"calibration": {
|
||||
"regions": {"flanders": {"f1": 0.3, "precision": 0.35, "recall": 0.27}},
|
||||
"samples": {"cal-industry": {"f1": 0.2, "precision": 0.3, "recall": 0.15}},
|
||||
},
|
||||
}
|
||||
|
||||
paths, metadata = MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
|
||||
)
|
||||
|
||||
assert paths.count(str(Path("/tmp/industry-pos.png").resolve())) == 5
|
||||
assert paths.count(str(Path("/tmp/industry-neg.png").resolve())) == 1
|
||||
assert paths.count(str(Path("/tmp/suburban-pos.png").resolve())) == 3
|
||||
assert not any("protected" in path for path in paths)
|
||||
assert metadata["weak_recall_contexts"] == ["flanders:industrial"]
|
||||
assert metadata["weak_precision_contexts"] == ["flanders:industrial"]
|
||||
assert metadata["recall_dominant_regions"] == ["flanders"]
|
||||
|
||||
|
||||
def test_sampling_rejects_background_feedback_after_a_protected_background_opening() -> None:
|
||||
manifest = {"samples": [
|
||||
{"sample_slug": "positive", "split": "train", "region": "flanders", "context": "industrial"},
|
||||
{"sample_slug": "negative", "split": "train", "region": "flanders", "context": "industrial-hard-negative"},
|
||||
]}
|
||||
summary = {"tiles": [
|
||||
{"sample_slug": "positive", "split": "train", "label_count": 1, "image_path": "/tmp/positive.png"},
|
||||
{"sample_slug": "negative", "split": "train", "label_count": 0, "image_path": "/tmp/negative.png"},
|
||||
]}
|
||||
assessment = {
|
||||
"status": "continue_training_loop",
|
||||
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
|
||||
"max_pure_empty_false_positives": 0},
|
||||
"calibration": {"regions": {
|
||||
"flanders": {"f1": .25, "precision": .4, "recall": .2},
|
||||
}},
|
||||
"background": {"pure_empty_false_positives": 1},
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError, match="protected background evidence"):
|
||||
MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0,
|
||||
)
|
||||
|
||||
|
||||
def test_region_cap_drops_only_repeats_and_preserves_every_unique_tile() -> None:
|
||||
manifest = {"samples": [
|
||||
{"sample_slug": "fl", "split": "train", "region": "flanders", "context": "industrial"},
|
||||
{"sample_slug": "wa", "split": "train", "region": "wallonia", "context": "rural-town"},
|
||||
{"sample_slug": "br", "split": "train", "region": "brussels", "context": "dense-urban"},
|
||||
]}
|
||||
summary = {"tiles": [
|
||||
{"sample_slug": "fl", "split": "train", "label_count": 2, "image_path": f"/tmp/fl-{index}.png"}
|
||||
for index in range(4)
|
||||
] + [
|
||||
{"sample_slug": "wa", "split": "train", "label_count": 2, "image_path": f"/tmp/wa-{index}.png"}
|
||||
for index in range(2)
|
||||
] + [
|
||||
{"sample_slug": "br", "split": "train", "label_count": 2, "image_path": f"/tmp/br-{index}.png"}
|
||||
for index in range(2)
|
||||
]}
|
||||
assessment = {
|
||||
"status": "continue_training_loop",
|
||||
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
|
||||
"max_pure_empty_false_positives": 0},
|
||||
"calibration": {"regions": {
|
||||
"flanders": {"f1": .2, "precision": .3, "recall": .2},
|
||||
"wallonia": {"f1": .6, "precision": .6, "recall": .6},
|
||||
"brussels": {"f1": .6, "precision": .6, "recall": .6},
|
||||
}},
|
||||
}
|
||||
|
||||
paths, metadata = MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment,
|
||||
positive_repeat=5, max_region_share=.65,
|
||||
)
|
||||
|
||||
assert all(str(Path(f"/tmp/fl-{index}.png").resolve()) in paths for index in range(4))
|
||||
assert metadata["pre_cap_entries_by_region"]["flanders"] == 20
|
||||
assert metadata["sampled_entries_by_region"]["flanders"] == 7
|
||||
assert metadata["dropped_region_repeat_count"] == 13
|
||||
assert metadata["sampled_entries_by_region"]["flanders"] / len(paths) <= .65
|
||||
|
||||
|
||||
def test_region_cap_rotates_repeats_between_sampling_rounds() -> None:
|
||||
manifest = {"samples": [
|
||||
{"sample_slug": "fl", "split": "train", "region": "flanders", "context": "industrial"},
|
||||
{"sample_slug": "wa", "split": "train", "region": "wallonia", "context": "rural-town"},
|
||||
{"sample_slug": "br", "split": "train", "region": "brussels", "context": "dense-urban"},
|
||||
]}
|
||||
summary = {"tiles": [
|
||||
{"sample_slug": "fl", "split": "train", "label_count": 2, "image_path": f"/tmp/fl-{index}.png"}
|
||||
for index in range(4)
|
||||
] + [
|
||||
{"sample_slug": "wa", "split": "train", "label_count": 2, "image_path": f"/tmp/wa-{index}.png"}
|
||||
for index in range(2)
|
||||
] + [
|
||||
{"sample_slug": "br", "split": "train", "label_count": 2, "image_path": f"/tmp/br-{index}.png"}
|
||||
for index in range(2)
|
||||
]}
|
||||
assessment = {
|
||||
"status": "continue_training_loop",
|
||||
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
|
||||
"max_pure_empty_false_positives": 0},
|
||||
"calibration": {"regions": {
|
||||
"flanders": {"f1": .2, "precision": .3, "recall": .2},
|
||||
"wallonia": {"f1": .6, "precision": .6, "recall": .6},
|
||||
"brussels": {"f1": .6, "precision": .6, "recall": .6},
|
||||
}},
|
||||
}
|
||||
|
||||
first, first_metadata = MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment,
|
||||
positive_repeat=5, max_region_share=.65, sampling_round=1,
|
||||
)
|
||||
second, second_metadata = MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment,
|
||||
positive_repeat=5, max_region_share=.65, sampling_round=2,
|
||||
)
|
||||
|
||||
assert first != second
|
||||
assert set(first) == set(second)
|
||||
assert first_metadata["sampling_round"] == 1
|
||||
assert second_metadata["sampling_round"] == 2
|
||||
|
||||
|
||||
def test_region_cap_preserves_failed_context_positive_before_hard_negative() -> None:
|
||||
manifest = {"samples": [
|
||||
{"sample_slug": "target", "split": "train", "region": "flanders", "context": "industrial"},
|
||||
{"sample_slug": "negative", "split": "train", "region": "flanders", "context": "industrial-hard-negative"},
|
||||
{"sample_slug": "wa", "split": "train", "region": "wallonia", "context": "rural-town"},
|
||||
{"sample_slug": "br", "split": "train", "region": "brussels", "context": "dense-urban"},
|
||||
]}
|
||||
summary = {"tiles": [
|
||||
{"sample_slug": "target", "split": "train", "label_count": 2, "image_path": "/tmp/target.png"},
|
||||
{"sample_slug": "negative", "split": "train", "label_count": 0, "image_path": "/tmp/negative.png"},
|
||||
{"sample_slug": "wa", "split": "train", "label_count": 1, "image_path": "/tmp/wa.png"},
|
||||
{"sample_slug": "br", "split": "train", "label_count": 1, "image_path": "/tmp/br.png"},
|
||||
]}
|
||||
assessment = {
|
||||
"status": "continue_training_loop",
|
||||
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
|
||||
"max_pure_empty_false_positives": 0},
|
||||
"calibration": {
|
||||
"regions": {
|
||||
"flanders": {"f1": .2, "precision": .2, "recall": .3},
|
||||
"wallonia": {"f1": .6, "precision": .6, "recall": .6},
|
||||
"brussels": {"f1": .6, "precision": .6, "recall": .6},
|
||||
},
|
||||
"samples": {"target": {"f1": .2, "precision": .2, "recall": .3}},
|
||||
},
|
||||
}
|
||||
|
||||
paths, metadata = MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=.65,
|
||||
)
|
||||
|
||||
assert paths.count(str(Path("/tmp/target.png").resolve())) == 2
|
||||
assert paths.count(str(Path("/tmp/negative.png").resolve())) == 1
|
||||
assert metadata["priority_positive_repeat_count"] == 4
|
||||
|
||||
|
||||
def test_precision_guard_band_keeps_near_gate_region_stabilized() -> None:
|
||||
manifest = {"samples": [
|
||||
{"sample_slug": "wa-positive", "split": "train", "region": "wallonia", "context": "rural-town"},
|
||||
{"sample_slug": "wa-negative", "split": "train", "region": "wallonia", "context": "farmland-hard-negative"},
|
||||
]}
|
||||
summary = {"tiles": [
|
||||
{"sample_slug": "wa-positive", "split": "train", "label_count": 1, "image_path": "/tmp/wa-positive.png"},
|
||||
{"sample_slug": "wa-negative", "split": "train", "label_count": 0, "image_path": "/tmp/wa-negative.png"},
|
||||
]}
|
||||
assessment = {
|
||||
"status": "continue_training_loop",
|
||||
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
|
||||
"max_pure_empty_false_positives": 0},
|
||||
"calibration": {"regions": {
|
||||
"wallonia": {"f1": .6, "precision": .52, "recall": .7},
|
||||
}},
|
||||
}
|
||||
|
||||
paths, metadata = MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0,
|
||||
)
|
||||
|
||||
assert "wallonia" in metadata["weak_precision_regions"]
|
||||
assert paths.count(str(Path("/tmp/wa-negative.png").resolve())) == 4
|
||||
assert metadata["precision_guard_band"] == .03
|
||||
|
||||
|
||||
def test_coastal_precision_failure_targets_port_and_dunes_negatives() -> None:
|
||||
manifest = {"samples": [
|
||||
{"sample_slug": "coastal-train", "split": "train", "region": "flanders", "context": "coastal-urban"},
|
||||
{"sample_slug": "port-negative", "split": "train", "region": "flanders", "context": "port-hard-negative"},
|
||||
{"sample_slug": "dunes-negative", "split": "train", "region": "flanders", "context": "dunes-negative"},
|
||||
{"sample_slug": "coastal-cal", "split": "calibration", "region": "flanders", "context": "coastal-urban"},
|
||||
]}
|
||||
summary = {"tiles": [
|
||||
{"sample_slug": "coastal-train", "split": "train", "label_count": 2, "image_path": "/tmp/coastal.png"},
|
||||
{"sample_slug": "port-negative", "split": "train", "label_count": 0, "image_path": "/tmp/port.png"},
|
||||
{"sample_slug": "dunes-negative", "split": "train", "label_count": 0, "image_path": "/tmp/dunes.png"},
|
||||
]}
|
||||
assessment = {
|
||||
"status": "continue_training_loop",
|
||||
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
|
||||
"max_pure_empty_false_positives": 0},
|
||||
"calibration": {
|
||||
"regions": {"flanders": {"f1": .3, "precision": .2, "recall": .4}},
|
||||
"samples": {"coastal-cal": {"f1": .1, "precision": .05, "recall": .2}},
|
||||
},
|
||||
}
|
||||
|
||||
paths, metadata = MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
|
||||
)
|
||||
|
||||
assert paths.count(str(Path("/tmp/port.png").resolve())) == 6
|
||||
assert paths.count(str(Path("/tmp/dunes.png").resolve())) == 6
|
||||
assert "flanders:port-hard-negative" in metadata["targeted_negative_contexts"]
|
||||
assert "flanders:dunes-negative" in metadata["targeted_negative_contexts"]
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Flood risk must be a share of what was modelled, not of what was drawn.
|
||||
|
||||
The share and fraction divided the inundated cells by every cell whose centre
|
||||
fell inside the selection, including cells where the VMM raster holds nodata
|
||||
because the area lies outside the modelled extent. An operator drawing a
|
||||
rectangle that reaches past the model coverage read "3% at risk" where the
|
||||
honest answer is "of the 40% we have a model for, 7.5% is at risk, and for the
|
||||
rest there is no model at all".
|
||||
|
||||
Terrain, bathymetry and thematic raster analysis already divide by valid cells
|
||||
and report a coverage ratio; this brings flood hazard in line.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
np = pytest.importorskip("numpy")
|
||||
|
||||
from app.services.flood_hazard_analysis_service import ( # noqa: E402 - optional NumPy gate precedes service import
|
||||
FloodHazardCellStatistics,
|
||||
)
|
||||
|
||||
|
||||
NODATA = -9999.0
|
||||
|
||||
|
||||
def _stats(values, selected) -> FloodHazardCellStatistics:
|
||||
return FloodHazardCellStatistics.from_cells(
|
||||
np.asarray(values, dtype="float64"),
|
||||
np.asarray(selected, dtype=bool),
|
||||
nodata=NODATA,
|
||||
)
|
||||
|
||||
|
||||
def test_share_ignores_cells_the_model_does_not_cover() -> None:
|
||||
# Ten selected cells: four modelled (one of them wet), six nodata.
|
||||
values = [1.5, 0.0, 0.0, 0.0] + [NODATA] * 6
|
||||
selected = [True] * 10
|
||||
|
||||
stats = _stats(values, selected)
|
||||
|
||||
assert stats.selected_cell_count == 10
|
||||
assert stats.valid_cell_count == 4
|
||||
assert stats.no_data_cell_count == 6
|
||||
assert stats.inundated_cell_count == 1
|
||||
# 1 of 4 modelled cells, not 1 of 10 drawn cells.
|
||||
assert stats.inundated_fraction == pytest.approx(0.25)
|
||||
assert stats.data_coverage_ratio == pytest.approx(0.4)
|
||||
|
||||
|
||||
def test_cells_outside_the_drawn_selection_are_not_counted() -> None:
|
||||
values = [1.5, 1.5, 0.0, 0.0]
|
||||
selected = [True, False, True, False]
|
||||
|
||||
stats = _stats(values, selected)
|
||||
|
||||
assert stats.selected_cell_count == 2
|
||||
assert stats.valid_cell_count == 2
|
||||
assert stats.inundated_cell_count == 1
|
||||
assert stats.inundated_fraction == pytest.approx(0.5)
|
||||
|
||||
|
||||
def test_a_selection_without_any_model_data_reports_zero_coverage() -> None:
|
||||
stats = _stats([NODATA] * 4, [True] * 4)
|
||||
|
||||
assert stats.valid_cell_count == 0
|
||||
assert stats.no_data_cell_count == 4
|
||||
assert stats.data_coverage_ratio == 0.0
|
||||
# No model, so no risk figure may be invented.
|
||||
assert stats.inundated_fraction is None
|
||||
|
||||
|
||||
def test_nan_is_treated_as_missing_model_data() -> None:
|
||||
stats = _stats([float("nan"), 2.0], [True, True])
|
||||
|
||||
assert stats.valid_cell_count == 1
|
||||
assert stats.no_data_cell_count == 1
|
||||
assert stats.inundated_cell_count == 1
|
||||
|
||||
|
||||
def test_negative_depths_are_data_but_not_inundation() -> None:
|
||||
"""A modelled zero or negative depth means dry, not unknown."""
|
||||
|
||||
stats = _stats([0.0, 0.0, 3.0], [True, True, True])
|
||||
|
||||
assert stats.valid_cell_count == 3
|
||||
assert stats.inundated_cell_count == 1
|
||||
assert stats.inundated_fraction == pytest.approx(1 / 3)
|
||||
|
||||
|
||||
def test_depth_statistics_use_only_inundated_cells() -> None:
|
||||
stats = _stats([0.0, 2.0, 4.0, NODATA], [True] * 4)
|
||||
|
||||
assert stats.depth_values.tolist() == [2.0, 4.0]
|
||||
assert stats.depth_values.mean() == pytest.approx(3.0)
|
||||
|
||||
|
||||
def test_areas_are_derived_from_the_matching_cell_populations() -> None:
|
||||
stats = _stats([1.0, 1.0, 0.0, NODATA], [True] * 4)
|
||||
|
||||
# 100 m2 cells: 2 inundated, 3 modelled, 4 drawn.
|
||||
assert stats.inundated_area_ha(100.0) == pytest.approx(2 * 100.0 / 10_000.0)
|
||||
assert stats.analysed_area_ha(100.0) == pytest.approx(3 * 100.0 / 10_000.0)
|
||||
assert stats.selected_area_ha(100.0) == pytest.approx(4 * 100.0 / 10_000.0)
|
||||
@@ -0,0 +1,22 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_frontend_api_client_accepts_top_level_error_contract() -> None:
|
||||
client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8")
|
||||
|
||||
assert 'typeof payload?.error === "string"' in client
|
||||
assert "payload?.message" in client
|
||||
assert "payload?.details" in client
|
||||
|
||||
|
||||
def test_frontend_api_client_remains_legacy_error_tolerant() -> None:
|
||||
client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8")
|
||||
|
||||
assert "legacyError?.code" in client
|
||||
assert "legacyError?.message" in client
|
||||
assert "legacyError?.details" in client
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Keep the frontend-contract tests from drifting back to formatting checks.
|
||||
|
||||
211 of the backend test files read frontend sources and assert on literal
|
||||
substrings. That reds the suite on renames and copy changes without proving
|
||||
anything about behaviour, and it is why the suite was failing on main.
|
||||
|
||||
This guard blocks the two patterns that caused the failures: counting
|
||||
occurrences of a code fragment (a formatting rule) and asserting an exact
|
||||
number of hook calls. New wiring assertions belong in
|
||||
``tests/frontend_contract.py``; anything about what a user sees belongs in a
|
||||
frontend component test, where the rendered output can be checked.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
TESTS_DIR = Path(__file__).resolve().parent
|
||||
|
||||
# Counting a quoted *code* fragment — one containing a bracket, an arrow or a
|
||||
# semicolon — is the pattern to block. Counting values in a list, or counting
|
||||
# how often a URL was requested, are ordinary behavioural assertions.
|
||||
COUNT_ASSERTION = re.compile(
|
||||
r"""assert\s+\w+\.count\(\s*["'][^"']*[(){}=;][^"']*["']\s*\)\s*[=!<>]=""",
|
||||
)
|
||||
|
||||
# Tests that predate the guard and still count call sites. The list must only
|
||||
# ever shrink; a new entry means a new formatting test was written.
|
||||
KNOWN_COUNT_ASSERTIONS: set[str] = set()
|
||||
|
||||
|
||||
def _test_sources() -> list[tuple[Path, str]]:
|
||||
return [
|
||||
(path, path.read_text(encoding="utf-8"))
|
||||
for path in sorted(TESTS_DIR.glob("test_*.py"))
|
||||
if path.name != Path(__file__).name
|
||||
]
|
||||
|
||||
|
||||
def test_no_test_counts_occurrences_of_a_source_fragment() -> None:
|
||||
offenders = {
|
||||
path.name
|
||||
for path, source in _test_sources()
|
||||
if COUNT_ASSERTION.search(source)
|
||||
}
|
||||
|
||||
unexpected = sorted(offenders - KNOWN_COUNT_ASSERTIONS)
|
||||
assert not unexpected, (
|
||||
"These tests assert on how often a code fragment appears, which is a "
|
||||
f"formatting rule rather than a contract: {unexpected}. Use "
|
||||
"tests/frontend_contract.py to assert on wiring instead."
|
||||
)
|
||||
|
||||
|
||||
def test_the_known_offender_list_does_not_grow_stale() -> None:
|
||||
"""A file listed as an exception must still exist and still offend."""
|
||||
|
||||
offenders = {
|
||||
path.name
|
||||
for path, source in _test_sources()
|
||||
if COUNT_ASSERTION.search(source)
|
||||
}
|
||||
|
||||
stale = sorted(KNOWN_COUNT_ASSERTIONS - offenders)
|
||||
assert not stale, f"Remove these from KNOWN_COUNT_ASSERTIONS; they are clean now: {stale}"
|
||||
|
||||
|
||||
def test_frontend_contract_helpers_are_available() -> None:
|
||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired
|
||||
|
||||
source = "const total = analyzeSelection(bbox, areaIdForSelection(bbox))"
|
||||
assert_wired(source, "analyzeSelection")
|
||||
assert_calls(source, "analyzeSelection", first_argument="bbox")
|
||||
assert_mentions(source, "AREAIDFORSELECTION")
|
||||
|
||||
|
||||
FRONTEND_READ = re.compile(
|
||||
r'(?P<var>\w+)\s*=\s*\(?\s*(?:ROOT|root|REPO_ROOT)\s*/\s*'
|
||||
r'(?P<path>"[^"]+"(?:\s*/\s*"[^"]+")*)\s*\)?\.read_text\('
|
||||
)
|
||||
NEGATIVE_ASSERT = re.compile(r"assert\s+[^\n]*not in\s+(\w+)")
|
||||
|
||||
|
||||
def _feature_owner() -> dict[str, str]:
|
||||
from tests.frontend_contract import FEATURE_SOURCES
|
||||
|
||||
return {source: feature for feature, sources in FEATURE_SOURCES.items() for source in sources}
|
||||
|
||||
|
||||
def test_a_positive_contract_reads_the_feature_not_one_file() -> None:
|
||||
"""Moving code between sibling modules must not red the suite.
|
||||
|
||||
A single-file read is right for a *negative* contract — "this component
|
||||
performs no transport" is a statement about that file, and widening it
|
||||
would quietly weaken the check. For a positive contract it pins the
|
||||
contract to whichever file happens to hold it today.
|
||||
"""
|
||||
|
||||
owner = _feature_owner()
|
||||
offenders: list[str] = []
|
||||
|
||||
for path, source in _test_sources():
|
||||
if "frontend" not in source:
|
||||
continue
|
||||
negatives = set(NEGATIVE_ASSERT.findall(source))
|
||||
for match in FRONTEND_READ.finditer(source):
|
||||
if match.group("var") in negatives:
|
||||
continue
|
||||
joined = re.sub(r'["\s/]+', "/", match.group("path")).strip("/")
|
||||
if "frontend/src/" not in joined:
|
||||
continue
|
||||
relative = joined.split("frontend/src/", 1)[1]
|
||||
if relative in owner:
|
||||
offenders.append(f"{path.name}: {match.group('var')} -> {relative}")
|
||||
|
||||
assert not offenders, (
|
||||
"These read one file of a multi-module feature for a positive contract. "
|
||||
f"Use read_feature() from tests/frontend_contract.py instead: {sorted(offenders)}"
|
||||
)
|
||||
@@ -0,0 +1,188 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.services import geojson_service
|
||||
from app.services.dataset_service import DatasetService
|
||||
|
||||
|
||||
def test_parse_geojson_payload_extracts_metadata() -> None:
|
||||
payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [4.5, 51.3],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
metadata = geojson_service.parse_geojson_payload(payload)
|
||||
|
||||
assert metadata["feature_count"] == 1
|
||||
assert metadata["feature_geometry_count"] == 1
|
||||
assert metadata["bounds_json"] == {
|
||||
"min_x": 4.5,
|
||||
"min_y": 51.3,
|
||||
"max_x": 4.5,
|
||||
"max_y": 51.3,
|
||||
}
|
||||
assert metadata["geometry_types"] == ["Point"]
|
||||
|
||||
|
||||
def test_parse_geojson_payload_rejects_non_feature_collection() -> None:
|
||||
payload = {"type": "Feature", "features": []}
|
||||
|
||||
try:
|
||||
geojson_service.parse_geojson_payload(payload)
|
||||
except ValueError as exc:
|
||||
assert "FeatureCollection" in str(exc)
|
||||
else:
|
||||
raise AssertionError("Invalid GeoJSON should raise ValueError")
|
||||
|
||||
|
||||
def test_get_dataset_geojson_reads_stored_payload(tmp_path, monkeypatch) -> None:
|
||||
file_path = tmp_path / "dataset.geojson"
|
||||
file_path.write_text(
|
||||
json.dumps({"type": "FeatureCollection", "features": []}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
dataset = SimpleNamespace(dataset_type="vector", storage_path=str(file_path))
|
||||
monkeypatch.setattr(DatasetService, "get_dataset", lambda _db, _id: dataset)
|
||||
|
||||
payload = DatasetService.get_dataset_geojson(Path("."), uuid4())
|
||||
|
||||
assert payload["type"] == "FeatureCollection"
|
||||
|
||||
|
||||
def test_get_dataset_geojson_rejects_invalid_stored_json(tmp_path, monkeypatch) -> None:
|
||||
file_path = tmp_path / "invalid.geojson"
|
||||
file_path.write_text("not-json", encoding="utf-8")
|
||||
|
||||
dataset = SimpleNamespace(dataset_type="vector", storage_path=str(file_path))
|
||||
monkeypatch.setattr(DatasetService, "get_dataset", lambda _db, _id: dataset)
|
||||
|
||||
try:
|
||||
DatasetService.get_dataset_geojson(Path("."), uuid4())
|
||||
except AppError as exc:
|
||||
assert exc.code == "INVALID_GEOJSON"
|
||||
else:
|
||||
raise AssertionError("Invalid stored payload should raise AppError")
|
||||
|
||||
|
||||
def test_parse_geojson_payload_returns_vector_metadata() -> None:
|
||||
payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[4.3, 51.2],
|
||||
[4.4, 51.2],
|
||||
[4.4, 51.3],
|
||||
[4.3, 51.3],
|
||||
[4.3, 51.2],
|
||||
]
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:31370"}},
|
||||
}
|
||||
|
||||
metadata = geojson_service.parse_geojson_payload(payload)
|
||||
|
||||
assert metadata["feature_count"] == 1
|
||||
assert metadata["feature_geometry_count"] == 1
|
||||
assert metadata["geometry_types"] == ["Polygon"]
|
||||
assert metadata["bounds_json"] == {
|
||||
"min_x": 4.3,
|
||||
"min_y": 51.2,
|
||||
"max_x": 4.4,
|
||||
"max_y": 51.3,
|
||||
}
|
||||
assert metadata["crs"] == "EPSG:31370"
|
||||
assert metadata["approximate_area_m2"] is not None
|
||||
assert metadata["approximate_area_m2"] >= 0.0
|
||||
|
||||
|
||||
def test_parse_geojson_payload_reports_z_dimension_for_canonical_2d_storage() -> None:
|
||||
metadata = geojson_service.parse_geojson_payload(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.08, 51.18, 0.0]},
|
||||
"properties": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
assert metadata["z_dimension_feature_count"] == 1
|
||||
assert metadata["canonical_storage_dimension"] == "2D"
|
||||
|
||||
|
||||
def test_parse_geojson_payload_rejects_invalid_geometry() -> None:
|
||||
payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": "invalid",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
try:
|
||||
geojson_service.parse_geojson_payload(payload)
|
||||
except ValueError as exc:
|
||||
assert "Invalid feature geometry" in str(exc)
|
||||
else:
|
||||
raise AssertionError("Invalid geometry should raise ValueError")
|
||||
|
||||
|
||||
def test_get_dataset_geojson_accepts_legacy_geojson_type(tmp_path, monkeypatch) -> None:
|
||||
file_path = tmp_path / "legacy.geojson"
|
||||
file_path.write_text(
|
||||
json.dumps({"type": "FeatureCollection", "features": []}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
dataset = SimpleNamespace(dataset_type="geojson", storage_path=str(file_path))
|
||||
monkeypatch.setattr(DatasetService, "get_dataset", lambda _db, _id: dataset)
|
||||
|
||||
payload = DatasetService.get_dataset_geojson(Path("."), uuid4())
|
||||
assert payload["type"] == "FeatureCollection"
|
||||
|
||||
|
||||
def test_vector_summary_supports_legacy_geojson_type(monkeypatch) -> None:
|
||||
dataset = SimpleNamespace(
|
||||
dataset_type="geojson",
|
||||
metadata_json={
|
||||
"feature_count": 7,
|
||||
"geometry_types": ["Point"],
|
||||
"bounds_json": {"min_x": 0.0, "min_y": 0.0, "max_x": 1.0, "max_y": 1.0},
|
||||
},
|
||||
storage_path="",
|
||||
)
|
||||
monkeypatch.setattr(DatasetService, "get_dataset", lambda _db, _id: dataset)
|
||||
|
||||
summary = DatasetService.vector_summary(Path("."), uuid4())
|
||||
assert summary["feature_count"] == 7
|
||||
assert summary["geometry_types"] == ["Point"]
|
||||
@@ -0,0 +1,765 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
from pyproj import Transformer
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import (
|
||||
Area,
|
||||
Dataset,
|
||||
DatasetQuarantine,
|
||||
DatasetVersion,
|
||||
Project,
|
||||
SourceRegistry,
|
||||
SourceSnapshot,
|
||||
VectorFeature,
|
||||
)
|
||||
from app.services.dataset_service import DatasetService, _PartitionedGeoJsonRecords
|
||||
from app.services.vector_operations_service import VectorOperationsService
|
||||
|
||||
|
||||
class _Query:
|
||||
def __init__(self, session: "_Session", model: type) -> None:
|
||||
self.session = session
|
||||
self.model = model
|
||||
self.predicates = []
|
||||
|
||||
def filter(self, *predicates):
|
||||
self.predicates.extend(predicates)
|
||||
return self
|
||||
|
||||
def one_or_none(self):
|
||||
matches = self._matches()
|
||||
if len(matches) > 1:
|
||||
raise AssertionError(
|
||||
f"expected one {self.model.__name__}, found {len(matches)}"
|
||||
)
|
||||
return matches[0] if matches else None
|
||||
|
||||
def all(self):
|
||||
return self._matches()
|
||||
|
||||
def _matches(self):
|
||||
matches = list(self.session.rows.get(self.model, []))
|
||||
for predicate in self.predicates:
|
||||
field_name = predicate.left.key
|
||||
expected = predicate.right.value
|
||||
operator_name = getattr(predicate.operator, "__name__", "")
|
||||
if operator_name == "in_op":
|
||||
matches = [
|
||||
item for item in matches if getattr(item, field_name) in expected
|
||||
]
|
||||
else:
|
||||
matches = [
|
||||
item for item in matches if getattr(item, field_name) == expected
|
||||
]
|
||||
return matches
|
||||
|
||||
|
||||
class _Session:
|
||||
"""Small ORM-shaped harness that exercises the real governed path."""
|
||||
|
||||
def __init__(self, project: Project) -> None:
|
||||
self.rows: dict[type, list[object]] = {Project: [project]}
|
||||
self.commits = 0
|
||||
self.rollbacks = 0
|
||||
self.flushes = 0
|
||||
|
||||
def get(self, model: type, item_id: UUID):
|
||||
return next(
|
||||
(
|
||||
item
|
||||
for item in self.rows.get(model, [])
|
||||
if getattr(item, "id", None) == item_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
def query(self, model: type) -> _Query:
|
||||
return _Query(self, model)
|
||||
|
||||
def add(self, item: object) -> None:
|
||||
if getattr(item, "id", None) is None:
|
||||
setattr(item, "id", uuid4())
|
||||
self.rows.setdefault(type(item), []).append(item)
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flushes += 1
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def rollback(self) -> None:
|
||||
self.rollbacks += 1
|
||||
|
||||
def refresh(self, _item: object) -> None:
|
||||
return None
|
||||
|
||||
def expunge(self, _item: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _storage_info(tmp_path: Path, content: bytes) -> dict[str, object]:
|
||||
path = tmp_path / "grb-buildings.geojson"
|
||||
path.write_bytes(content)
|
||||
return {
|
||||
"storage_path": str(path),
|
||||
"original_filename": path.name,
|
||||
"stored_filename": path.name,
|
||||
"content_type": "application/geo+json",
|
||||
"size_bytes": len(content),
|
||||
"checksum_sha256": sha256(content).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def _valid_payload() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": "gbg-1",
|
||||
"properties": {"id": "gbg-1"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _grb_payload_without_required_id() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {"unrelated": "not a GRB identity"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _lambert_grb_payload() -> tuple[bytes, tuple[float, float, float, float]]:
|
||||
"""Create a valid GRB-shaped source artifact in its declared native CRS."""
|
||||
|
||||
longitude, latitude = 4.70, 51.10
|
||||
max_longitude, max_latitude = 4.7001, 51.1001
|
||||
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
lambert_ring = [
|
||||
to_lambert.transform(longitude, latitude),
|
||||
to_lambert.transform(max_longitude, latitude),
|
||||
to_lambert.transform(max_longitude, max_latitude),
|
||||
to_lambert.transform(longitude, latitude),
|
||||
]
|
||||
return (
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:31370"}},
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": "GBG.lambert.1",
|
||||
"properties": {"id": "GBG.lambert.1"},
|
||||
"geometry": {"type": "Polygon", "coordinates": [lambert_ring]},
|
||||
}
|
||||
],
|
||||
}
|
||||
).encode("utf-8"),
|
||||
(longitude, latitude, max_longitude, max_latitude),
|
||||
)
|
||||
|
||||
|
||||
def test_governed_vector_import_persists_snapshot_contract_and_queryable_features(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
project = Project(id=uuid4(), name="Phase 2 governed ingest")
|
||||
db = _Session(project)
|
||||
raw = _valid_payload()
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **_kwargs: _storage_info(tmp_path, raw),
|
||||
)
|
||||
|
||||
result = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-buildings.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
"license": "Open data",
|
||||
"source_url": "https://example.invalid/grb",
|
||||
},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:2026-08",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||
snapshot = db.rows[SourceSnapshot][0]
|
||||
source = db.rows[SourceRegistry][0]
|
||||
assert result.status == "ready"
|
||||
assert dataset.source_name == "grb"
|
||||
assert dataset.source_registry_id == source.id
|
||||
assert dataset.source_snapshot_id == snapshot.id
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.provenance_status == "complete"
|
||||
assert dataset.lineage_status == "complete"
|
||||
assert dataset.quarantine_status == "not_quarantined"
|
||||
assert dataset.crs == "EPSG:4326"
|
||||
assert snapshot.checksum_sha256 == sha256(raw).hexdigest()
|
||||
assert len(db.rows[VectorFeature]) == 1
|
||||
assert db.commits == 1
|
||||
|
||||
# A retry with identical governed evidence is idempotent and does not
|
||||
# create a second source snapshot, dataset or vector feature.
|
||||
repeated = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-buildings.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:2026-08",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
assert repeated.id == result.id
|
||||
assert len(db.rows[Dataset]) == 1
|
||||
assert len(db.rows[SourceSnapshot]) == 1
|
||||
assert len(db.rows[VectorFeature]) == 1
|
||||
|
||||
|
||||
def test_governed_lambert_geojson_persists_canonical_consumption_bytes_and_provenance_evidence(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Projected source bytes must never be the file that vector operations consume."""
|
||||
|
||||
project = Project(id=uuid4(), name="Canonical GeoJSON storage")
|
||||
db = _Session(project)
|
||||
raw, (longitude, latitude, max_longitude, max_latitude) = _lambert_grb_payload()
|
||||
consumption_path = tmp_path / "consumption" / "grb-buildings.geojson"
|
||||
provenance_path = tmp_path / "provenance" / "grb-buildings.geojson"
|
||||
|
||||
def _persist_dataset_file(**kwargs):
|
||||
stored = kwargs["content"]
|
||||
consumption_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
consumption_path.write_bytes(stored)
|
||||
return _storage_info(consumption_path.parent, stored)
|
||||
|
||||
def _persist_file(storage_path, content, original_filename, content_type):
|
||||
del storage_path, original_filename, content_type
|
||||
provenance_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
provenance_path.write_bytes(content)
|
||||
return _storage_info(provenance_path.parent, content)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
_persist_dataset_file,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_file",
|
||||
_persist_file,
|
||||
)
|
||||
|
||||
result = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-lambert.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:lambert:2026-08",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01-lambert",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||
dataset_version = db.rows[DatasetVersion][0]
|
||||
snapshot = db.rows[SourceSnapshot][0]
|
||||
canonical_bytes = Path(str(dataset.storage_path)).read_bytes()
|
||||
canonical_payload = json.loads(canonical_bytes)
|
||||
source_artifact = dataset.provenance_metadata["source_artifact"]
|
||||
|
||||
assert result.status == "ready"
|
||||
assert dataset.crs == "EPSG:4326"
|
||||
assert canonical_payload["crs"]["properties"]["name"] == "EPSG:4326"
|
||||
assert canonical_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
|
||||
[longitude, latitude], abs=0.000001
|
||||
)
|
||||
assert sha256(canonical_bytes).hexdigest() == dataset.checksum_sha256
|
||||
assert dataset_version.checksum_sha256 == dataset.checksum_sha256
|
||||
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
||||
assert source_artifact["retention"] == "provenance_evidence_only"
|
||||
assert source_artifact["checksum_sha256"] == sha256(raw).hexdigest()
|
||||
assert source_artifact["storage_path"] != dataset.storage_path
|
||||
assert Path(source_artifact["storage_path"]).read_bytes() == raw
|
||||
assert dataset.provenance_metadata["canonical_consumption_artifact"] == {
|
||||
"checksum_sha256": dataset.checksum_sha256,
|
||||
"crs": "EPSG:4326",
|
||||
"storage_role": "dataset_consumption",
|
||||
}
|
||||
|
||||
inspection = VectorOperationsService.inspect(db, dataset.id)
|
||||
assert inspection.crs == "EPSG:4326"
|
||||
assert inspection.bounds_json == {
|
||||
"min_x": pytest.approx(longitude, abs=0.000001),
|
||||
"min_y": pytest.approx(latitude, abs=0.000001),
|
||||
"max_x": pytest.approx(max_longitude, abs=0.000001),
|
||||
"max_y": pytest.approx(max_latitude, abs=0.000001),
|
||||
}
|
||||
response_payload = DatasetService.get_dataset_geojson(db, dataset.id)
|
||||
assert response_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
|
||||
[longitude, latitude], abs=0.000001
|
||||
)
|
||||
|
||||
# The storage identity is enforced at the operation boundary too; a
|
||||
# replacement with different canonical bytes is not silently processed.
|
||||
Path(str(dataset.storage_path)).write_bytes(canonical_bytes + b"\n")
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
VectorOperationsService.inspect(db, dataset.id)
|
||||
assert exc_info.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH"
|
||||
|
||||
|
||||
def test_metadata_refresh_refuses_mutated_governed_artifact(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""A passed snapshot cannot be silently re-described from mutable storage."""
|
||||
|
||||
project = Project(id=uuid4(), name="Phase 2 immutable refresh")
|
||||
db = _Session(project)
|
||||
raw = _valid_payload()
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **_kwargs: _storage_info(tmp_path, raw),
|
||||
)
|
||||
result = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-buildings.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:2026-08",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||
original_checksum = dataset.checksum_sha256
|
||||
original_metadata = dict(dataset.metadata_json or {})
|
||||
original_commit_count = db.commits
|
||||
|
||||
# Simulate an out-of-band storage replacement at the same path. The
|
||||
# refresh endpoint must not parse it into an already-passed contract row.
|
||||
Path(str(dataset.storage_path)).write_bytes(_grb_payload_without_required_id())
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetService.refresh_metadata(db, dataset.id)
|
||||
|
||||
assert exc_info.value.code == "GOVERNED_DATASET_REINGEST_REQUIRED"
|
||||
assert dataset.status == "ready"
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.checksum_sha256 == original_checksum
|
||||
assert dataset.metadata_json == original_metadata
|
||||
assert db.commits == original_commit_count
|
||||
|
||||
|
||||
def test_governed_import_quarantines_bad_artifacts_and_refuses_unknown_source(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
project = Project(id=uuid4(), name="Phase 2 quarantine")
|
||||
db = _Session(project)
|
||||
raw = b'{"type":"FeatureCollection","features":[]}'
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **_kwargs: _storage_info(tmp_path, raw),
|
||||
)
|
||||
|
||||
quarantined = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="empty.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={},
|
||||
temporal_series_key="grb:2026-08-empty",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01-empty",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
assert quarantined.status == "quarantined"
|
||||
assert quarantined.validation_status == "failed"
|
||||
assert quarantined.quarantine_status == "quarantined"
|
||||
assert len(db.rows[DatasetQuarantine]) == 1
|
||||
assert db.rows[SourceSnapshot][0].ingest_status == "quarantined"
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="unregistered.geojson",
|
||||
content=_valid_payload(),
|
||||
source="caller_controlled",
|
||||
source_name="caller_claimed_grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={},
|
||||
)
|
||||
assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND"
|
||||
|
||||
|
||||
def test_governed_grb_vector_quarantines_missing_server_owned_required_attribute(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
project = Project(id=uuid4(), name="Phase 2 source schema")
|
||||
db = _Session(project)
|
||||
raw = _grb_payload_without_required_id()
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **kwargs: _storage_info(tmp_path, kwargs["content"]),
|
||||
)
|
||||
|
||||
quarantined = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-missing-id.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:missing-id",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01-missing-id",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == quarantined.id)
|
||||
assert quarantined.status == "quarantined"
|
||||
assert dataset.validation_status == "failed"
|
||||
assert dataset.quarantine_status == "quarantined"
|
||||
issue = dataset.validation_report_json["issues"][0]
|
||||
assert issue["code"] == "SOURCE_SCHEMA_REQUIRED_ATTRIBUTE_MISSING"
|
||||
assert issue["category"] == "source_schema"
|
||||
assert len(db.rows[DatasetQuarantine]) == 1
|
||||
|
||||
|
||||
def test_partitioned_vector_ingest_is_idempotent_and_quarantines_noncanonical_partition_coordinates(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
project = Project(id=uuid4(), name="Partitioned governed ingest")
|
||||
area = Area(id=uuid4(), project_id=project.id, name="Partitioned AOI")
|
||||
db = _Session(project)
|
||||
db.rows[Area] = [area]
|
||||
|
||||
feature = {
|
||||
"type": "Feature",
|
||||
"id": "GBG.1",
|
||||
"properties": {"id": "GBG.1", "source_feature_id": "GBG.1"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
||||
],
|
||||
},
|
||||
}
|
||||
partition_payload = {"type": "FeatureCollection", "features": [feature]}
|
||||
partition_path = tmp_path / "partition-01.geojson"
|
||||
partition_path.write_text(json.dumps(partition_payload), encoding="utf-8")
|
||||
artifact_payload = {
|
||||
"type": "FeatureCollection",
|
||||
"crs": "EPSG:4326",
|
||||
"features": [feature],
|
||||
}
|
||||
artifact_path = tmp_path / "grb-partitioned.geojson"
|
||||
artifact_raw = json.dumps(artifact_payload).encode("utf-8")
|
||||
artifact_path.write_bytes(artifact_raw)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
||||
lambda **_kwargs: _storage_info(tmp_path, artifact_raw),
|
||||
)
|
||||
|
||||
result = DatasetService.import_partitioned_vector_artifact(
|
||||
db,
|
||||
project_id=project.id,
|
||||
area_id=area.id,
|
||||
artifact_path=artifact_path,
|
||||
partition_paths=[partition_path],
|
||||
original_filename="grb-partitioned.geojson",
|
||||
source="operator_official_import",
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
metadata_json={
|
||||
"feature_count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds_json": {
|
||||
"min_x": 4.69,
|
||||
"min_y": 51.09,
|
||||
"max_x": 4.70,
|
||||
"max_y": 51.10,
|
||||
},
|
||||
},
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={
|
||||
"artifact_sha256": sha256(artifact_raw).hexdigest(),
|
||||
"partition_checksums": {
|
||||
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
||||
},
|
||||
},
|
||||
temporal_series_key="grb:partitioned:test",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
)
|
||||
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||
assert result.status == "ready"
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.provenance_status == "complete"
|
||||
assert dataset.source_name == "grb"
|
||||
assert len(db.rows[SourceSnapshot]) == 1
|
||||
assert len(db.rows[VectorFeature]) == 1
|
||||
assert dataset.metadata_json["partitioned_geometry_audit"][
|
||||
"partition_checksums_sha256"
|
||||
] == {partition_path.name: sha256(partition_path.read_bytes()).hexdigest()}
|
||||
assert dataset.provenance_metadata["partition_checksum_manifest_sha256"]
|
||||
assert dataset.provenance_metadata["partitioned_artifact_binding_sha256"]
|
||||
|
||||
repeated = DatasetService.import_partitioned_vector_artifact(
|
||||
db,
|
||||
project_id=project.id,
|
||||
area_id=area.id,
|
||||
artifact_path=artifact_path,
|
||||
partition_paths=[partition_path],
|
||||
original_filename="grb-partitioned.geojson",
|
||||
source="operator_official_import",
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
metadata_json={
|
||||
"feature_count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds_json": {
|
||||
"min_x": 4.69,
|
||||
"min_y": 51.09,
|
||||
"max_x": 4.70,
|
||||
"max_y": 51.10,
|
||||
},
|
||||
},
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={
|
||||
"artifact_sha256": sha256(artifact_raw).hexdigest(),
|
||||
"partition_checksums": {
|
||||
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
||||
},
|
||||
},
|
||||
temporal_series_key="grb:partitioned:test",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
)
|
||||
assert repeated.id == result.id
|
||||
assert len(db.rows[Dataset]) == 1
|
||||
assert len(db.rows[VectorFeature]) == 1
|
||||
|
||||
lambert_feature = {
|
||||
**feature,
|
||||
"id": "GBG.lambert",
|
||||
"properties": {"id": "GBG.lambert"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[150000, 170000], [150010, 170000], [150010, 170010], [150000, 170000]]
|
||||
],
|
||||
},
|
||||
}
|
||||
lambert_partition = tmp_path / "partition-lambert.geojson"
|
||||
lambert_partition.write_text(
|
||||
json.dumps({"type": "FeatureCollection", "features": [lambert_feature]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
lambert_artifact = tmp_path / "grb-lambert.geojson"
|
||||
lambert_raw = json.dumps(
|
||||
{"type": "FeatureCollection", "features": [lambert_feature]}
|
||||
).encode("utf-8")
|
||||
lambert_artifact.write_bytes(lambert_raw)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
||||
lambda **_kwargs: _storage_info(tmp_path, lambert_raw),
|
||||
)
|
||||
|
||||
quarantined = DatasetService.import_partitioned_vector_artifact(
|
||||
db,
|
||||
project_id=project.id,
|
||||
area_id=area.id,
|
||||
artifact_path=lambert_artifact,
|
||||
partition_paths=[lambert_partition],
|
||||
original_filename="grb-lambert.geojson",
|
||||
source="operator_official_import",
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
metadata_json={
|
||||
"feature_count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds_json": {
|
||||
"min_x": 150000,
|
||||
"min_y": 170000,
|
||||
"max_x": 150010,
|
||||
"max_y": 170010,
|
||||
},
|
||||
},
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={
|
||||
"artifact_sha256": sha256(lambert_raw).hexdigest(),
|
||||
"partition_checksums": {
|
||||
lambert_partition.name: sha256(
|
||||
lambert_partition.read_bytes()
|
||||
).hexdigest()
|
||||
},
|
||||
},
|
||||
temporal_series_key="grb:partitioned:lambert",
|
||||
observed_at=datetime(2026, 8, 2, tzinfo=UTC),
|
||||
source_version="2026-08-02",
|
||||
)
|
||||
assert quarantined.status == "quarantined"
|
||||
assert quarantined.validation_status == "failed"
|
||||
assert quarantined.quarantine_status == "quarantined"
|
||||
|
||||
missing_manifest_partition = tmp_path / "partition-missing-manifest.geojson"
|
||||
missing_manifest_partition.write_text(
|
||||
json.dumps(partition_payload), encoding="utf-8"
|
||||
)
|
||||
missing_manifest_artifact = tmp_path / "grb-missing-manifest.geojson"
|
||||
missing_manifest_raw = json.dumps(
|
||||
{"type": "FeatureCollection", "features": [feature]}
|
||||
).encode("utf-8")
|
||||
missing_manifest_artifact.write_bytes(missing_manifest_raw)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
||||
lambda **_kwargs: _storage_info(tmp_path, missing_manifest_raw),
|
||||
)
|
||||
|
||||
missing_manifest = DatasetService.import_partitioned_vector_artifact(
|
||||
db,
|
||||
project_id=project.id,
|
||||
area_id=area.id,
|
||||
artifact_path=missing_manifest_artifact,
|
||||
partition_paths=[missing_manifest_partition],
|
||||
original_filename="grb-missing-manifest.geojson",
|
||||
source="operator_official_import",
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
metadata_json={
|
||||
"feature_count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds_json": {
|
||||
"min_x": 4.69,
|
||||
"min_y": 51.09,
|
||||
"max_x": 4.70,
|
||||
"max_y": 51.10,
|
||||
},
|
||||
},
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={
|
||||
"artifact_sha256": sha256(missing_manifest_raw).hexdigest()
|
||||
},
|
||||
temporal_series_key="grb:partitioned:missing-manifest",
|
||||
observed_at=datetime(2026, 8, 3, tzinfo=UTC),
|
||||
source_version="2026-08-03",
|
||||
)
|
||||
assert missing_manifest.status == "quarantined"
|
||||
assert (
|
||||
missing_manifest.validation_report_json["issues"][0]["code"]
|
||||
== "PARTITION_CHECKSUM_MANIFEST_REQUIRED"
|
||||
)
|
||||
|
||||
|
||||
def test_partitioned_geometry_audit_handles_more_than_generic_topology_limit_without_materializing_geometries(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
feature_count = 10_001
|
||||
partition_path = tmp_path / "large-partition.geojson"
|
||||
partition_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": f"GBG.{index}",
|
||||
"properties": {"id": f"GBG.{index}"},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [4.0 + index / 10_000_000, 51.0],
|
||||
},
|
||||
}
|
||||
for index in range(feature_count)
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
audit = _PartitionedGeoJsonRecords(
|
||||
[partition_path],
|
||||
expected_feature_count=feature_count,
|
||||
declared_partition_checksums={
|
||||
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
||||
},
|
||||
).audit()
|
||||
|
||||
assert audit.feature_count == feature_count
|
||||
assert audit.bounds_json["min_x"] == 4.0
|
||||
assert audit.bounds_json["max_x"] > audit.bounds_json["min_x"]
|
||||
assert audit.representative_record.geometry.geom_type == "MultiPoint"
|
||||
@@ -0,0 +1,127 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "build_grayscale_yolo_dataset.py"
|
||||
|
||||
|
||||
def test_grayscale_builder_preserves_labels_and_split(tmp_path: Path) -> None:
|
||||
source = tmp_path / "source"
|
||||
entries = []
|
||||
for split, sample_slug, colour in (("train", "fixture-train", (255, 0, 0)), ("val", "fixture-val", (0, 255, 0))):
|
||||
image = source / "images" / split / f"{sample_slug}.png"
|
||||
label = source / "labels" / split / f"{sample_slug}.txt"
|
||||
image.parent.mkdir(parents=True, exist_ok=True)
|
||||
label.parent.mkdir(parents=True, exist_ok=True)
|
||||
Image.new("RGB", (8, 8), colour).save(image)
|
||||
label.write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8")
|
||||
entries.append((split, sample_slug, image, label))
|
||||
manifest = source / "operator_samples_manifest.json"
|
||||
policy = "geointel-training-source-eligibility/v1"
|
||||
manifest.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"immutable": True,
|
||||
"training_eligibility": {"policy_version": policy, "status": "eligible", "fixture_mode": True},
|
||||
"samples": [
|
||||
{
|
||||
"sample_slug": sample_slug,
|
||||
"split": split,
|
||||
"raster_dataset_id": f"raster:{sample_slug}",
|
||||
"reference_dataset_id": f"reference:{sample_slug}",
|
||||
"training_eligibility": {
|
||||
"policy_version": policy,
|
||||
"eligible": True,
|
||||
"fixture_mode": True,
|
||||
"raster": {"eligible": True, "reasons": [], "evidence": {"dataset_id": f"raster:{sample_slug}", "checksum_sha256": "a" * 64, "source_registry_id": "fixture-raster", "source_snapshot_id": "fixture-raster-snapshot"}},
|
||||
"reference": {"eligible": True, "reasons": [], "evidence": {"dataset_id": f"reference:{sample_slug}", "checksum_sha256": "b" * 64, "source_registry_id": "fixture-reference", "source_snapshot_id": "fixture-reference-snapshot"}},
|
||||
},
|
||||
}
|
||||
for split, sample_slug, _image, _label in entries
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(source / "corpus-freeze.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 2,
|
||||
"immutable": True,
|
||||
"fixture_mode": True,
|
||||
"training_eligibility_policy": policy,
|
||||
"manifest_sha256": sha256(manifest.read_bytes()).hexdigest(),
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
dataset_yaml = source / "dataset.yaml"
|
||||
dataset_yaml.write_text(
|
||||
f"path: {source}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
release_script = SCRIPT.parent / "training_release_manifest.py"
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(release_script),
|
||||
"create",
|
||||
"--train-yaml",
|
||||
str(dataset_yaml),
|
||||
"--corpus-manifest",
|
||||
str(manifest),
|
||||
"--fixture-mode",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
release_path = dataset_yaml.with_name(dataset_yaml.name + ".geointel-training-release.json")
|
||||
asset_path = dataset_yaml.with_name(dataset_yaml.name + ".geointel-training-assets.json")
|
||||
assets = json.loads(asset_path.read_text(encoding="utf-8"))
|
||||
summary = source / "yolo_tile_dataset_summary.json"
|
||||
summary.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"dataset_yaml": str(dataset_yaml.resolve()),
|
||||
"training_release_manifest": str(release_path.resolve()),
|
||||
"training_release_manifest_sha256": sha256(release_path.read_bytes()).hexdigest(),
|
||||
"training_asset_manifest": str(asset_path.resolve()),
|
||||
"source_manifest_sha256": sha256(manifest.read_bytes()).hexdigest(),
|
||||
"tiles": [
|
||||
{"split": entry["split"], "image_path": entry["image_path"], "label_path": entry["label_path"]}
|
||||
for entry in assets["entries"]
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
output = tmp_path / "gray"
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--summary",
|
||||
str(summary),
|
||||
"--train-yaml",
|
||||
str(dataset_yaml),
|
||||
"--corpus-manifest",
|
||||
str(manifest),
|
||||
"--fixture-mode",
|
||||
"--output-dir",
|
||||
str(output),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
converted = Image.open(output / "images" / "train" / "fixture-train.png")
|
||||
r, g, b = converted.getpixel((0, 0))
|
||||
assert r == g == b
|
||||
assert (output / "labels" / "train" / "fixture-train.txt").read_text() == "0 0.5 0.5 0.5 0.5\n"
|
||||
evidence = json.loads((output / "grayscale-dataset-evidence.json").read_text())
|
||||
assert evidence["converted_tile_count"] == 2
|
||||
assert evidence["training_eligible"] is False
|
||||
@@ -0,0 +1,348 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.public_demo import PUBLIC_DEMO_PROJECT_ID
|
||||
from app.db.session import get_db
|
||||
from app.main import create_app
|
||||
from app.models import AnalysisRun, Dataset, Detection, Export, Job, Segmentation
|
||||
from app.schemas import (
|
||||
DetectionRunListResponse,
|
||||
DetectionRunResponse,
|
||||
SegmentationRunListResponse,
|
||||
SegmentationRunResponse,
|
||||
)
|
||||
from app.services.auth_service import AuthService
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
|
||||
|
||||
GUEST_PROJECT_ID = PUBLIC_DEMO_PROJECT_ID
|
||||
OTHER_PROJECT_ID = UUID("00000000-0000-0000-0000-000000000999")
|
||||
DATASET_ID = UUID("00000000-0000-0000-0000-000000000201")
|
||||
DETECTION_RUN_ID = UUID("00000000-0000-0000-0000-000000000202")
|
||||
SEGMENTATION_RUN_ID = UUID("00000000-0000-0000-0000-000000000203")
|
||||
DETECTION_ID = UUID("00000000-0000-0000-0000-000000000204")
|
||||
SEGMENTATION_ID = UUID("00000000-0000-0000-0000-000000000205")
|
||||
EXPORT_ID = UUID("00000000-0000-0000-0000-000000000206")
|
||||
JOB_ID = UUID("00000000-0000-0000-0000-000000000207")
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, objects: dict[tuple[type, UUID], object]) -> None:
|
||||
self.objects = objects
|
||||
|
||||
def get(self, model, row_id):
|
||||
return self.objects.get((model, row_id))
|
||||
|
||||
|
||||
def _guest_client(monkeypatch, db: FakeSession) -> TestClient:
|
||||
password_hash = AuthService.hash_password(
|
||||
"operator-password",
|
||||
salt=b"guest-scope-test-salt",
|
||||
iterations=100_000,
|
||||
)
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_ENABLED", "true")
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_USERNAME", "operator")
|
||||
monkeypatch.setenv("GEOINTEL_AUTH_PASSWORD_HASH", password_hash)
|
||||
monkeypatch.setenv(
|
||||
"GEOINTEL_AUTH_SESSION_SECRET",
|
||||
"guest-scope-test-session-secret-value",
|
||||
)
|
||||
monkeypatch.setenv("GEOINTEL_GUEST_ACCESS_ENABLED", "true")
|
||||
monkeypatch.setenv("GEOINTEL_GUEST_DISPLAY_NAME", "Gast")
|
||||
|
||||
client = TestClient(create_app())
|
||||
|
||||
def fake_db():
|
||||
yield db
|
||||
|
||||
client.app.dependency_overrides[get_db] = fake_db
|
||||
token = AuthService.create_session_token(
|
||||
"Gast",
|
||||
get_settings(),
|
||||
role="guest",
|
||||
project_id=GUEST_PROJECT_ID,
|
||||
)
|
||||
client.cookies.set("geointel_session", token)
|
||||
return client
|
||||
|
||||
|
||||
def _project_objects(project_id: UUID, export_path: Path) -> dict[tuple[type, UUID], object]:
|
||||
return {
|
||||
(Dataset, DATASET_ID): Dataset(
|
||||
id=DATASET_ID,
|
||||
project_id=project_id,
|
||||
name="scope-test.tif",
|
||||
dataset_type="raster",
|
||||
source="fixture",
|
||||
),
|
||||
(AnalysisRun, DETECTION_RUN_ID): AnalysisRun(
|
||||
id=DETECTION_RUN_ID,
|
||||
project_id=project_id,
|
||||
dataset_id=DATASET_ID,
|
||||
analysis_type="detection",
|
||||
status="success",
|
||||
parameters_json={},
|
||||
),
|
||||
(AnalysisRun, SEGMENTATION_RUN_ID): AnalysisRun(
|
||||
id=SEGMENTATION_RUN_ID,
|
||||
project_id=project_id,
|
||||
dataset_id=DATASET_ID,
|
||||
analysis_type="segmentation",
|
||||
status="success",
|
||||
parameters_json={},
|
||||
),
|
||||
(Detection, DETECTION_ID): Detection(
|
||||
id=DETECTION_ID,
|
||||
project_id=project_id,
|
||||
dataset_id=DATASET_ID,
|
||||
analysis_run_id=DETECTION_RUN_ID,
|
||||
model_name="fixture-detector",
|
||||
class_name="building",
|
||||
confidence=0.9,
|
||||
geometry="SRID=4326;POINT (5 51)",
|
||||
),
|
||||
(Segmentation, SEGMENTATION_ID): Segmentation(
|
||||
id=SEGMENTATION_ID,
|
||||
project_id=project_id,
|
||||
dataset_id=DATASET_ID,
|
||||
analysis_run_id=SEGMENTATION_RUN_ID,
|
||||
model_name="fixture-segmenter",
|
||||
class_name="building",
|
||||
confidence=0.9,
|
||||
geometry="SRID=4326;MULTIPOLYGON (((5 51, 5.1 51, 5.1 51.1, 5 51)))",
|
||||
),
|
||||
(Export, EXPORT_ID): Export(
|
||||
id=EXPORT_ID,
|
||||
project_id=project_id,
|
||||
export_type="dataset_geojson",
|
||||
storage_path=str(export_path),
|
||||
metadata_json={},
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
[
|
||||
f"/api/v1/detection/runs/{DETECTION_RUN_ID}",
|
||||
f"/api/v1/detection/runs/{DETECTION_RUN_ID}/detections",
|
||||
f"/api/v1/detection/runs/{DETECTION_RUN_ID}/geojson",
|
||||
f"/api/v1/detection/datasets/{DATASET_ID}/detections",
|
||||
f"/api/v1/detection/datasets/{DATASET_ID}/geojson",
|
||||
f"/api/v1/detection/detections/{DETECTION_ID}",
|
||||
f"/api/v1/segmentation/runs/{SEGMENTATION_RUN_ID}",
|
||||
f"/api/v1/segmentation/runs/{SEGMENTATION_RUN_ID}/segmentations",
|
||||
f"/api/v1/segmentation/runs/{SEGMENTATION_RUN_ID}/geojson",
|
||||
f"/api/v1/segmentation/datasets/{DATASET_ID}/segmentations",
|
||||
f"/api/v1/segmentation/datasets/{DATASET_ID}/geojson",
|
||||
f"/api/v1/segmentation/segmentations/{SEGMENTATION_ID}",
|
||||
f"/api/v1/exports/{EXPORT_ID}",
|
||||
f"/api/v1/exports/{EXPORT_ID}/content",
|
||||
f"/api/v1/exports/{EXPORT_ID}/download",
|
||||
f"/api/v1/exports/projects/{OTHER_PROJECT_ID}/exports",
|
||||
],
|
||||
)
|
||||
def test_matching_guest_query_cannot_authorize_another_projects_resource(
|
||||
path: str,
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
artifact = tmp_path / "other-project.geojson"
|
||||
artifact.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
|
||||
client = _guest_client(monkeypatch, FakeSession(_project_objects(OTHER_PROJECT_ID, artifact)))
|
||||
|
||||
response = client.get(f"{path}?project_id={GUEST_PROJECT_ID}")
|
||||
|
||||
assert response.status_code == 403
|
||||
assert response.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("path", "payload"),
|
||||
[
|
||||
(
|
||||
"/api/v1/detection/run",
|
||||
{"project_id": str(OTHER_PROJECT_ID), "dataset_id": str(DATASET_ID), "model_id": "fixture"},
|
||||
),
|
||||
(
|
||||
"/api/v1/detection/run-async",
|
||||
{"project_id": str(OTHER_PROJECT_ID), "dataset_id": str(DATASET_ID), "model_id": "fixture"},
|
||||
),
|
||||
(
|
||||
"/api/v1/segmentation/run",
|
||||
{"project_id": str(OTHER_PROJECT_ID), "dataset_id": str(DATASET_ID), "model_id": "fixture"},
|
||||
),
|
||||
(
|
||||
"/api/v1/segmentation/run-async",
|
||||
{"project_id": str(OTHER_PROJECT_ID), "dataset_id": str(DATASET_ID), "model_id": "fixture"},
|
||||
),
|
||||
(
|
||||
"/api/v1/detection/runs/{run_id}/qa/reference".format(run_id=DETECTION_RUN_ID),
|
||||
{"reference_dataset_id": str(DATASET_ID)},
|
||||
),
|
||||
(
|
||||
"/api/v1/segmentation/runs/{run_id}/qa/reference".format(run_id=SEGMENTATION_RUN_ID),
|
||||
{"reference_dataset_id": str(DATASET_ID)},
|
||||
),
|
||||
(
|
||||
"/api/v1/exports/geojson",
|
||||
{"export_kind": "dataset", "dataset_id": str(DATASET_ID)},
|
||||
),
|
||||
(
|
||||
"/api/v1/exports/geojson",
|
||||
{"export_kind": "detection_run", "analysis_run_id": str(DETECTION_RUN_ID)},
|
||||
),
|
||||
(
|
||||
"/api/v1/exports/geojson",
|
||||
{"export_kind": "segmentation_run", "analysis_run_id": str(SEGMENTATION_RUN_ID)},
|
||||
),
|
||||
(
|
||||
"/api/v1/exports/metadata",
|
||||
{"project_id": str(OTHER_PROJECT_ID)},
|
||||
),
|
||||
(
|
||||
"/api/v1/exports/report",
|
||||
{"project_id": str(OTHER_PROJECT_ID)},
|
||||
),
|
||||
(
|
||||
"/api/v1/exports/map-result",
|
||||
{
|
||||
"project_id": str(OTHER_PROJECT_ID),
|
||||
"mode": "current",
|
||||
"dataset_id": str(DATASET_ID),
|
||||
"bbox": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.1, "max_y": 51.1, "crs": "EPSG:4326"},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_matching_guest_query_cannot_override_post_body_or_target_scope(
|
||||
path: str,
|
||||
payload: dict,
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
artifact = tmp_path / "other-project.geojson"
|
||||
artifact.write_text("{}", encoding="utf-8")
|
||||
client = _guest_client(monkeypatch, FakeSession(_project_objects(OTHER_PROJECT_ID, artifact)))
|
||||
|
||||
response = client.post(f"{path}?project_id={GUEST_PROJECT_ID}", json=payload)
|
||||
|
||||
assert response.status_code == 403
|
||||
assert response.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
|
||||
|
||||
|
||||
def test_guest_can_still_read_and_download_its_own_resources(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
artifact = tmp_path / "demo.geojson"
|
||||
artifact.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
|
||||
client = _guest_client(monkeypatch, FakeSession(_project_objects(GUEST_PROJECT_ID, artifact)))
|
||||
suffix = f"?project_id={GUEST_PROJECT_ID}"
|
||||
|
||||
detection = client.get(f"/api/v1/detection/runs/{DETECTION_RUN_ID}{suffix}")
|
||||
segmentation = client.get(f"/api/v1/segmentation/runs/{SEGMENTATION_RUN_ID}{suffix}")
|
||||
export = client.get(f"/api/v1/exports/{EXPORT_ID}{suffix}")
|
||||
download = client.get(f"/api/v1/exports/{EXPORT_ID}/download{suffix}")
|
||||
|
||||
assert detection.status_code == 200
|
||||
assert segmentation.status_code == 200
|
||||
assert export.status_code == 200
|
||||
assert download.status_code == 200
|
||||
assert download.json()["type"] == "FeatureCollection"
|
||||
|
||||
|
||||
def test_guest_run_lists_and_new_runs_remain_bound_to_the_session_project(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
artifact = tmp_path / "demo.geojson"
|
||||
artifact.write_text("{}", encoding="utf-8")
|
||||
client = _guest_client(monkeypatch, FakeSession(_project_objects(GUEST_PROJECT_ID, artifact)))
|
||||
observed: list[UUID] = []
|
||||
|
||||
def detection_list(_db, *, project_id, **_kwargs):
|
||||
observed.append(project_id)
|
||||
return DetectionRunListResponse(items=[], total=0, limit=50, offset=0, truncated=False)
|
||||
|
||||
def segmentation_list(_db, *, project_id, **_kwargs):
|
||||
observed.append(project_id)
|
||||
return SegmentationRunListResponse(items=[], total=0, limit=50, offset=0, truncated=False)
|
||||
|
||||
def detection_run(**kwargs):
|
||||
observed.append(kwargs["project_id"])
|
||||
return DetectionRunResponse(
|
||||
analysis_run_id=DETECTION_RUN_ID,
|
||||
job_id=JOB_ID,
|
||||
project_id=kwargs["project_id"],
|
||||
dataset_id=kwargs["dataset_id"],
|
||||
model_id=kwargs["model_id"],
|
||||
status="success",
|
||||
detection_count=0,
|
||||
message="Demo run completed",
|
||||
)
|
||||
|
||||
def segmentation_run(**kwargs):
|
||||
observed.append(kwargs["project_id"])
|
||||
return SegmentationRunResponse(
|
||||
analysis_run_id=SEGMENTATION_RUN_ID,
|
||||
job_id=JOB_ID,
|
||||
project_id=kwargs["project_id"],
|
||||
dataset_id=kwargs["dataset_id"],
|
||||
model_id=kwargs["model_id"],
|
||||
status="success",
|
||||
segmentation_count=0,
|
||||
message="Demo run completed",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(DetectionService, "list_runs", detection_list)
|
||||
monkeypatch.setattr(SegmentationService, "list_runs", segmentation_list)
|
||||
monkeypatch.setattr(DetectionService, "run_detection", detection_run)
|
||||
monkeypatch.setattr(SegmentationService, "run_segmentation", segmentation_run)
|
||||
|
||||
def enqueue_detection(**kwargs):
|
||||
observed.append(kwargs["project_id"])
|
||||
return Job(
|
||||
id=JOB_ID,
|
||||
job_type="detection.run",
|
||||
status="queued",
|
||||
project_id=kwargs["project_id"],
|
||||
dataset_id=kwargs["dataset_id"],
|
||||
parameters_json={},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(DetectionService, "enqueue_detection", enqueue_detection)
|
||||
|
||||
def enqueue_segmentation(**kwargs):
|
||||
observed.append(kwargs["project_id"])
|
||||
return Job(
|
||||
id=JOB_ID,
|
||||
job_type="segmentation.run",
|
||||
status="queued",
|
||||
project_id=kwargs["project_id"],
|
||||
dataset_id=kwargs["dataset_id"],
|
||||
parameters_json={},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(SegmentationService, "enqueue_segmentation", enqueue_segmentation)
|
||||
query = f"?project_id={GUEST_PROJECT_ID}"
|
||||
payload = {"project_id": str(GUEST_PROJECT_ID), "dataset_id": str(DATASET_ID), "model_id": "fixture"}
|
||||
|
||||
responses = [
|
||||
client.get(f"/api/v1/detection/runs{query}"),
|
||||
client.get(f"/api/v1/segmentation/runs{query}"),
|
||||
client.post(f"/api/v1/detection/run{query}", json=payload),
|
||||
client.post(f"/api/v1/detection/run-async{query}", json=payload),
|
||||
client.post(f"/api/v1/segmentation/run{query}", json=payload),
|
||||
client.post(f"/api/v1/segmentation/run-async{query}", json=payload),
|
||||
]
|
||||
|
||||
assert all(response.status_code == 200 for response in responses)
|
||||
assert observed == [GUEST_PROJECT_ID] * 6
|
||||
@@ -0,0 +1,100 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.api.routes import health
|
||||
from app.main import app
|
||||
|
||||
|
||||
READY_DATABASE = {
|
||||
"database": "ok",
|
||||
"postgis": "ok:3.4 USE_GEOS=1 USE_PROJ=1",
|
||||
"migration": "ok:202607160001",
|
||||
}
|
||||
|
||||
|
||||
def test_liveness_is_independent_from_database(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
health,
|
||||
"_database_checks",
|
||||
lambda: (_ for _ in ()).throw(AssertionError("must not query DB")),
|
||||
)
|
||||
|
||||
response = TestClient(app).get("/health/live")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "ok"
|
||||
|
||||
|
||||
def test_readiness_returns_ok_only_when_all_checks_pass(monkeypatch) -> None:
|
||||
monkeypatch.setattr(health, "_database_checks", lambda: READY_DATABASE.copy())
|
||||
monkeypatch.setattr(health, "_storage_check", lambda _: "ok")
|
||||
|
||||
response = TestClient(app).get("/health/ready")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["status"] == "ok"
|
||||
assert payload["service"] == "geointel-backend"
|
||||
assert payload["database"] == "ok"
|
||||
assert payload["postgis"].startswith("ok:")
|
||||
assert payload["migration"] == "ok:202607160001"
|
||||
assert payload["storage"] == "ok"
|
||||
|
||||
|
||||
def test_compatibility_health_is_fail_closed(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
health,
|
||||
"_database_checks",
|
||||
lambda: {
|
||||
"database": "degraded",
|
||||
"postgis": "degraded",
|
||||
"migration": "degraded",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(health, "_storage_check", lambda _: "ok")
|
||||
|
||||
response = TestClient(app).get("/health")
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json()["status"] == "degraded"
|
||||
|
||||
|
||||
def test_system_capabilities_report_runtime_state(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
health,
|
||||
"_dependency_enabled",
|
||||
lambda module_name: module_name in {"rasterio", "geopandas"},
|
||||
)
|
||||
monkeypatch.setattr(health, "_database_checks", lambda: READY_DATABASE.copy())
|
||||
monkeypatch.setattr(
|
||||
health.ModelRegistryService,
|
||||
"get_model_capability",
|
||||
lambda *args, **kwargs: SimpleNamespace(
|
||||
configured=True,
|
||||
status="configured",
|
||||
),
|
||||
)
|
||||
|
||||
response = TestClient(app).get("/api/v1/system/capabilities")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()["data"]
|
||||
assert payload["postgis"] is True
|
||||
assert payload["rasterio"] is True
|
||||
assert payload["geopandas"] is True
|
||||
assert payload["yolo"] is True
|
||||
assert payload["yolo_status"] == "configured"
|
||||
assert payload["version"]
|
||||
|
||||
|
||||
def test_requests_receive_a_correlation_id() -> None:
|
||||
client = TestClient(app)
|
||||
|
||||
generated = client.get("/health/live")
|
||||
retained = client.get("/health/live", headers={"x-request-id": "test-request"})
|
||||
|
||||
assert generated.headers["x-request-id"]
|
||||
assert retained.headers["x-request-id"] == "test-request"
|
||||
@@ -0,0 +1,34 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_live_migration_smoke_checks_postgis_after_migrations() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "live_migration_smoke.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
upgrade_index = content.index("-m alembic upgrade head")
|
||||
postgis_index = content.index("PostGIS_Version()")
|
||||
|
||||
assert upgrade_index < postgis_index
|
||||
|
||||
|
||||
def test_live_migration_smoke_checks_required_runtime_schema_objects() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "live_migration_smoke.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "to_regclass(:object_name)" in content
|
||||
assert '"public.projects"' in content
|
||||
assert '"public.datasets"' in content
|
||||
assert '"public.vector_features"' in content
|
||||
assert '"public.detections"' in content
|
||||
assert '"public.segmentations"' in content
|
||||
assert '"public.ix_segmentations_geometry"' in content
|
||||
|
||||
|
||||
def test_live_migration_smoke_reports_collation_version_mismatch_without_failing() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "live_migration_smoke.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "pg_database_collation_actual_version(oid)" in content
|
||||
assert "COLLATION_VERSION_MISMATCH" in content
|
||||
assert "REFRESH COLLATION VERSION" in content
|
||||
assert "Database collation version: ok" in content
|
||||
@@ -0,0 +1,153 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.schemas.bathymetry import MdkBathymetryAcquireRequest
|
||||
from app.schemas.operations import VectorSelectionBBox
|
||||
from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisitionService
|
||||
|
||||
CAPABILITIES_XML = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<WCS_Capabilities version="1.0.0" xmlns="http://www.opengis.net/wcs">
|
||||
<ContentMetadata>
|
||||
<CoverageOfferingBrief>
|
||||
<name>depth_model_20m_lat</name>
|
||||
<label>Belgian Continental Shelf depth model</label>
|
||||
</CoverageOfferingBrief>
|
||||
</ContentMetadata>
|
||||
</WCS_Capabilities>
|
||||
"""
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, content: bytes, content_type: str = "application/xml") -> None:
|
||||
self._stream = io.BytesIO(content)
|
||||
self.headers = {"Content-Type": content_type}
|
||||
|
||||
def read(self, limit: int = -1) -> bytes:
|
||||
return self._stream.read(limit)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
|
||||
def _payload(**overrides) -> MdkBathymetryAcquireRequest:
|
||||
values = {
|
||||
"bbox": VectorSelectionBBox(min_x=2.5, min_y=51.3, max_x=2.6, max_y=51.4),
|
||||
"force_refresh": True,
|
||||
}
|
||||
values.update(overrides)
|
||||
return MdkBathymetryAcquireRequest(**values)
|
||||
|
||||
|
||||
def _settings(**overrides) -> Settings:
|
||||
values = {
|
||||
"mdk_bathymetry_acquisition_enabled": True,
|
||||
"mdk_bathymetry_coverage_id": "depth_model_20m_lat",
|
||||
}
|
||||
values.update(overrides)
|
||||
return Settings(**values)
|
||||
|
||||
|
||||
def test_acquisition_fails_closed_when_disabled() -> None:
|
||||
settings = _settings(mdk_bathymetry_acquisition_enabled=False)
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings)
|
||||
|
||||
assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_ACQUISITION_DISABLED"
|
||||
|
||||
|
||||
def test_acquisition_fails_closed_without_coverage_id() -> None:
|
||||
settings = _settings(mdk_bathymetry_coverage_id=None)
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings)
|
||||
|
||||
assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_COVERAGE_NOT_CONFIGURED"
|
||||
|
||||
|
||||
def test_acquisition_rejects_oversized_bbox() -> None:
|
||||
settings = _settings(mdk_bathymetry_max_bbox_deg2=0.001)
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings)
|
||||
|
||||
assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_BBOX_TOO_LARGE"
|
||||
|
||||
|
||||
def test_acquisition_requires_reachable_probe() -> None:
|
||||
settings = _settings()
|
||||
|
||||
def failing_opener(request, timeout=None):
|
||||
raise OSError("connection refused")
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings, opener=failing_opener)
|
||||
|
||||
assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_ENDPOINT_NOT_READY"
|
||||
|
||||
|
||||
def test_acquisition_requires_advertised_coverage_id() -> None:
|
||||
settings = _settings(mdk_bathymetry_coverage_id="not_advertised_coverage")
|
||||
|
||||
def opener(request, timeout=None):
|
||||
return FakeResponse(CAPABILITIES_XML)
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings, opener=opener)
|
||||
|
||||
assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_COVERAGE_NOT_ADVERTISED"
|
||||
|
||||
|
||||
def test_acquisition_rejects_non_geotiff_coverage_response() -> None:
|
||||
settings = _settings()
|
||||
responses = []
|
||||
|
||||
def opener(request, timeout=None):
|
||||
url = request.full_url if hasattr(request, "full_url") else str(request)
|
||||
responses.append(url)
|
||||
if "GetCapabilities" in url:
|
||||
return FakeResponse(CAPABILITIES_XML)
|
||||
return FakeResponse(b"<ServiceExceptionReport>boom</ServiceExceptionReport>", "application/xml")
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings, opener=opener)
|
||||
|
||||
assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_INVALID_RESPONSE"
|
||||
assert any("GetCoverage" in url for url in responses)
|
||||
coverage_urls = [url for url in responses if "GetCoverage" in url]
|
||||
assert "coverage=depth_model_20m_lat" in coverage_urls[0]
|
||||
assert "format=GeoTIFF" in coverage_urls[0]
|
||||
|
||||
|
||||
def test_get_coverage_url_is_bounded_and_pinned() -> None:
|
||||
settings = _settings()
|
||||
bbox = [2.5, 51.3, 2.6, 51.4]
|
||||
|
||||
url = MdkBathymetryAcquisitionService._get_coverage_url(settings, "depth_model_20m_lat", bbox)
|
||||
|
||||
assert url.startswith("https://")
|
||||
assert "request=GetCoverage" in url
|
||||
assert "version=1.0.0" in url
|
||||
assert "crs=EPSG%3A4326" in url or "crs=EPSG:4326" in url
|
||||
width, height = MdkBathymetryAcquisitionService._pixel_dimensions(bbox)
|
||||
assert 1 <= width <= MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE
|
||||
assert 1 <= height <= MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE
|
||||
|
||||
|
||||
def test_source_module_never_disables_tls_verification() -> None:
|
||||
source = (
|
||||
Path(__file__).resolve().parents[1] / "app" / "services" / "mdk_bathymetry_acquisition_service.py"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "_create_unverified_context" not in source
|
||||
assert "CERT_NONE" not in source
|
||||
assert "check_hostname = False" not in source
|
||||
@@ -0,0 +1,355 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.main import app
|
||||
from app.models import (
|
||||
AnalysisRun,
|
||||
Dataset,
|
||||
DatasetVersion,
|
||||
Detection,
|
||||
Job,
|
||||
Project,
|
||||
SourceRegistry,
|
||||
SourceSnapshot,
|
||||
)
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
from app.services.tile_manifest_service import TileManifestService
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, objects=None) -> None:
|
||||
self.objects = objects or {}
|
||||
self.added = []
|
||||
self.commits = 0
|
||||
self.refreshes = []
|
||||
|
||||
def get(self, model, item_id):
|
||||
return self.objects.get((model, item_id))
|
||||
|
||||
def add(self, item) -> None:
|
||||
self.added.append(item)
|
||||
if getattr(item, "id", None) is not None:
|
||||
self.objects[(item.__class__, item.id)] = item
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def refresh(self, item) -> None:
|
||||
self.refreshes.append(item)
|
||||
|
||||
|
||||
class MockYoloAdapter:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self.settings = settings
|
||||
|
||||
@staticmethod
|
||||
def dependencies_available() -> bool:
|
||||
return True
|
||||
|
||||
def load_model(self, model_path: Path):
|
||||
return {"model_path": str(model_path)}
|
||||
|
||||
def predict_tiles(self, model, tile_paths, confidence_threshold: float) -> list[list[dict]]:
|
||||
# The service batches tiles; this double still answers per tile.
|
||||
return [self.predict_tile(model, tile_path, confidence_threshold) for tile_path in tile_paths]
|
||||
|
||||
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
|
||||
assert model["model_path"].endswith("building-detector.pt")
|
||||
return [
|
||||
{
|
||||
"class_name": "building",
|
||||
"confidence": 0.9,
|
||||
"bbox": [10.0, 20.0, 30.0, 40.0],
|
||||
"properties": {"adapter": "mock"},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _project_and_raster_dataset():
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
checksum = "a" * 64
|
||||
project = Project(id=project_id, name="Geel")
|
||||
source_registry = SourceRegistry(
|
||||
id=source_registry_id,
|
||||
source_key="test-derived-raster",
|
||||
display_name="Governed test-derived raster",
|
||||
classification="derived",
|
||||
authority_name="GeoIntel test fixture",
|
||||
usage_policy_json={"ground_truth_allowed": False},
|
||||
)
|
||||
source_snapshot = SourceSnapshot(
|
||||
id=source_snapshot_id,
|
||||
source_registry_id=source_registry_id,
|
||||
snapshot_key="test-derived-raster-v1",
|
||||
checksum_sha256=checksum,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="source.tif",
|
||||
dataset_type="raster",
|
||||
source="test-derived-raster",
|
||||
source_name="test-derived-raster",
|
||||
storage_path="storage/uploads/source.tif",
|
||||
checksum_sha256=checksum,
|
||||
crs="EPSG:4326",
|
||||
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
data_contract_key="geointel.raster.geotiff",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
status="ready",
|
||||
)
|
||||
dataset.source_registry = source_registry
|
||||
dataset.source_snapshot = source_snapshot
|
||||
dataset.versions.append(
|
||||
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
|
||||
)
|
||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||
return db, project_id, dataset_id
|
||||
|
||||
|
||||
def _manifest(tmp_path: Path, db: FakeSession, dataset: Dataset) -> Path:
|
||||
tile_path = tmp_path / "tile_0000.tif"
|
||||
tile_path.write_bytes(b"tile")
|
||||
binding = TileManifestService.dataset_binding(db, dataset)
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
**binding,
|
||||
"tile_set_id": "tiles-fixture",
|
||||
"count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||
"tiles": [
|
||||
{
|
||||
"path": str(tile_path),
|
||||
"pixel_window": [0, 0, 100, 100],
|
||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
|
||||
"crs": "EPSG:4326",
|
||||
"index": 0,
|
||||
**TileManifestService.tile_integrity(tile_path),
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return manifest_path
|
||||
|
||||
|
||||
def _write_model_sidecar(
|
||||
model_path: Path,
|
||||
settings: Settings,
|
||||
*,
|
||||
db: FakeSession | None = None,
|
||||
) -> None:
|
||||
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
source_version = settings.yolo_model_version or "test-v1"
|
||||
if db is not None:
|
||||
source_registry = SourceRegistry(
|
||||
id=source_registry_id,
|
||||
source_key="model",
|
||||
display_name="Governed test model artifact",
|
||||
classification="experimental",
|
||||
authority_name="GeoIntel test fixture",
|
||||
freshness_status="current",
|
||||
ingest_status="configured",
|
||||
)
|
||||
source_snapshot = SourceSnapshot(
|
||||
id=source_snapshot_id,
|
||||
source_registry_id=source_registry_id,
|
||||
snapshot_key=f"model-{source_version}",
|
||||
source_version=source_version,
|
||||
checksum_sha256=model_sha256,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
db.objects[(SourceRegistry, source_registry_id)] = source_registry
|
||||
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
|
||||
payload = {
|
||||
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
|
||||
"model": {
|
||||
"model_id": settings.yolo_model_id,
|
||||
"task_type": "object_detection",
|
||||
"sha256": model_sha256,
|
||||
"model_format": "pytorch",
|
||||
"framework": "ultralytics/pytorch",
|
||||
"class_mapping": {"0": "building"},
|
||||
"source_version": source_version,
|
||||
},
|
||||
"source": {
|
||||
"source_registry_id": str(source_registry_id),
|
||||
"source_snapshot_id": str(source_snapshot_id),
|
||||
"source_registry_key": "model",
|
||||
"source_snapshot_checksum_sha256": model_sha256,
|
||||
},
|
||||
"lineage": {
|
||||
"upstream_asset_ids": ["test-training-corpus"],
|
||||
"upstream_checksums_sha256": ["a" * 64],
|
||||
"transformations": [
|
||||
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
|
||||
],
|
||||
},
|
||||
"metadata": {"training_manifest_sha256": "c" * 64},
|
||||
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||
}
|
||||
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
|
||||
json.dumps(payload, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def test_model_asset_catalog_lists_supported_local_model_files(tmp_path: Path) -> None:
|
||||
model_file = tmp_path / "building-detector.pt"
|
||||
model_file.write_bytes(b"local model")
|
||||
ignored_file = tmp_path / "notes.txt"
|
||||
ignored_file.write_text("ignore me", encoding="utf-8")
|
||||
settings = Settings(yolo_models_dir=str(tmp_path), yolo_model_path=str(model_file), yolo_enabled=True)
|
||||
|
||||
response = ModelAssetCatalogService.list_assets(settings=settings)
|
||||
|
||||
assert response.total == 1
|
||||
asset = response.items[0]
|
||||
assert asset.model_asset_id == "building-detector-pt"
|
||||
assert asset.filename == "building-detector.pt"
|
||||
assert asset.display_name == "building-detector"
|
||||
assert asset.model_path == str(model_file)
|
||||
assert asset.size_bytes == len(b"local model")
|
||||
assert len(asset.sha256) == 64
|
||||
assert asset.active is True
|
||||
assert asset.runtime_available is True
|
||||
assert asset.runtime_status == "active"
|
||||
assert asset.governed_validation_status == "not_verified_by_catalog"
|
||||
assert asset.promotion_status == "not_verified_by_catalog"
|
||||
assert asset.status == "runtime_active"
|
||||
assert asset.will_download_models is False
|
||||
|
||||
|
||||
def test_model_asset_catalog_resolves_known_asset(tmp_path: Path) -> None:
|
||||
model_file = tmp_path / "building-detector.pt"
|
||||
model_file.write_bytes(b"local model")
|
||||
settings = Settings(yolo_models_dir=str(tmp_path), yolo_enabled=True)
|
||||
|
||||
asset = ModelAssetCatalogService.resolve_asset("building-detector-pt", settings=settings)
|
||||
|
||||
assert asset.filename == "building-detector.pt"
|
||||
assert asset.model_path == str(model_file)
|
||||
|
||||
|
||||
def test_model_asset_catalog_only_exposes_explicit_active_asset_in_runtime(tmp_path: Path) -> None:
|
||||
active_file = tmp_path / "approved-building-detector.pt"
|
||||
active_file.write_bytes(b"approved")
|
||||
(tmp_path / "training-smoke.pt").write_bytes(b"experiment")
|
||||
(tmp_path / "partial-checkpoint.pt").write_bytes(b"partial")
|
||||
settings = Settings(
|
||||
yolo_models_dir=str(tmp_path),
|
||||
yolo_model_path=str(active_file),
|
||||
yolo_enabled=True,
|
||||
)
|
||||
|
||||
response = ModelAssetCatalogService.list_assets(settings=settings)
|
||||
|
||||
assert response.total == 1
|
||||
assert response.items[0].filename == active_file.name
|
||||
assert response.items[0].active is True
|
||||
assert response.items[0].runtime_status == "active"
|
||||
assert response.items[0].governed_validation_status == "not_verified_by_catalog"
|
||||
assert response.items[0].promotion_status == "not_verified_by_catalog"
|
||||
assert response.items[0].status == "runtime_active"
|
||||
|
||||
|
||||
def test_model_asset_catalog_rejects_unknown_asset(tmp_path: Path) -> None:
|
||||
settings = Settings(yolo_models_dir=str(tmp_path), yolo_enabled=True)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
ModelAssetCatalogService.resolve_asset("missing-model", settings=settings)
|
||||
|
||||
assert exc_info.value.code == "DETECTION_MODEL_ASSET_NOT_FOUND"
|
||||
assert exc_info.value.status_code == 404
|
||||
|
||||
|
||||
def test_model_assets_api_returns_canonical_envelope(monkeypatch, tmp_path: Path) -> None:
|
||||
model_file = tmp_path / "building-detector.pt"
|
||||
model_file.write_bytes(b"local model")
|
||||
monkeypatch.setenv("YOLO_MODELS_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("YOLO_MODEL_PATH", str(model_file))
|
||||
|
||||
response = TestClient(app).get("/api/v1/detection/model-assets")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert set(payload) == {"data"}
|
||||
assert payload["data"]["total"] == 1
|
||||
assert payload["data"]["items"][0]["model_asset_id"] == "building-detector-pt"
|
||||
assert payload["data"]["items"][0]["active"] is True
|
||||
assert payload["data"]["items"][0]["runtime_status"] == "active"
|
||||
assert payload["data"]["items"][0]["governed_validation_status"] == "not_verified_by_catalog"
|
||||
assert payload["data"]["items"][0]["promotion_status"] == "not_verified_by_catalog"
|
||||
assert payload["data"]["items"][0]["will_download_models"] is False
|
||||
|
||||
|
||||
def test_detection_run_persists_selected_model_asset_parameters(tmp_path, monkeypatch: Path) -> None:
|
||||
# A manifest written into tmp_path is only a governed artifact if
|
||||
# tmp_path is the storage root.
|
||||
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
|
||||
model_file = tmp_path / "building-detector.pt"
|
||||
model_file.write_bytes(b"local model")
|
||||
db, project_id, dataset_id = _project_and_raster_dataset()
|
||||
settings = Settings(
|
||||
yolo_enabled=True,
|
||||
yolo_model_path=str(tmp_path / "default.pt"),
|
||||
yolo_models_dir=str(tmp_path),
|
||||
yolo_max_tiles=4,
|
||||
)
|
||||
_write_model_sidecar(model_file, settings, db=db)
|
||||
|
||||
result = DetectionService.run_detection(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-configured",
|
||||
model_asset_id="building-detector-pt",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(_manifest(tmp_path, db, db.get(Dataset, dataset_id))),
|
||||
settings=settings,
|
||||
yolo_adapter_class=MockYoloAdapter,
|
||||
)
|
||||
|
||||
jobs = [item for item in db.added if isinstance(item, Job)]
|
||||
runs = [item for item in db.added if isinstance(item, AnalysisRun)]
|
||||
detections = [item for item in db.added if isinstance(item, Detection)]
|
||||
|
||||
assert result.status == "success"
|
||||
assert result.detection_count == 1
|
||||
assert jobs[0].parameters_json["model_asset_id"] == "building-detector-pt"
|
||||
assert jobs[0].parameters_json["model_asset_path"] == str(model_file)
|
||||
assert len(jobs[0].parameters_json["model_asset_sha256"]) == 64
|
||||
assert runs[0].parameters_json["model_asset_id"] == "building-detector-pt"
|
||||
assert detections[0].model_name == "yolo-configured"
|
||||
@@ -0,0 +1,22 @@
|
||||
from scripts.render_operator_polygon_label_qa import geometry_rings
|
||||
|
||||
|
||||
def test_geometry_rings_yields_polygon_exterior_and_hole() -> None:
|
||||
exterior = [[0, 0], [1, 0], [1, 1], [0, 0]]
|
||||
hole = [[0.2, 0.2], [0.4, 0.2], [0.2, 0.2]]
|
||||
assert list(geometry_rings({"type": "Polygon", "coordinates": [exterior, hole]})) == [
|
||||
exterior,
|
||||
hole,
|
||||
]
|
||||
|
||||
|
||||
def test_geometry_rings_flattens_multipolygon_rings() -> None:
|
||||
first = [[0, 0], [1, 0], [0, 0]]
|
||||
second = [[2, 2], [3, 2], [2, 2]]
|
||||
assert list(
|
||||
geometry_rings({"type": "MultiPolygon", "coordinates": [[first], [second]]})
|
||||
) == [first, second]
|
||||
|
||||
|
||||
def test_geometry_rings_ignores_non_polygon_geometry() -> None:
|
||||
assert list(geometry_rings({"type": "Point", "coordinates": [0, 0]})) == []
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Bounded acquisition must stay bounded to the official host.
|
||||
|
||||
Every acquisition service builds its URL from configured settings, so the
|
||||
request payload cannot point the runtime anywhere. The redirect chain can:
|
||||
``urlopen`` follows redirects by default, so a misconfigured or compromised
|
||||
upstream can send the runtime to ``127.0.0.1``, to the container network, or to
|
||||
a cloud metadata endpoint — and the response is then persisted as if it were
|
||||
official source data.
|
||||
|
||||
The product's stated rule is that acquisition fails closed and never
|
||||
substitutes fabricated data for official data. A redirect off the configured
|
||||
host is exactly that substitution.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.services.outbound_request_guard import (
|
||||
_ValidatedRedirects,
|
||||
assert_public_http_url,
|
||||
assert_same_origin_redirect,
|
||||
validated_redirect_opener,
|
||||
)
|
||||
|
||||
|
||||
class TestUrlShape:
|
||||
def test_an_official_https_endpoint_is_accepted(self) -> None:
|
||||
assert_public_http_url("https://geo.api.vlaanderen.be/dhmv/wcs?SERVICE=WCS")
|
||||
|
||||
def test_a_non_http_scheme_is_refused(self) -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
assert_public_http_url("file:///etc/passwd")
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url",
|
||||
[
|
||||
"http://127.0.0.1:8000/internal",
|
||||
"http://localhost/internal",
|
||||
"http://10.1.2.3/internal",
|
||||
"http://192.168.123.45/internal",
|
||||
"http://172.16.0.9/internal",
|
||||
"http://169.254.169.254/latest/meta-data/",
|
||||
"http://[::1]/internal",
|
||||
],
|
||||
)
|
||||
def test_private_and_loopback_destinations_are_refused(self, url: str) -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
assert_public_http_url(url)
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
|
||||
|
||||
def test_a_url_without_a_host_is_refused(self) -> None:
|
||||
with pytest.raises(AppError):
|
||||
assert_public_http_url("https:///no-host")
|
||||
|
||||
|
||||
class TestRedirects:
|
||||
def test_a_redirect_within_the_same_origin_is_allowed(self) -> None:
|
||||
assert_same_origin_redirect(
|
||||
"https://geo.api.vlaanderen.be/dhmv/wcs",
|
||||
"https://geo.api.vlaanderen.be/dhmv/wcs/v2?x=1",
|
||||
)
|
||||
|
||||
def test_a_redirect_to_another_host_is_refused(self) -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
assert_same_origin_redirect(
|
||||
"https://geo.api.vlaanderen.be/dhmv/wcs",
|
||||
"https://cdn.example.net/payload.tif",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
|
||||
assert "cdn.example.net" in str(exc_info.value.details)
|
||||
|
||||
def test_a_downgrade_to_plain_http_is_refused(self) -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
assert_same_origin_redirect(
|
||||
"https://geo.api.vlaanderen.be/wcs",
|
||||
"http://geo.api.vlaanderen.be/wcs",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
|
||||
|
||||
def test_a_redirect_to_the_loopback_is_refused_even_on_the_same_scheme(self) -> None:
|
||||
with pytest.raises(AppError):
|
||||
assert_same_origin_redirect("https://geo.api.vlaanderen.be/wcs", "https://127.0.0.1/wcs")
|
||||
|
||||
def test_an_upgrade_to_https_stays_allowed(self) -> None:
|
||||
assert_same_origin_redirect("http://geo.example.be/wcs", "https://geo.example.be/wcs")
|
||||
|
||||
def test_a_redirect_to_another_port_is_refused(self) -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
assert_same_origin_redirect(
|
||||
"https://geo.api.vlaanderen.be/wcs",
|
||||
"https://geo.api.vlaanderen.be:8443/wcs",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
|
||||
|
||||
def test_embedded_credentials_are_refused(self) -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
assert_public_http_url("https://operator:secret@geo.example.be/wcs")
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
|
||||
|
||||
def test_a_redirect_with_an_invalid_port_fails_closed(self) -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
assert_same_origin_redirect(
|
||||
"https://geo.api.vlaanderen.be/wcs",
|
||||
"https://geo.api.vlaanderen.be:not-a-port/wcs",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
|
||||
|
||||
|
||||
def test_the_guard_opener_refuses_a_cross_host_redirect() -> None:
|
||||
"""The opener is what the acquisition services actually call."""
|
||||
|
||||
from app.services.outbound_request_guard import guarded_opener
|
||||
|
||||
opener = guarded_opener("https://geo.api.vlaanderen.be/wcs")
|
||||
|
||||
class _Redirecting:
|
||||
def __init__(self, location: str) -> None:
|
||||
self.url = location
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
with opener(
|
||||
type("Req", (), {"full_url": "https://geo.api.vlaanderen.be/wcs"})(),
|
||||
timeout=1,
|
||||
_transport=lambda *_a, **_k: _Redirecting("https://evil.example.net/x"),
|
||||
):
|
||||
pass
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
|
||||
|
||||
|
||||
class TestTheGuardIsWiredIntoAcquisition:
|
||||
"""Behavioural, not a grep: each service is called on its real fetch path.
|
||||
|
||||
Every existing acquisition test injects an ``opener``, which bypasses the
|
||||
guard by design — that is how those tests stub the network. These call the
|
||||
production default instead.
|
||||
"""
|
||||
|
||||
def _settings(self):
|
||||
from app.core.config import Settings
|
||||
|
||||
return Settings(_env_file=None)
|
||||
|
||||
def test_dhmv_refuses_a_loopback_endpoint(self) -> None:
|
||||
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DhmvAcquisitionService._fetch("http://127.0.0.1:9/wcs", self._settings())
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
|
||||
|
||||
def test_flood_hazard_refuses_a_link_local_endpoint(self) -> None:
|
||||
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
FloodHazardAcquisitionService._fetch("http://169.254.169.254/latest/", self._settings())
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
|
||||
|
||||
def test_thematic_raster_refuses_a_private_endpoint(self) -> None:
|
||||
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
ThematicRasterAcquisitionService._fetch("http://10.0.0.5/product.tif", self._settings())
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
|
||||
|
||||
def test_orthophoto_refuses_a_private_endpoint(self) -> None:
|
||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
OrthophotoAcquisitionService._fetch("http://192.168.123.45/wms", self._settings())
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
|
||||
|
||||
|
||||
class TestOneRedirectPolicy:
|
||||
"""Two acquisition services rejected every redirect through their own
|
||||
opener while eight allowed a same-origin one through this guard. Two
|
||||
policies with no stated reason, and only one of them checked where the
|
||||
response actually came from."""
|
||||
|
||||
def test_the_strict_policy_refuses_any_redirect(self) -> None:
|
||||
from app.services.outbound_request_guard import guarded_opener
|
||||
|
||||
opener = guarded_opener("https://geo.api.vlaanderen.be/GRB/wfs", allow_redirect=False)
|
||||
|
||||
class _Redirected:
|
||||
url = "https://geo.api.vlaanderen.be/GRB/wfs/v2"
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
with opener(object(), timeout=1, _transport=lambda *_a, **_k: _Redirected()):
|
||||
pass
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
|
||||
|
||||
def test_the_strict_policy_still_allows_the_response_it_asked_for(self) -> None:
|
||||
from app.services.outbound_request_guard import guarded_opener
|
||||
|
||||
url = "https://geo.api.vlaanderen.be/GRB/wfs"
|
||||
opener = guarded_opener(url, allow_redirect=False)
|
||||
|
||||
class _Direct:
|
||||
def __init__(self) -> None:
|
||||
self.url = url
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
with opener(object(), timeout=1, _transport=lambda *_a, **_k: _Direct()) as response:
|
||||
assert response.url == url
|
||||
|
||||
def test_both_policies_refuse_a_private_destination(self) -> None:
|
||||
from app.services.outbound_request_guard import guarded_opener
|
||||
|
||||
for allow_redirect in (True, False):
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
guarded_opener("http://10.0.0.5/wfs", allow_redirect=allow_redirect)
|
||||
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
|
||||
|
||||
def test_the_strict_services_use_the_shared_guard(self) -> None:
|
||||
"""Behavioural: their own fetch paths refuse a private endpoint, which
|
||||
the hand-rolled opener never checked."""
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.services.grb_acquisition_service import GrbAcquisitionService
|
||||
from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService
|
||||
|
||||
settings = Settings(_env_file=None)
|
||||
for service in (GrbAcquisitionService, OfficialVectorAcquisitionService):
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
service._read_page("http://127.0.0.1:9/wfs", settings, None)
|
||||
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
|
||||
|
||||
|
||||
def test_a_refused_redirect_is_never_requested() -> None:
|
||||
"""Rejecting after the fact still sends the request.
|
||||
|
||||
Checking ``response.url`` means urllib has already followed the chain: the
|
||||
connection to the redirect target was opened and the response read. For a
|
||||
destination like a metadata endpoint that is the whole attack. The strict
|
||||
policy must refuse to follow, not refuse afterwards.
|
||||
"""
|
||||
|
||||
from app.services.outbound_request_guard import no_redirect_opener
|
||||
|
||||
opener = no_redirect_opener()
|
||||
handlers = [type(handler).__name__ for handler in opener.handlers]
|
||||
|
||||
assert "_RejectRedirects" in handlers
|
||||
|
||||
|
||||
def test_the_default_guard_validates_before_following_a_redirect() -> None:
|
||||
opener = validated_redirect_opener("https://geo.api.vlaanderen.be/wcs")
|
||||
handlers = [type(handler).__name__ for handler in opener.handlers]
|
||||
|
||||
assert "_ValidatedRedirects" in handlers
|
||||
|
||||
handler = _ValidatedRedirects("https://geo.api.vlaanderen.be/wcs")
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
handler.redirect_request(
|
||||
None,
|
||||
None,
|
||||
302,
|
||||
"Found",
|
||||
{},
|
||||
"http://169.254.169.254/latest/meta-data/",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
|
||||
|
||||
|
||||
def test_the_rejecting_handler_returns_no_new_request() -> None:
|
||||
from app.services.outbound_request_guard import _RejectRedirects
|
||||
|
||||
handler = _RejectRedirects()
|
||||
|
||||
assert handler.redirect_request(None, None, 302, "Found", {}, "http://169.254.169.254/") is None
|
||||
@@ -0,0 +1,120 @@
|
||||
"""The paged readers' loop protections, exercised rather than assumed.
|
||||
|
||||
GRB and official vector both refuse a repeated page URL and bound the page
|
||||
count, and GRB deduplicates on feature identity. None of that had a test, so
|
||||
none of it was known to work — the same category as the redirect handler that
|
||||
turned out to be dead code while looking like protection.
|
||||
|
||||
A provider that answers every page with a "next" link pointing back at itself
|
||||
is not hypothetical: it is what a misconfigured cursor or a caching proxy in
|
||||
front of an OGC endpoint produces.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import Polygon
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.services.grb_acquisition_service import GrbAcquisitionService
|
||||
from tests.test_sprint239_bounded_grb_acquisition import JsonResponse, polygon_feature
|
||||
|
||||
COLLECTION_ITEMS = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
|
||||
SCOPE = Polygon([(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)])
|
||||
|
||||
|
||||
def _building(feature_id: str) -> dict:
|
||||
return polygon_feature(
|
||||
feature_id,
|
||||
[(5.155, 51.185), (5.175, 51.185), (5.175, 51.195), (5.155, 51.195), (5.155, 51.185)],
|
||||
)
|
||||
|
||||
|
||||
def _fetch(opener):
|
||||
return GrbAcquisitionService._fetch_features(
|
||||
GrbAcquisitionService._product("buildings"),
|
||||
SCOPE,
|
||||
SCOPE.bounds,
|
||||
"bounded_selection",
|
||||
Settings(_env_file=None),
|
||||
opener,
|
||||
)
|
||||
|
||||
|
||||
def test_a_next_link_pointing_at_itself_is_refused() -> None:
|
||||
requests: list[str] = []
|
||||
|
||||
def opener(request, timeout): # noqa: ARG001
|
||||
requests.append(request.full_url)
|
||||
return JsonResponse(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [_building("GBG.1")],
|
||||
"links": [{"rel": "next", "href": f"{COLLECTION_ITEMS}?cursor=stuck"}],
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
_fetch(opener)
|
||||
|
||||
assert exc_info.value.code == "GRB_PROVIDER_PAGINATION_LOOP"
|
||||
# Refused on the second sighting, not after exhausting the page budget.
|
||||
assert len(requests) == 2
|
||||
|
||||
|
||||
def test_an_endless_chain_of_fresh_pages_stops_at_the_page_limit() -> None:
|
||||
"""Distinct URLs defeat the loop check, so the page budget is the backstop."""
|
||||
|
||||
settings = Settings(_env_file=None)
|
||||
requests: list[str] = []
|
||||
|
||||
def opener(request, timeout): # noqa: ARG001
|
||||
requests.append(request.full_url)
|
||||
cursor = len(requests)
|
||||
return JsonResponse(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [_building(f"GBG.{cursor}")],
|
||||
"links": [{"rel": "next", "href": f"{COLLECTION_ITEMS}?cursor=p{cursor}"}],
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
_fetch(opener)
|
||||
|
||||
assert exc_info.value.code == "GRB_SELECTION_TOO_LARGE"
|
||||
assert exc_info.value.status_code == 422
|
||||
assert len(requests) == settings.grb_max_pages
|
||||
|
||||
|
||||
def test_a_repeated_feature_across_pages_is_counted_once() -> None:
|
||||
"""Two pages, distinct URLs, overlapping content.
|
||||
|
||||
Unlike a repeated URL this is not necessarily provider misbehaviour — a
|
||||
cursor over a changing table can hand back a record twice — so the reader
|
||||
keeps it once rather than failing the acquisition.
|
||||
"""
|
||||
|
||||
def opener(request, timeout): # noqa: ARG001
|
||||
if "cursor=next" in request.full_url:
|
||||
return JsonResponse(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [_building("GBG.1"), _building("GBG.2")],
|
||||
"links": [],
|
||||
}
|
||||
)
|
||||
return JsonResponse(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [_building("GBG.1")],
|
||||
"links": [{"rel": "next", "href": f"{COLLECTION_ITEMS}?cursor=next"}],
|
||||
}
|
||||
)
|
||||
|
||||
features, transfer = _fetch(opener)
|
||||
|
||||
assert {feature["id"] for feature in features} == {"GBG:GBG.1", "GBG:GBG.2"}
|
||||
assert transfer["candidate_feature_count"] == 3
|
||||
assert transfer["feature_count"] == 2
|
||||
@@ -0,0 +1,61 @@
|
||||
"""A rectangle across a municipal boundary must not return the same object twice.
|
||||
|
||||
Partitioned selection de-duplicated ``total_feature_count`` on
|
||||
``source_feature_id`` but returned the raw rows. A feature present in two
|
||||
municipal partitions was therefore drawn twice on the map and counted once in
|
||||
the headline, so the number on the panel disagreed with the geometry beside it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
class _Row:
|
||||
def __init__(self, source_feature_id, row_id=None, dataset_id=None):
|
||||
self.source_feature_id = source_feature_id
|
||||
self.id = row_id or uuid4()
|
||||
self.dataset_id = dataset_id or uuid4()
|
||||
|
||||
|
||||
def _ids(rows):
|
||||
return [row.source_feature_id or str(row.id) for row in rows]
|
||||
|
||||
|
||||
def test_a_feature_in_two_partitions_is_returned_once() -> None:
|
||||
shared = "grb-building-42"
|
||||
rows = [_Row(shared), _Row("grb-building-7"), _Row(shared)]
|
||||
|
||||
kept = VectorFeatureService.deduplicate_rows(rows)
|
||||
|
||||
assert _ids(kept) == [shared, "grb-building-7"]
|
||||
|
||||
|
||||
def test_the_first_occurrence_wins_so_the_result_is_stable() -> None:
|
||||
first = _Row("dup")
|
||||
second = _Row("dup")
|
||||
|
||||
assert VectorFeatureService.deduplicate_rows([first, second])[0] is first
|
||||
assert VectorFeatureService.deduplicate_rows([second, first])[0] is second
|
||||
|
||||
|
||||
def test_rows_without_a_source_id_fall_back_to_their_own_identity() -> None:
|
||||
"""Two distinct rows with no source id are two distinct features."""
|
||||
|
||||
rows = [_Row(None), _Row(None)]
|
||||
|
||||
assert len(VectorFeatureService.deduplicate_rows(rows)) == 2
|
||||
|
||||
|
||||
def test_an_empty_source_id_is_not_treated_as_a_shared_identity() -> None:
|
||||
rows = [_Row(""), _Row("")]
|
||||
|
||||
assert len(VectorFeatureService.deduplicate_rows(rows)) == 2
|
||||
|
||||
|
||||
def test_deduplication_leaves_a_clean_population_untouched() -> None:
|
||||
rows = [_Row("a"), _Row("b"), _Row("c")]
|
||||
|
||||
assert VectorFeatureService.deduplicate_rows(rows) == rows
|
||||
@@ -0,0 +1,644 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
from app.models import Dataset, DatasetVersion, SourceRegistry, SourceSnapshot
|
||||
from app.services.data_contract_validation import (
|
||||
build_vector_ingest_input,
|
||||
validate_registered_asset,
|
||||
)
|
||||
from app.services.demo_workflow_service import DemoWorkflowService
|
||||
from app.services.derived_dataset_governance_service import (
|
||||
DerivedDatasetGovernanceService,
|
||||
)
|
||||
from app.services.raster_operations_service import RasterOperationsService
|
||||
from app.services.storage_service import StorageService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
from app.services.vector_operations_service import VectorOperationsService
|
||||
|
||||
|
||||
_CHECKSUM = "a" * 64
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
def __init__(self, rows=None) -> None:
|
||||
self.rows = rows or {}
|
||||
self.added = []
|
||||
|
||||
def get(self, model, row_id):
|
||||
row = self.rows.get((model, row_id))
|
||||
if row is not None:
|
||||
return row
|
||||
return next(
|
||||
(
|
||||
item
|
||||
for item in self.added
|
||||
if isinstance(item, model) and item.id == row_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
def add(self, row) -> None:
|
||||
self.added.append(row)
|
||||
|
||||
def commit(self) -> None:
|
||||
return None
|
||||
|
||||
def refresh(self, row) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _GovernedSession:
|
||||
"""Small ORM-shaped session for the real governance branch.
|
||||
|
||||
Registry persistence is monkeypatched below; the test exercises the
|
||||
service's orchestration and report decisions without needing PostGIS.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.added = []
|
||||
self.flushes = 0
|
||||
|
||||
class _EmptyQuery:
|
||||
def filter(self, *_args, **_kwargs):
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def all():
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def one_or_none():
|
||||
return None
|
||||
|
||||
def query(self, *_args, **_kwargs):
|
||||
# Governance now performs a bounded lineage traversal during quarantine.
|
||||
# This focused harness intentionally has no persisted siblings/edges.
|
||||
return self._EmptyQuery()
|
||||
|
||||
def add(self, row) -> None:
|
||||
self.added.append(row)
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flushes += 1
|
||||
|
||||
|
||||
def _governed_parent() -> Dataset:
|
||||
source_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
source = SourceRegistry(
|
||||
id=source_id,
|
||||
source_key="grb",
|
||||
display_name="GRB parent fixture",
|
||||
classification="authoritative",
|
||||
authority_name="Digitaal Vlaanderen",
|
||||
authority_scope_json={"zone": "Flanders"},
|
||||
usage_policy_json={"ground_truth_allowed": True},
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=source_id,
|
||||
snapshot_key="governed-parent",
|
||||
checksum_sha256=_CHECKSUM,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="governed.geojson",
|
||||
dataset_type="vector",
|
||||
source="grb",
|
||||
source_name="grb",
|
||||
status="ready",
|
||||
checksum_sha256=_CHECKSUM,
|
||||
data_contract_key="geointel.vector.geojson",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
source_registry_id=source_id,
|
||||
source_snapshot_id=snapshot_id,
|
||||
)
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
return dataset
|
||||
|
||||
|
||||
def test_lineage_evidence_quarantines_ungoverned_parent_without_inventing_a_checksum() -> (
|
||||
None
|
||||
):
|
||||
parent = _governed_parent()
|
||||
valid_lineage = DerivedDatasetGovernanceService._lineage_evidence(
|
||||
parent, "vector.clip", {"area_id": "a"}
|
||||
)
|
||||
valid_report = validate_registered_asset(
|
||||
build_vector_ingest_input(
|
||||
asset_id="derived-valid",
|
||||
source_crs="EPSG:4326",
|
||||
storage_crs="EPSG:4326",
|
||||
feature_collection={
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
}
|
||||
],
|
||||
},
|
||||
checksum_sha256=_CHECKSUM,
|
||||
computed_checksum_sha256=_CHECKSUM,
|
||||
source_registry_id="derived-source",
|
||||
source_snapshot_id="derived-snapshot",
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
metadata={
|
||||
"license": "internal derived artifact",
|
||||
"bounds_json": {
|
||||
"min_x": 5.0,
|
||||
"min_y": 51.0,
|
||||
"max_x": 5.0,
|
||||
"max_y": 51.0,
|
||||
},
|
||||
},
|
||||
temporal_unknown_reason="derived input has no precise observation timestamp",
|
||||
source_version_unknown_reason="transform version is recorded separately",
|
||||
lineage=valid_lineage,
|
||||
)
|
||||
)
|
||||
|
||||
assert valid_report.validation_status.value == "passed"
|
||||
assert valid_lineage.upstream_asset_ids == (str(parent.id),)
|
||||
assert valid_lineage.upstream_checksums_sha256 == (_CHECKSUM,)
|
||||
|
||||
parent.validation_status = "not_validated"
|
||||
rejected_lineage = DerivedDatasetGovernanceService._lineage_evidence(
|
||||
parent, "vector.clip", {}
|
||||
)
|
||||
rejected_report = validate_registered_asset(
|
||||
build_vector_ingest_input(
|
||||
asset_id="derived-rejected",
|
||||
source_crs="EPSG:4326",
|
||||
storage_crs="EPSG:4326",
|
||||
feature_collection={
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
}
|
||||
],
|
||||
},
|
||||
checksum_sha256=_CHECKSUM,
|
||||
computed_checksum_sha256=_CHECKSUM,
|
||||
source_registry_id="derived-source",
|
||||
source_snapshot_id="derived-snapshot",
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
metadata={
|
||||
"license": "internal derived artifact",
|
||||
"bounds_json": {
|
||||
"min_x": 5.0,
|
||||
"min_y": 51.0,
|
||||
"max_x": 5.0,
|
||||
"max_y": 51.0,
|
||||
},
|
||||
},
|
||||
temporal_unknown_reason="derived input has no precise observation timestamp",
|
||||
source_version_unknown_reason="transform version is recorded separately",
|
||||
lineage=rejected_lineage,
|
||||
)
|
||||
)
|
||||
|
||||
assert rejected_lineage.upstream_checksums_sha256 == (
|
||||
"parent_dataset_not_governed",
|
||||
)
|
||||
assert rejected_report.validation_status.value == "failed"
|
||||
assert rejected_report.quarantine_status.value == "quarantined"
|
||||
assert any(
|
||||
issue.code == "UPSTREAM_CHECKSUM_FORMAT_INVALID"
|
||||
for issue in rejected_report.issues
|
||||
)
|
||||
|
||||
|
||||
def test_govern_vector_binds_snapshot_contract_and_lineage_before_marking_ready(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
from app.services.source_registry_service import SourceRegistryService
|
||||
|
||||
db = _GovernedSession()
|
||||
parent = _governed_parent()
|
||||
parent_version_id = uuid4()
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=parent.project_id,
|
||||
name="derived.geojson",
|
||||
dataset_type="vector",
|
||||
source="operation:clip",
|
||||
source_name="derived",
|
||||
checksum_sha256=_CHECKSUM,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
crs="EPSG:4326",
|
||||
metadata_json={
|
||||
"bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0}
|
||||
},
|
||||
status="validating",
|
||||
)
|
||||
version = DatasetVersion(
|
||||
id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM
|
||||
)
|
||||
source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact")
|
||||
snapshot = SimpleNamespace(id=uuid4(), source_registry_id=source.id)
|
||||
edges = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
SourceRegistryService,
|
||||
"ensure_server_owned_source",
|
||||
lambda *_args, **_kwargs: source,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DerivedDatasetGovernanceService,
|
||||
"_latest_parent_version_id",
|
||||
lambda *_args: parent_version_id,
|
||||
)
|
||||
|
||||
def _bind(target, **kwargs):
|
||||
target.source_registry_id = kwargs["source"].id
|
||||
target.source_snapshot_id = kwargs["snapshot"].id
|
||||
target.data_contract_key = kwargs["data_contract_key"]
|
||||
target.data_contract_version = kwargs["data_contract_version"]
|
||||
target.validation_status = kwargs["validation_status"]
|
||||
target.provenance_status = kwargs["provenance_status"]
|
||||
target.lineage_status = kwargs["lineage_status"]
|
||||
return target
|
||||
|
||||
monkeypatch.setattr(SourceRegistryService, "bind_dataset_provenance", _bind)
|
||||
monkeypatch.setattr(SourceRegistryService, "bind_dataset_version_provenance", _bind)
|
||||
monkeypatch.setattr(
|
||||
SourceRegistryService,
|
||||
"record_lineage_edge",
|
||||
lambda *_args, **kwargs: edges.append(kwargs),
|
||||
)
|
||||
|
||||
ready = DerivedDatasetGovernanceService.govern_vector(
|
||||
db,
|
||||
dataset=dataset,
|
||||
dataset_version=version,
|
||||
feature_collection={
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
}
|
||||
],
|
||||
},
|
||||
source_key="derived",
|
||||
operation="vector.clip",
|
||||
parent_dataset=parent,
|
||||
operation_parameters={"area_id": "a"},
|
||||
)
|
||||
|
||||
assert ready is True
|
||||
assert dataset.status == "ready"
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.provenance_status == "complete"
|
||||
assert dataset.source_registry_id == source.id
|
||||
assert version.source_snapshot_id == snapshot.id
|
||||
assert db.flushes >= 1
|
||||
assert edges[0]["parent_dataset_id"] == parent.id
|
||||
assert edges[0]["parent_dataset_version_id"] == parent_version_id
|
||||
assert edges[0]["child_dataset_version_id"] == version.id
|
||||
|
||||
|
||||
def test_govern_vector_quarantines_output_when_parent_is_manual_or_experimental(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
from app.services.source_registry_service import SourceRegistryService
|
||||
|
||||
db = _GovernedSession()
|
||||
parent = _governed_parent()
|
||||
parent.source = "manual"
|
||||
parent.source_name = "manual"
|
||||
parent.source_registry.source_key = "manual"
|
||||
parent.source_registry.classification = "experimental"
|
||||
parent.source_registry.usage_policy_json = {"ground_truth_allowed": False}
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=parent.project_id,
|
||||
name="manual-derived.geojson",
|
||||
dataset_type="vector",
|
||||
source="operation:clip",
|
||||
source_name="derived",
|
||||
checksum_sha256=_CHECKSUM,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
crs="EPSG:4326",
|
||||
metadata_json={
|
||||
"bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0}
|
||||
},
|
||||
status="validating",
|
||||
)
|
||||
version = DatasetVersion(
|
||||
id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM
|
||||
)
|
||||
source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact")
|
||||
snapshot = SimpleNamespace(
|
||||
id=uuid4(), source_registry_id=source.id, ingest_status="ingested"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
SourceRegistryService,
|
||||
"ensure_server_owned_source",
|
||||
lambda *_args, **_kwargs: source,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DerivedDatasetGovernanceService,
|
||||
"_latest_parent_version_id",
|
||||
lambda *_args: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SourceRegistryService, "record_lineage_edge", lambda *_args, **_kwargs: None
|
||||
)
|
||||
|
||||
ready = DerivedDatasetGovernanceService.govern_vector(
|
||||
db,
|
||||
dataset=dataset,
|
||||
dataset_version=version,
|
||||
feature_collection={
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
}
|
||||
],
|
||||
},
|
||||
source_key="derived",
|
||||
operation="vector.clip",
|
||||
parent_dataset=parent,
|
||||
)
|
||||
|
||||
assert ready is False
|
||||
assert dataset.status == "quarantined"
|
||||
assert dataset.quarantine_status == "quarantined"
|
||||
assert dataset.validation_status == "failed"
|
||||
assert any(
|
||||
issue["code"] == "PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING"
|
||||
for issue in dataset.validation_report_json["issues"]
|
||||
)
|
||||
assert snapshot.ingest_status == "quarantined"
|
||||
|
||||
|
||||
def test_vector_selection_uses_map_selection_registry_and_skips_features_when_quarantined(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
source = _governed_parent()
|
||||
source.area_id = None
|
||||
source.storage_path = str(tmp_path / "source.geojson")
|
||||
db = _FakeSession({(Dataset, source.id): source})
|
||||
output_path = tmp_path / "selection.geojson"
|
||||
calls = []
|
||||
persisted_features = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
VectorFeatureService,
|
||||
"select_features_by_bbox",
|
||||
lambda *_args, **_kwargs: {
|
||||
"selection_bbox": {
|
||||
"min_x": 4.9,
|
||||
"min_y": 50.9,
|
||||
"max_x": 5.2,
|
||||
"max_y": 51.2,
|
||||
"crs": "EPSG:4326",
|
||||
},
|
||||
"feature_count": 1,
|
||||
"limit": 250,
|
||||
"truncated": False,
|
||||
"geojson": {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": "source-feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {
|
||||
"vector_feature_id": "source-feature",
|
||||
"dataset_id": str(source.id),
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
def _persist_dataset_file(**kwargs):
|
||||
output_path.write_bytes(kwargs["content"])
|
||||
return {
|
||||
"original_filename": kwargs["original_filename"],
|
||||
"stored_filename": output_path.name,
|
||||
"content_type": kwargs["content_type"],
|
||||
"size_bytes": len(kwargs["content"]),
|
||||
"checksum_sha256": _CHECKSUM,
|
||||
"storage_path": str(output_path),
|
||||
}
|
||||
|
||||
monkeypatch.setattr(StorageService, "persist_dataset_file", _persist_dataset_file)
|
||||
|
||||
def _quarantine(db, **kwargs):
|
||||
calls.append(kwargs)
|
||||
kwargs["dataset"].status = "quarantined"
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine)
|
||||
monkeypatch.setattr(
|
||||
VectorFeatureService,
|
||||
"persist_geojson_features",
|
||||
lambda **kwargs: persisted_features.append(kwargs),
|
||||
)
|
||||
|
||||
response = VectorOperationsService.derive_selection_dataset(
|
||||
db=db,
|
||||
dataset_id=source.id,
|
||||
bbox={
|
||||
"min_x": 4.9,
|
||||
"min_y": 50.9,
|
||||
"max_x": 5.2,
|
||||
"max_y": 51.2,
|
||||
"crs": "EPSG:4326",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status == "quarantined"
|
||||
assert calls[0]["source_key"] == "map_selection"
|
||||
assert calls[0]["parent_dataset"] is source
|
||||
assert calls[0]["operation"] == "vector.selection"
|
||||
assert persisted_features == []
|
||||
|
||||
|
||||
def test_vector_buffer_uses_projected_metres_instead_of_wgs84_degrees(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
source = _governed_parent()
|
||||
source.storage_path = str(tmp_path / "source.geojson")
|
||||
source.crs = "EPSG:4326"
|
||||
Path(source.storage_path).write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"crs": "EPSG:4326",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
},
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
# A governed consumption artifact must carry the checksum of these exact
|
||||
# bytes; vector operations deliberately refuse a stale fixture checksum.
|
||||
source.checksum_sha256 = sha256(Path(source.storage_path).read_bytes()).hexdigest()
|
||||
source.source_snapshot.checksum_sha256 = source.checksum_sha256
|
||||
db = _FakeSession({(Dataset, source.id): source})
|
||||
captured = {}
|
||||
|
||||
def _persist(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return uuid4()
|
||||
|
||||
monkeypatch.setattr(VectorOperationsService, "_persist_derived_dataset", _persist)
|
||||
VectorOperationsService.buffer(
|
||||
db, source.id, distance_m=100.0, dissolve=False, output_name=None
|
||||
)
|
||||
|
||||
coordinates = captured["feature_collection"]["features"][0]["geometry"][
|
||||
"coordinates"
|
||||
][0]
|
||||
longitudes = [coordinate[0] for coordinate in coordinates]
|
||||
latitudes = [coordinate[1] for coordinate in coordinates]
|
||||
assert max(longitudes) - min(longitudes) < 0.01
|
||||
assert max(latitudes) - min(latitudes) < 0.01
|
||||
|
||||
|
||||
def test_raster_operation_uses_derived_registry_before_commit(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
source = _governed_parent()
|
||||
source.dataset_type = "raster"
|
||||
source.storage_path = str(tmp_path / "source.tif")
|
||||
output_path = tmp_path / "derived.tif"
|
||||
output_path.write_bytes(b"derived-raster")
|
||||
db = _FakeSession()
|
||||
calls = []
|
||||
|
||||
def _govern(db, **kwargs):
|
||||
calls.append(kwargs)
|
||||
kwargs["dataset"].status = "quarantined"
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_raster", _govern)
|
||||
|
||||
result = RasterOperationsService._persist_derived_dataset(
|
||||
db,
|
||||
source_dataset=source,
|
||||
source_dataset_id=source.id,
|
||||
operation="ndvi",
|
||||
output_path=str(output_path),
|
||||
output_name="derived.tif",
|
||||
metadata={
|
||||
"crs": "EPSG:31370",
|
||||
"bounds": [100000.0, 100000.0, 100001.0, 100001.0],
|
||||
"resolution": [1.0, 1.0],
|
||||
"width": 1,
|
||||
"height": 1,
|
||||
"band_count": 1,
|
||||
"dtype": ["float32"],
|
||||
"operation_parameters": {"nir_band": 4, "red_band": 3},
|
||||
},
|
||||
)
|
||||
|
||||
derived = next(item for item in db.added if isinstance(item, Dataset))
|
||||
assert result == derived.id
|
||||
assert derived.status == "quarantined"
|
||||
assert derived.source_name == "derived"
|
||||
assert calls[0]["source_key"] == "derived"
|
||||
assert calls[0]["parent_dataset"] is source
|
||||
assert calls[0]["operation"] == "raster.ndvi"
|
||||
|
||||
|
||||
def test_demo_fixture_creation_is_governed_and_does_not_persist_features_when_rejected(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
project_id = uuid4()
|
||||
area_id = uuid4()
|
||||
db = _FakeSession()
|
||||
calls = []
|
||||
features = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
StorageService,
|
||||
"persist_dataset_file",
|
||||
lambda **kwargs: {
|
||||
"storage_path": str(tmp_path / kwargs["original_filename"]),
|
||||
"original_filename": kwargs["original_filename"],
|
||||
"stored_filename": kwargs["original_filename"],
|
||||
"content_type": kwargs["content_type"],
|
||||
"size_bytes": len(kwargs["content"]),
|
||||
"checksum_sha256": _CHECKSUM,
|
||||
},
|
||||
)
|
||||
|
||||
def _quarantine(db, **kwargs):
|
||||
calls.append(kwargs)
|
||||
kwargs["dataset"].status = "quarantined"
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine)
|
||||
monkeypatch.setattr(
|
||||
VectorFeatureService,
|
||||
"persist_geojson_features",
|
||||
lambda **kwargs: features.append(kwargs),
|
||||
)
|
||||
payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
dataset = DemoWorkflowService._create_dataset(
|
||||
db,
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
filename="fixture.geojson",
|
||||
payload=payload,
|
||||
raw=json.dumps(payload).encode("utf-8"),
|
||||
role="source",
|
||||
source_name="fixture",
|
||||
reference_layer_name=None,
|
||||
)
|
||||
|
||||
assert dataset.status == "quarantined"
|
||||
assert calls[0]["source_key"] == "fixture"
|
||||
assert calls[0]["operation"] == "demo.fixture_vector"
|
||||
assert features == []
|
||||
@@ -0,0 +1,453 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from geoalchemy2.shape import from_shape
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import MultiPolygon, Polygon
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.schemas.official_vector import OfficialVectorAcquireRequest
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.official_vector_acquisition_service import (
|
||||
OfficialVectorAcquisitionService,
|
||||
_TO_LAMBERT72,
|
||||
)
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def __init__(self, result=None):
|
||||
self.result = result
|
||||
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
|
||||
def order_by(self, *_args):
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return self.result if isinstance(self.result, list) else []
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, rows=None, query_result=None):
|
||||
self.rows = rows or {}
|
||||
self.query_result = query_result
|
||||
|
||||
def get(self, model, row_id):
|
||||
return self.rows.get((model, row_id))
|
||||
|
||||
def query(self, _model):
|
||||
return FakeQuery(self.query_result)
|
||||
|
||||
|
||||
class JsonResponse:
|
||||
def __init__(self, payload, content_type="application/geo+json"):
|
||||
self.content = json.dumps(payload).encode("utf-8")
|
||||
self.content_type = content_type
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self, size=-1):
|
||||
return self.content if size < 0 else self.content[:size]
|
||||
|
||||
def getheader(self, name):
|
||||
return self.content_type if name.lower() == "content-type" else None
|
||||
|
||||
|
||||
def request(product_key: str, bbox: tuple[float, float, float, float], area_id=None):
|
||||
return OfficialVectorAcquireRequest(
|
||||
bbox={
|
||||
"min_x": bbox[0],
|
||||
"min_y": bbox[1],
|
||||
"max_x": bbox[2],
|
||||
"max_y": bbox[3],
|
||||
"crs": "EPSG:4326",
|
||||
},
|
||||
area_id=area_id,
|
||||
product_key=product_key,
|
||||
force_refresh=True,
|
||||
)
|
||||
|
||||
|
||||
def area(project_id, name: str, bounds: tuple[float, float, float, float]):
|
||||
min_x, min_y, max_x, max_y = bounds
|
||||
geometry = MultiPolygon(
|
||||
[
|
||||
Polygon(
|
||||
[
|
||||
(min_x, min_y),
|
||||
(max_x, min_y),
|
||||
(max_x, max_y),
|
||||
(min_x, max_y),
|
||||
(min_x, min_y),
|
||||
]
|
||||
)
|
||||
]
|
||||
)
|
||||
return Area(
|
||||
id=uuid4(),
|
||||
project_id=project_id,
|
||||
name=name,
|
||||
geometry=from_shape(geometry, srid=4326),
|
||||
)
|
||||
|
||||
|
||||
def test_regional_product_registry_is_explicit_and_source_specific() -> None:
|
||||
products = {
|
||||
item["key"]: item
|
||||
for item in OfficialVectorAcquisitionService.list_products()
|
||||
}
|
||||
|
||||
assert products["spw_picc_buildings"]["coverage_zones"] == ["wallonia"]
|
||||
assert products["spw_picc_roads"]["geometry_types"] == [
|
||||
"LineString",
|
||||
"MultiLineString",
|
||||
]
|
||||
assert products["spw_picc_waterways"]["collection"] == "28"
|
||||
assert products["spw_picc_water_surfaces"]["collection"] == "30"
|
||||
assert products["spw_flood_hazard_2021"]["collection"] == "2"
|
||||
assert products["spw_flood_hazard_2021"]["theme"] == "flood_hazard"
|
||||
assert products["spw_flood_hazard_2021"]["coverage_zones"] == ["wallonia"]
|
||||
assert products["urbis_buildings"]["coverage_zones"] == ["brussels"]
|
||||
assert products["urbis_buildings"]["license_note"] == "Buildings are published under CC0."
|
||||
assert "FPS Finance" in products["urbis_cadastral_parcels"]["license_note"]
|
||||
# urbis_street_axes is live-validated against the UrbIS WFS capabilities:
|
||||
# urbisvector:StreetAxes exposes INSPIRE_ID and LineString geometry. The
|
||||
# same capabilities document advertises no hydrography feature type, so
|
||||
# Brussels surface water intentionally stays not_configured.
|
||||
assert products["urbis_street_axes"]["coverage_zones"] == ["brussels"]
|
||||
assert products["urbis_street_axes"]["collection"] == "urbisvector:StreetAxes"
|
||||
assert products["urbis_street_axes"]["geometry_types"] == [
|
||||
"LineString",
|
||||
"MultiLineString",
|
||||
]
|
||||
assert products["urbis_street_axes"]["theme"] == "roads"
|
||||
assert products["urbis_land_cover_blocks"]["collection"] == "urbisvector:Blocks"
|
||||
assert products["urbis_land_cover_blocks"]["theme"] == "space_occupation"
|
||||
assert products["urbis_forest_parks"]["theme"] == "forest"
|
||||
assert products["urbis_water_surfaces"]["theme"] == "water"
|
||||
|
||||
|
||||
def test_urbis_land_cover_products_filter_only_documented_block_classes() -> None:
|
||||
scope_wgs84 = Polygon(
|
||||
[(4.35, 50.84), (4.36, 50.84), (4.36, 50.85), (4.35, 50.85), (4.35, 50.84)]
|
||||
)
|
||||
scope_metric = Polygon([_TO_LAMBERT72.transform(x, y) for x, y in scope_wgs84.exterior.coords])
|
||||
min_x, min_y, max_x, max_y = scope_metric.bounds
|
||||
|
||||
def block(block_type: str):
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": f"Blocks.{block_type}",
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[
|
||||
[min_x + 10, min_y + 10],
|
||||
[min_x + 100, min_y + 10],
|
||||
[min_x + 100, min_y + 100],
|
||||
[min_x + 10, min_y + 100],
|
||||
[min_x + 10, min_y + 10],
|
||||
]],
|
||||
},
|
||||
"properties": {
|
||||
"INSPIRE_ID": f"https://databrussels.be/id/block/{block_type}",
|
||||
"TYPE": block_type,
|
||||
},
|
||||
}
|
||||
|
||||
forest_product = OfficialVectorAcquisitionService._product("urbis_forest_parks")
|
||||
water_product = OfficialVectorAcquisitionService._product("urbis_water_surfaces")
|
||||
land_cover_product = OfficialVectorAcquisitionService._product("urbis_land_cover_blocks")
|
||||
|
||||
assert OfficialVectorAcquisitionService._normalize_regional_feature(
|
||||
forest_product, block("FO"), scope_metric, "brussels"
|
||||
) is not None
|
||||
assert OfficialVectorAcquisitionService._normalize_regional_feature(
|
||||
forest_product, block("CB"), scope_metric, "brussels"
|
||||
) is None
|
||||
assert OfficialVectorAcquisitionService._normalize_regional_feature(
|
||||
water_product, block("WB"), scope_metric, "brussels"
|
||||
) is not None
|
||||
assert OfficialVectorAcquisitionService._normalize_regional_feature(
|
||||
water_product, block("GB"), scope_metric, "brussels"
|
||||
) is None
|
||||
normalized = OfficialVectorAcquisitionService._normalize_regional_feature(
|
||||
land_cover_product, block("CB"), scope_metric, "brussels"
|
||||
)
|
||||
assert normalized is not None
|
||||
assert normalized["properties"]["TYPE"] == "CB"
|
||||
assert normalized["properties"]["clipped_area_ha"] > 0
|
||||
|
||||
|
||||
def test_spw_arcgis_paging_is_bounded_stable_and_clipped() -> None:
|
||||
product = OfficialVectorAcquisitionService._product("spw_picc_buildings")
|
||||
scope = Polygon(
|
||||
[(4.55, 50.58), (4.56, 50.58), (4.56, 50.59), (4.55, 50.59), (4.55, 50.58)]
|
||||
)
|
||||
scope_metric = Polygon(
|
||||
[
|
||||
_TO_LAMBERT72.transform(x, y)
|
||||
for x, y in scope.exterior.coords
|
||||
]
|
||||
)
|
||||
offsets = []
|
||||
|
||||
def feature(object_id: int, min_x: float):
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": object_id,
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[
|
||||
[min_x, 50.581],
|
||||
[min_x + 0.002, 50.581],
|
||||
[min_x + 0.002, 50.583],
|
||||
[min_x, 50.583],
|
||||
[min_x, 50.581],
|
||||
]],
|
||||
},
|
||||
"properties": {"OBJECTID": object_id, "GEOREF_ID": f"wallonia-{object_id}"},
|
||||
}
|
||||
|
||||
def opener(raw_request, timeout):
|
||||
assert timeout == 180
|
||||
query = parse_qs(urlparse(raw_request.full_url).query)
|
||||
assert query["orderByFields"] == ["OBJECTID"]
|
||||
assert query["f"] == ["geojson"]
|
||||
offset = int(query["resultOffset"][0])
|
||||
offsets.append(offset)
|
||||
return JsonResponse(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [feature(offset + 1, 4.551 + offset * 0.0001)],
|
||||
"exceededTransferLimit": offset == 0,
|
||||
}
|
||||
)
|
||||
|
||||
features, transfer = OfficialVectorAcquisitionService._fetch_features(
|
||||
product,
|
||||
scope,
|
||||
scope_metric,
|
||||
"wallonia",
|
||||
Settings(_env_file=None, OFFICIAL_VECTOR_PAGE_SIZE=1),
|
||||
opener,
|
||||
)
|
||||
|
||||
assert offsets == [0, 1]
|
||||
assert transfer["page_count"] == 2
|
||||
assert transfer["reference_truncated"] is False
|
||||
assert {item["properties"]["source_feature_id"] for item in features} == {
|
||||
"11:wallonia-1",
|
||||
"11:wallonia-2",
|
||||
}
|
||||
assert all(item["properties"]["coverage_scope"] == "wallonia" for item in features)
|
||||
assert all(item["properties"]["clipped_area_ha"] > 0 for item in features)
|
||||
|
||||
|
||||
def test_spw_flood_hazard_uses_separate_governed_endpoint_and_persists_classification() -> None:
|
||||
product = OfficialVectorAcquisitionService._product("spw_flood_hazard_2021")
|
||||
scope = Polygon(
|
||||
[(4.55, 50.58), (4.56, 50.58), (4.56, 50.59), (4.55, 50.59), (4.55, 50.58)]
|
||||
)
|
||||
scope_metric = Polygon(
|
||||
[_TO_LAMBERT72.transform(x, y) for x, y in scope.exterior.coords]
|
||||
)
|
||||
|
||||
def opener(raw_request, timeout):
|
||||
assert timeout == 180
|
||||
parsed = urlparse(raw_request.full_url)
|
||||
assert parsed.path.endswith("/EAU/ALEA_INOND/MapServer/2/query")
|
||||
query = parse_qs(parsed.query)
|
||||
assert query["outSR"] == ["4326"]
|
||||
return JsonResponse(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": 7,
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[
|
||||
[4.551, 50.581],
|
||||
[4.559, 50.581],
|
||||
[4.559, 50.589],
|
||||
[4.551, 50.589],
|
||||
[4.551, 50.581],
|
||||
]],
|
||||
},
|
||||
"properties": {
|
||||
"OBJECTID": 7,
|
||||
"LOCALID": "ALEA-7",
|
||||
"TYPEALEA": "Debordement",
|
||||
"CLASSEMENT": 130,
|
||||
"MILLESIME": 2021,
|
||||
},
|
||||
}
|
||||
],
|
||||
"exceededTransferLimit": False,
|
||||
}
|
||||
)
|
||||
|
||||
features, transfer = OfficialVectorAcquisitionService._fetch_features(
|
||||
product,
|
||||
scope,
|
||||
scope_metric,
|
||||
"wallonia",
|
||||
Settings(_env_file=None),
|
||||
opener,
|
||||
)
|
||||
|
||||
assert transfer["feature_count"] == 1
|
||||
assert features[0]["id"] == "2:ALEA-7"
|
||||
assert features[0]["properties"]["CLASSEMENT"] == 130
|
||||
assert features[0]["properties"]["source_name"] == "spw_flood_hazard"
|
||||
assert features[0]["properties"]["clipped_area_ha"] > 0
|
||||
|
||||
|
||||
def test_spw_flood_hazard_can_be_disabled_independently() -> None:
|
||||
project_id = uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
OfficialVectorAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
request("spw_flood_hazard_2021", (4.55, 50.58, 4.56, 50.59)),
|
||||
settings=Settings(_env_file=None, SPW_FLOOD_HAZARD_ENABLED=False),
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "SPW_FLOOD_HAZARD_NOT_CONFIGURED"
|
||||
|
||||
|
||||
def test_regional_products_require_the_persisted_authoritative_coverage_area() -> None:
|
||||
project_id = uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
OfficialVectorAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
request("spw_picc_buildings", (4.55, 50.58, 4.56, 50.59)),
|
||||
settings=Settings(_env_file=None),
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "OFFICIAL_VECTOR_COVERAGE_NOT_READY"
|
||||
|
||||
|
||||
def test_urbis_wfs_transforms_lambert72_and_persists_through_dataset_service(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
project_id, dataset_id = uuid4(), uuid4()
|
||||
brussels = area(project_id, "Brussels-Capital Region", (4.25, 50.75, 4.5, 50.95))
|
||||
db = FakeSession(
|
||||
{
|
||||
(Project, project_id): Project(id=project_id, name="Belgium"),
|
||||
(Area, brussels.id): brussels,
|
||||
},
|
||||
query_result=[brussels],
|
||||
)
|
||||
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
min_x, min_y = to_lambert.transform(4.35, 50.84)
|
||||
max_x, max_y = to_lambert.transform(4.351, 50.841)
|
||||
captured = {}
|
||||
|
||||
def opener(raw_request, timeout):
|
||||
assert timeout == 180
|
||||
query = parse_qs(urlparse(raw_request.full_url).query)
|
||||
assert query["typeNames"] == ["urbisvector:Buildings"]
|
||||
assert query["srsName"] == ["EPSG:31370"]
|
||||
assert query["sortBy"] == ["INSPIRE_ID"]
|
||||
return JsonResponse(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"numberMatched": 1,
|
||||
"numberReturned": 1,
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": "Buildings.1",
|
||||
"geometry": {
|
||||
"type": "MultiPolygon",
|
||||
"coordinates": [[[
|
||||
[min_x, min_y],
|
||||
[max_x, min_y],
|
||||
[max_x, max_y],
|
||||
[min_x, max_y],
|
||||
[min_x, min_y],
|
||||
]]],
|
||||
},
|
||||
"properties": {
|
||||
"INSPIRE_ID": "https://databrussels.be/id/building/1",
|
||||
"AREA": 75,
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
"application/json",
|
||||
)
|
||||
|
||||
def persist(_db, **kwargs):
|
||||
captured.update(kwargs)
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
area_id=brussels.id,
|
||||
name=kwargs["filename"],
|
||||
dataset_type="vector",
|
||||
source=kwargs["source"],
|
||||
dataset_role=kwargs["dataset_role"],
|
||||
source_name=kwargs["source_name"],
|
||||
reference_layer_name=kwargs["reference_layer_name"],
|
||||
temporal_series_key=kwargs["temporal_series_key"],
|
||||
observed_at=kwargs["observed_at"],
|
||||
source_version=kwargs["source_version"],
|
||||
source_metadata=kwargs["source_metadata"],
|
||||
provenance_metadata=kwargs["provenance_metadata"],
|
||||
metadata_json={"feature_count": 1},
|
||||
status="ready",
|
||||
)
|
||||
db.rows[(Dataset, dataset_id)] = dataset
|
||||
return SimpleNamespace(id=dataset_id)
|
||||
|
||||
monkeypatch.setattr(DatasetService, "import_vector_bytes", persist)
|
||||
result = OfficialVectorAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
request(
|
||||
"urbis_buildings",
|
||||
(4.349, 50.839, 4.352, 50.842),
|
||||
area_id=brussels.id,
|
||||
),
|
||||
settings=Settings(_env_file=None),
|
||||
opener=opener,
|
||||
)
|
||||
|
||||
assert result["output_dataset_id"] == str(dataset_id)
|
||||
assert captured["source_name"] == "urbis"
|
||||
assert captured["reference_layer_name"] == "buildings"
|
||||
assert captured["source_metadata"]["coverage_zones"] == ["brussels"]
|
||||
assert captured["source_metadata"]["selection_aggregation"]["metric_key"] == (
|
||||
"building_footprint_area"
|
||||
)
|
||||
collection = json.loads(captured["content"])
|
||||
geometry = collection["features"][0]["geometry"]
|
||||
assert geometry["type"] in {"Polygon", "MultiPolygon"}
|
||||
first_coordinate = (
|
||||
geometry["coordinates"][0][0][0]
|
||||
if geometry["type"] == "MultiPolygon"
|
||||
else geometry["coordinates"][0][0]
|
||||
)
|
||||
assert 4.34999 <= first_coordinate[0] <= 4.35101
|
||||
assert 50.83999 <= first_coordinate[1] <= 50.84101
|
||||
@@ -0,0 +1,144 @@
|
||||
"""What is promoted must be what the workbench then runs.
|
||||
|
||||
The candidate evaluation freezes its post-processing before the protected test
|
||||
— NMS IoU and a containment threshold selected during calibration. The runtime
|
||||
applied its own hardcoded containment value, so a model gated at one setting
|
||||
was served at another and suppressed detections the gate had counted. The
|
||||
difference is invisible in both reports.
|
||||
|
||||
Containment is therefore configuration, recorded with every run, and two runs
|
||||
that post-processed differently are not comparable however good their numbers
|
||||
look.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.services.detection_comparison_service import DetectionComparisonService
|
||||
from app.services.detection_service import DetectionService
|
||||
|
||||
|
||||
def _candidate(name: str, geometry, confidence: float):
|
||||
return {
|
||||
"class_name": "building",
|
||||
"confidence": confidence,
|
||||
"geometry": geometry,
|
||||
"bbox": [0.0, 0.0, 1.0, 1.0],
|
||||
"source_tile_path": f"/tiles/{name}.tif",
|
||||
"properties": {"name": name},
|
||||
}
|
||||
|
||||
|
||||
class TestConfigurableContainment:
|
||||
def test_the_runtime_threshold_comes_from_settings(self) -> None:
|
||||
assert Settings(_env_file=None).yolo_containment_nms_threshold == pytest.approx(0.85)
|
||||
assert Settings(
|
||||
_env_file=None, yolo_containment_nms_threshold=1.0
|
||||
).yolo_containment_nms_threshold == pytest.approx(1.0)
|
||||
|
||||
def test_a_strict_threshold_suppresses_only_a_fully_nested_box(self) -> None:
|
||||
outer = _candidate("outer", box(0, 0, 10, 10), 0.9)
|
||||
# 90% of the smaller box lies inside the larger one, but their IoU is
|
||||
# only 0.09 — so only the containment rule can act on this pair.
|
||||
mostly_nested = _candidate("mostly", box(8.2, 1, 10.2, 6), 0.5)
|
||||
|
||||
kept = DetectionService._suppress_duplicate_candidates(
|
||||
[outer, mostly_nested], iou_threshold=0.5, containment_threshold=1.0
|
||||
)
|
||||
|
||||
assert [item["properties"]["name"] for item in kept] == ["outer", "mostly"]
|
||||
|
||||
def test_a_looser_threshold_suppresses_it(self) -> None:
|
||||
outer = _candidate("outer", box(0, 0, 10, 10), 0.9)
|
||||
mostly_nested = _candidate("mostly", box(8.2, 1, 10.2, 6), 0.5)
|
||||
|
||||
kept = DetectionService._suppress_duplicate_candidates(
|
||||
[outer, mostly_nested], iou_threshold=0.5, containment_threshold=0.7
|
||||
)
|
||||
|
||||
assert [item["properties"]["name"] for item in kept] == ["outer"]
|
||||
|
||||
def test_the_default_matches_the_documented_runtime_value(self) -> None:
|
||||
outer = _candidate("outer", box(0, 0, 10, 10), 0.9)
|
||||
nested = _candidate("nested", box(1, 1, 9, 9), 0.5)
|
||||
|
||||
kept = DetectionService._suppress_duplicate_candidates([outer, nested], iou_threshold=0.5)
|
||||
|
||||
assert [item["properties"]["name"] for item in kept] == ["outer"]
|
||||
|
||||
|
||||
class TestComparabilityOfPostProcessing:
|
||||
def _entry(self, *, dataset_id, reference_id, containment: float, duplicate_iou: float = 0.5):
|
||||
from uuid import uuid4
|
||||
|
||||
return {
|
||||
"analysis_run_id": uuid4(),
|
||||
"dataset_id": dataset_id,
|
||||
"model_id": "yolo-configured",
|
||||
"model_asset_id": "asset",
|
||||
"reference_dataset_id": reference_id,
|
||||
"coverage_mode": "persisted_tile_manifest_union",
|
||||
"reference_evaluated_count": 100,
|
||||
"containment_suppression_threshold": containment,
|
||||
"duplicate_iou_threshold": duplicate_iou,
|
||||
}
|
||||
|
||||
def test_runs_with_the_same_post_processing_stay_comparable(self) -> None:
|
||||
from uuid import uuid4
|
||||
|
||||
dataset_id, reference_id = uuid4(), uuid4()
|
||||
report = DetectionComparisonService.assess_comparability(
|
||||
[
|
||||
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85),
|
||||
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85),
|
||||
]
|
||||
)
|
||||
|
||||
assert report["comparable"] is True
|
||||
|
||||
def test_a_different_containment_threshold_blocks_the_comparison(self) -> None:
|
||||
from uuid import uuid4
|
||||
|
||||
dataset_id, reference_id = uuid4(), uuid4()
|
||||
report = DetectionComparisonService.assess_comparability(
|
||||
[
|
||||
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85),
|
||||
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=1.0),
|
||||
]
|
||||
)
|
||||
|
||||
assert report["comparable"] is False
|
||||
assert "different_post_processing" in report["blocking_reasons"]
|
||||
|
||||
def test_a_different_duplicate_iou_blocks_the_comparison(self) -> None:
|
||||
from uuid import uuid4
|
||||
|
||||
dataset_id, reference_id = uuid4(), uuid4()
|
||||
report = DetectionComparisonService.assess_comparability(
|
||||
[
|
||||
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85, duplicate_iou=0.5),
|
||||
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85, duplicate_iou=0.7),
|
||||
]
|
||||
)
|
||||
|
||||
assert report["comparable"] is False
|
||||
assert "different_post_processing" in report["blocking_reasons"]
|
||||
|
||||
def test_runs_from_before_the_setting_existed_do_not_block(self) -> None:
|
||||
"""Older runs recorded no threshold; absence is not a difference."""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
dataset_id, reference_id = uuid4(), uuid4()
|
||||
entries = [
|
||||
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85),
|
||||
self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85),
|
||||
]
|
||||
for entry in entries:
|
||||
entry.pop("containment_suppression_threshold")
|
||||
entry.pop("duplicate_iou_threshold")
|
||||
|
||||
assert DetectionComparisonService.assess_comparability(entries)["comparable"] is True
|
||||
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_publication_hygiene_gate_passes_for_tracked_tree() -> None:
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(ROOT / "scripts" / "check_repository_hygiene.py")],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_checkpoint_inspector_uses_weights_only_deserialization() -> None:
|
||||
inspector = (ROOT / "scripts" / "inspect_torch_checkpoint.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert "weights_only=True" in inspector
|
||||
assert "weights_only=False" not in inspector
|
||||
@@ -0,0 +1,88 @@
|
||||
"""QA matching must be reproducible and must credit the best candidate.
|
||||
|
||||
The greedy IoU matcher decides which candidate is reported as a match and
|
||||
which becomes false-positive evidence for an operator. If that decision
|
||||
depends on database row order, the same run produces different scores and
|
||||
points reviewers at the wrong geometry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.services.qa_service import QaService
|
||||
|
||||
|
||||
REFERENCE = [({"id": "R1"}, box(0.0, 0.0, 10.0, 10.0))]
|
||||
|
||||
# Two detections of the same building. ``sloppy`` is far too tall (IoU 0.51),
|
||||
# ``accurate`` is nearly exact (IoU 0.96).
|
||||
SLOPPY = box(0.0, 0.0, 10.0, 19.5)
|
||||
ACCURATE = box(0.0, 0.0, 10.0, 10.4)
|
||||
|
||||
|
||||
def _candidates(order: list[tuple[str, float]]) -> list[tuple[dict, object]]:
|
||||
geometries = {"sloppy": SLOPPY, "accurate": ACCURATE}
|
||||
return [
|
||||
({"id": name, "confidence": confidence}, geometries[name])
|
||||
for name, confidence in order
|
||||
]
|
||||
|
||||
|
||||
def test_matching_is_independent_of_candidate_row_order() -> None:
|
||||
forward = QaService._match_io_u_evidence(
|
||||
_candidates([("sloppy", 0.42), ("accurate", 0.91)]), REFERENCE, 0.5
|
||||
)
|
||||
reverse = QaService._match_io_u_evidence(
|
||||
_candidates([("accurate", 0.91), ("sloppy", 0.42)]), REFERENCE, 0.5
|
||||
)
|
||||
|
||||
assert forward.matches == reverse.matches
|
||||
assert forward.false_positives == reverse.false_positives
|
||||
assert forward.false_negatives == reverse.false_negatives
|
||||
assert forward.match_iou_values == reverse.match_iou_values
|
||||
assert forward.match_evidence == reverse.match_evidence
|
||||
assert forward.false_positive_evidence == reverse.false_positive_evidence
|
||||
|
||||
|
||||
def test_highest_confidence_candidate_claims_the_reference() -> None:
|
||||
evidence = QaService._match_io_u_evidence(
|
||||
_candidates([("sloppy", 0.42), ("accurate", 0.91)]), REFERENCE, 0.5
|
||||
)
|
||||
|
||||
assert evidence.matches == 1
|
||||
assert evidence.match_evidence[0]["candidate_feature_id"] == "accurate"
|
||||
assert evidence.false_positive_evidence == [{"candidate_feature_id": "sloppy"}]
|
||||
assert evidence.match_iou_values[0] > 0.9
|
||||
|
||||
|
||||
def test_matching_without_confidence_is_still_deterministic() -> None:
|
||||
"""Vector-vs-vector QA has no confidence; identity keeps it reproducible."""
|
||||
|
||||
left = [
|
||||
({"id": "b-second"}, SLOPPY),
|
||||
({"id": "a-first"}, ACCURATE),
|
||||
]
|
||||
right = list(reversed(left))
|
||||
|
||||
assert QaService._match_io_u_evidence(left, REFERENCE, 0.5).match_evidence == (
|
||||
QaService._match_io_u_evidence(right, REFERENCE, 0.5).match_evidence
|
||||
)
|
||||
|
||||
|
||||
def test_evidence_is_ordered_by_confidence_for_review() -> None:
|
||||
evidence = QaService._match_io_u_evidence(
|
||||
[
|
||||
({"id": "low", "confidence": 0.30}, box(30.0, 30.0, 31.0, 31.0)),
|
||||
({"id": "high", "confidence": 0.95}, box(40.0, 40.0, 41.0, 41.0)),
|
||||
({"id": "mid", "confidence": 0.60}, box(50.0, 50.0, 51.0, 51.0)),
|
||||
],
|
||||
REFERENCE,
|
||||
0.5,
|
||||
)
|
||||
|
||||
assert [item["candidate_feature_id"] for item in evidence.false_positive_evidence] == [
|
||||
"high",
|
||||
"mid",
|
||||
"low",
|
||||
]
|
||||
@@ -0,0 +1,205 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from app.models import Dataset, SourceRegistry, SourceSnapshot
|
||||
from app.services.qa_service import QaService
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, datasets=None, areas=None):
|
||||
self.datasets = {item.id: item for item in (datasets or [])}
|
||||
self.areas = {item.id: item for item in (areas or [])}
|
||||
|
||||
def get(self, model, item_id):
|
||||
if model.__name__ == "Dataset":
|
||||
return self.datasets.get(item_id)
|
||||
if model.__name__ == "Area":
|
||||
return self.areas.get(item_id)
|
||||
return None
|
||||
|
||||
|
||||
def _feature(feature_id: str, coordinates: list[list[list[float]]]) -> dict:
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": feature_id,
|
||||
"properties": {"source_feature_id": feature_id},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": coordinates,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _write_dataset(path: Path, coordinates: list[list[list[float]]]) -> None:
|
||||
payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [_feature("feature-1", coordinates)],
|
||||
}
|
||||
path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
|
||||
def _write_features(path: Path, features: list[dict]) -> None:
|
||||
path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8")
|
||||
|
||||
|
||||
def _authoritative_reference(dataset: Dataset) -> Dataset:
|
||||
"""Give QA reference fixtures the same durable authority proof as GRB."""
|
||||
|
||||
source_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
checksum = sha256(Path(str(dataset.storage_path)).read_bytes()).hexdigest()
|
||||
source = SourceRegistry(
|
||||
id=source_id,
|
||||
source_key="grb",
|
||||
display_name="GRB test reference",
|
||||
classification="authoritative",
|
||||
authority_name="Digitaal Vlaanderen",
|
||||
authority_scope_json={"zone": "Flanders"},
|
||||
usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}},
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=source_id,
|
||||
snapshot_key=f"qa-grb-{dataset.id}",
|
||||
checksum_sha256=checksum,
|
||||
ingest_status="ingested",
|
||||
freshness_status="current",
|
||||
)
|
||||
dataset.source = "grb"
|
||||
dataset.source_name = "grb"
|
||||
dataset.dataset_role = "reference"
|
||||
dataset.status = "ready"
|
||||
dataset.checksum_sha256 = checksum
|
||||
dataset.source_registry_id = source_id
|
||||
dataset.source_snapshot_id = snapshot_id
|
||||
dataset.data_contract_key = "geointel.vector.geojson"
|
||||
dataset.data_contract_version = "1.0.0"
|
||||
dataset.validation_status = "passed"
|
||||
dataset.provenance_status = "complete"
|
||||
dataset.lineage_status = "complete"
|
||||
dataset.quarantine_status = "not_quarantined"
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
return dataset
|
||||
|
||||
|
||||
def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
|
||||
project_id = uuid4()
|
||||
candidate_id = uuid4()
|
||||
reference_id = uuid4()
|
||||
candidate_path = tmp_path / "candidate.geojson"
|
||||
reference_path = tmp_path / "reference.geojson"
|
||||
polygon = [[[4.0, 51.0], [4.1, 51.0], [4.1, 51.1], [4.0, 51.1], [4.0, 51.0]]]
|
||||
_write_dataset(candidate_path, polygon)
|
||||
_write_dataset(reference_path, polygon)
|
||||
|
||||
candidate = Dataset(
|
||||
id=candidate_id,
|
||||
project_id=project_id,
|
||||
name="candidate.geojson",
|
||||
dataset_type="vector",
|
||||
source="test",
|
||||
storage_path=str(candidate_path),
|
||||
crs="EPSG:4326",
|
||||
metadata_json={"crs_assumed": False},
|
||||
)
|
||||
reference = _authoritative_reference(Dataset(
|
||||
id=reference_id,
|
||||
project_id=project_id,
|
||||
name="reference.geojson",
|
||||
dataset_type="vector",
|
||||
source="test",
|
||||
storage_path=str(reference_path),
|
||||
crs="EPSG:4326",
|
||||
metadata_json={"crs_assumed": False},
|
||||
))
|
||||
|
||||
result = QaService.compare_candidate_with_reference(
|
||||
db=FakeSession([candidate, reference]),
|
||||
project_id=project_id,
|
||||
candidate_dataset_id=candidate_id,
|
||||
reference_dataset_id=reference_id,
|
||||
iou_threshold=0.5,
|
||||
)
|
||||
|
||||
assert result.status == "ok"
|
||||
assert result.matches == 1
|
||||
assert result.false_positives == 0
|
||||
assert result.false_negatives == 0
|
||||
assert result.precision == 1.0
|
||||
assert result.recall == 1.0
|
||||
assert result.f1_score == 1.0
|
||||
|
||||
|
||||
def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_path) -> None:
|
||||
project_id = uuid4()
|
||||
candidate_id = uuid4()
|
||||
reference_id = uuid4()
|
||||
candidate_path = tmp_path / "candidate.geojson"
|
||||
reference_path = tmp_path / "reference.geojson"
|
||||
matched_candidate = [[[4.0, 51.0], [4.1, 51.0], [4.1, 51.1], [4.0, 51.1], [4.0, 51.0]]]
|
||||
matched_reference = [[[4.0, 51.0], [4.1, 51.0], [4.1, 51.1], [4.0, 51.1], [4.0, 51.0]]]
|
||||
false_positive = [[[4.5, 51.5], [4.6, 51.5], [4.6, 51.6], [4.5, 51.6], [4.5, 51.5]]]
|
||||
false_negative = [[[4.8, 51.8], [4.9, 51.8], [4.9, 51.9], [4.8, 51.9], [4.8, 51.8]]]
|
||||
_write_features(candidate_path, [_feature("candidate-match", matched_candidate), _feature("candidate-extra", false_positive)])
|
||||
_write_features(reference_path, [_feature("reference-match", matched_reference), _feature("reference-missing", false_negative)])
|
||||
|
||||
candidate = Dataset(
|
||||
id=candidate_id,
|
||||
project_id=project_id,
|
||||
name="candidate.geojson",
|
||||
dataset_type="vector",
|
||||
source="test",
|
||||
storage_path=str(candidate_path),
|
||||
crs="EPSG:4326",
|
||||
metadata_json={"crs_assumed": False},
|
||||
)
|
||||
reference = _authoritative_reference(Dataset(
|
||||
id=reference_id,
|
||||
project_id=project_id,
|
||||
name="reference.geojson",
|
||||
dataset_type="vector",
|
||||
source="test",
|
||||
storage_path=str(reference_path),
|
||||
crs="EPSG:4326",
|
||||
metadata_json={"crs_assumed": False},
|
||||
))
|
||||
|
||||
result = QaService.compare_candidate_with_reference(
|
||||
db=FakeSession([candidate, reference]),
|
||||
project_id=project_id,
|
||||
candidate_dataset_id=candidate_id,
|
||||
reference_dataset_id=reference_id,
|
||||
iou_threshold=0.5,
|
||||
)
|
||||
|
||||
assert result.matches == 1
|
||||
assert result.false_positives == 1
|
||||
assert result.false_negatives == 1
|
||||
assert result.match_evidence == [
|
||||
{
|
||||
"candidate_feature_id": "candidate-match",
|
||||
"reference_feature_id": "reference-match",
|
||||
"iou": 1.0,
|
||||
}
|
||||
]
|
||||
assert result.false_positive_evidence == [{"candidate_feature_id": "candidate-extra"}]
|
||||
assert result.false_negative_evidence == [{"reference_feature_id": "reference-missing"}]
|
||||
|
||||
|
||||
def test_dataset_reference_metadata_migration_declares_required_columns() -> None:
|
||||
migration_path = Path(__file__).parents[1] / "alembic" / "versions" / "202606120001_add_dataset_reference_metadata.py"
|
||||
migration_text = migration_path.read_text(encoding="utf-8")
|
||||
for column_name in (
|
||||
"dataset_role",
|
||||
"source_name",
|
||||
"reference_layer_name",
|
||||
"source_metadata",
|
||||
"provenance_metadata",
|
||||
"imported_at",
|
||||
):
|
||||
assert column_name in migration_text
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Evidence review must stay usable on a regional run.
|
||||
|
||||
evidence_geojson emitted one feature per false positive, one per false
|
||||
negative and *two* per match, with no limit. A regional QA run of 40k
|
||||
detections against 45k reference footprints produced well over a hundred
|
||||
thousand features in a single response, plus one warning string per
|
||||
unresolvable identifier. The endpoint the whole review workflow depends on
|
||||
therefore stopped working exactly where review matters most.
|
||||
|
||||
The budget goes to what a reviewer must act on — misses and false positives —
|
||||
before confirmations, and the response says what it left out.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.services.quality_evidence_service import QualityEvidenceService
|
||||
|
||||
|
||||
def _features(role: str, count: int) -> list[dict]:
|
||||
return [{"properties": {"evidence_role": role}, "id": f"{role}-{index}"} for index in range(count)]
|
||||
|
||||
|
||||
def test_missing_identifiers_collapse_into_one_statement() -> None:
|
||||
warnings = QualityEvidenceService.summarize_missing(
|
||||
candidate_ids=["a", "b", "c"],
|
||||
reference_ids=["r1"],
|
||||
)
|
||||
|
||||
assert len(warnings) == 1
|
||||
assert "3" in warnings[0]
|
||||
assert "1" in warnings[0]
|
||||
|
||||
|
||||
def test_nothing_missing_produces_no_warning() -> None:
|
||||
assert QualityEvidenceService.summarize_missing(candidate_ids=[], reference_ids=[]) == []
|
||||
|
||||
|
||||
def test_the_plan_is_capped_before_any_geometry_is_fetched() -> None:
|
||||
"""Resolving 130k geometries to draw 5k of them is work for nothing."""
|
||||
|
||||
findings = {
|
||||
"match_evidence": [{"candidate_feature_id": f"c{i}", "reference_feature_id": f"r{i}"} for i in range(100)],
|
||||
"false_positive_evidence": [{"candidate_feature_id": f"fp{i}"} for i in range(10)],
|
||||
"false_negative_evidence": [{"reference_feature_id": f"fn{i}"} for i in range(10)],
|
||||
}
|
||||
|
||||
plan = QualityEvidenceService.plan_evidence(findings, limit=8)
|
||||
|
||||
assert plan.truncated is True
|
||||
assert len(plan.items) == 8
|
||||
# Both error classes are represented; confirmations do not get a share
|
||||
# while errors are still waiting.
|
||||
assert {item.role for item in plan.items} == {"false_negative", "false_positive"}
|
||||
# Only the identifiers that will actually be drawn need resolving.
|
||||
assert len(plan.candidate_ids) + len(plan.reference_ids) == 8
|
||||
assert plan.candidate_ids <= {f"fp{i}" for i in range(10)}
|
||||
assert plan.reference_ids <= {f"fn{i}" for i in range(10)}
|
||||
|
||||
|
||||
def test_a_rare_error_class_is_never_crowded_out() -> None:
|
||||
"""50.000 misses must not hide the three false positives."""
|
||||
|
||||
findings = {
|
||||
"false_negative_evidence": [{"reference_feature_id": f"fn{i}"} for i in range(5_000)],
|
||||
"false_positive_evidence": [{"candidate_feature_id": f"fp{i}"} for i in range(3)],
|
||||
}
|
||||
|
||||
plan = QualityEvidenceService.plan_evidence(findings, limit=100)
|
||||
|
||||
roles = [item.role for item in plan.items]
|
||||
assert roles.count("false_positive") >= 1
|
||||
assert roles.count("false_negative") >= 90
|
||||
assert len(plan.items) == 100
|
||||
|
||||
|
||||
def test_the_plan_reports_the_complete_population_not_the_capped_one() -> None:
|
||||
findings = {
|
||||
"match_evidence": [{"candidate_feature_id": f"c{i}", "reference_feature_id": f"r{i}"} for i in range(100)],
|
||||
"false_negative_evidence": [{"reference_feature_id": "fn"}],
|
||||
}
|
||||
|
||||
plan = QualityEvidenceService.plan_evidence(findings, limit=2)
|
||||
|
||||
assert plan.total_feature_count == 201
|
||||
assert plan.role_counts == {"match_candidate": 100, "match_reference": 100, "false_negative": 1}
|
||||
|
||||
|
||||
def test_an_uncapped_plan_keeps_everything() -> None:
|
||||
findings = {"false_negative_evidence": [{"reference_feature_id": f"fn{i}"} for i in range(30)]}
|
||||
|
||||
plan = QualityEvidenceService.plan_evidence(findings, limit=0)
|
||||
|
||||
assert plan.truncated is False
|
||||
assert len(plan.items) == 30
|
||||
assert plan.reference_ids == {f"fn{i}" for i in range(30)}
|
||||
|
||||
|
||||
def test_evidence_without_identifiers_is_skipped_not_planned() -> None:
|
||||
findings = {
|
||||
"false_positive_evidence": [{"candidate_feature_id": None}, {"candidate_feature_id": "fp"}],
|
||||
"false_negative_evidence": [{}],
|
||||
}
|
||||
|
||||
plan = QualityEvidenceService.plan_evidence(findings, limit=0)
|
||||
|
||||
assert [item.role for item in plan.items] == ["false_positive"]
|
||||
assert plan.candidate_ids == {"fp"}
|
||||
|
||||
|
||||
|
||||
def test_a_planned_match_keeps_its_candidate_and_reference_together() -> None:
|
||||
"""Half a match is not reviewable evidence."""
|
||||
|
||||
findings = {
|
||||
"match_evidence": [
|
||||
{"candidate_feature_id": "c1", "reference_feature_id": "r1"},
|
||||
{"candidate_feature_id": "c2", "reference_feature_id": "r2"},
|
||||
]
|
||||
}
|
||||
|
||||
plan = QualityEvidenceService.plan_evidence(findings, limit=3)
|
||||
|
||||
assert plan.truncated is True
|
||||
# An odd budget drops the second pair rather than showing one side of it.
|
||||
assert len(plan.items) == 1
|
||||
assert plan.candidate_ids == {"c1"}
|
||||
assert plan.reference_ids == {"r1"}
|
||||
@@ -0,0 +1,84 @@
|
||||
"""A selection smaller than a raster cell must not silently read as zero.
|
||||
|
||||
``geometry_mask`` selects a cell when its *centre* falls inside the geometry.
|
||||
A rectangle smaller than one cell, or one that lands between four centres,
|
||||
therefore selects nothing at all — and the analysis returned zeros, which on
|
||||
screen is indistinguishable from "we looked and there is nothing here". On a
|
||||
100 m population raster a 40 m rectangle over a city block reported no
|
||||
inhabitants.
|
||||
|
||||
Selection now falls back to every touched cell and says that it did, so the
|
||||
value is readable as "at least one whole cell", not as an empty area.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
np = pytest.importorskip("numpy")
|
||||
rasterio = pytest.importorskip("rasterio")
|
||||
|
||||
from rasterio.transform import from_origin # noqa: E402 - optional rasterio gate precedes imports
|
||||
from shapely.geometry import box # noqa: E402 - optional rasterio gate precedes imports
|
||||
|
||||
from app.services.raster_cell_selection import select_cells # noqa: E402 - optional rasterio gate precedes service import
|
||||
|
||||
|
||||
# 100 m cells, origin at the top-left corner of a 3x3 grid.
|
||||
TRANSFORM = from_origin(200_000, 210_000, 100.0, 100.0)
|
||||
SHAPE = (3, 3)
|
||||
|
||||
|
||||
def test_a_normal_selection_uses_cell_centres() -> None:
|
||||
selection = select_cells(box(200_000, 209_700, 200_300, 210_000), out_shape=SHAPE, transform=TRANSFORM)
|
||||
|
||||
assert selection.mask.sum() == 9
|
||||
assert selection.mode == "cell_centre"
|
||||
assert selection.expanded_to_touched_cells is False
|
||||
assert selection.warning is None
|
||||
|
||||
|
||||
def test_a_rectangle_smaller_than_one_cell_still_returns_that_cell() -> None:
|
||||
selection = select_cells(box(200_010, 209_960, 200_050, 209_990), out_shape=SHAPE, transform=TRANSFORM)
|
||||
|
||||
assert selection.mask.sum() == 1
|
||||
assert selection.mode == "all_touched"
|
||||
assert selection.expanded_to_touched_cells is True
|
||||
assert "cel" in selection.warning
|
||||
|
||||
|
||||
def test_a_rectangle_between_four_cell_centres_returns_all_four() -> None:
|
||||
selection = select_cells(box(200_080, 209_880, 200_120, 209_920), out_shape=SHAPE, transform=TRANSFORM)
|
||||
|
||||
assert selection.mask.sum() == 4
|
||||
assert selection.expanded_to_touched_cells is True
|
||||
|
||||
|
||||
def test_a_selection_entirely_off_the_raster_selects_nothing() -> None:
|
||||
"""Falling back must not invent coverage where the geometry does not reach."""
|
||||
|
||||
selection = select_cells(box(300_000, 300_000, 300_100, 300_100), out_shape=SHAPE, transform=TRANSFORM)
|
||||
|
||||
assert selection.mask.sum() == 0
|
||||
assert selection.expanded_to_touched_cells is False
|
||||
assert selection.mode == "cell_centre"
|
||||
|
||||
|
||||
def test_the_warning_states_how_much_larger_the_analysed_area_is() -> None:
|
||||
selection = select_cells(
|
||||
box(200_010, 209_960, 200_050, 209_990),
|
||||
out_shape=SHAPE,
|
||||
transform=TRANSFORM,
|
||||
cell_area_m2=100.0 * 100.0,
|
||||
)
|
||||
|
||||
# One 100x100 m cell was analysed for a 40x30 m request.
|
||||
assert "1 rastercel" in selection.warning
|
||||
assert "1.0 ha" in selection.warning
|
||||
|
||||
|
||||
def test_the_mask_shape_always_matches_the_raster_window() -> None:
|
||||
selection = select_cells(box(200_010, 209_960, 200_050, 209_990), out_shape=SHAPE, transform=TRANSFORM)
|
||||
|
||||
assert selection.mask.shape == SHAPE
|
||||
assert selection.mask.dtype == np.bool_
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,67 @@
|
||||
from app.core.errors import AppError
|
||||
from app.services.raster_service import extract_raster_metadata
|
||||
|
||||
|
||||
def test_extract_raster_metadata_returns_dependency_aware_error(monkeypatch, tmp_path) -> None:
|
||||
monkeypatch.setattr("app.services.raster_service._import_rasterio", lambda: (_ for _ in ()).throw(ImportError("rasterio not installed")))
|
||||
|
||||
file_path = tmp_path / "missing.tif"
|
||||
file_path.write_bytes(b"\x00\x01\x02")
|
||||
|
||||
try:
|
||||
extract_raster_metadata(str(file_path))
|
||||
except AppError as exc:
|
||||
assert exc.code == "RASTER_PROCESSING_UNAVAILABLE"
|
||||
else:
|
||||
raise AssertionError("Missing rasterio should raise AppError code RASTER_PROCESSING_UNAVAILABLE")
|
||||
|
||||
|
||||
def test_extract_raster_metadata_maps_basic_profile_fields(monkeypatch, tmp_path) -> None:
|
||||
file_path = tmp_path / "sample.tif"
|
||||
file_path.write_bytes(b"fake")
|
||||
|
||||
class FakeDataset:
|
||||
width = 1024
|
||||
height = 768
|
||||
count = 4
|
||||
driver = "GTiff"
|
||||
crs = "EPSG:31370"
|
||||
bounds = (100.0, 200.0, 500.0, 800.0)
|
||||
res = (0.25, 0.25)
|
||||
dtypes = ["uint16", "uint16", "uint16", "uint16"]
|
||||
nodata = -9999
|
||||
|
||||
class transform:
|
||||
@staticmethod
|
||||
def to_gdal():
|
||||
return (0.25, 0.0, 100.0, 0.0, -0.25, 800.0, 0.0, 0.0, 1.0)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
class FakeRasterio:
|
||||
class errors:
|
||||
class RasterioIOError(Exception):
|
||||
...
|
||||
|
||||
def open(self, *_):
|
||||
return FakeDataset()
|
||||
|
||||
class FakeErrors:
|
||||
RasterioIOError = FakeRasterio.errors.RasterioIOError
|
||||
|
||||
monkeypatch.setattr("app.services.raster_service._import_rasterio", lambda: (FakeRasterio(), FakeErrors()))
|
||||
|
||||
metadata = extract_raster_metadata(str(file_path))
|
||||
assert metadata["driver"] == "GTiff"
|
||||
assert metadata["width"] == 1024
|
||||
assert metadata["height"] == 768
|
||||
assert metadata["band_count"] == 4
|
||||
assert metadata["crs"] == "EPSG:31370"
|
||||
assert metadata["bounds"] == [100.0, 200.0, 500.0, 800.0]
|
||||
assert metadata["resolution"] == [0.25, 0.25]
|
||||
assert metadata["dtype"] == ["uint16", "uint16", "uint16", "uint16"]
|
||||
assert metadata["nodata"] == -9999.0
|
||||
@@ -0,0 +1,569 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
|
||||
|
||||
def load_script(name: str):
|
||||
path = SCRIPTS / name
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha256") -> None:
|
||||
root.mkdir(parents=True)
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"release_id": "rc10-test",
|
||||
"created_at": created_at.isoformat(),
|
||||
"read_only_source": True,
|
||||
"database_password_secure": True,
|
||||
"inventory_mode": inventory_mode,
|
||||
"storage_inventory_requested": True,
|
||||
"storage_snapshot_requested": True,
|
||||
"models_inventory_requested": False,
|
||||
"models_snapshot_requested": False,
|
||||
"git_commit": "0123456789abcdef",
|
||||
}
|
||||
files = {
|
||||
"manifest.json": json.dumps(manifest),
|
||||
"database.dump": "database",
|
||||
"database.list": "list",
|
||||
"database-metadata.tsv": "alembic_head\t202607160001",
|
||||
"table-counts.tsv": "datasets\t1",
|
||||
"storage-manifest.tsv": "relative_path\tsize_bytes\tmtime_ns\tsha256",
|
||||
}
|
||||
for name, content in files.items():
|
||||
(root / name).write_text(content, encoding="utf-8")
|
||||
(root / "storage-snapshot").mkdir()
|
||||
checksums = []
|
||||
for name in sorted(files):
|
||||
digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
|
||||
checksums.append(f"{digest} {name}")
|
||||
(root / "CHECKSUMS.sha256").write_text("\n".join(checksums) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def test_backup_guard_requires_recent_complete_sha256_storage_backup(tmp_path: Path) -> None:
|
||||
guard = load_script("release_backup_guard.py")
|
||||
now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
|
||||
backup = tmp_path / "backup"
|
||||
write_backup(backup, created_at=now - timedelta(hours=2))
|
||||
|
||||
verified = guard.verify_current_backup(backup, now=now)
|
||||
|
||||
assert verified.release_id == "rc10-test"
|
||||
assert verified.age_hours == pytest.approx(2)
|
||||
|
||||
|
||||
def test_backup_guard_rejects_stale_or_tampered_backup(tmp_path: Path) -> None:
|
||||
guard = load_script("release_backup_guard.py")
|
||||
now = datetime(2026, 7, 18, 12, tzinfo=timezone.utc)
|
||||
stale = tmp_path / "stale"
|
||||
write_backup(stale, created_at=now - timedelta(hours=30))
|
||||
with pytest.raises(RuntimeError, match="maximum allowed age"):
|
||||
guard.verify_current_backup(stale, now=now)
|
||||
|
||||
current = tmp_path / "tampered"
|
||||
write_backup(current, created_at=now)
|
||||
(current / "database.dump").write_text("tampered", encoding="utf-8")
|
||||
with pytest.raises(RuntimeError, match="checksum mismatch"):
|
||||
guard.verify_current_backup(current, now=now)
|
||||
|
||||
|
||||
def test_storage_lifecycle_is_fail_closed_and_protects_release_evidence() -> None:
|
||||
audit = load_script("audit_data_operations.py")
|
||||
|
||||
assert audit.classify_relative_path("release-evidence/rc11/manifest.json") == (
|
||||
"release-evidence",
|
||||
True,
|
||||
False,
|
||||
)
|
||||
assert audit.classify_relative_path("operator-evidence/source/raw.json")[1:] == (True, False)
|
||||
assert audit.classify_relative_path("uploads/project/data.geojson")[1:] == (True, False)
|
||||
assert audit.classify_relative_path("exports/project/old.json")[1:] == (False, True)
|
||||
assert audit.classify_relative_path("unknown/value.bin")[1:] == (True, False)
|
||||
|
||||
|
||||
def test_storage_audit_only_selects_old_unreferenced_allowlisted_files(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
audit = load_script("audit_data_operations.py")
|
||||
storage = tmp_path / "storage"
|
||||
old_orphan = storage / "exports" / "project" / "old.json"
|
||||
referenced = storage / "exports" / "project" / "kept.json"
|
||||
protected = storage / "release-evidence" / "rc" / "manifest.json"
|
||||
unknown = storage / "misc" / "unknown.bin"
|
||||
for path in (old_orphan, referenced, protected, unknown):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(path.name, encoding="utf-8")
|
||||
old_timestamp = (datetime.now(timezone.utc) - timedelta(days=30)).timestamp()
|
||||
for path in (old_orphan, referenced, protected, unknown):
|
||||
path.touch()
|
||||
Path(path).chmod(0o644)
|
||||
import os
|
||||
|
||||
os.utime(path, (old_timestamp, old_timestamp))
|
||||
|
||||
monkeypatch.setattr(
|
||||
audit,
|
||||
"collect_database_state",
|
||||
lambda _db, _root: {
|
||||
"references": {referenced.resolve()},
|
||||
"counts": {},
|
||||
"source_families": {"national": [], "regional": [], "maritime": []},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
audit,
|
||||
"disk_pressure",
|
||||
lambda _root: {
|
||||
"status": "ok",
|
||||
"total_bytes": 100,
|
||||
"used_bytes": 50,
|
||||
"free_bytes": 50,
|
||||
"free_percent": 50.0,
|
||||
"acquisition_allowed": True,
|
||||
},
|
||||
)
|
||||
|
||||
report, candidates = audit.build_report(storage, SimpleNamespace(), minimum_age_days=7)
|
||||
|
||||
assert [candidate.relative_path for candidate in candidates] == ["exports/project/old.json"]
|
||||
assert report["cleanup"]["candidate_count"] == 1
|
||||
assert "release-evidence" in report["cleanup"]["protected_prefixes"]
|
||||
assert report["integrity"]["missing_referenced_path_count"] == 0
|
||||
assert report["integrity"]["missing_manifest_artifact_count"] == 0
|
||||
|
||||
|
||||
def test_referenced_tile_manifest_protects_its_tiles(tmp_path: Path) -> None:
|
||||
audit = load_script("audit_data_operations.py")
|
||||
storage = tmp_path / "storage"
|
||||
manifest = storage / "tiles" / "dataset" / "set" / "manifest.json"
|
||||
tile = manifest.parent / "tile_0000.tif"
|
||||
tile.parent.mkdir(parents=True)
|
||||
tile.write_bytes(b"tile")
|
||||
manifest.write_text(json.dumps({"tiles": [{"path": "tile_0000.tif"}]}), encoding="utf-8")
|
||||
|
||||
expanded = audit.expand_manifest_references({manifest.resolve()}, storage)
|
||||
|
||||
assert manifest.resolve() in expanded
|
||||
assert tile.resolve() in expanded
|
||||
|
||||
|
||||
def test_ordinary_json_export_is_not_treated_as_an_artifact_manifest(tmp_path: Path) -> None:
|
||||
audit = load_script("audit_data_operations.py")
|
||||
storage = tmp_path / "storage"
|
||||
export = storage / "exports" / "project" / "report.json"
|
||||
export.parent.mkdir(parents=True)
|
||||
export.write_text(json.dumps({"dataset_id": "not-a-file"}), encoding="utf-8")
|
||||
|
||||
expanded = audit.expand_manifest_references({export.resolve()}, storage)
|
||||
|
||||
assert expanded == {export.resolve()}
|
||||
|
||||
|
||||
def test_manifest_ids_dates_and_labels_are_not_treated_as_paths(tmp_path: Path) -> None:
|
||||
audit = load_script("audit_data_operations.py")
|
||||
storage = tmp_path / "storage"
|
||||
manifest = storage / "operator-data" / "scope" / "scope.manifest.json"
|
||||
actual = manifest.parent / "scope.geojson"
|
||||
manifest.parent.mkdir(parents=True)
|
||||
actual.write_text("{}", encoding="utf-8")
|
||||
manifest.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"municipality_ids": ["13025", "11001"],
|
||||
"generated_at": "2026-07-18T00:00:00Z",
|
||||
"label": "Belgium",
|
||||
"output_path": "scope.geojson",
|
||||
"output_checksum_sha256": "a" * 64,
|
||||
"output_crs": "EPSG:4326",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
expanded = audit.expand_manifest_references({manifest.resolve()}, storage)
|
||||
|
||||
assert expanded == {manifest.resolve(), actual.resolve()}
|
||||
|
||||
|
||||
def test_disk_pressure_uses_absolute_headroom_for_large_arrays(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
audit = load_script("audit_data_operations.py")
|
||||
monkeypatch.setattr(
|
||||
audit.shutil,
|
||||
"disk_usage",
|
||||
lambda _path: SimpleNamespace(
|
||||
total=56 * 1024**4,
|
||||
used=(56 * 1024**4) - (700 * 1024**3),
|
||||
free=700 * 1024**3,
|
||||
),
|
||||
)
|
||||
|
||||
pressure = audit.disk_pressure(tmp_path)
|
||||
|
||||
assert pressure["free_percent"] < 2
|
||||
assert pressure["status"] == "ok"
|
||||
assert pressure["acquisition_allowed"] is True
|
||||
|
||||
|
||||
def test_source_family_report_covers_national_regional_and_maritime(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
audit = load_script("audit_data_operations.py")
|
||||
national_id = uuid4()
|
||||
regional_id = uuid4()
|
||||
now = datetime.now(timezone.utc)
|
||||
rows = {
|
||||
audit.Project: [
|
||||
SimpleNamespace(id=national_id, name=audit.NATIONAL_PROJECT_NAME, status="active"),
|
||||
SimpleNamespace(id=regional_id, name="Wallonia operator", status="active"),
|
||||
],
|
||||
audit.Dataset: [
|
||||
SimpleNamespace(
|
||||
project_id=national_id,
|
||||
source_name="ngi_adminvector",
|
||||
source="ngi",
|
||||
name="Belgium boundary",
|
||||
source_metadata={"coverage_zones": ["belgium"]},
|
||||
source_version="2026",
|
||||
imported_at=now,
|
||||
status="ready",
|
||||
storage_path=None,
|
||||
metadata_json=None,
|
||||
provenance_metadata=None,
|
||||
),
|
||||
SimpleNamespace(
|
||||
project_id=national_id,
|
||||
source_name="rbins_marine_reporting_units",
|
||||
source="rbins",
|
||||
name="Belgian North Sea",
|
||||
source_metadata={"coverage_zones": ["belgian_north_sea"]},
|
||||
source_version="2024",
|
||||
imported_at=now,
|
||||
status="ready",
|
||||
storage_path=None,
|
||||
metadata_json=None,
|
||||
provenance_metadata=None,
|
||||
),
|
||||
SimpleNamespace(
|
||||
project_id=regional_id,
|
||||
source_name="wallonia_manual",
|
||||
source="manual",
|
||||
name="Wallonia source",
|
||||
source_metadata={},
|
||||
source_version="1",
|
||||
imported_at=now,
|
||||
status="ready",
|
||||
storage_path=None,
|
||||
metadata_json=None,
|
||||
provenance_metadata=None,
|
||||
),
|
||||
],
|
||||
audit.DatasetVersion: [],
|
||||
audit.Export: [],
|
||||
audit.Detection: [],
|
||||
audit.Segmentation: [],
|
||||
audit.Job: [],
|
||||
audit.AnalysisRun: [],
|
||||
}
|
||||
|
||||
class Query:
|
||||
def __init__(self, values):
|
||||
self.values = values
|
||||
|
||||
def all(self):
|
||||
return self.values
|
||||
|
||||
class Session:
|
||||
def query(self, model, *_fields):
|
||||
if model in rows:
|
||||
return Query(rows[model])
|
||||
owner = getattr(model, "class_", None)
|
||||
if owner in rows:
|
||||
return Query(rows[owner])
|
||||
raise AssertionError(f"Unexpected query entity: {model!r}")
|
||||
|
||||
monkeypatch.setattr(audit, "query_count", lambda _db, model, *_conditions: len(rows[model]))
|
||||
monkeypatch.setattr(audit, "query_distinct_nonnull", lambda _db, _column: [])
|
||||
state = audit.collect_database_state(Session(), tmp_path)
|
||||
|
||||
assert {item["source_name"] for item in state["source_families"]["national"]} == {
|
||||
"ngi_adminvector",
|
||||
"rbins_marine_reporting_units",
|
||||
}
|
||||
assert {item["source_name"] for item in state["source_families"]["maritime"]} == {
|
||||
"rbins_marine_reporting_units"
|
||||
}
|
||||
assert {item["source_name"] for item in state["source_families"]["regional"]} == {
|
||||
"wallonia_manual"
|
||||
}
|
||||
|
||||
|
||||
def test_national_and_maritime_sources_have_explicit_freshness_policies() -> None:
|
||||
from app.services.source_freshness_service import SOURCE_POLICIES
|
||||
|
||||
for source_name in (
|
||||
"ngi_adminvector",
|
||||
"rbins_marine_reporting_units",
|
||||
"rbins_msp_2026",
|
||||
):
|
||||
assert SOURCE_POLICIES[source_name].refresh_policy == "edition"
|
||||
|
||||
|
||||
def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> None:
|
||||
generic = (SCRIPTS / "cleanup_storage_artifacts.py").read_text(encoding="utf-8")
|
||||
demo = (ROOT / "backend/scripts/cleanup_demo_artifacts.py").read_text(encoding="utf-8")
|
||||
compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
|
||||
dockerman = (ROOT / "deploy/unraid/run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "QUARANTINE_STORAGE_ARTIFACTS" in generic
|
||||
assert "verify_current_backup" in generic
|
||||
assert "os.link" in generic
|
||||
assert 'entry["status"] = "linked"' in generic
|
||||
assert "cleanup-quarantine" in generic
|
||||
assert "DELETE_DEMO_EXPORTS" in demo
|
||||
assert "verify_current_backup" in demo
|
||||
assert "/app/backups:ro" in compose
|
||||
assert '/app/backups:ro"' in dockerman
|
||||
for name in (
|
||||
"release_backup_guard.py",
|
||||
"audit_data_operations.py",
|
||||
"cleanup_storage_artifacts.py",
|
||||
"restore_storage_quarantine.py",
|
||||
"release_backup_snapshot.py",
|
||||
):
|
||||
assert f"COPY scripts/{name}" in dockerfile
|
||||
assert f"py_compile scripts/{name}" in readiness
|
||||
assert "bash -n scripts/run_rc10_data_operations_audit.sh" in readiness
|
||||
assert "--apply" not in live_audit
|
||||
assert "table-counts-before.tsv" in live_audit
|
||||
assert "table-counts-after.tsv" in live_audit
|
||||
assert "deleted_count" in live_audit
|
||||
assert "missing_manifest_artifact_count" in live_audit
|
||||
|
||||
|
||||
def test_cleanup_apply_moves_bytes_to_protected_traceable_quarantine(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
monkeypatch.syspath_prepend(str(SCRIPTS))
|
||||
cleanup = load_script("cleanup_storage_artifacts.py")
|
||||
storage = tmp_path / "storage"
|
||||
source = storage / "derived" / "orphan.bin"
|
||||
source.parent.mkdir(parents=True)
|
||||
source.write_bytes(b"recoverable-derived-artifact")
|
||||
candidate = SimpleNamespace(
|
||||
path=source.resolve(),
|
||||
relative_path="derived/orphan.bin",
|
||||
size_bytes=source.stat().st_size,
|
||||
)
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
class SessionContext:
|
||||
def __enter__(self):
|
||||
return SimpleNamespace()
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(
|
||||
cleanup,
|
||||
"parse_args",
|
||||
lambda: SimpleNamespace(
|
||||
storage_root=storage,
|
||||
minimum_age_days=7,
|
||||
max_delete=1,
|
||||
apply=True,
|
||||
confirm="QUARANTINE_STORAGE_ARTIFACTS",
|
||||
backup_dir=tmp_path / "backup",
|
||||
backup_max_age_hours=24.0,
|
||||
quarantine_root=None,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(cleanup, "SessionLocal", lambda: SessionContext())
|
||||
monkeypatch.setattr(
|
||||
cleanup,
|
||||
"build_report",
|
||||
lambda *_args, **_kwargs: ({"cleanup": {"protected_prefixes": []}}, [candidate]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
cleanup,
|
||||
"verify_current_backup",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(
|
||||
release_id="predeploy-test",
|
||||
created_at=now,
|
||||
age_hours=0.1,
|
||||
backup_tool_revision="0123456789abcdef",
|
||||
),
|
||||
)
|
||||
|
||||
assert cleanup.main() == 0
|
||||
payload = json.loads(capsys.readouterr().out)
|
||||
manifest_path = Path(payload["quarantine_manifest"])
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
quarantined_path = storage / payload["quarantined"][0]["quarantine_relative_path"]
|
||||
|
||||
assert not source.exists()
|
||||
assert quarantined_path.read_bytes() == b"recoverable-derived-artifact"
|
||||
assert manifest["state"] == "complete"
|
||||
assert manifest["backup_release_id"] == "predeploy-test"
|
||||
assert manifest["entries"][0]["status"] == "quarantined"
|
||||
assert payload["deleted_count"] == 0
|
||||
|
||||
restore = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPTS / "restore_storage_quarantine.py"),
|
||||
"--storage-root",
|
||||
str(storage),
|
||||
"--manifest",
|
||||
str(manifest_path),
|
||||
"--confirm",
|
||||
"RESTORE_QUARANTINED_ARTIFACTS",
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert restore.returncode == 0, restore.stderr
|
||||
assert source.read_bytes() == b"recoverable-derived-artifact"
|
||||
assert not quarantined_path.exists()
|
||||
restored_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
assert restored_manifest["state"] == "restored"
|
||||
assert restored_manifest["entries"][0]["status"] == "restored"
|
||||
|
||||
|
||||
def _write_interrupted_quarantine(
|
||||
storage: Path,
|
||||
*,
|
||||
original_exists: bool,
|
||||
quarantine_exists: bool,
|
||||
hard_linked: bool = False,
|
||||
) -> tuple[Path, Path, Path]:
|
||||
original = storage / "derived" / "interrupted.bin"
|
||||
operation = storage / "operator-evidence" / "cleanup-quarantine" / "cleanup-interrupted"
|
||||
quarantined = operation / "files" / "derived" / "interrupted.bin"
|
||||
original.parent.mkdir(parents=True, exist_ok=True)
|
||||
quarantined.parent.mkdir(parents=True, exist_ok=True)
|
||||
retained = b"interrupted-retained-bytes"
|
||||
if original_exists:
|
||||
original.write_bytes(retained)
|
||||
if quarantine_exists:
|
||||
if hard_linked:
|
||||
os.link(original, quarantined)
|
||||
else:
|
||||
quarantined.write_bytes(retained)
|
||||
manifest = operation / "manifest.json"
|
||||
manifest.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"state": "in_progress",
|
||||
"entries": [
|
||||
{
|
||||
"relative_path": "derived/interrupted.bin",
|
||||
"quarantine_relative_path": quarantined.relative_to(storage).as_posix(),
|
||||
"size_bytes": len(retained),
|
||||
"sha256": hashlib.sha256(retained).hexdigest(),
|
||||
"status": "linked" if hard_linked else "planned",
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return manifest, original, quarantined
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("original_exists", "quarantine_exists", "hard_linked"),
|
||||
((False, True, False), (True, True, True)),
|
||||
)
|
||||
def test_quarantine_restore_recovers_each_interrupted_move_window(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
original_exists: bool,
|
||||
quarantine_exists: bool,
|
||||
hard_linked: bool,
|
||||
) -> None:
|
||||
restore = load_script("restore_storage_quarantine.py")
|
||||
storage = tmp_path / "storage"
|
||||
manifest, original, quarantined = _write_interrupted_quarantine(
|
||||
storage,
|
||||
original_exists=original_exists,
|
||||
quarantine_exists=quarantine_exists,
|
||||
hard_linked=hard_linked,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
restore,
|
||||
"parse_args",
|
||||
lambda: SimpleNamespace(
|
||||
storage_root=storage,
|
||||
manifest=manifest,
|
||||
confirm="RESTORE_QUARANTINED_ARTIFACTS",
|
||||
),
|
||||
)
|
||||
|
||||
assert restore.main() == 0
|
||||
assert original.read_bytes() == b"interrupted-retained-bytes"
|
||||
assert not quarantined.exists()
|
||||
assert json.loads(manifest.read_text(encoding="utf-8"))["state"] == "restored"
|
||||
|
||||
|
||||
def test_quarantine_restore_never_clobbers_recreated_destination(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
restore = load_script("restore_storage_quarantine.py")
|
||||
storage = tmp_path / "storage"
|
||||
manifest, original, quarantined = _write_interrupted_quarantine(
|
||||
storage,
|
||||
original_exists=False,
|
||||
quarantine_exists=True,
|
||||
)
|
||||
original.write_bytes(b"new-runtime-bytes")
|
||||
monkeypatch.setattr(
|
||||
restore,
|
||||
"parse_args",
|
||||
lambda: SimpleNamespace(
|
||||
storage_root=storage,
|
||||
manifest=manifest,
|
||||
confirm="RESTORE_QUARANTINED_ARTIFACTS",
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="different bytes"):
|
||||
restore.main()
|
||||
assert original.read_bytes() == b"new-runtime-bytes"
|
||||
assert quarantined.read_bytes() == b"interrupted-retained-bytes"
|
||||
@@ -0,0 +1,135 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "build_release_package.py"
|
||||
|
||||
|
||||
def load_script():
|
||||
spec = importlib.util.spec_from_file_location("build_release_package", SCRIPT)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_release_version_is_consistent_across_runtime_packages() -> None:
|
||||
version = (ROOT / "VERSION").read_text(encoding="utf-8").strip()
|
||||
config = (ROOT / "backend" / "app" / "core" / "config.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
pyproject = (ROOT / "backend" / "pyproject.toml").read_text(encoding="utf-8")
|
||||
frontend = json.loads(
|
||||
(ROOT / "frontend" / "package.json").read_text(encoding="utf-8")
|
||||
)
|
||||
package_lock = json.loads(
|
||||
(ROOT / "frontend" / "package-lock.json").read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
assert version == "1.0.0"
|
||||
assert f'default="{version}"' in config
|
||||
assert "GEOINTEL_APP_VERSION" in config
|
||||
assert 'version = "1.0.0"' in pyproject
|
||||
assert frontend["version"] == version
|
||||
assert package_lock["version"] == version
|
||||
assert package_lock["packages"][""]["version"] == version
|
||||
|
||||
|
||||
def test_release_image_carries_semantic_version_identity() -> None:
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
deploy = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert "ARG GEOINTEL_APP_VERSION=1.0.0" in dockerfile
|
||||
assert 'org.opencontainers.image.version="${GEOINTEL_APP_VERSION}"' in dockerfile
|
||||
assert "GEOINTEL_APP_VERSION=\"$(tr -d '[:space:]' < VERSION)\"" in deploy
|
||||
assert "--build-arg GEOINTEL_APP_VERSION=" in deploy
|
||||
assert "stored_version" in deploy
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("ssh-keygen") is None, reason="ssh-keygen unavailable")
|
||||
def test_release_package_signature_and_checksums_fail_closed(tmp_path: Path) -> None:
|
||||
module = load_script()
|
||||
key = tmp_path / "release-key"
|
||||
result = subprocess.run(
|
||||
["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(key)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
package = tmp_path / "package"
|
||||
package.mkdir()
|
||||
evidence = package / "readiness.txt"
|
||||
evidence.write_text("passed\n", encoding="utf-8")
|
||||
nested_checksums = package / "backup" / module.CHECKSUMS_NAME
|
||||
nested_checksums.parent.mkdir()
|
||||
nested_checksums.write_text("backup evidence\n", encoding="utf-8")
|
||||
identity = "geointel-release"
|
||||
namespace = "geointel-release"
|
||||
(package / module.SIGNERS_NAME).write_text(
|
||||
f"{identity} {module.public_key(key)}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"release_id": "v1.0.0",
|
||||
"version": "1.0.0",
|
||||
"scope": "Belgium and the Belgian North Sea",
|
||||
"signature": {"identity": identity, "namespace": namespace},
|
||||
"evidence": [
|
||||
{
|
||||
"path": evidence.name,
|
||||
"size_bytes": evidence.stat().st_size,
|
||||
"sha256": module.sha256(evidence),
|
||||
}
|
||||
],
|
||||
}
|
||||
manifest_path = package / module.MANIFEST_NAME
|
||||
manifest_path.write_text(json.dumps(manifest) + "\n", encoding="utf-8")
|
||||
module.run(
|
||||
(
|
||||
"ssh-keygen",
|
||||
"-Y",
|
||||
"sign",
|
||||
"-f",
|
||||
str(key),
|
||||
"-n",
|
||||
namespace,
|
||||
str(manifest_path),
|
||||
)
|
||||
)
|
||||
module.write_checksums(package)
|
||||
|
||||
verified = module.verify_package(package)
|
||||
assert verified["release_id"] == "v1.0.0"
|
||||
|
||||
evidence.write_text("tampered\n", encoding="utf-8")
|
||||
with pytest.raises(RuntimeError, match="Checksum mismatch"):
|
||||
module.verify_package(package)
|
||||
|
||||
|
||||
def test_release_package_cli_requires_tagged_clean_revision() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert 'run(("git", "status", "--porcelain=v1"))' in source
|
||||
assert 'run(("git", "rev-list", "-n", "1", release_id))' in source
|
||||
assert "Image revision must equal the tagged Git commit" in source
|
||||
assert "ssh-keygen" in source
|
||||
assert "verify_checksums(package_dir)" in source
|
||||
assert "py_compile scripts/build_release_package.py" in readiness
|
||||
@@ -0,0 +1,488 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.schemas.coverage import CoverageBBox
|
||||
from app.services.coverage_registry_service import CoverageRegistryService, THEMES, ZONES
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
from tests.frontend_contract import read_map_workspace, read_feature
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
|
||||
def filter(self, *_args, **_kwargs):
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return list(self.rows)
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, *, project, areas, datasets):
|
||||
self.project = project
|
||||
self.areas = areas
|
||||
self.datasets = datasets
|
||||
|
||||
def get(self, model, object_id):
|
||||
if model is Project and str(self.project.id) == str(object_id):
|
||||
return self.project
|
||||
return None
|
||||
|
||||
def query(self, model):
|
||||
if model is Area:
|
||||
return FakeQuery(self.areas)
|
||||
if model is Dataset:
|
||||
return FakeQuery(self.datasets)
|
||||
raise AssertionError(f"Unexpected query model: {model}")
|
||||
|
||||
|
||||
def scope_area(name: str, geometry):
|
||||
return SimpleNamespace(name=name, geometry=geometry)
|
||||
|
||||
|
||||
def governed_materialization(
|
||||
*,
|
||||
source_name: str,
|
||||
reference_layer_name: str | None,
|
||||
source_metadata: dict[str, object],
|
||||
dataset_id=None,
|
||||
) -> SimpleNamespace:
|
||||
"""Build a complete authoritative materialization for coverage tests.
|
||||
|
||||
Coverage is a production-facing statement. These fixtures must therefore
|
||||
carry the same registry, immutable snapshot, checksum and freshness state
|
||||
that a materialized official dataset needs in production.
|
||||
"""
|
||||
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
checksum_sha256 = "a" * 64
|
||||
return SimpleNamespace(
|
||||
id=dataset_id or uuid4(),
|
||||
status="ready",
|
||||
source=source_name,
|
||||
source_name=source_name,
|
||||
reference_layer_name=reference_layer_name,
|
||||
source_metadata=dict(source_metadata),
|
||||
checksum_sha256=checksum_sha256,
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
data_contract_key="geointel.vector.geojson",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
source_registry=SimpleNamespace(
|
||||
source_key=source_name,
|
||||
classification="authoritative",
|
||||
authority_scope_json={"scope": "coverage test"},
|
||||
usage_policy_json={},
|
||||
),
|
||||
source_snapshot=SimpleNamespace(
|
||||
source_registry_id=source_registry_id,
|
||||
checksum_sha256=checksum_sha256,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider_registry() -> None:
|
||||
catalog = CoverageRegistryService.catalog()
|
||||
|
||||
assert set(catalog.themes) == set(THEMES)
|
||||
assert set(catalog.zones) == set(ZONES)
|
||||
assert catalog.statuses == ["unsupported", "not_configured", "partial", "operational"]
|
||||
assert {source.source_name for source in catalog.sources} >= {
|
||||
"ngi_adminvector",
|
||||
"statbel",
|
||||
"digitaal_vlaanderen",
|
||||
"spw_geoportail",
|
||||
"urbis",
|
||||
"rbins_marine_reporting_units",
|
||||
"rbins_msp_2026",
|
||||
"mdk_bathymetry",
|
||||
"vmm_vha_bathymetry_profiles",
|
||||
}
|
||||
assert next(source for source in catalog.sources if source.source_name == "ngi_adminvector").license_note == "CC BY 4.0"
|
||||
assert next(source for source in catalog.sources if source.source_name == "mdk_bathymetry").integration_status == "not_configured"
|
||||
assert next(
|
||||
source for source in catalog.sources if source.source_name == "vmm_vha_bathymetry_profiles"
|
||||
).integration_status == "operational"
|
||||
|
||||
response = TestClient(app).get("/api/v1/external/coverage/catalog")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["data"]["themes"] == list(THEMES)
|
||||
|
||||
|
||||
def test_national_and_maritime_reference_layers_are_selection_analyzable() -> None:
|
||||
cases = (
|
||||
("ngi_adminvector", "belgium_municipalities", "administrative"),
|
||||
("rbins_marine_reporting_units", "marine_legal_scopes", "marine_environment"),
|
||||
("rbins_msp_2026", "marine_spatial_plan_2026", "maritime_planning"),
|
||||
)
|
||||
for source_name, layer_name, expected_theme in cases:
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name=f"{layer_name}.geojson",
|
||||
dataset_type="vector",
|
||||
source="operator_official_import",
|
||||
source_name=source_name,
|
||||
reference_layer_name=layer_name,
|
||||
source_metadata={"authority_level": "authoritative"},
|
||||
status="ready",
|
||||
)
|
||||
|
||||
assert VectorFeatureService._dataset_theme(dataset) == expected_theme
|
||||
assert VectorFeatureService.supports_selection_summary(dataset) is True
|
||||
|
||||
|
||||
def test_national_scope_operator_assigns_explicit_map_themes() -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
operator = (root / "scripts" / "provision_belgium_north_sea_scope.py").read_text(encoding="utf-8")
|
||||
map_workspace = read_map_workspace()
|
||||
|
||||
assert '"belgium_municipalities": "administrative"' in operator
|
||||
assert '"marine_legal_scopes": "marine_environment"' in operator
|
||||
assert '"marine_spatial_plan_2026": "maritime_planning"' in operator
|
||||
assert "id: 'administrative'" in map_workspace
|
||||
assert "id: 'maritime_planning'" in map_workspace
|
||||
assert "id: 'marine_environment'" in map_workspace
|
||||
|
||||
|
||||
def test_coverage_resolver_only_reports_operational_for_materialized_ready_dataset() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
areas = [
|
||||
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
|
||||
scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)),
|
||||
]
|
||||
bbox = CoverageBBox(minx=4.9, miny=50.9, maxx=5.0, maxy=51.0)
|
||||
|
||||
without_materialized = CoverageRegistryService.resolve(
|
||||
FakeSession(project=project, areas=areas, datasets=[]),
|
||||
project_id,
|
||||
bbox,
|
||||
["admin"],
|
||||
)
|
||||
assert without_materialized.intersected_zones == ["flanders"]
|
||||
assert without_materialized.items[0].status == "partial"
|
||||
assert without_materialized.items[0].materialized_dataset_ids == []
|
||||
|
||||
dataset_id = uuid4()
|
||||
materialized = governed_materialization(
|
||||
dataset_id=dataset_id,
|
||||
source_name="ngi_adminvector",
|
||||
reference_layer_name="belgium_regions",
|
||||
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
|
||||
)
|
||||
with_materialized = CoverageRegistryService.resolve(
|
||||
FakeSession(project=project, areas=areas, datasets=[materialized]),
|
||||
project_id,
|
||||
bbox,
|
||||
["admin"],
|
||||
)
|
||||
admin_item = next(item for item in with_materialized.items if item.zone == "flanders")
|
||||
assert admin_item.status == "operational"
|
||||
assert admin_item.materialized_dataset_ids == [dataset_id]
|
||||
|
||||
|
||||
def test_statbel_population_materialization_does_not_masquerade_as_admin_data() -> None:
|
||||
project_id = uuid4()
|
||||
statbel_id = uuid4()
|
||||
statbel = governed_materialization(
|
||||
dataset_id=statbel_id,
|
||||
source_name="statbel",
|
||||
reference_layer_name="population",
|
||||
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
|
||||
)
|
||||
result = CoverageRegistryService.resolve(
|
||||
FakeSession(
|
||||
project=SimpleNamespace(id=project_id),
|
||||
areas=[
|
||||
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
|
||||
scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)),
|
||||
],
|
||||
datasets=[statbel],
|
||||
),
|
||||
project_id,
|
||||
CoverageBBox(minx=4.9, miny=50.9, maxx=5.0, maxy=51.0),
|
||||
["admin", "population"],
|
||||
)
|
||||
|
||||
admin = next(item for item in result.items if item.theme == "admin")
|
||||
population = next(item for item in result.items if item.theme == "population")
|
||||
assert admin.materialized_dataset_ids == []
|
||||
assert population.status == "operational"
|
||||
assert population.materialized_dataset_ids == [statbel_id]
|
||||
|
||||
|
||||
def test_bounded_api_materialization_only_covers_its_persisted_bbox() -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
dataset = governed_materialization(
|
||||
dataset_id=dataset_id,
|
||||
source_name="spw_picc",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
"coverage_zones": ["wallonia"],
|
||||
"bbox_epsg4326": [4.55, 50.58, 4.56, 50.59],
|
||||
},
|
||||
)
|
||||
session = FakeSession(
|
||||
project=SimpleNamespace(id=project_id),
|
||||
areas=[
|
||||
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
|
||||
scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8)),
|
||||
],
|
||||
datasets=[dataset],
|
||||
)
|
||||
|
||||
inside = CoverageRegistryService.resolve(
|
||||
session,
|
||||
project_id,
|
||||
CoverageBBox(minx=4.551, miny=50.581, maxx=4.559, maxy=50.589),
|
||||
["buildings"],
|
||||
)
|
||||
outside = CoverageRegistryService.resolve(
|
||||
session,
|
||||
project_id,
|
||||
CoverageBBox(minx=4.7, miny=50.6, maxx=4.71, maxy=50.61),
|
||||
["buildings"],
|
||||
)
|
||||
|
||||
assert inside.items[0].status == "operational"
|
||||
assert inside.items[0].materialized_dataset_ids == [dataset_id]
|
||||
assert outside.items[0].status == "partial"
|
||||
assert outside.items[0].materialized_dataset_ids == []
|
||||
|
||||
|
||||
def test_bounded_partition_union_can_be_operational() -> None:
|
||||
project_id = uuid4()
|
||||
left_id = uuid4()
|
||||
right_id = uuid4()
|
||||
datasets = [
|
||||
governed_materialization(
|
||||
dataset_id=left_id,
|
||||
source_name="spw_picc",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
"coverage_zones": ["wallonia"],
|
||||
"bbox_epsg4326": [4.50, 50.50, 4.60, 50.60],
|
||||
},
|
||||
),
|
||||
governed_materialization(
|
||||
dataset_id=right_id,
|
||||
source_name="spw_picc",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
"coverage_zones": ["wallonia"],
|
||||
"bbox_epsg4326": [4.60, 50.50, 4.70, 50.60],
|
||||
},
|
||||
),
|
||||
]
|
||||
session = FakeSession(
|
||||
project=SimpleNamespace(id=project_id),
|
||||
areas=[scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)), scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8))],
|
||||
datasets=datasets,
|
||||
)
|
||||
|
||||
result = CoverageRegistryService.resolve(session, project_id, CoverageBBox(minx=4.51, miny=50.51, maxx=4.69, maxy=50.59), ["buildings"])
|
||||
|
||||
assert result.items[0].status == "operational"
|
||||
assert result.items[0].materialized_dataset_ids == [left_id, right_id]
|
||||
|
||||
|
||||
def test_spw_bathymetry_materialization_is_source_specific() -> None:
|
||||
project_id = uuid4()
|
||||
scope = [
|
||||
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
|
||||
scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8)),
|
||||
]
|
||||
selection = CoverageBBox(minx=4.851, miny=50.451, maxx=4.869, maxy=50.469)
|
||||
spw_picc = governed_materialization(
|
||||
source_name="spw_picc",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
"coverage_zones": ["wallonia"],
|
||||
"bbox_epsg4326": [4.85, 50.45, 4.87, 50.47],
|
||||
},
|
||||
)
|
||||
without_bathymetry = CoverageRegistryService.resolve(
|
||||
FakeSession(project=SimpleNamespace(id=project_id), areas=scope, datasets=[spw_picc]),
|
||||
project_id,
|
||||
selection,
|
||||
["bathymetry"],
|
||||
)
|
||||
assert without_bathymetry.items[0].status == "partial"
|
||||
assert without_bathymetry.items[0].materialized_dataset_ids == []
|
||||
|
||||
bathymetry_id = uuid4()
|
||||
bathymetry = governed_materialization(
|
||||
dataset_id=bathymetry_id,
|
||||
source_name="spw_bathymetry",
|
||||
reference_layer_name=None,
|
||||
source_metadata={
|
||||
"coverage_zones": ["wallonia"],
|
||||
"bbox_epsg4326": [4.85, 50.45, 4.87, 50.47],
|
||||
},
|
||||
)
|
||||
with_bathymetry = CoverageRegistryService.resolve(
|
||||
FakeSession(project=SimpleNamespace(id=project_id), areas=scope, datasets=[spw_picc, bathymetry]),
|
||||
project_id,
|
||||
selection,
|
||||
["bathymetry"],
|
||||
)
|
||||
assert with_bathymetry.items[0].status == "operational"
|
||||
assert with_bathymetry.items[0].materialized_dataset_ids == [bathymetry_id]
|
||||
|
||||
|
||||
def test_vha_bathymetry_profiles_are_operational_only_inside_the_persisted_selection() -> None:
|
||||
project_id = uuid4()
|
||||
scope = [
|
||||
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
|
||||
scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)),
|
||||
]
|
||||
selection = CoverageBBox(minx=5.101, miny=51.171, maxx=5.109, maxy=51.179)
|
||||
without_profiles = CoverageRegistryService.resolve(
|
||||
FakeSession(project=SimpleNamespace(id=project_id), areas=scope, datasets=[]),
|
||||
project_id,
|
||||
selection,
|
||||
["bathymetry"],
|
||||
)
|
||||
assert without_profiles.items[0].status == "partial"
|
||||
assert without_profiles.items[0].source_names == ["vmm_vha_bathymetry_profiles"]
|
||||
|
||||
profile_id = uuid4()
|
||||
profiles = governed_materialization(
|
||||
dataset_id=profile_id,
|
||||
source_name="vmm_vha_bathymetry_profiles",
|
||||
reference_layer_name="bathymetry_profile_points",
|
||||
source_metadata={
|
||||
"coverage_zones": ["flanders"],
|
||||
"bbox_epsg4326": [5.1, 51.17, 5.11, 51.18],
|
||||
},
|
||||
)
|
||||
with_profiles = CoverageRegistryService.resolve(
|
||||
FakeSession(project=SimpleNamespace(id=project_id), areas=scope, datasets=[profiles]),
|
||||
project_id,
|
||||
selection,
|
||||
["bathymetry"],
|
||||
)
|
||||
assert with_profiles.items[0].status == "operational"
|
||||
assert with_profiles.items[0].materialized_dataset_ids == [profile_id]
|
||||
|
||||
|
||||
def test_mixed_land_and_north_sea_selection_remains_split() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
areas = [
|
||||
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
|
||||
scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)),
|
||||
scope_area("Belgian part of the North Sea", box(2.2, 51.1, 3.4, 51.9)),
|
||||
scope_area("Belgian territorial sea (0-12 nautical miles)", box(2.7, 51.1, 3.4, 51.5)),
|
||||
scope_area("Belgian exclusive economic zone beyond territorial sea", box(2.2, 51.5, 3.1, 51.9)),
|
||||
scope_area("Belgian continental shelf beyond territorial sea", box(2.2, 51.5, 3.1, 51.9)),
|
||||
]
|
||||
|
||||
result = CoverageRegistryService.resolve(
|
||||
FakeSession(project=project, areas=areas, datasets=[]),
|
||||
project_id,
|
||||
CoverageBBox(minx=2.65, miny=51.05, maxx=2.85, maxy=51.2),
|
||||
["admin", "bathymetry"],
|
||||
)
|
||||
|
||||
assert result.intersected_zones == ["flanders", "territorial_sea"]
|
||||
assert len(result.items) == 4
|
||||
assert any("crosses coverage zones" in warning for warning in result.warnings)
|
||||
bathymetry = next(item for item in result.items if item.zone == "territorial_sea" and item.theme == "bathymetry")
|
||||
assert bathymetry.status == "not_configured"
|
||||
assert bathymetry.materialized_dataset_ids == []
|
||||
|
||||
|
||||
def test_flemish_materialization_is_theme_specific() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
orthophoto_id = uuid4()
|
||||
orthophoto = governed_materialization(
|
||||
dataset_id=orthophoto_id,
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
reference_layer_name="orthophoto",
|
||||
source_metadata={"coverage_zones": ["flanders"]},
|
||||
)
|
||||
result = CoverageRegistryService.resolve(
|
||||
FakeSession(
|
||||
project=project,
|
||||
areas=[scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5))],
|
||||
datasets=[orthophoto],
|
||||
),
|
||||
project_id,
|
||||
CoverageBBox(minx=4.9, miny=50.9, maxx=5.0, maxy=51.0),
|
||||
["orthophoto", "roads"],
|
||||
)
|
||||
|
||||
assert next(item for item in result.items if item.theme == "orthophoto").status == "operational"
|
||||
assert next(item for item in result.items if item.theme == "roads").status == "partial"
|
||||
|
||||
|
||||
def test_outside_scope_and_unknown_theme_are_explicit() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
db = FakeSession(
|
||||
project=project,
|
||||
areas=[scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5))],
|
||||
datasets=[],
|
||||
)
|
||||
result = CoverageRegistryService.resolve(
|
||||
db,
|
||||
project_id,
|
||||
CoverageBBox(minx=7.0, miny=52.0, maxx=7.1, maxy=52.1),
|
||||
["admin"],
|
||||
)
|
||||
assert result.intersected_zones == []
|
||||
assert result.outside_supported_scope is True
|
||||
assert result.items == []
|
||||
|
||||
try:
|
||||
CoverageRegistryService.resolve(
|
||||
db,
|
||||
project_id,
|
||||
CoverageBBox(minx=4.0, miny=50.0, maxx=4.1, maxy=50.1),
|
||||
["invented_metric"],
|
||||
)
|
||||
except AppError as exc:
|
||||
assert exc.code == "COVERAGE_THEME_UNSUPPORTED"
|
||||
assert exc.status_code == 422
|
||||
assert exc.details["unsupported_themes"] == ["invented_metric"]
|
||||
else:
|
||||
raise AssertionError("Unknown coverage theme was accepted")
|
||||
|
||||
|
||||
def test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbox() -> None:
|
||||
root = Path(__file__).parents[2]
|
||||
focus = (root / "frontend" / "src" / "config" / "primaryFocus.ts").read_text(encoding="utf-8")
|
||||
workspace_hook = read_feature("shell")
|
||||
coverage_hook = read_feature("map_workspace")
|
||||
map_workspace = read_map_workspace()
|
||||
|
||||
assert "Belgium and North Sea Workbench" in focus
|
||||
assert "nationalProject" in workspace_hook
|
||||
assert "return nationalProject.id" in workspace_hook
|
||||
assert "NATIONAL_WORKSPACE_REGION" in workspace_hook
|
||||
assert "externalApi.resolveCoverage" in coverage_hook
|
||||
assert "coverage.outside_supported_scope" in map_workspace
|
||||
assert "coverageStatusLabel" in map_workspace
|
||||
assert "activeThemeSupportsCurrentSelection" in map_workspace
|
||||
assert "activeThemeAvailable && !regionalPartitionedThemeActive" in map_workspace
|
||||
@@ -0,0 +1,183 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box, mapping, shape
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import provision_belgium_north_sea_scope as operator # noqa: E402
|
||||
from app.utils.geometry import normalize_to_multipolygon # noqa: E402
|
||||
|
||||
|
||||
def marine_feature(identifier: str, geometry):
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": identifier,
|
||||
"geometry": mapping(geometry),
|
||||
"properties": {"MarineReportingUnitId": identifier},
|
||||
}
|
||||
|
||||
|
||||
def test_marine_legal_scopes_are_derived_from_official_reporting_units() -> None:
|
||||
reporting_units = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
marine_feature("ANS-BE-MS-1", box(0, 0, 10, 10)),
|
||||
marine_feature("ANS-BE-AA-CW", box(0, 0, 2, 2)),
|
||||
marine_feature("ANS-BE-AA-TEW", box(0, 2, 2, 4)),
|
||||
marine_feature("ANS-BE-AA-OFFSHORE", box(2, 0, 10, 10)),
|
||||
],
|
||||
}
|
||||
|
||||
payload = operator.derive_marine_scope_payload(reporting_units)
|
||||
by_zone = {
|
||||
feature["properties"]["coverage_zone"]: feature
|
||||
for feature in payload["features"]
|
||||
}
|
||||
|
||||
assert set(by_zone) == {
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
}
|
||||
assert shape(by_zone["territorial_sea"]["geometry"]).area == pytest.approx(8.0)
|
||||
assert shape(by_zone["exclusive_economic_zone"]["geometry"]).equals(
|
||||
shape(by_zone["continental_shelf"]["geometry"])
|
||||
)
|
||||
assert (
|
||||
by_zone["exclusive_economic_zone"]["properties"]["legal_domain"]
|
||||
!= by_zone["continental_shelf"]["properties"]["legal_domain"]
|
||||
)
|
||||
assert by_zone["territorial_sea"]["properties"]["derived_from_reporting_unit_ids"] == [
|
||||
"ANS-BE-AA-CW",
|
||||
"ANS-BE-AA-TEW",
|
||||
]
|
||||
|
||||
|
||||
def test_marine_scope_derivation_fails_when_a_required_unit_is_missing() -> None:
|
||||
with pytest.raises(RuntimeError, match="ANS-BE-AA-CW"):
|
||||
operator.derive_marine_scope_payload(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
marine_feature("ANS-BE-MS-1", box(0, 0, 10, 10)),
|
||||
marine_feature("ANS-BE-AA-TEW", box(0, 2, 2, 4)),
|
||||
marine_feature("ANS-BE-AA-OFFSHORE", box(2, 0, 10, 10)),
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_area_geometry_normalization_drops_source_z_dimension() -> None:
|
||||
geometry = normalize_to_multipolygon(
|
||||
{
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[2.5, 49.5, 0],
|
||||
[6.4, 49.5, 0],
|
||||
[6.4, 51.5, 0],
|
||||
[2.5, 51.5, 0],
|
||||
[2.5, 49.5, 0],
|
||||
]
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
assert geometry.has_z is False
|
||||
assert geometry.geom_type == "MultiPolygon"
|
||||
|
||||
|
||||
def test_archive_extraction_accepts_one_safe_geopackage_and_rejects_traversal(tmp_path: Path) -> None:
|
||||
archive = tmp_path / "adminvector.zip"
|
||||
with zipfile.ZipFile(archive, "w") as handle:
|
||||
handle.writestr("release/adminvector.gpkg", b"sqlite-bytes")
|
||||
|
||||
result = operator.extract_single_geopackage(archive, tmp_path / "output")
|
||||
assert result.read_bytes() == b"sqlite-bytes"
|
||||
|
||||
unsafe = tmp_path / "unsafe.zip"
|
||||
with zipfile.ZipFile(unsafe, "w") as handle:
|
||||
handle.writestr("../adminvector.gpkg", b"unsafe")
|
||||
with pytest.raises(RuntimeError, match="unsafe"):
|
||||
operator.extract_single_geopackage(unsafe, tmp_path / "unsafe-output")
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
self.content = json.dumps(payload).encode("utf-8")
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, pages):
|
||||
self.pages = list(pages)
|
||||
self.calls = []
|
||||
|
||||
def get(self, url, params, timeout):
|
||||
self.calls.append({"url": url, "params": params, "timeout": timeout})
|
||||
return FakeResponse(self.pages.pop(0))
|
||||
|
||||
|
||||
def test_wfs_fetch_is_allowlisted_paginated_and_complete() -> None:
|
||||
pages = [
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"numberMatched": 2,
|
||||
"features": [{"type": "Feature", "id": "unit.1", "geometry": None, "properties": {}}],
|
||||
},
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"numberMatched": 2,
|
||||
"features": [{"type": "Feature", "id": "unit.2", "geometry": None, "properties": {}}],
|
||||
},
|
||||
]
|
||||
session = FakeSession(pages)
|
||||
payload = operator.fetch_wfs_layer(
|
||||
session,
|
||||
service_url=operator.RBINS_MRU_WFS_URL,
|
||||
layer_name=operator.RBINS_MRU_LAYER,
|
||||
timeout=30,
|
||||
page_size=1,
|
||||
)
|
||||
|
||||
assert [feature["id"] for feature in payload["features"]] == ["unit.1", "unit.2"]
|
||||
assert [call["params"]["startIndex"] for call in session.calls] == [0, 1]
|
||||
assert all(call["params"]["srsName"] == "EPSG:4326" for call in session.calls)
|
||||
|
||||
with pytest.raises(RuntimeError, match="allowlist"):
|
||||
operator.fetch_wfs_layer(
|
||||
FakeSession([]),
|
||||
service_url=operator.RBINS_MSP_WFS_URL,
|
||||
layer_name="untrusted:layer",
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
|
||||
def test_operator_is_packaged_and_guarded_by_readiness() -> None:
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
source = (ROOT / "scripts" / "provision_belgium_north_sea_scope.py").read_text(encoding="utf-8")
|
||||
|
||||
assert "COPY scripts/provision_belgium_north_sea_scope.py /app/scripts/" in dockerfile
|
||||
assert "py_compile scripts/provision_belgium_north_sea_scope.py" in readiness
|
||||
assert "/datasets/upload" in source
|
||||
assert "frame.to_json(drop_id=False, default=str)" in source
|
||||
assert "from app.models" not in source
|
||||
assert "INSERT INTO vector_features" not in source
|
||||
@@ -0,0 +1,219 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_build_identity_does_not_invalidate_dependency_layers() -> None:
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
|
||||
dependency_install = dockerfile.index("/usr/bin/python3.11 -m venv /opt/geointel/venv")
|
||||
source_copy = dockerfile.index("COPY backend/ /app/")
|
||||
build_identity = dockerfile.index("ARG GEOINTEL_BUILD_SHA=unknown")
|
||||
|
||||
assert build_identity > dependency_install
|
||||
assert build_identity > source_copy
|
||||
assert 'org.opencontainers.image.revision="${GEOINTEL_BUILD_SHA}"' in dockerfile
|
||||
assert 'org.opencontainers.image.created="${GEOINTEL_BUILD_TIME}"' in dockerfile
|
||||
assert 'io.geointel.ai.enabled="${GEOINTEL_INSTALL_AI}"' in dockerfile
|
||||
|
||||
|
||||
def test_release_deploy_preserves_immutable_and_backup_specific_rollback_images() -> None:
|
||||
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert 'GEOINTEL_RELEASE_VARIANT="ai"' in script
|
||||
assert 'GEOINTEL_RELEASE_VARIANT="gis"' in script
|
||||
assert 'GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"' in script
|
||||
assert 'GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"' in script
|
||||
assert 'docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script
|
||||
assert '--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script
|
||||
assert "GEOINTEL_PREVIOUS_IMAGE" not in script
|
||||
assert 'if docker image inspect "$GEOINTEL_RELEASE_IMAGE"' in script
|
||||
assert "Immutable release tag has conflicting metadata" in script
|
||||
assert "Reusing existing immutable image" in script
|
||||
assert "rollback_previous()" in script
|
||||
assert "Deployed immutable image" in script
|
||||
|
||||
|
||||
def test_release_creates_verified_backup_before_candidate_migrations() -> None:
|
||||
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||
|
||||
backup_index = script.index("create_predeploy_backup\n")
|
||||
scan_index = script.index("scan_release_image\n")
|
||||
candidate_start_index = script.index('if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"')
|
||||
assert scan_index < backup_index
|
||||
assert backup_index < candidate_start_index
|
||||
assert script.index("docker build") < backup_index
|
||||
assert "/mnt/user/appdata/geointel/backups" in script
|
||||
assert "--inventory-mode sha256" in script
|
||||
assert "scripts/verify_release_backup.sh" in script
|
||||
assert "refusing an unbacked migration" in script
|
||||
assert "Quiescing the current backend" in script
|
||||
assert "restarting the unchanged current release" in script
|
||||
assert "preflight_backup_capacity" in script
|
||||
assert "select_verified_link_dest" in script
|
||||
assert "release_backup_snapshot.py" in script
|
||||
assert "run_low_impact()" in script
|
||||
assert "ionice -c 2 -n 7" in script
|
||||
assert "nice -n 10" in script
|
||||
assert "run_low_impact bash scripts/backup_release_state.sh" in script
|
||||
assert "run_low_impact bash scripts/verify_release_backup.sh" in script
|
||||
|
||||
|
||||
def test_gitea_deploy_waits_for_the_mandatory_large_snapshot() -> None:
|
||||
workflow = (ROOT / ".gitea" / "workflows" / "release-gates.yml").read_text(encoding="utf-8")
|
||||
|
||||
assert "cancel-in-progress: false" in workflow
|
||||
deploy_job = workflow.split("\n deploy:\n", maxsplit=1)[1]
|
||||
assert "timeout-minutes: 720" in deploy_job
|
||||
assert "docker exec gitea-deploy-control" in deploy_job
|
||||
|
||||
|
||||
def test_release_starts_only_the_locally_attested_ai_image() -> None:
|
||||
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"' in script
|
||||
assert "Production release deployment requires the gated AI image" in script
|
||||
assert 'bash scripts/generate_container_sbom.sh "$scanned_image_id"' in script
|
||||
assert 'bash scripts/scan_container_image.sh "$scanned_image_id"' in script
|
||||
assert 'running_image_id="$(docker inspect --format \'{{.Image}}\' geointel)"' in script
|
||||
assert 'if [ "$running_image_id" != "$image" ]' in script
|
||||
assert "deployment-attestation.json" in script
|
||||
assert "artifacts/release-evidence/deploy" in script
|
||||
assert "GITEA_COMMIT_SHA" in script
|
||||
assert "GITHUB_SHA" in script
|
||||
assert "must contain one full 40-character Git commit SHA" in script
|
||||
assert 'marker_path="$ROOT/.gitea-deploy/revision"' in script
|
||||
assert "git rev-parse --show-toplevel" in script
|
||||
assert "Prepared source revision marker does not match" in script
|
||||
assert "neither an exact Git checkout nor bound" in script
|
||||
assert 'running_revision" != "$GEOINTEL_BUILD_SHA"' in script
|
||||
assert 'running_ai" != "true"' in script
|
||||
assert '"revision": revision' in script
|
||||
|
||||
|
||||
def test_release_and_container_replacement_are_serialized() -> None:
|
||||
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
rollback_script = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
restore_script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert "GEOINTEL_DEPLOY_LOCK_FILE" in release_script
|
||||
assert "flock -n 9" in release_script
|
||||
assert "GEOINTEL_DEPLOY_LOCK_FILE" in rollback_script
|
||||
assert "flock -n 9" in rollback_script
|
||||
assert "GEOINTEL_DEPLOY_LOCK_FILE" in restore_script
|
||||
assert "flock -n 9" in restore_script
|
||||
assert "GEOINTEL_DEPLOY_LOCK_HELD=true" in release_script
|
||||
assert "GEOINTEL_CONTAINER_LOCK_FILE" in run_script
|
||||
assert "flock -w 300 8" in run_script
|
||||
assert "GeoIntel container removal did not complete within 60 seconds" in run_script
|
||||
|
||||
|
||||
def test_release_waits_for_large_postgis_volume_recovery() -> None:
|
||||
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||
start_script = (ROOT / "deploy" / "unraid" / "all-in-one-start.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "for attempt in $(seq 1 480)" in release_script
|
||||
assert "for attempt in $(seq 1 450)" in start_script
|
||||
assert "PostGIS did not become ready within 15 minutes." in start_script
|
||||
assert 'chown postgres:postgres "$PGDATA"' in start_script
|
||||
assert 'chown -R postgres:postgres "$PGDATA"' not in start_script
|
||||
|
||||
|
||||
def test_runtime_configuration_is_validated_before_container_replacement() -> None:
|
||||
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
|
||||
validation_index = run_script.index("validate_runtime_config")
|
||||
# The script replaces the container with "docker rm -f"; what matters is
|
||||
# that no removal happens before the runtime config has been validated.
|
||||
replacement_index = run_script.index("docker rm -f geointel")
|
||||
|
||||
assert validation_index < replacement_index
|
||||
assert "known-default PostGIS password" in run_script
|
||||
assert "GEOINTEL_MAX_UPLOAD_MB must be between 1 and 2048" in run_script
|
||||
assert 'docker image inspect "$GEOINTEL_IMAGE"' in run_script
|
||||
|
||||
|
||||
def test_fresh_install_smoke_is_isolated_and_cleans_only_its_temp_path() -> None:
|
||||
script = (ROOT / "scripts" / "verify_release_fresh_install.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "mktemp -d" in script
|
||||
assert "geointel-fresh-smoke.*" in script
|
||||
assert "-p 127.0.0.1::80" in script
|
||||
assert "GEOINTEL_POSTGRES_PASSWORD=" in script
|
||||
assert "/health/ready" in script
|
||||
assert "/api/v1/system/capabilities" in script
|
||||
assert "docker exec" in script
|
||||
assert "python -m alembic heads" in script
|
||||
|
||||
|
||||
def test_manual_rollback_reuses_persistent_paths_and_requires_existing_image() -> None:
|
||||
rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "Backup manifest lacks an immutable rollback image ID" in rollback
|
||||
assert 'get("image_id", "")' in rollback
|
||||
assert 'docker image inspect "$GEOINTEL_ROLLBACK_IMAGE"' in rollback
|
||||
assert 'GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE"' in rollback
|
||||
assert "restore-predeploy-database.sh" in rollback
|
||||
assert "--confirm-production-database-restore" in rollback
|
||||
assert "Image-only rollback" in rollback
|
||||
assert '-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data"' in run_script
|
||||
assert '-v "${GEOINTEL_STORAGE_PATH}:/app/storage"' in run_script
|
||||
|
||||
|
||||
def test_same_revision_redeploy_rolls_back_by_backup_bound_image_id() -> None:
|
||||
release = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||
rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
restore = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
backup = (ROOT / "scripts" / "backup_release_state.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "current_image_id" in release
|
||||
assert "release_image_id" not in release
|
||||
assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in backup
|
||||
assert 'if [ -z "$RESTORE_IMAGE" ]; then\n RESTORE_IMAGE="$BACKUP_IMAGE_ID"' in restore
|
||||
assert 'GEOINTEL_ROLLBACK_IMAGE="$(python3 - "$BACKUP_DIR/manifest.json"' in rollback
|
||||
assert "geointel-all-in-one:previous" not in release + rollback + restore
|
||||
|
||||
|
||||
def test_readiness_checks_all_release_shell_entrypoints() -> None:
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
|
||||
for path in (
|
||||
"scripts/deploy_tower.sh",
|
||||
"scripts/verify_release_fresh_install.sh",
|
||||
"scripts/verify_release_upgrade_smoke.sh",
|
||||
"deploy/unraid/all-in-one-start.sh",
|
||||
"deploy/unraid/run-dockerman-container.sh",
|
||||
"deploy/unraid/deploy-release.sh",
|
||||
"deploy/unraid/rollback-dockerman-container.sh",
|
||||
"deploy/unraid/restore-predeploy-database.sh",
|
||||
):
|
||||
assert f"bash -n {path}" in readiness
|
||||
|
||||
|
||||
def test_upgrade_smoke_restores_and_upgrades_only_an_isolated_database() -> None:
|
||||
script = (ROOT / "scripts" / "verify_release_upgrade_smoke.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "--confirm-isolated-upgrade" in script
|
||||
assert "restore_release_backup_smoke.sh" in script
|
||||
assert "--keep-database" in script
|
||||
assert "geointel_restore_verify_" in script
|
||||
assert "from sqlalchemy import URL" in script
|
||||
assert 'username=os.environ["GEOINTEL_POSTGRES_USER"]' in script
|
||||
assert 'password=os.environ["GEOINTEL_POSTGRES_PASSWORD"]' in script
|
||||
assert 'database=os.environ["TARGET_DB"]' in script
|
||||
assert '"$CONTAINER" sh -c' in script
|
||||
assert '"$CONTAINER" sh -lc' not in script
|
||||
assert "python -m alembic upgrade head" in script
|
||||
assert "production_database_untouched" in script
|
||||
assert 'dropdb --if-exists -U "$db_user" "$TARGET_DB"' in script
|
||||
@@ -0,0 +1,198 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def read(path: str) -> str:
|
||||
return (ROOT / path).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_python_ci_lock_is_hashed_linux_311_and_excludes_ai() -> None:
|
||||
for lock_path in (
|
||||
"backend/requirements-runtime.lock",
|
||||
"backend/requirements-ci.lock",
|
||||
):
|
||||
lock = read(lock_path)
|
||||
assert "pip-compile with Python 3.11" in lock
|
||||
assert "# geointel-input-sha256: " in lock
|
||||
assert "--generate-hashes" in lock
|
||||
assert "\ntorch==" not in lock
|
||||
assert "\nultralytics==" not in lock
|
||||
|
||||
|
||||
def test_lock_generator_uses_pinned_linux_runtime_and_verifies_policy() -> None:
|
||||
generator = read("scripts/generate_python_lock.sh")
|
||||
|
||||
assert "python:3.11-bookworm@sha256:" in generator
|
||||
assert 'PIP_TOOLS_VERSION="7.5.3"' in generator
|
||||
assert "--extra gis" in generator
|
||||
assert "--extra dev" in generator
|
||||
assert "requirements-runtime.lock" in generator
|
||||
assert "requirements-ci.lock" in generator
|
||||
assert "--generate-hashes" in generator
|
||||
assert "verify_python_lock.py --stamp" in generator
|
||||
|
||||
|
||||
def test_ci_runs_complete_release_and_supply_chain_gates() -> None:
|
||||
for workflow_path, context in (
|
||||
(".github/workflows/release-gates.yml", "github.sha"),
|
||||
(".gitea/workflows/release-gates.yml", "gitea.sha"),
|
||||
):
|
||||
workflow = read(workflow_path)
|
||||
assert "backend/requirements-ci.lock" in workflow
|
||||
assert "scripts/verify_python_lock.py" in workflow
|
||||
assert "scripts/run_readiness_check.sh" in workflow
|
||||
assert "python -m alembic upgrade head --sql" in workflow
|
||||
assert "docker compose config" in workflow
|
||||
assert "pip-audit==2.10.1" in workflow
|
||||
assert "audit_python_dependencies.sh" in workflow
|
||||
assert "npm audit --audit-level=high" in workflow
|
||||
assert "GEOINTEL_INSTALL_AI=true" in workflow
|
||||
assert "geointel-ci:$RELEASE_SHA-ai" in workflow
|
||||
assert "artifacts/image-id.txt" in workflow
|
||||
assert 'scan_container_image.sh "$IMAGE_ID"' in workflow
|
||||
assert "generate_container_sbom.sh" in workflow
|
||||
assert "scan_container_image.sh" in workflow
|
||||
assert "GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar" in workflow
|
||||
assert "Remove temporary image archive" in workflow
|
||||
assert context in workflow
|
||||
|
||||
assert (
|
||||
"actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02"
|
||||
in read(".github/workflows/release-gates.yml")
|
||||
)
|
||||
assert (
|
||||
"actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de"
|
||||
in read(".gitea/workflows/release-gates.yml")
|
||||
)
|
||||
|
||||
|
||||
def test_gitea_production_deploy_depends_on_every_release_gate() -> None:
|
||||
release = read(".gitea/workflows/release-gates.yml")
|
||||
legacy_deploy = ROOT / ".gitea" / "workflows" / "unraid-deploy.yml"
|
||||
|
||||
assert "pull_request:" in release
|
||||
assert "needs: [quality, dependency-audit, container]" in release
|
||||
assert "gitea.event_name == 'push'" in release
|
||||
assert "gitea.ref == 'refs/heads/main'" in release
|
||||
assert "/opt/gitea-deploy/deploy.py deploy" in release
|
||||
assert not legacy_deploy.exists()
|
||||
assert "workflow_dispatch:" in release
|
||||
|
||||
|
||||
def test_release_workflows_pin_third_party_actions_to_reviewed_commits() -> None:
|
||||
shared = (
|
||||
"actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683",
|
||||
"actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065",
|
||||
"actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020",
|
||||
)
|
||||
for path in (".gitea/workflows/release-gates.yml", ".github/workflows/release-gates.yml"):
|
||||
workflow = read(path)
|
||||
for action in shared:
|
||||
assert action in workflow
|
||||
|
||||
assert (
|
||||
"actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02"
|
||||
in read(".github/workflows/release-gates.yml")
|
||||
)
|
||||
assert (
|
||||
"actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de"
|
||||
in read(".gitea/workflows/release-gates.yml")
|
||||
)
|
||||
|
||||
|
||||
def test_managed_validation_targets_the_actual_backend_and_frontend_projects() -> None:
|
||||
workflow = read(".gitea/workflows/managed-validation.yml")
|
||||
|
||||
assert "backend/requirements-ci.lock" in workflow
|
||||
assert "frontend/package-lock.json" in workflow
|
||||
assert "python -m pytest -W error::DeprecationWarning" in workflow
|
||||
assert "cd frontend && npm run test:unit" in workflow
|
||||
assert "python -m ruff check backend scripts tests" in workflow
|
||||
assert "python scripts/verify_repository_layout.py" in workflow
|
||||
assert "[[ -f pyproject.toml" not in workflow
|
||||
|
||||
|
||||
def test_scanner_images_are_versioned_and_digest_pinned() -> None:
|
||||
sbom = read("scripts/generate_container_sbom.sh")
|
||||
scan = read("scripts/scan_container_image.sh")
|
||||
|
||||
assert "anchore/syft:v1.44.0@sha256:" in sbom
|
||||
assert "--user 0:0" in sbom
|
||||
assert 'docker save "$IMAGE_ID"' in sbom
|
||||
assert '"docker-archive:$WORKDIR/$IMAGE_ARCHIVE"' in sbom
|
||||
assert 'SYFT_PARALLELISM=${SYFT_PARALLELISM:-1}' in sbom
|
||||
assert 'GOMEMLIMIT=${SYFT_GOMEMLIMIT:-4GiB}' in sbom
|
||||
assert 'GOGC=${SYFT_GOGC:-25}' in sbom
|
||||
assert "--select-catalogers=-binary" in sbom
|
||||
assert '--volumes-from "$HOSTNAME"' in sbom
|
||||
assert 'ARCHIVE_ID_FILE="${IMAGE_ARCHIVE}.image-id"' in sbom
|
||||
assert "/var/run/docker.sock" not in sbom
|
||||
assert "aquasec/trivy:0.70.0@sha256:" in scan
|
||||
assert "--severity HIGH,CRITICAL" in scan
|
||||
assert "--ignore-unfixed" in scan
|
||||
assert "--timeout 20m" in scan
|
||||
assert "--scanners vuln" in scan
|
||||
assert 'docker save "$IMAGE_ID"' in scan
|
||||
assert '--input "$WORKDIR/$IMAGE_ARCHIVE"' in scan
|
||||
assert '--volumes-from "$HOSTNAME"' in scan
|
||||
assert '--ignorefile "$CONTAINER_IGNORE_FILE"' in scan
|
||||
assert "/var/run/docker.sock" not in scan
|
||||
assert "--skip-files /usr/local/bin/gosu" in scan
|
||||
assert "final filesystem replaces it with the audited setpriv shell wrapper" in scan
|
||||
assert "geointel-container-vulnerabilities.json" in scan
|
||||
|
||||
|
||||
def test_readiness_guards_lock_and_supply_chain_entrypoints() -> None:
|
||||
readiness = read("scripts/run_readiness_check.sh")
|
||||
|
||||
assert "scripts/verify_python_lock.py" in readiness
|
||||
assert "scripts/verify_security_exceptions.py" in readiness
|
||||
for path in (
|
||||
"scripts/generate_python_lock.sh",
|
||||
"scripts/generate_container_sbom.sh",
|
||||
"scripts/scan_container_image.sh",
|
||||
"scripts/audit_python_dependencies.sh",
|
||||
):
|
||||
assert f"bash -n {path}" in readiness
|
||||
|
||||
|
||||
def test_python_audit_policy_has_no_active_exceptions_and_keeps_full_evidence() -> None:
|
||||
policy = read("security/pip-audit-exceptions.json")
|
||||
audit_script = read("scripts/audit_python_dependencies.sh")
|
||||
|
||||
assert '"schema_version": 1' in policy
|
||||
assert '"advisories": []' in policy
|
||||
assert "pip-audit-full.json" in audit_script
|
||||
assert "pip-audit-policy.json" in audit_script
|
||||
assert "--ignore-vuln" in audit_script
|
||||
assert "verify_security_exceptions.py" in audit_script
|
||||
|
||||
|
||||
def test_release_image_uses_locked_non_ai_dependencies_and_npm_ci() -> None:
|
||||
dockerfile = read("deploy/unraid/Dockerfile.all-in-one")
|
||||
|
||||
assert "RUN npm ci" in dockerfile
|
||||
assert "COPY backend/requirements-runtime.lock /app/" in dockerfile
|
||||
assert "COPY backend/requirements-ai-linux.lock /app/" in dockerfile
|
||||
assert "COPY backend/requirements-build-tools.lock /app/" in dockerfile
|
||||
assert "pip install --no-cache-dir --require-hashes -r requirements-runtime.lock" in dockerfile
|
||||
assert "-r requirements-ai-linux.lock" in dockerfile
|
||||
assert "-r requirements-build-tools.lock" in dockerfile
|
||||
assert "--require-hashes" in dockerfile
|
||||
assert "ultralytics==8.4.99 --hash=sha256:" in read("backend/requirements-ai-linux.lock")
|
||||
assert "torch==2.11.0+cu128 --hash=sha256:" in read("backend/requirements-ai-linux.lock")
|
||||
assert "COPY deploy/unraid/gosu-setpriv /usr/local/bin/gosu" in dockerfile
|
||||
assert "&& pip check" in dockerfile
|
||||
|
||||
|
||||
def test_gosu_compatibility_wrapper_uses_exec_and_setpriv() -> None:
|
||||
wrapper = read("deploy/unraid/gosu-setpriv")
|
||||
readiness = read("scripts/run_readiness_check.sh")
|
||||
|
||||
assert "exec setpriv" in wrapper
|
||||
assert '--reuid="$target_user"' in wrapper
|
||||
assert '--regid="$target_user"' in wrapper
|
||||
assert "--init-groups" in wrapper
|
||||
assert "bash -n deploy/unraid/gosu-setpriv" in readiness
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
BACKEND = ROOT / "backend"
|
||||
|
||||
|
||||
def test_no_untyped_fastapi_response_models_remain() -> None:
|
||||
route_root = BACKEND / "app" / "api" / "routes"
|
||||
route_sources = "\n".join(
|
||||
path.read_text(encoding="utf-8")
|
||||
for path in sorted(route_root.glob("*.py"))
|
||||
)
|
||||
|
||||
assert "response_model=dict" not in route_sources
|
||||
|
||||
|
||||
def test_every_json_success_response_has_a_concrete_canonical_schema() -> None:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(BACKEND))
|
||||
from app.main import create_app
|
||||
|
||||
from scripts.audit_api_contracts import (
|
||||
ALLOWED_NON_ENVELOPE_ENDPOINTS,
|
||||
_validate_response_contracts,
|
||||
)
|
||||
|
||||
openapi = create_app().openapi()
|
||||
assert _validate_response_contracts(openapi) == []
|
||||
|
||||
untyped_successes: set[tuple[str, str]] = set()
|
||||
for path, path_item in openapi["paths"].items():
|
||||
for method, operation in path_item.items():
|
||||
if method.upper() not in {"GET", "POST", "PATCH", "DELETE"}:
|
||||
continue
|
||||
has_json_schema = any(
|
||||
response.get("content", {})
|
||||
.get("application/json", {})
|
||||
.get("schema")
|
||||
for code, response in operation.get("responses", {}).items()
|
||||
if str(code).startswith("2")
|
||||
)
|
||||
if not has_json_schema:
|
||||
untyped_successes.add((method.upper(), path))
|
||||
|
||||
assert untyped_successes == ALLOWED_NON_ENVELOPE_ENDPOINTS - {
|
||||
("GET", "/health"),
|
||||
("GET", "/health/live"),
|
||||
("GET", "/health/ready"),
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT_PATH = ROOT / "scripts" / "provision_release_golden_areas.py"
|
||||
|
||||
|
||||
def load_operator():
|
||||
spec = importlib.util.spec_from_file_location("provision_release_golden_areas_test", SCRIPT_PATH)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_release_golden_area_contract_covers_belgium_and_north_sea() -> None:
|
||||
module = load_operator()
|
||||
created_keys = {item["key"] for item in module.GOLDEN_AREAS}
|
||||
source_keys = {item["key"] for item in module.SOURCE_AREAS}
|
||||
|
||||
assert created_keys == {
|
||||
"wallonia_urban_rural",
|
||||
"brussels_urban",
|
||||
"language_boundary",
|
||||
"coast_land_sea",
|
||||
"north_sea_multi_zone",
|
||||
}
|
||||
assert source_keys == {"mol_municipality", "kempen_region"}
|
||||
assert len(created_keys | source_keys) == 7
|
||||
assert all(item["source_project"] != module.NATIONAL_PROJECT for item in module.SOURCE_AREAS)
|
||||
|
||||
expected_zones = {
|
||||
zone
|
||||
for definition in (*module.GOLDEN_AREAS, *module.SOURCE_AREAS)
|
||||
for zone in definition["expected_zones"]
|
||||
}
|
||||
assert {
|
||||
"flanders",
|
||||
"wallonia",
|
||||
"brussels",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
} <= expected_zones
|
||||
|
||||
|
||||
def test_release_golden_area_geometries_are_bounded_and_fingerprintable() -> None:
|
||||
module = load_operator()
|
||||
hashes = set()
|
||||
for definition in module.GOLDEN_AREAS:
|
||||
bbox = module.geometry_bbox(definition["geometry"])
|
||||
assert -180 <= bbox["minx"] < bbox["maxx"] <= 180
|
||||
assert -90 <= bbox["miny"] < bbox["maxy"] <= 90
|
||||
assert bbox["maxx"] - bbox["minx"] <= 0.25
|
||||
assert bbox["maxy"] - bbox["miny"] <= 0.20
|
||||
digest = module.canonical_hash(definition["geometry"])
|
||||
assert len(digest) == 64
|
||||
hashes.add(digest)
|
||||
assert len(hashes) == len(module.GOLDEN_AREAS)
|
||||
|
||||
|
||||
def test_rc8_runner_and_container_operator_are_release_wired() -> None:
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
package = (ROOT / "frontend" / "package.json").read_text(encoding="utf-8")
|
||||
|
||||
assert "npm run test:unit" in readiness
|
||||
assert '--check frontend/e2e/releaseJourneys.mjs' in readiness
|
||||
assert "bash -n scripts/run_rc8_release_journeys.sh" in readiness
|
||||
assert "COPY scripts/provision_release_golden_areas.py" in dockerfile
|
||||
assert '"test:e2e": "node e2e/releaseJourneys.mjs"' in package
|
||||
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from tests.frontend_contract import read_map_workspace, read_feature
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_rc9_ux_audit_is_wired_into_frontend_and_readiness() -> None:
|
||||
package = json.loads((ROOT / "frontend" / "package.json").read_text(encoding="utf-8"))
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
wrapper = ROOT / "scripts" / "run_rc9_ux_audit.sh"
|
||||
|
||||
assert package["scripts"]["test:e2e:ux"] == "node e2e/uxAudit.mjs"
|
||||
assert '"${NODE_BIN}" --check frontend/e2e/uxAudit.mjs' in readiness
|
||||
assert "bash -n scripts/run_rc9_ux_audit.sh" in readiness
|
||||
assert wrapper.is_file()
|
||||
|
||||
|
||||
def test_rc9_loading_and_accessibility_states_are_explicit() -> None:
|
||||
app = read_feature("shell")
|
||||
map_workspace = read_map_workspace()
|
||||
geo_map = (ROOT / "frontend" / "src" / "components" / "GeoMap.tsx").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert "workspaceDataLoading" in app
|
||||
assert 'role="status" aria-live="polite"' in app
|
||||
assert "Databronnen worden gecontroleerd" in map_workspace
|
||||
# Copy changes; the requirement is that source availability is stated.
|
||||
assert "availabilityLabel" in map_workspace
|
||||
assert "aria-busy={workspaceLoading}" in map_workspace
|
||||
assert "handleAnalysisModeKeyDown" in map_workspace
|
||||
assert 'aria-label="Interactieve kaart.' in geo_map
|
||||
|
||||
|
||||
def test_rc9_performance_budgets_are_documented_and_visible() -> None:
|
||||
budget = (
|
||||
ROOT / "frontend" / "src" / "lib" / "performanceBudget.ts"
|
||||
).read_text(encoding="utf-8")
|
||||
docs = (ROOT / "docs" / "UX_PERFORMANCE_BUDGETS.md").read_text(encoding="utf-8")
|
||||
map_workspace = read_map_workspace()
|
||||
|
||||
assert "COVERAGE_RESPONSE_BUDGET_MS = 4_000" in budget
|
||||
assert "MAP_ANALYSIS_BUDGET_MS = 15_000" in budget
|
||||
assert "4 seconds" in docs
|
||||
assert "15 seconds" in docs
|
||||
assert "overschrijdt het releasebudget" in map_workspace
|
||||
@@ -0,0 +1,142 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
|
||||
|
||||
def read(name: str) -> str:
|
||||
return (SCRIPTS / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_backup_is_atomic_read_only_and_checksum_bound() -> None:
|
||||
script = read("backup_release_state.sh")
|
||||
|
||||
assert "pg_dump" in script
|
||||
assert "-Fc" in script
|
||||
assert "--no-owner" in script
|
||||
assert "CHECKSUMS.sha256" in script
|
||||
assert "database-password" not in script.lower()
|
||||
assert 'for required in docker python3 sha256sum; do' in script
|
||||
assert 'for required in docker python3 sha256sum git; do' not in script
|
||||
assert "GITEA_COMMIT_SHA" in script
|
||||
assert "GITHUB_SHA" in script
|
||||
assert "GEOINTEL_BUILD_SHA" in script
|
||||
assert 'if command -v git >/dev/null 2>&1' in script
|
||||
assert "mv \"$PARTIAL\" \"$FINAL\"" in script
|
||||
assert "rm -rf -- \"$PARTIAL\"" in script
|
||||
assert "DROP DATABASE" not in script
|
||||
assert "pg_restore --clean" not in script
|
||||
assert "/mnt/user/appdata/geointel/backups" in script
|
||||
assert "release_backup_snapshot.py" in script
|
||||
assert "storage-snapshot" not in script # labels are composed without unsafe path interpolation
|
||||
assert "--link-dest-backup" in script
|
||||
assert "--rollback-image-tag" in script
|
||||
assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in script
|
||||
|
||||
|
||||
def test_backup_binds_prepared_source_without_requiring_dot_git() -> None:
|
||||
script = read("backup_release_state.sh")
|
||||
|
||||
controller_resolution = script.index('local gitea_sha="${GITEA_COMMIT_SHA:-}"')
|
||||
optional_git_fallback = script.index('if command -v git >/dev/null 2>&1')
|
||||
docker_access = script.index("docker inspect -f '{{.State.Running}}'")
|
||||
assert controller_resolution < optional_git_fallback < docker_access
|
||||
assert 'git -C "$ROOT" rev-parse --show-toplevel' in script
|
||||
assert '"$(cd "$git_top" && pwd -P)" = "$(cd "$ROOT" && pwd -P)"' in script
|
||||
assert 'SOURCE_REVISION="$explicit_sha"' in script
|
||||
assert '"backup_tool_revision": ${SOURCE_REVISION@Q}' in script
|
||||
assert '"running_image_revision": ${RUNNING_IMAGE_REVISION@Q}' in script
|
||||
assert "Cannot bind backup to a source revision" in script
|
||||
|
||||
|
||||
def test_backup_verification_is_read_only() -> None:
|
||||
script = read("verify_release_backup.sh")
|
||||
|
||||
assert "sha256sum -c CHECKSUMS.sha256" in script
|
||||
assert "pg_restore --list" in script
|
||||
assert "createdb" not in script
|
||||
assert "dropdb" not in script
|
||||
assert "pg_restore --clean" not in script
|
||||
|
||||
|
||||
def test_restore_smoke_is_forced_to_generated_isolated_database() -> None:
|
||||
script = read("restore_release_backup_smoke.sh")
|
||||
|
||||
assert "--confirm-isolated-restore" in script
|
||||
assert "geointel_restore_verify_" in script
|
||||
assert 'if [ "$TARGET_DB" = "$DB_NAME" ]' in script
|
||||
assert "createdb" in script
|
||||
assert "dropdb --if-exists" in script
|
||||
assert "pg_restore \\\n --clean" not in script
|
||||
assert '"production_database_untouched": True' in script
|
||||
|
||||
|
||||
def test_release_safety_scripts_have_valid_bash_syntax() -> None:
|
||||
for name in (
|
||||
"backup_release_state.sh",
|
||||
"verify_release_backup.sh",
|
||||
"restore_release_backup_smoke.sh",
|
||||
"../deploy/unraid/restore-predeploy-database.sh",
|
||||
):
|
||||
script_path = f"scripts/{name}" if not name.startswith("../") else name.removeprefix("../")
|
||||
result = subprocess.run(
|
||||
["bash", "-n", script_path],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, f"{name}: {result.stderr}"
|
||||
|
||||
|
||||
def test_production_restore_is_explicit_bounded_and_verified() -> None:
|
||||
script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert "--confirm-production-database-restore" in script
|
||||
assert "/mnt/user/appdata/geointel/backups" in script
|
||||
assert "backup.relative_to(root)" in script
|
||||
assert "sha256sum -c CHECKSUMS.sha256" in script
|
||||
assert '"$RESTORE_PROOF_DB"' in script
|
||||
assert "pg_restore" in script
|
||||
assert "Restored Alembic head" in script
|
||||
assert "Restored count mismatch" in script
|
||||
assert "pg_restore --clean" not in script
|
||||
assert "geointel_restore_proof_" in script
|
||||
assert "Isolated predeploy restore proof passed" in script
|
||||
assert "ALTER DATABASE" in script
|
||||
assert "Pre-restore production database retained" in script
|
||||
drop_start = script.index("dropdb --if-exists --force")
|
||||
drop_command = script[drop_start : script.index("\n fi", drop_start)]
|
||||
assert '"$RESTORE_PROOF_DB"' in drop_command
|
||||
assert '"$GEOINTEL_POSTGRES_DB"' not in drop_command
|
||||
|
||||
|
||||
def test_readiness_gate_checks_release_safety_scripts() -> None:
|
||||
readiness = read("run_readiness_check.sh")
|
||||
|
||||
for name in (
|
||||
"backup_release_state.sh",
|
||||
"verify_release_backup.sh",
|
||||
"restore_release_backup_smoke.sh",
|
||||
):
|
||||
assert f"bash -n scripts/{name}" in readiness
|
||||
|
||||
|
||||
def test_password_rotation_never_prints_or_persists_generated_secret() -> None:
|
||||
script = read("rotate_postgres_password.sh")
|
||||
|
||||
assert "openssl rand -hex 32" in script
|
||||
assert 'echo "$NEW_PASSWORD"' not in script
|
||||
assert 'printf "%s" "$NEW_PASSWORD"' not in script
|
||||
assert "GEOINTEL_ROTATED_DATABASE_PASSWORD" in script
|
||||
assert "NamedTemporaryFile" in script
|
||||
assert "temporary.replace(path)" in script
|
||||
assert "ALTER ROLE %s PASSWORD" in script
|
||||
assert "run-dockerman-container.sh" in script
|
||||
assert "/health/ready" not in script
|
||||
@@ -0,0 +1,137 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset
|
||||
from app.services.temporal_compatibility_service import TemporalCompatibilityService
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
|
||||
|
||||
def dataset(
|
||||
*,
|
||||
dataset_type: str,
|
||||
source_name: str,
|
||||
observed_at: datetime | None = None,
|
||||
valid_from: datetime | None = None,
|
||||
valid_to: datetime | None = None,
|
||||
temporal_granularity: str | None = None,
|
||||
source_metadata: dict | None = None,
|
||||
) -> Dataset:
|
||||
return Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="temporal-source",
|
||||
dataset_type=dataset_type,
|
||||
source=source_name,
|
||||
source_name=source_name,
|
||||
observed_at=observed_at,
|
||||
valid_from=valid_from,
|
||||
valid_to=valid_to,
|
||||
temporal_granularity=temporal_granularity,
|
||||
source_metadata=source_metadata,
|
||||
)
|
||||
|
||||
|
||||
def test_historical_orthophoto_is_rejected_for_detection() -> None:
|
||||
historical = dataset(
|
||||
dataset_type="raster",
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
observed_at=datetime(2020, 1, 1, tzinfo=UTC),
|
||||
valid_from=datetime(2020, 1, 1, tzinfo=UTC),
|
||||
valid_to=datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC),
|
||||
temporal_granularity="year",
|
||||
source_metadata={"product_key": "2020", "supports_detection": False},
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
TemporalCompatibilityService.ensure_detection_source_supported(historical)
|
||||
|
||||
assert exc_info.value.code == "DETECTION_SOURCE_TEMPORALLY_UNSUPPORTED"
|
||||
assert exc_info.value.status_code == 422
|
||||
|
||||
|
||||
def test_historical_detection_qa_rejects_current_reference() -> None:
|
||||
historical = dataset(
|
||||
dataset_type="raster",
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
valid_from=datetime(2020, 1, 1, tzinfo=UTC),
|
||||
valid_to=datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC),
|
||||
temporal_granularity="year",
|
||||
source_metadata={"product_key": "2020", "supports_detection": False},
|
||||
)
|
||||
current_reference = dataset(
|
||||
dataset_type="vector",
|
||||
source_name="grb",
|
||||
valid_from=datetime(2026, 7, 1, tzinfo=UTC),
|
||||
valid_to=datetime(2026, 7, 31, 23, 59, 59, tzinfo=UTC),
|
||||
temporal_granularity="month",
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
TemporalCompatibilityService.assess_detection_qa(historical, current_reference)
|
||||
|
||||
assert exc_info.value.code == "DETECTION_QA_TEMPORAL_MISMATCH"
|
||||
assert exc_info.value.status_code == 422
|
||||
|
||||
|
||||
def test_historical_detection_qa_accepts_overlapping_reference_edition() -> None:
|
||||
historical = dataset(
|
||||
dataset_type="raster",
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
valid_from=datetime(2020, 1, 1, tzinfo=UTC),
|
||||
valid_to=datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC),
|
||||
temporal_granularity="year",
|
||||
source_metadata={"product_key": "2020", "supports_detection": False},
|
||||
)
|
||||
historical_reference = dataset(
|
||||
dataset_type="vector",
|
||||
source_name="manual",
|
||||
valid_from=datetime(2020, 6, 1, tzinfo=UTC),
|
||||
valid_to=datetime(2020, 6, 30, 23, 59, 59, tzinfo=UTC),
|
||||
temporal_granularity="month",
|
||||
)
|
||||
|
||||
result = TemporalCompatibilityService.assess_detection_qa(historical, historical_reference)
|
||||
|
||||
assert result["status"] == "compatible"
|
||||
assert result["candidate_historical"] is True
|
||||
assert result["candidate_interval"]["start"].startswith("2020-01-01")
|
||||
assert result["reference_interval"]["start"].startswith("2020-06-01")
|
||||
|
||||
|
||||
def test_current_source_with_unbounded_current_reference_remains_supported() -> None:
|
||||
current = dataset(
|
||||
dataset_type="raster",
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
observed_at=datetime(2026, 7, 17, tzinfo=UTC),
|
||||
valid_from=datetime(2026, 7, 17, tzinfo=UTC),
|
||||
temporal_granularity="snapshot",
|
||||
source_metadata={"product_key": "most_recent", "supports_detection": True},
|
||||
)
|
||||
current_reference = dataset(
|
||||
dataset_type="vector",
|
||||
source_name="grb",
|
||||
observed_at=datetime(2026, 7, 16, tzinfo=UTC),
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
|
||||
TemporalCompatibilityService.ensure_detection_source_supported(current)
|
||||
result = TemporalCompatibilityService.assess_detection_qa(current, current_reference)
|
||||
|
||||
assert result["status"] == "compatible"
|
||||
assert result["candidate_historical"] is False
|
||||
|
||||
|
||||
def test_detection_frontend_has_no_implicit_first_raster_fallback() -> None:
|
||||
source = (ROOT / "frontend" / "src" / "hooks" / "useDetectionWorkflow.ts").read_text(encoding="utf-8")
|
||||
|
||||
assert "rasterDatasets[0]" not in source
|
||||
assert "setSelectedDetectionDatasetId(rasterDatasets" not in source
|
||||
assert "const datasetId = selectedDetectionDatasetId" in source
|
||||
@@ -0,0 +1,85 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "capture_release_evidence.py"
|
||||
|
||||
|
||||
def load_script():
|
||||
spec = importlib.util.spec_from_file_location("capture_release_evidence", SCRIPT)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_release_evidence_manifest_is_secret_free_and_read_only(tmp_path: Path) -> None:
|
||||
module = load_script()
|
||||
args = module.parse_args(
|
||||
[
|
||||
"--output",
|
||||
str(tmp_path / "evidence.json"),
|
||||
"--release-id",
|
||||
"test-rc",
|
||||
]
|
||||
)
|
||||
|
||||
manifest = module.build_manifest(args)
|
||||
|
||||
assert manifest["schema_version"] == 1
|
||||
assert manifest["release_id"] == "test-rc"
|
||||
assert manifest["version"] == "1.0.0"
|
||||
assert manifest["read_only"] is True
|
||||
assert manifest["scope"] == "Belgium and the Belgian North Sea"
|
||||
assert "DATABASE_URL" not in json.dumps(manifest).replace(
|
||||
'"DATABASE_URL": false',
|
||||
"",
|
||||
).replace(
|
||||
'"DATABASE_URL": true',
|
||||
"",
|
||||
)
|
||||
assert manifest["storage"] == {"requested": False}
|
||||
assert manifest["live"] == {"requested": False}
|
||||
|
||||
|
||||
def test_release_evidence_cli_writes_single_head_manifest(tmp_path: Path) -> None:
|
||||
output = tmp_path / "baseline.json"
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--output",
|
||||
str(output),
|
||||
"--release-id",
|
||||
"test-cli",
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(output.read_text(encoding="utf-8"))
|
||||
assert payload["git"]["commit"]
|
||||
assert payload["migrations"]["single_head"] is True
|
||||
assert payload["files"]["docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md"]["sha256"]
|
||||
assert payload["files"]["docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md"]["sha256"]
|
||||
assert payload["files"]["docs/RELEASE_RUNBOOK.md"]["sha256"]
|
||||
assert payload["files"]["docs/KNOWN_LIMITATIONS.md"]["sha256"]
|
||||
|
||||
|
||||
def test_readiness_gate_compiles_release_evidence_command() -> None:
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert "py_compile scripts/capture_release_evidence.py" in readiness
|
||||
@@ -0,0 +1,53 @@
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.main import app
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
|
||||
|
||||
def test_valid_request_id_is_returned() -> None:
|
||||
response = TestClient(app).get("/health/live", headers={"x-request-id": "rc3-check.123"})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers["x-request-id"] == "rc3-check.123"
|
||||
|
||||
|
||||
def test_unsafe_request_id_is_replaced() -> None:
|
||||
response = TestClient(app).get("/health/live", headers={"x-request-id": "unsafe request/id"})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers["x-request-id"] != "unsafe request/id"
|
||||
assert " " not in response.headers["x-request-id"]
|
||||
|
||||
|
||||
def test_runtime_report_is_read_only_by_default_and_requires_confirmation() -> None:
|
||||
source = (ROOT / "scripts" / "runtime_state_report.py").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
|
||||
assert '"mode": "read_only"' in source
|
||||
assert 'IMPORT_ROOT = ROOT if (ROOT / "app").is_dir() else BACKEND' in source
|
||||
assert "if args.reconcile and args.confirm != RECONCILE_CONFIRMATION" in source
|
||||
assert "RuntimeReconciliationService.reconcile(db)" in source
|
||||
assert "COPY scripts/runtime_state_report.py /app/scripts/runtime_state_report.py" in dockerfile
|
||||
|
||||
|
||||
def test_all_in_one_deploy_embeds_immutable_build_identity() -> None:
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||
deploy_powershell = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8")
|
||||
deploy_shell = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "ARG GEOINTEL_BUILD_SHA=unknown" in dockerfile
|
||||
assert 'GEOINTEL_BUILD_SHA="${GEOINTEL_BUILD_SHA}"' in dockerfile
|
||||
assert 'GEOINTEL_BUILD_TIME="${GEOINTEL_BUILD_TIME}"' in dockerfile
|
||||
assert 'org.opencontainers.image.revision="${GEOINTEL_BUILD_SHA}"' in dockerfile
|
||||
# However it is factored, the build SHA must come from git HEAD.
|
||||
assert "git rev-parse HEAD" in release_script
|
||||
assert "GEOINTEL_BUILD_SHA" in release_script
|
||||
assert "--build-arg GEOINTEL_BUILD_SHA=" in release_script
|
||||
assert "--build-arg GEOINTEL_BUILD_TIME=" in release_script
|
||||
for deploy_source in (deploy_powershell, deploy_shell):
|
||||
assert "bash deploy/unraid/deploy-release.sh" in deploy_source
|
||||
@@ -0,0 +1,141 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_backend_uses_patched_starlette_and_explicit_httpx2_test_client() -> None:
|
||||
pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml"
|
||||
content = pyproject.read_text(encoding="utf-8")
|
||||
|
||||
assert '"starlette>=1.3.1,<2.0.0"' in content
|
||||
assert '"httpx2>=2.0.0"' in content
|
||||
|
||||
|
||||
def test_readiness_gate_treats_deprecation_warnings_as_errors() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "-W error::DeprecationWarning" in content
|
||||
|
||||
|
||||
def test_readiness_gate_runs_contract_smoke() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "scripts/smoke_contracts.py" in content
|
||||
|
||||
|
||||
def test_readiness_gate_checks_demo_export_workflow_script_syntax() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "bash -n scripts/verify_demo_export_workflow.sh" in content
|
||||
assert "bash -n scripts/verify_workbench_default_state.sh" in content
|
||||
|
||||
|
||||
def test_readiness_gate_runs_golden_qa_benchmark() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "scripts/run_golden_qa_benchmark.py --json" in content
|
||||
assert "bash -n scripts/verify_golden_qa_benchmark.sh" in content
|
||||
|
||||
|
||||
def test_readiness_gate_compiles_demo_cleanup_script() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "-m py_compile scripts/cleanup_demo_artifacts.py" in content
|
||||
assert "-m py_compile backend/scripts/cleanup_demo_artifacts.py" in content
|
||||
|
||||
|
||||
def test_readiness_gate_checks_demo_cleanup_dry_run_script_syntax() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "bash -n scripts/verify_demo_cleanup_dry_run.sh" in content
|
||||
|
||||
|
||||
def test_demo_cleanup_dry_run_script_is_dry_run_only() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "verify_demo_cleanup_dry_run.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "--apply" not in content
|
||||
assert "deleted_export_count=0" in content
|
||||
assert "dry_run=true" in content
|
||||
assert "candidate_exports" in content
|
||||
assert "CLEANUP_MODE" in content
|
||||
assert "project_report_html" in content
|
||||
|
||||
|
||||
def test_readiness_gate_checks_workbench_screenshot_capture_syntax() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "bash -n scripts/capture_workbench_screenshots.sh" in content
|
||||
|
||||
|
||||
def test_workbench_screenshot_capture_is_artifact_based_and_non_mutating() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "capture_workbench_screenshots.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "artifacts/screenshots" in content
|
||||
assert "/api/v1/demo/workflow" in content
|
||||
assert "['overview', 'Overview']" in content
|
||||
assert "['system', 'System']" in content
|
||||
assert "workspace-nav-${workspaceKey}" in content
|
||||
assert "manifest.json" in content
|
||||
assert "page.screenshot" in content
|
||||
assert "fullPage: false" in content
|
||||
assert "Playwright is required" in content
|
||||
assert "--apply" not in content
|
||||
|
||||
|
||||
def test_readiness_gate_compiles_yolo_preflight_script() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "-m py_compile scripts/yolo_preflight.py" in content
|
||||
assert "-m py_compile backend/scripts/yolo_preflight.py" in content
|
||||
|
||||
|
||||
def test_demo_export_workflow_script_verifies_export_endpoints() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "verify_demo_export_workflow.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "/api/v1/demo/workflow" in content
|
||||
assert "/areas" in content
|
||||
assert "/datasets" in content
|
||||
assert "/content" in content
|
||||
assert "/vector/summary" in content
|
||||
assert "GeoJSON Polygon/MultiPolygon geometry" in content
|
||||
assert "precision" in content
|
||||
assert "false_negative_count" in content
|
||||
assert "fixtures/golden/expected_qa_metrics.json" in content
|
||||
assert "QA/QC metric {key} drifted" in content
|
||||
assert "Seeded QA/QC match count does not match golden baseline" in content
|
||||
assert "/api/v1/exports/metadata" in content
|
||||
assert "/api/v1/exports/report" in content
|
||||
assert "/api/v1/exports/geojson" in content
|
||||
assert "/download" in content
|
||||
|
||||
|
||||
def test_workbench_default_state_script_verifies_populated_demo_start_state() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "verify_workbench_default_state.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "/api/v1/demo/workflow" in content
|
||||
assert "GeoIntel Demo - Building QA" in content
|
||||
assert "/areas" in content
|
||||
assert "/datasets" in content
|
||||
assert "/quality-checks" in content
|
||||
assert "data.items" in content
|
||||
assert "Demo AOI - Geel buildings" in content
|
||||
assert "3/3 ready" in content
|
||||
|
||||
|
||||
def test_pass_end_check_excludes_vendor_and_build_outputs() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "codex_pass_end_check.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "--exclude-dir=node_modules" in content
|
||||
assert "--exclude-dir=dist" in content
|
||||
assert "--exclude-dir=__pycache__" in content
|
||||
@@ -0,0 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "build_regional_yolo_dataset.py"
|
||||
SPEC = importlib.util.spec_from_file_location("regional_yolo_dataset", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def test_select_paths_is_region_and_split_safe() -> None:
|
||||
manifest = {
|
||||
"samples": [
|
||||
{"sample_slug": "f-train", "region": "flanders", "split": "train"},
|
||||
{"sample_slug": "f-val", "region": "flanders", "split": "val"},
|
||||
{"sample_slug": "f-test", "region": "flanders", "split": "test"},
|
||||
{"sample_slug": "w-train", "region": "wallonia", "split": "train"},
|
||||
]
|
||||
}
|
||||
summary = {
|
||||
"tiles": [
|
||||
{"sample_slug": "f-train", "split": "train", "image_path": "/f-train.png"},
|
||||
{"sample_slug": "f-val", "split": "val", "image_path": "/f-val.png"},
|
||||
{"sample_slug": "f-test", "split": "val", "image_path": "/f-test.png"},
|
||||
{"sample_slug": "w-train", "split": "train", "image_path": "/w-train.png"},
|
||||
]
|
||||
}
|
||||
train, val = MODULE.select_paths(summary, manifest, "flanders")
|
||||
assert train == ["/f-train.png"]
|
||||
assert val == ["/f-val.png"]
|
||||
@@ -0,0 +1,83 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "release_backup_snapshot.py"
|
||||
|
||||
|
||||
def load_snapshot_module():
|
||||
spec = importlib.util.spec_from_file_location("release_backup_snapshot_test", SCRIPT)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_snapshot_is_byte_complete_and_reuses_only_verified_backup_bytes(tmp_path: Path) -> None:
|
||||
snapshot = load_snapshot_module()
|
||||
source = tmp_path / "source"
|
||||
source.mkdir()
|
||||
(source / "same.bin").write_bytes(b"unchanged")
|
||||
(source / "changed.bin").write_bytes(b"before")
|
||||
(source / "empty").mkdir()
|
||||
|
||||
prior = tmp_path / "prior"
|
||||
prior_manifest = tmp_path / "prior.tsv"
|
||||
snapshot.create_snapshot(source, prior, prior_manifest, label="storage")
|
||||
snapshot.verify_snapshot(prior, prior_manifest)
|
||||
|
||||
(source / "changed.bin").write_bytes(b"after")
|
||||
current = tmp_path / "current"
|
||||
current_manifest = tmp_path / "current.tsv"
|
||||
snapshot.create_snapshot(
|
||||
source,
|
||||
current,
|
||||
current_manifest,
|
||||
label="storage",
|
||||
link_dest_snapshot=prior,
|
||||
link_dest_manifest=prior_manifest,
|
||||
)
|
||||
snapshot.verify_snapshot(current, current_manifest)
|
||||
|
||||
assert os.path.samefile(prior / "same.bin", current / "same.bin")
|
||||
assert not os.path.samefile(prior / "changed.bin", current / "changed.bin")
|
||||
assert (current / "changed.bin").read_bytes() == b"after"
|
||||
assert (current / "empty").is_dir()
|
||||
|
||||
|
||||
def test_snapshot_rejects_symlinked_content(tmp_path: Path) -> None:
|
||||
snapshot = load_snapshot_module()
|
||||
source = tmp_path / "source"
|
||||
source.mkdir()
|
||||
target = source / "target.bin"
|
||||
target.write_bytes(b"target")
|
||||
try:
|
||||
(source / "link.bin").symlink_to(target)
|
||||
except OSError:
|
||||
pytest.skip("Symlink creation is unavailable on this host")
|
||||
|
||||
with pytest.raises(RuntimeError, match="refuses symlinked content"):
|
||||
snapshot.create_snapshot(source, tmp_path / "snapshot", tmp_path / "manifest.tsv", label="storage")
|
||||
|
||||
|
||||
def test_snapshot_verification_rejects_changed_retained_bytes(tmp_path: Path) -> None:
|
||||
snapshot = load_snapshot_module()
|
||||
source = tmp_path / "source"
|
||||
source.mkdir()
|
||||
(source / "artifact.bin").write_bytes(b"retained")
|
||||
retained = tmp_path / "snapshot"
|
||||
manifest = tmp_path / "manifest.tsv"
|
||||
snapshot.create_snapshot(source, retained, manifest, label="storage")
|
||||
(retained / "artifact.bin").chmod(0o644)
|
||||
(retained / "artifact.bin").write_bytes(b"tampered")
|
||||
|
||||
with pytest.raises(RuntimeError, match="checksum differs"):
|
||||
snapshot.verify_snapshot(retained, manifest)
|
||||
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "verify_repository_layout.py"
|
||||
SPEC = importlib.util.spec_from_file_location("verify_repository_layout", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def test_nested_repository_mirror_is_absent() -> None:
|
||||
assert MODULE.nested_mirror_markers(ROOT) == []
|
||||
|
||||
|
||||
def test_nested_repository_mirror_is_detected(tmp_path: Path) -> None:
|
||||
for marker in MODULE.CANONICAL_MARKERS:
|
||||
path = tmp_path / "geointel" / marker
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("fixture", encoding="utf-8")
|
||||
|
||||
assert MODULE.nested_mirror_markers(tmp_path) == list(MODULE.CANONICAL_MARKERS)
|
||||
@@ -0,0 +1,55 @@
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.main import app
|
||||
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"host",
|
||||
[
|
||||
"trusted.example/@admin",
|
||||
"trusted.example?shadow=admin",
|
||||
"trusted.example#shadow",
|
||||
],
|
||||
)
|
||||
def test_invalid_host_request_target_is_rejected_canonically(host: str) -> None:
|
||||
response = client.get("/health/live", headers={"host": host})
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.headers["x-request-id"]
|
||||
assert response.json()["error"] == "INVALID_REQUEST_TARGET"
|
||||
assert response.json()["request_id"] == response.headers["x-request-id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"host",
|
||||
["localhost:1202", "127.0.0.1:8000", "[::1]:8000", "testserver"],
|
||||
)
|
||||
def test_normal_host_forms_remain_available(host: str) -> None:
|
||||
response = client.get("/health/live", headers={"host": host})
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
def test_urlencoded_form_body_is_rejected_before_starlette_form_parsing() -> None:
|
||||
response = client.post(
|
||||
"/api/v1/datasets/upload",
|
||||
headers={"content-type": "application/x-www-form-urlencoded"},
|
||||
content="dataset_type=vector&field=" + ("x" * 10_000),
|
||||
)
|
||||
|
||||
assert response.status_code == 415
|
||||
assert response.json()["error"] == "UNSUPPORTED_CONTENT_TYPE"
|
||||
|
||||
|
||||
def test_multipart_upload_contract_remains_available() -> None:
|
||||
response = client.post(
|
||||
"/health/live",
|
||||
files={"file": ("empty.geojson", b"{}", "application/geo+json")},
|
||||
data={"dataset_type": "vector"},
|
||||
)
|
||||
|
||||
assert response.status_code == 405
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "retile_yolo_dataset.py"
|
||||
SPEC = importlib.util.spec_from_file_location("retile_yolo", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def test_tile_starts_cover_edges_with_overlap() -> None:
|
||||
assert MODULE.tile_starts(640, 384, 128) == [0, 256]
|
||||
assert MODULE.tile_starts(700, 384, 128) == [0, 256, 316]
|
||||
|
||||
|
||||
def test_tile_starts_reject_image_smaller_than_tile() -> None:
|
||||
assert MODULE.tile_starts(320, 384, 128) == []
|
||||
@@ -0,0 +1,254 @@
|
||||
"""An operator's adjudication must reach the score.
|
||||
|
||||
The review vocabulary already distinguishes a model error from a reference gap
|
||||
— the product's own position is that official footprints are not automatically
|
||||
perfect ground truth. But the reviews were only counted. An operator who
|
||||
inspects forty false positives and establishes that twelve are buildings the
|
||||
reference simply lacks still sees a precision that counts all forty against the
|
||||
model, and that they have personally disproved.
|
||||
|
||||
Because part of the evidence is usually still unreviewed, the honest answer is
|
||||
an interval, not a single corrected number: pessimistic assumes every
|
||||
unreviewed item is a model error, optimistic assumes none is.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.reviewed_metrics_service import ReviewedMetricsService
|
||||
|
||||
|
||||
def _counts(**decisions: int) -> dict[str, int]:
|
||||
return decisions
|
||||
|
||||
|
||||
class TestAdjudication:
|
||||
def test_a_reference_gap_stops_counting_against_precision(self) -> None:
|
||||
result = ReviewedMetricsService.adjudicate(
|
||||
matches=80,
|
||||
false_positives=20,
|
||||
false_negatives=10,
|
||||
false_positive_decisions=_counts(reference_gap_or_change=20),
|
||||
false_negative_decisions={},
|
||||
)
|
||||
|
||||
# Every false positive was the reference missing a real building.
|
||||
assert result["adjudicated"]["false_positives"] == 0
|
||||
assert result["adjudicated"]["precision"] == pytest.approx(1.0)
|
||||
|
||||
def test_a_confirmed_model_error_keeps_counting(self) -> None:
|
||||
result = ReviewedMetricsService.adjudicate(
|
||||
matches=80,
|
||||
false_positives=20,
|
||||
false_negatives=0,
|
||||
false_positive_decisions=_counts(confirmed_model_false_positive=20),
|
||||
false_negative_decisions={},
|
||||
)
|
||||
|
||||
assert result["adjudicated"]["false_positives"] == 20
|
||||
assert result["adjudicated"]["precision"] == pytest.approx(0.8)
|
||||
|
||||
def test_an_alignment_mismatch_is_not_a_model_error(self) -> None:
|
||||
"""Both the detection and the footprint were right; the matching failed."""
|
||||
|
||||
result = ReviewedMetricsService.adjudicate(
|
||||
matches=80,
|
||||
false_positives=20,
|
||||
false_negatives=0,
|
||||
false_positive_decisions=_counts(qa_alignment_mismatch=20),
|
||||
false_negative_decisions={},
|
||||
)
|
||||
|
||||
assert result["adjudicated"]["false_positives"] == 0
|
||||
|
||||
def test_a_reference_gap_on_a_miss_stops_counting_against_recall(self) -> None:
|
||||
result = ReviewedMetricsService.adjudicate(
|
||||
matches=80,
|
||||
false_positives=0,
|
||||
false_negatives=20,
|
||||
false_positive_decisions={},
|
||||
false_negative_decisions=_counts(reference_gap_or_change=20),
|
||||
)
|
||||
|
||||
# The reference held twenty footprints that are not there.
|
||||
assert result["adjudicated"]["false_negatives"] == 0
|
||||
assert result["adjudicated"]["recall"] == pytest.approx(1.0)
|
||||
|
||||
def test_an_uncertain_verdict_keeps_counting_against_the_model(self) -> None:
|
||||
"""Not being able to judge is not evidence in the model's favour."""
|
||||
|
||||
result = ReviewedMetricsService.adjudicate(
|
||||
matches=80,
|
||||
false_positives=20,
|
||||
false_negatives=0,
|
||||
false_positive_decisions=_counts(uncertain=10, imagery_obscured_or_uncertain=10),
|
||||
false_negative_decisions={},
|
||||
)
|
||||
|
||||
assert result["adjudicated"]["false_positives"] == 20
|
||||
|
||||
|
||||
class TestBounds:
|
||||
def test_a_partly_reviewed_check_reports_an_interval(self) -> None:
|
||||
result = ReviewedMetricsService.adjudicate(
|
||||
matches=80,
|
||||
false_positives=20,
|
||||
false_negatives=0,
|
||||
false_positive_decisions=_counts(reference_gap_or_change=10),
|
||||
false_negative_decisions={},
|
||||
)
|
||||
|
||||
# Ten unreviewed: pessimistically all model errors, optimistically none.
|
||||
assert result["pessimistic"]["precision"] == pytest.approx(80 / 90)
|
||||
assert result["optimistic"]["precision"] == pytest.approx(1.0)
|
||||
assert result["review_complete"] is False
|
||||
|
||||
def test_a_fully_reviewed_check_collapses_the_interval(self) -> None:
|
||||
result = ReviewedMetricsService.adjudicate(
|
||||
matches=80,
|
||||
false_positives=20,
|
||||
false_negatives=5,
|
||||
false_positive_decisions=_counts(reference_gap_or_change=12, confirmed_model_false_positive=8),
|
||||
false_negative_decisions=_counts(confirmed_model_false_negative=5),
|
||||
)
|
||||
|
||||
assert result["review_complete"] is True
|
||||
assert result["pessimistic"]["precision"] == pytest.approx(result["optimistic"]["precision"])
|
||||
assert result["adjudicated"]["precision"] == pytest.approx(80 / 88)
|
||||
|
||||
def test_an_unreviewed_check_reports_the_raw_numbers_unchanged(self) -> None:
|
||||
result = ReviewedMetricsService.adjudicate(
|
||||
matches=80,
|
||||
false_positives=20,
|
||||
false_negatives=10,
|
||||
false_positive_decisions={},
|
||||
false_negative_decisions={},
|
||||
)
|
||||
|
||||
assert result["review_complete"] is False
|
||||
assert result["adjudicated"]["precision"] == pytest.approx(result["raw"]["precision"])
|
||||
assert result["adjudicated"]["recall"] == pytest.approx(result["raw"]["recall"])
|
||||
|
||||
def test_the_raw_score_is_always_reported_alongside(self) -> None:
|
||||
result = ReviewedMetricsService.adjudicate(
|
||||
matches=80,
|
||||
false_positives=20,
|
||||
false_negatives=0,
|
||||
false_positive_decisions=_counts(reference_gap_or_change=20),
|
||||
false_negative_decisions={},
|
||||
)
|
||||
|
||||
assert result["raw"]["precision"] == pytest.approx(0.8)
|
||||
assert result["adjudicated"]["precision"] == pytest.approx(1.0)
|
||||
|
||||
|
||||
class TestEdges:
|
||||
def test_a_check_without_findings_makes_no_claim(self) -> None:
|
||||
result = ReviewedMetricsService.adjudicate(
|
||||
matches=0,
|
||||
false_positives=0,
|
||||
false_negatives=0,
|
||||
false_positive_decisions={},
|
||||
false_negative_decisions={},
|
||||
)
|
||||
|
||||
assert result["adjudicated"]["precision"] is None
|
||||
assert result["adjudicated"]["recall"] is None
|
||||
assert result["review_complete"] is True
|
||||
|
||||
def test_more_decisions_than_findings_cannot_invent_a_negative_count(self) -> None:
|
||||
result = ReviewedMetricsService.adjudicate(
|
||||
matches=10,
|
||||
false_positives=2,
|
||||
false_negatives=0,
|
||||
false_positive_decisions=_counts(reference_gap_or_change=99),
|
||||
false_negative_decisions={},
|
||||
)
|
||||
|
||||
assert result["adjudicated"]["false_positives"] == 0
|
||||
|
||||
def test_an_unknown_decision_is_treated_as_no_judgement(self) -> None:
|
||||
result = ReviewedMetricsService.adjudicate(
|
||||
matches=10,
|
||||
false_positives=5,
|
||||
false_negatives=0,
|
||||
false_positive_decisions=_counts(something_new_from_a_later_release=5),
|
||||
false_negative_decisions={},
|
||||
)
|
||||
|
||||
assert result["adjudicated"]["false_positives"] == 5
|
||||
assert result["review_complete"] is False
|
||||
|
||||
|
||||
class TestThroughTheReviewPanel:
|
||||
"""The score the panel shows, not just the arithmetic behind it."""
|
||||
|
||||
def _quality_check(self, quality_check_id, project_id):
|
||||
from app.models import QualityCheck
|
||||
|
||||
return QualityCheck(
|
||||
id=quality_check_id,
|
||||
project_id=project_id,
|
||||
reference_dataset_id=__import__("uuid").uuid4(),
|
||||
check_type="detections_vs_reference",
|
||||
status="ok",
|
||||
findings_json={
|
||||
"matches": 80,
|
||||
"false_positives": 20,
|
||||
"false_negatives": 0,
|
||||
"false_positive_evidence": [
|
||||
{"candidate_feature_id": f"detection-{index}"} for index in range(20)
|
||||
],
|
||||
"false_negative_evidence": [],
|
||||
},
|
||||
)
|
||||
|
||||
def test_adjudicated_reference_gaps_raise_the_reported_precision(self) -> None:
|
||||
import uuid
|
||||
|
||||
from app.models import DetectionReview, QualityCheck
|
||||
from app.services.detection_review_service import DetectionReviewService
|
||||
|
||||
quality_check_id, project_id = uuid.uuid4(), uuid.uuid4()
|
||||
quality_check = self._quality_check(quality_check_id, project_id)
|
||||
reviews = [
|
||||
DetectionReview(
|
||||
id=uuid.uuid4(),
|
||||
quality_check_id=quality_check_id,
|
||||
evidence_role="false_positive",
|
||||
evidence_feature_id=f"detection-{index}",
|
||||
decision="reference_gap_or_change",
|
||||
)
|
||||
for index in range(12)
|
||||
]
|
||||
|
||||
class _Query:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return self.rows
|
||||
|
||||
class _Session:
|
||||
def get(self, model, item_id):
|
||||
return quality_check if model is QualityCheck and item_id == quality_check_id else None
|
||||
|
||||
def query(self, _model):
|
||||
return _Query(reviews)
|
||||
|
||||
result = DetectionReviewService.list_reviews(
|
||||
_Session(), project_id=project_id, quality_check_id=quality_check_id
|
||||
)
|
||||
metrics = result.summary.reviewed_metrics
|
||||
|
||||
assert metrics is not None
|
||||
assert metrics["raw"]["precision"] == pytest.approx(0.8)
|
||||
# Twelve of the twenty were the reference missing a building.
|
||||
assert metrics["adjudicated"]["precision"] == pytest.approx(80 / 88)
|
||||
assert metrics["review_complete"] is False
|
||||
assert metrics["false_positive_breakdown"]["exonerated"] == 12
|
||||
assert metrics["false_positive_breakdown"]["unreviewed"] == 8
|
||||
@@ -0,0 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "rotate_belgium_building_holdouts.py"
|
||||
SPEC = importlib.util.spec_from_file_location("rotate_building_holdouts", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def test_rotated_corpus_version_is_explicit_and_canonical() -> None:
|
||||
assert MODULE.validate_dataset_version("building-be-v31-rotated-holdouts-r1") == (
|
||||
"building-be-v31-rotated-holdouts-r1"
|
||||
)
|
||||
with pytest.raises(ValueError, match="canonical slug"):
|
||||
MODULE.validate_dataset_version("Building BE v31")
|
||||
@@ -0,0 +1,144 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import AnalysisRun, Dataset, Job, Project
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.job_service import JobService
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
|
||||
|
||||
class FakeSession:
|
||||
"""Minimal session double without rollback support, mirroring existing test doubles."""
|
||||
|
||||
def __init__(self, objects=None) -> None:
|
||||
self.objects = objects or {}
|
||||
self.added = []
|
||||
self.commits = 0
|
||||
|
||||
def get(self, model, item_id):
|
||||
return self.objects.get((model, item_id))
|
||||
|
||||
def add(self, item) -> None:
|
||||
self.added.append(item)
|
||||
if getattr(item, "id", None) is not None:
|
||||
self.objects[(item.__class__, item.id)] = item
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def refresh(self, item) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _project_and_dataset():
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
project = Project(id=project_id, name="Mol")
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="ortho.tif",
|
||||
dataset_type="raster",
|
||||
source="fixture",
|
||||
source_name="fixture",
|
||||
storage_path="storage/uploads/ortho.tif",
|
||||
source_metadata={"fixture": True},
|
||||
)
|
||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||
return db, project_id, dataset_id
|
||||
|
||||
|
||||
def _statuses(db: FakeSession) -> tuple[list[str], list[str]]:
|
||||
runs = [item.status for item in db.added if isinstance(item, AnalysisRun)]
|
||||
jobs = [item.status for item in db.added if isinstance(item, Job)]
|
||||
return runs, jobs
|
||||
|
||||
|
||||
def test_invalid_fixture_detections_mark_run_and_job_failed() -> None:
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DetectionService.run_detection(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="manual-fixture-detector",
|
||||
confidence_threshold=0.5,
|
||||
parameters_json={"fixture_mode": True, "fixture_detections": "not-a-list"},
|
||||
settings=Settings(_env_file=None),
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "INVALID_FIXTURE_DETECTIONS"
|
||||
run_statuses, job_statuses = _statuses(db)
|
||||
assert run_statuses and all(status == "failed" for status in run_statuses)
|
||||
assert job_statuses and all(status == "failed" for status in job_statuses)
|
||||
|
||||
|
||||
def test_invalid_fixture_segmentations_mark_run_and_job_failed() -> None:
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
SegmentationService.run_segmentation(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="fixture-segmenter",
|
||||
confidence_threshold=0.5,
|
||||
parameters_json={"fixture_mode": True, "fixture_segmentations": "not-a-list"},
|
||||
settings=Settings(_env_file=None),
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "INVALID_FIXTURE_SEGMENTATIONS"
|
||||
run_statuses, job_statuses = _statuses(db)
|
||||
assert run_statuses and all(status == "failed" for status in run_statuses)
|
||||
assert job_statuses and all(status == "failed" for status in job_statuses)
|
||||
|
||||
|
||||
def test_unexpected_error_in_sync_job_marks_job_failed() -> None:
|
||||
project_id = uuid4()
|
||||
db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Mol")})
|
||||
|
||||
def exploding_operation():
|
||||
raise RuntimeError("unexpected internal failure")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
JobService.run_sync_job(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
job_type="test.unexpected",
|
||||
parameters={},
|
||||
operation=exploding_operation,
|
||||
)
|
||||
|
||||
jobs = [item for item in db.added if isinstance(item, Job)]
|
||||
assert jobs
|
||||
final_job = jobs[-1]
|
||||
assert final_job.status == "failed"
|
||||
assert "Unexpected internal error" in (final_job.error_message or "")
|
||||
|
||||
|
||||
def test_app_error_in_sync_job_still_marks_job_failed() -> None:
|
||||
project_id = uuid4()
|
||||
db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Mol")})
|
||||
|
||||
def failing_operation():
|
||||
raise AppError(code="SOME_DOMAIN_ERROR", message="Bounded failure", status_code=422)
|
||||
|
||||
with pytest.raises(AppError):
|
||||
JobService.run_sync_job(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
job_type="test.bounded",
|
||||
parameters={},
|
||||
operation=failing_operation,
|
||||
)
|
||||
|
||||
jobs = [item for item in db.added if isinstance(item, Job)]
|
||||
assert jobs
|
||||
assert jobs[-1].status == "failed"
|
||||
assert jobs[-1].error_message == "Bounded failure"
|
||||
@@ -0,0 +1,217 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "migrate_runtime_model_provenance.py"
|
||||
SPEC = importlib.util.spec_from_file_location("migrate_runtime_model_provenance", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
module = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(module)
|
||||
|
||||
|
||||
def _sha(value: bytes) -> str:
|
||||
return sha256(value).hexdigest()
|
||||
|
||||
|
||||
def _args(tmp_path: Path):
|
||||
model = tmp_path / "active.pt"
|
||||
checkpoint = tmp_path / "best.pt"
|
||||
base_model = tmp_path / "base.pt"
|
||||
training_args = tmp_path / "args.yaml"
|
||||
training_results = tmp_path / "results.csv"
|
||||
dataset_summary = tmp_path / "dataset-summary.json"
|
||||
dataset_yaml = tmp_path / "dataset.yaml"
|
||||
training_summary = tmp_path / "training-summary.json"
|
||||
|
||||
model.write_bytes(b"exact promoted model bytes")
|
||||
checkpoint.write_bytes(model.read_bytes())
|
||||
base_model.write_bytes(b"exact base model bytes")
|
||||
training_args.write_text("epochs: 30\nseed: 0\n", encoding="utf-8")
|
||||
training_results.write_text("epoch,metric\n1,0.1\n", encoding="utf-8")
|
||||
dataset_yaml.write_text("names:\n 0: building\n", encoding="utf-8")
|
||||
dataset_summary.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"class_names": ["building"],
|
||||
"tile_count": 198,
|
||||
"train_tile_count": 180,
|
||||
"val_tile_count": 18,
|
||||
"label_count": 58_820,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
training_summary.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"trained_model_sha256": _sha(model.read_bytes()),
|
||||
"base_model_sha256": _sha(base_model.read_bytes()),
|
||||
"dataset_summary_sha256": _sha(dataset_summary.read_bytes()),
|
||||
"dataset_yaml_sha256": _sha(dataset_yaml.read_bytes()),
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return module.parse_args(
|
||||
[
|
||||
"--model-path",
|
||||
str(model),
|
||||
"--checkpoint-path",
|
||||
str(checkpoint),
|
||||
"--base-model-path",
|
||||
str(base_model),
|
||||
"--training-summary-path",
|
||||
str(training_summary),
|
||||
"--training-args-path",
|
||||
str(training_args),
|
||||
"--training-results-path",
|
||||
str(training_results),
|
||||
"--dataset-summary-path",
|
||||
str(dataset_summary),
|
||||
"--dataset-yaml-path",
|
||||
str(dataset_yaml),
|
||||
"--source-version",
|
||||
"sprint174-smallbld-minpx3-img640-ft30",
|
||||
"--framework-version",
|
||||
"8.4.93",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_recovered_evidence_requires_byte_identical_checkpoint_and_recorded_hashes(tmp_path: Path) -> None:
|
||||
args = _args(tmp_path)
|
||||
|
||||
evidence = module.inspect_evidence(args)
|
||||
|
||||
assert evidence["checksums"]["model"] == evidence["checksums"]["checkpoint"]
|
||||
assert evidence["checksums"]["training_summary"] == _sha(
|
||||
Path(args.training_summary_path).read_bytes()
|
||||
)
|
||||
assert evidence["class_mapping"] == {"0": "building"}
|
||||
|
||||
|
||||
def test_recovered_evidence_rejects_changed_checkpoint(tmp_path: Path) -> None:
|
||||
args = _args(tmp_path)
|
||||
Path(args.checkpoint_path).write_bytes(b"other checkpoint")
|
||||
|
||||
exit_code, payload = module.migrate(args)
|
||||
|
||||
assert exit_code == 2
|
||||
assert payload["status"] == "evidence_invalid"
|
||||
assert "checkpoint/model SHA-256 mismatch" in payload["message"]
|
||||
|
||||
|
||||
def test_generated_sidecar_passes_exact_runtime_contract(tmp_path: Path) -> None:
|
||||
args = _args(tmp_path)
|
||||
evidence = module.inspect_evidence(args)
|
||||
payload = module._manifest_payload(
|
||||
args=args,
|
||||
evidence=evidence,
|
||||
source_registry_id=str(uuid4()),
|
||||
source_snapshot_id=str(uuid4()),
|
||||
imported_at="2026-08-23T21:00:00+00:00",
|
||||
)
|
||||
manifest_path = RuntimeModelProvenanceService.manifest_path_for_model(args.model_path)
|
||||
assert module._write_manifest_atomically(manifest_path, payload) is True
|
||||
|
||||
validated = RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=args.model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
expected_model_version="sprint174-smallbld-minpx3-img640-ft30",
|
||||
allowed_frameworks=("ultralytics/pytorch",),
|
||||
)
|
||||
|
||||
assert validated.model_sha256 == evidence["checksums"]["model"]
|
||||
assert validated.runtime_manifest_sha256 == payload["metadata"]["runtime_manifest_sha256"]
|
||||
assert module._write_manifest_atomically(manifest_path, payload) is False
|
||||
|
||||
|
||||
def test_model_registry_definition_is_runtime_ready_without_mutating_server_owned_row() -> None:
|
||||
from app.services.source_registry_service import SourceRegistryService
|
||||
|
||||
definition = SourceRegistryService.definition_for("model")
|
||||
|
||||
assert definition.ingest_status == "configured"
|
||||
assert definition.freshness_status == "current"
|
||||
script = SCRIPT.read_text(encoding="utf-8")
|
||||
assert 'source.ingest_status = "configured"' not in script
|
||||
assert 'source.freshness_status = "current"' not in script
|
||||
|
||||
|
||||
def test_model_registry_status_migration_is_narrow_and_restores_write_guard() -> None:
|
||||
migration = (
|
||||
ROOT
|
||||
/ "backend"
|
||||
/ "alembic"
|
||||
/ "versions"
|
||||
/ "202608230001_configure_model_source_registry.py"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "WHERE source_key = 'model'" in migration
|
||||
assert "registry_owner' = 'server'" in migration
|
||||
assert migration.count("DISABLE TRIGGER trg_source_registry_write_guard") == 1
|
||||
assert migration.count("ENABLE TRIGGER trg_source_registry_write_guard") == 1
|
||||
assert 'down_revision = "202608010001"' in migration
|
||||
|
||||
|
||||
def test_recovery_receipt_requires_hashed_backup_inventory(tmp_path: Path) -> None:
|
||||
args = _args(tmp_path)
|
||||
evidence = module.inspect_evidence(args)
|
||||
receipt = tmp_path / "dry-run.json"
|
||||
receipt.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ready_to_apply",
|
||||
"claim_boundary": module.CLAIM_BOUNDARY,
|
||||
"evidence": evidence,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
storage_manifest = tmp_path / "storage-manifest.tsv"
|
||||
missing_names = ("checkpoint", "training_summary", "training_args", "training_results")
|
||||
rows = ["relative_path\tsize_bytes\tmtime_ns\tsha256"]
|
||||
for name in missing_names:
|
||||
original = Path(evidence["paths"][name])
|
||||
evidence["paths"][name] = f"/app/storage/training/{original.name}"
|
||||
rows.append(f"training/{original.name}\t1\t0\t{evidence['checksums'][name]}")
|
||||
receipt.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ready_to_apply",
|
||||
"claim_boundary": module.CLAIM_BOUNDARY,
|
||||
"evidence": evidence,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
storage_manifest.write_text("\n".join(rows) + "\n", encoding="utf-8")
|
||||
backup_checksums = tmp_path / "CHECKSUMS.sha256"
|
||||
backup_checksums.write_text(
|
||||
f"{_sha(storage_manifest.read_bytes())} storage-manifest.tsv\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
args.evidence_receipt_path = str(receipt)
|
||||
args.backup_storage_manifest_path = str(storage_manifest)
|
||||
args.backup_checksums_path = str(backup_checksums)
|
||||
|
||||
recovered = module.inspect_evidence(args)
|
||||
|
||||
assert recovered["checksums"] == evidence["checksums"]
|
||||
assert recovered["recovery_receipt"]["missing_artifacts_not_recreated"] == list(missing_names)
|
||||
|
||||
storage_manifest.write_text(storage_manifest.read_text(encoding="utf-8") + "tampered\n", encoding="utf-8")
|
||||
exit_code, payload = module.migrate(args)
|
||||
assert exit_code == 2
|
||||
assert "backup storage manifest SHA-256 mismatch" in payload["message"]
|
||||
@@ -0,0 +1,325 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import DatasetQuarantine, SourceRegistry, SourceSnapshot
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
|
||||
|
||||
class FakeSession:
|
||||
"""Explicit database double for production-runtime provenance tests."""
|
||||
|
||||
def __init__(self, objects: dict[tuple[type, object], object] | None = None) -> None:
|
||||
self.objects = objects or {}
|
||||
|
||||
def get(self, model, item_id):
|
||||
return self.objects.get((model, item_id))
|
||||
|
||||
|
||||
def _write_sidecar(
|
||||
model_path: Path,
|
||||
*,
|
||||
model_id: str = "yolo-configured",
|
||||
task_type: str = "object_detection",
|
||||
framework: str = "ultralytics/pytorch",
|
||||
source_version: str = "test-v1",
|
||||
source_registry_id: str | None = None,
|
||||
source_snapshot_id: str | None = None,
|
||||
) -> Path:
|
||||
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||
payload = {
|
||||
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||
"data_contract": {
|
||||
"key": "geointel.model.pytorch",
|
||||
"version": "1.0.0",
|
||||
},
|
||||
"model": {
|
||||
"model_id": model_id,
|
||||
"task_type": task_type,
|
||||
"sha256": model_sha256,
|
||||
"model_format": "pytorch",
|
||||
"framework": framework,
|
||||
"class_mapping": {"0": "building"},
|
||||
"source_version": source_version,
|
||||
},
|
||||
"source": {
|
||||
"source_registry_id": source_registry_id or str(uuid4()),
|
||||
"source_snapshot_id": source_snapshot_id or str(uuid4()),
|
||||
"source_registry_key": "model",
|
||||
"source_snapshot_checksum_sha256": model_sha256,
|
||||
},
|
||||
"lineage": {
|
||||
"upstream_asset_ids": ["training-corpus:test-v1"],
|
||||
"upstream_checksums_sha256": ["a" * 64],
|
||||
"transformations": [
|
||||
{
|
||||
"name": "pytorch-training",
|
||||
"version": "1.0.0",
|
||||
"checksum_sha256": "b" * 64,
|
||||
}
|
||||
],
|
||||
},
|
||||
"metadata": {
|
||||
"training_manifest_sha256": "c" * 64,
|
||||
},
|
||||
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||
}
|
||||
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||
sidecar_path = RuntimeModelProvenanceService.manifest_path_for_model(model_path)
|
||||
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
|
||||
return sidecar_path
|
||||
|
||||
|
||||
def _governed_model_database(
|
||||
*,
|
||||
source_registry_id,
|
||||
source_snapshot_id,
|
||||
model_checksum: str,
|
||||
source_version: str = "test-v1",
|
||||
) -> tuple[FakeSession, SourceRegistry, SourceSnapshot]:
|
||||
registry = SourceRegistry(
|
||||
id=source_registry_id,
|
||||
source_key="model",
|
||||
display_name="Governed test model artifacts",
|
||||
classification="experimental",
|
||||
authority_name="GeoIntel test fixture",
|
||||
freshness_status="current",
|
||||
ingest_status="configured",
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=source_snapshot_id,
|
||||
source_registry_id=source_registry_id,
|
||||
snapshot_key=f"model-{source_version}",
|
||||
source_version=source_version,
|
||||
checksum_sha256=model_checksum,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
return (
|
||||
FakeSession(
|
||||
{
|
||||
(SourceRegistry, source_registry_id): registry,
|
||||
(SourceSnapshot, source_snapshot_id): snapshot,
|
||||
}
|
||||
),
|
||||
registry,
|
||||
snapshot,
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_model_provenance_accepts_byte_bound_pytorch_sidecar_for_structural_preflight(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"trusted local model bytes")
|
||||
sidecar_path = _write_sidecar(model_path, source_version="v1")
|
||||
|
||||
evidence = RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
expected_model_version="v1",
|
||||
allowed_frameworks=("ultralytics/pytorch",),
|
||||
)
|
||||
|
||||
assert evidence.model_sha256 == sha256(model_path.read_bytes()).hexdigest()
|
||||
assert evidence.manifest_path == str(sidecar_path.resolve())
|
||||
assert evidence.data_contract_key == "geointel.model.pytorch"
|
||||
assert evidence.data_contract_version == "1.0.0"
|
||||
assert len(evidence.validation_report_sha256) == 64
|
||||
|
||||
|
||||
def test_production_runtime_requires_db_bound_model_source_snapshot(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"governed local model bytes")
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
_write_sidecar(
|
||||
model_path,
|
||||
source_version="v1",
|
||||
source_registry_id=str(source_registry_id),
|
||||
source_snapshot_id=str(source_snapshot_id),
|
||||
)
|
||||
db, _, _ = _governed_model_database(
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
model_checksum=sha256(model_path.read_bytes()).hexdigest(),
|
||||
source_version="v1",
|
||||
)
|
||||
|
||||
evidence = RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||
db=db,
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
expected_model_version="v1",
|
||||
allowed_frameworks=("ultralytics/pytorch",),
|
||||
)
|
||||
|
||||
assert evidence.source_registry_id == str(source_registry_id)
|
||||
assert evidence.source_snapshot_id == str(source_snapshot_id)
|
||||
assert evidence.source_snapshot_checksum_sha256 == evidence.model_sha256
|
||||
|
||||
|
||||
def test_production_runtime_rejects_missing_database_source_binding(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"unbound model bytes")
|
||||
_write_sidecar(model_path)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||
db=FakeSession(),
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
|
||||
|
||||
|
||||
def test_production_runtime_requires_a_database_session(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"model bytes")
|
||||
_write_sidecar(model_path)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||
db=None,
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "MODEL_PROVENANCE_DATABASE_REQUIRED"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mutation", "expected_code"),
|
||||
(
|
||||
("registry_unsafe", "MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE"),
|
||||
("snapshot_registry_mismatch", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH"),
|
||||
("snapshot_missing", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND"),
|
||||
("snapshot_quarantined", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE"),
|
||||
("snapshot_checksum_mismatch", "MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH"),
|
||||
("active_quarantine", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED"),
|
||||
),
|
||||
)
|
||||
def test_production_runtime_rejects_unsafe_or_inconsistent_database_snapshot(
|
||||
tmp_path: Path,
|
||||
mutation: str,
|
||||
expected_code: str,
|
||||
) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"governed model bytes")
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
_write_sidecar(
|
||||
model_path,
|
||||
source_registry_id=str(source_registry_id),
|
||||
source_snapshot_id=str(source_snapshot_id),
|
||||
)
|
||||
db, registry, snapshot = _governed_model_database(
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
model_checksum=sha256(model_path.read_bytes()).hexdigest(),
|
||||
)
|
||||
if mutation == "registry_unsafe":
|
||||
registry.ingest_status = "quarantined"
|
||||
elif mutation == "snapshot_registry_mismatch":
|
||||
snapshot.source_registry_id = uuid4()
|
||||
elif mutation == "snapshot_missing":
|
||||
db.objects.pop((SourceSnapshot, source_snapshot_id))
|
||||
elif mutation == "snapshot_quarantined":
|
||||
snapshot.ingest_status = "quarantined"
|
||||
elif mutation == "snapshot_checksum_mismatch":
|
||||
snapshot.checksum_sha256 = "f" * 64
|
||||
elif mutation == "active_quarantine":
|
||||
snapshot.quarantines = [
|
||||
DatasetQuarantine(
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
stage="test",
|
||||
reason_code="test_active_quarantine",
|
||||
status="quarantined",
|
||||
)
|
||||
]
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||
db=db,
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == expected_code
|
||||
|
||||
|
||||
def test_runtime_model_provenance_rejects_missing_sidecar(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"unmanifested local model bytes")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_MISSING"
|
||||
|
||||
|
||||
def test_runtime_model_provenance_rejects_model_bytes_tampered_after_manifest(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"original local model bytes")
|
||||
_write_sidecar(model_path)
|
||||
model_path.write_bytes(b"tampered local model bytes")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH"
|
||||
|
||||
|
||||
def test_runtime_model_provenance_rejects_tampered_manifest_contents(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"original local model bytes")
|
||||
sidecar_path = _write_sidecar(model_path)
|
||||
payload = json.loads(sidecar_path.read_text(encoding="utf-8"))
|
||||
payload["model"]["class_mapping"]["1"] = "road"
|
||||
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH"
|
||||
|
||||
|
||||
def test_runtime_model_provenance_rejects_other_contract_even_if_structurally_valid(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"local model bytes")
|
||||
sidecar_path = _write_sidecar(model_path)
|
||||
payload = json.loads(sidecar_path.read_text(encoding="utf-8"))
|
||||
payload["data_contract"]["key"] = "geointel.vector.geojson"
|
||||
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_INVALID"
|
||||
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.models import AoiOperation, AoiOperationPartition, AnalysisRun, Job
|
||||
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
|
||||
|
||||
|
||||
def test_reconciliation_terminalizes_only_running_work() -> None:
|
||||
db = MagicMock()
|
||||
jobs = MagicMock()
|
||||
runs = MagicMock()
|
||||
resumable_partitions = MagicMock()
|
||||
exhausted_partitions = MagicMock()
|
||||
operations = MagicMock()
|
||||
jobs.filter.return_value.update.return_value = 5
|
||||
runs.filter.return_value.update.return_value = 2
|
||||
resumable_partitions.filter.return_value.update.return_value = 3
|
||||
exhausted_partitions.filter.return_value.update.return_value = 1
|
||||
operations.filter.return_value.update.return_value = 2
|
||||
db.query.side_effect = [jobs, runs, resumable_partitions, exhausted_partitions, operations]
|
||||
finished_at = datetime(2026, 7, 17, 20, 0, tzinfo=timezone.utc)
|
||||
|
||||
result = RuntimeReconciliationService.reconcile(
|
||||
db,
|
||||
finished_at=finished_at,
|
||||
)
|
||||
|
||||
assert result.interrupted_jobs == 5
|
||||
assert result.interrupted_analysis_runs == 2
|
||||
assert result.resumed_aoi_partitions == 3
|
||||
assert result.exhausted_aoi_partitions == 1
|
||||
jobs.filter.assert_called_once()
|
||||
runs.filter.assert_called_once()
|
||||
resumable_partitions.filter.assert_called_once()
|
||||
exhausted_partitions.filter.assert_called_once()
|
||||
operations.filter.assert_called_once()
|
||||
job_values = jobs.filter.return_value.update.call_args.args[0]
|
||||
run_values = runs.filter.return_value.update.call_args.args[0]
|
||||
assert job_values[Job.status] == "failed"
|
||||
assert job_values[Job.finished_at] == finished_at
|
||||
assert "PROCESS_INTERRUPTED" in job_values[Job.error_message]
|
||||
assert run_values[AnalysisRun.status] == "failed"
|
||||
assert run_values[AnalysisRun.finished_at] == finished_at
|
||||
assert "PROCESS_INTERRUPTED" in run_values[AnalysisRun.error_message]
|
||||
resumed_values = resumable_partitions.filter.return_value.update.call_args.args[0]
|
||||
exhausted_values = exhausted_partitions.filter.return_value.update.call_args.args[0]
|
||||
operation_values = operations.filter.return_value.update.call_args.args[0]
|
||||
assert resumed_values[AoiOperationPartition.status] == "queued"
|
||||
assert exhausted_values[AoiOperationPartition.status] == "failed"
|
||||
assert operation_values[AoiOperation.status] == "queued"
|
||||
db.commit.assert_called_once_with()
|
||||
|
||||
|
||||
def test_startup_reconciliation_is_enabled_only_in_all_in_one_runtime() -> None:
|
||||
start_script = (
|
||||
__import__("pathlib").Path(__file__).resolve().parents[2]
|
||||
/ "deploy"
|
||||
/ "unraid"
|
||||
/ "all-in-one-start.sh"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert (
|
||||
'GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true'
|
||||
in start_script
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "refine_yolo_labels_with_sam.py"
|
||||
SPEC = importlib.util.spec_from_file_location("sam_roof_refinement", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def test_plausible_refinement_is_fail_closed() -> None:
|
||||
source = (10.0, 10.0, 30.0, 30.0)
|
||||
limits = dict(
|
||||
min_iou=0.15,
|
||||
min_area_ratio=0.25,
|
||||
max_area_ratio=4.0,
|
||||
max_center_shift_ratio=0.75,
|
||||
min_dimension_ratio=0.5,
|
||||
max_dimension_ratio=2.0,
|
||||
)
|
||||
assert MODULE.plausible_refinement(source, (8.0, 9.0, 31.0, 32.0), **limits)
|
||||
assert not MODULE.plausible_refinement(source, (100.0, 100.0, 120.0, 120.0), **limits)
|
||||
assert not MODULE.plausible_refinement(source, (0.0, 0.0, 100.0, 100.0), **limits)
|
||||
assert not MODULE.plausible_refinement(source, (10.0, 10.0, 51.0, 20.0), **limits)
|
||||
assert not MODULE.plausible_refinement(source, (24.0, 24.0, 44.0, 44.0), **limits)
|
||||
|
||||
|
||||
def test_yolo_round_trip_shape() -> None:
|
||||
line = MODULE.yolo_line((10.0, 20.0, 30.0, 40.0), 100, 100)
|
||||
assert line == "0 0.20000000 0.30000000 0.20000000 0.20000000"
|
||||
|
||||
|
||||
def test_cli_exposes_explicit_fallback_policy() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
assert 'choices=("retain", "drop")' in source
|
||||
assert '"dropped_fallback_label_count"' in source
|
||||
@@ -0,0 +1,17 @@
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.schemas import detection, segmentation
|
||||
|
||||
|
||||
def test_model_prefixed_api_fields_are_explicitly_supported() -> None:
|
||||
schemas = [
|
||||
value
|
||||
for module in (detection, segmentation)
|
||||
for value in vars(module).values()
|
||||
if isinstance(value, type)
|
||||
and issubclass(value, BaseModel)
|
||||
and any(field_name.startswith("model_") for field_name in value.model_fields)
|
||||
]
|
||||
|
||||
assert schemas
|
||||
assert all(schema.model_config.get("protected_namespaces") == () for schema in schemas)
|
||||
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.services.segmentation_adapter import YoloSegmentationAdapter
|
||||
|
||||
|
||||
def _settings(*, require_cuda: bool, device: str) -> Settings:
|
||||
return Settings(
|
||||
_env_file=None,
|
||||
YOLO_REQUIRE_CUDA=require_cuda,
|
||||
YOLO_DEVICE=device,
|
||||
)
|
||||
|
||||
|
||||
def test_segmentation_runtime_allows_cpu_only_when_cuda_is_not_required() -> None:
|
||||
adapter = YoloSegmentationAdapter(_settings(require_cuda=False, device="cpu"))
|
||||
|
||||
adapter.validate_runtime()
|
||||
|
||||
|
||||
def test_segmentation_runtime_rejects_missing_cuda(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(
|
||||
__import__("sys").modules,
|
||||
"torch",
|
||||
SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)),
|
||||
)
|
||||
adapter = YoloSegmentationAdapter(_settings(require_cuda=True, device="cuda:0"))
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
adapter.validate_runtime()
|
||||
|
||||
assert exc_info.value.code == "SEGMENTATION_ACCELERATOR_UNAVAILABLE"
|
||||
|
||||
|
||||
def test_segmentation_runtime_rejects_cpu_device_when_cuda_is_required(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setitem(
|
||||
__import__("sys").modules,
|
||||
"torch",
|
||||
SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: True)),
|
||||
)
|
||||
adapter = YoloSegmentationAdapter(_settings(require_cuda=True, device="cpu"))
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
adapter.validate_runtime()
|
||||
|
||||
assert exc_info.value.code == "SEGMENTATION_ACCELERATOR_MISCONFIGURED"
|
||||
|
||||
|
||||
def test_segmentation_runtime_accepts_configured_cuda(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(
|
||||
__import__("sys").modules,
|
||||
"torch",
|
||||
SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: True)),
|
||||
)
|
||||
adapter = YoloSegmentationAdapter(_settings(require_cuda=True, device="cuda:0"))
|
||||
|
||||
adapter.validate_runtime()
|
||||
@@ -0,0 +1,567 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from geoalchemy2.shape import to_shape
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.models import Dataset, DatasetVersion, Project, Segmentation, SourceRegistry, SourceSnapshot
|
||||
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
from app.services.tile_manifest_service import TileManifestService
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, objects=None) -> None:
|
||||
self.objects = objects or {}
|
||||
self.added = []
|
||||
self.commits = 0
|
||||
self.refreshes = []
|
||||
|
||||
def get(self, model, item_id):
|
||||
return self.objects.get((model, item_id))
|
||||
|
||||
def add(self, item) -> None:
|
||||
self.added.append(item)
|
||||
if getattr(item, "id", None) is not None:
|
||||
self.objects[(item.__class__, item.id)] = item
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def refresh(self, item) -> None:
|
||||
self.refreshes.append(item)
|
||||
|
||||
|
||||
class AvailableSegAdapter:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self.settings = settings
|
||||
|
||||
@staticmethod
|
||||
def dependencies_available() -> bool:
|
||||
return True
|
||||
|
||||
def load_model(self, model_path: Path):
|
||||
return object()
|
||||
|
||||
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
|
||||
return [
|
||||
{
|
||||
"class_name": "building",
|
||||
"confidence": 0.91,
|
||||
"points": [[10.0, 20.0], [30.0, 20.0], [30.0, 40.0], [10.0, 40.0]],
|
||||
"bbox": [10.0, 20.0, 30.0, 40.0],
|
||||
"properties": {"class_id": 0},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
class ClassAgnosticSamAdapter(AvailableSegAdapter):
|
||||
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
|
||||
return [
|
||||
{
|
||||
"class_name": "segment",
|
||||
"confidence": None,
|
||||
"points": [[5.0, 5.0], [25.0, 5.0], [25.0, 25.0], [5.0, 25.0]],
|
||||
"bbox": [5.0, 5.0, 25.0, 25.0],
|
||||
"properties": {"class_id": -1},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
class MissingDependencySegAdapter(AvailableSegAdapter):
|
||||
@staticmethod
|
||||
def dependencies_available() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class NeverLoadSegAdapter(AvailableSegAdapter):
|
||||
load_calls = 0
|
||||
|
||||
def load_model(self, model_path: Path):
|
||||
type(self).load_calls += 1
|
||||
raise AssertionError("unmanifested weights must not reach adapter.load_model")
|
||||
|
||||
|
||||
def _project_and_dataset(dataset_type: str = "raster"):
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
source_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
checksum = "a" * 64
|
||||
project = Project(id=project_id, name="Mol")
|
||||
source = SourceRegistry(
|
||||
id=source_id,
|
||||
source_key="digitaal_vlaanderen_orthophoto",
|
||||
display_name="Governed orthophoto test source",
|
||||
classification="contextual",
|
||||
authority_name="Digitaal Vlaanderen",
|
||||
authority_scope_json={"zone": "Flanders", "role": "imagery"},
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=source_id,
|
||||
snapshot_key="configured-segmentation-orthophoto",
|
||||
checksum_sha256=checksum,
|
||||
ingest_status="ingested",
|
||||
freshness_status="current",
|
||||
)
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="ortho.tif",
|
||||
dataset_type=dataset_type,
|
||||
source="digitaal_vlaanderen_orthophoto",
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
storage_path="storage/uploads/ortho.tif",
|
||||
checksum_sha256=checksum,
|
||||
crs="EPSG:4326",
|
||||
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
|
||||
source_registry_id=source_id,
|
||||
source_snapshot_id=snapshot_id,
|
||||
data_contract_key="geointel.raster.geotiff",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
status="ready",
|
||||
)
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
dataset.versions.append(
|
||||
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
|
||||
)
|
||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||
return db, project_id, dataset_id
|
||||
|
||||
|
||||
def _settings(tmp_path: Path, **overrides) -> Settings:
|
||||
values = {
|
||||
# The runtime only consumes artifacts under the storage root, so a
|
||||
# test that writes tiles into tmp_path must say that is the root.
|
||||
"storage_root": str(tmp_path),
|
||||
"yolo_seg_enabled": True,
|
||||
"yolo_seg_model_path": str(tmp_path / "seg.pt"),
|
||||
"sam_enabled": True,
|
||||
"sam_model_path": str(tmp_path / "sam.pt"),
|
||||
"yolo_max_tiles": 4,
|
||||
}
|
||||
values.update(overrides)
|
||||
return Settings(**values)
|
||||
|
||||
|
||||
def _write_model_sidecar(
|
||||
model_path: Path,
|
||||
*,
|
||||
model_id: str,
|
||||
framework: str,
|
||||
source_version: str | None,
|
||||
db: FakeSession | None = None,
|
||||
) -> None:
|
||||
"""Create explicit local test evidence; no production code creates sidecars."""
|
||||
|
||||
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
resolved_source_version = source_version or "test-v1"
|
||||
if db is not None:
|
||||
source_registry = SourceRegistry(
|
||||
id=source_registry_id,
|
||||
source_key="model",
|
||||
display_name="Governed test model artifact",
|
||||
classification="experimental",
|
||||
authority_name="GeoIntel test fixture",
|
||||
freshness_status="current",
|
||||
ingest_status="configured",
|
||||
)
|
||||
source_snapshot = SourceSnapshot(
|
||||
id=source_snapshot_id,
|
||||
source_registry_id=source_registry_id,
|
||||
snapshot_key=f"model-{model_id}-{resolved_source_version}",
|
||||
source_version=resolved_source_version,
|
||||
checksum_sha256=model_sha256,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
db.objects[(SourceRegistry, source_registry_id)] = source_registry
|
||||
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
|
||||
payload = {
|
||||
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
|
||||
"model": {
|
||||
"model_id": model_id,
|
||||
"task_type": "segmentation",
|
||||
"sha256": model_sha256,
|
||||
"model_format": "pytorch",
|
||||
"framework": framework,
|
||||
"class_mapping": {"0": "segment"},
|
||||
"source_version": resolved_source_version,
|
||||
},
|
||||
"source": {
|
||||
"source_registry_id": str(source_registry_id),
|
||||
"source_snapshot_id": str(source_snapshot_id),
|
||||
"source_registry_key": "model",
|
||||
"source_snapshot_checksum_sha256": model_sha256,
|
||||
},
|
||||
"lineage": {
|
||||
"upstream_asset_ids": ["test-training-corpus"],
|
||||
"upstream_checksums_sha256": ["a" * 64],
|
||||
"transformations": [
|
||||
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
|
||||
],
|
||||
},
|
||||
"metadata": {"training_manifest_sha256": "c" * 64},
|
||||
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||
}
|
||||
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
|
||||
json.dumps(payload, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _write_configured_model_sidecars(
|
||||
tmp_path: Path,
|
||||
settings: Settings,
|
||||
*,
|
||||
include_yolo: bool = True,
|
||||
include_sam: bool = True,
|
||||
db: FakeSession | None = None,
|
||||
) -> None:
|
||||
if include_yolo:
|
||||
_write_model_sidecar(
|
||||
tmp_path / "seg.pt",
|
||||
model_id=settings.yolo_seg_model_id,
|
||||
framework="ultralytics/pytorch",
|
||||
source_version=settings.yolo_seg_model_version,
|
||||
db=db,
|
||||
)
|
||||
if include_sam:
|
||||
_write_model_sidecar(
|
||||
tmp_path / "sam.pt",
|
||||
model_id=settings.sam_model_id,
|
||||
framework="ultralytics/sam",
|
||||
source_version=settings.sam_model_version,
|
||||
db=db,
|
||||
)
|
||||
|
||||
|
||||
def _manifest(
|
||||
tmp_path: Path,
|
||||
tile_count: int = 1,
|
||||
*,
|
||||
db: FakeSession | None = None,
|
||||
dataset: Dataset | None = None,
|
||||
) -> Path:
|
||||
tiles = []
|
||||
for index in range(tile_count):
|
||||
tile_path = tmp_path / f"tile_{index:04d}.tif"
|
||||
tile_path.write_bytes(b"fixture")
|
||||
tile = {
|
||||
"path": str(tile_path),
|
||||
"pixel_window": [0, 0, 100, 100],
|
||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
|
||||
"crs": "EPSG:4326",
|
||||
"index": index,
|
||||
**TileManifestService.tile_integrity(tile_path),
|
||||
}
|
||||
tiles.append(tile)
|
||||
binding = TileManifestService.dataset_binding(db or FakeSession(), dataset) if dataset is not None else {}
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
**binding,
|
||||
"tile_set_id": "tiles-fixture",
|
||||
"source_dataset_id": binding.get("source_dataset_id", str(uuid4())),
|
||||
"source_raster_id": binding.get("source_raster_id", str(uuid4())),
|
||||
"crs": "EPSG:4326",
|
||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||
"tile_size": 100,
|
||||
"overlap": 0,
|
||||
"count": tile_count,
|
||||
"tiles": tiles,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return manifest_path
|
||||
|
||||
|
||||
def test_segmentation_models_report_not_configured_when_disabled(tmp_path: Path) -> None:
|
||||
settings = _settings(tmp_path, yolo_seg_enabled=False, sam_enabled=False)
|
||||
|
||||
models = {
|
||||
model.model_id: model
|
||||
for model in ModelRegistryService.list_segmentation_model_capabilities(settings=settings)
|
||||
}
|
||||
|
||||
assert models["yolo-seg-configured"].configured is False
|
||||
assert models["yolo-seg-configured"].status == "not_configured"
|
||||
assert models["sam-configured"].configured is False
|
||||
assert models["sam-configured"].status == "not_configured"
|
||||
|
||||
|
||||
def test_segmentation_models_report_dependency_unavailable(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"weights")
|
||||
(tmp_path / "sam.pt").write_bytes(b"weights")
|
||||
settings = _settings(tmp_path)
|
||||
|
||||
models = {
|
||||
model.model_id: model
|
||||
for model in ModelRegistryService.list_segmentation_model_capabilities(
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=MissingDependencySegAdapter,
|
||||
sam_adapter_class=MissingDependencySegAdapter,
|
||||
)
|
||||
}
|
||||
|
||||
assert models["yolo-seg-configured"].status == "dependency_unavailable"
|
||||
assert models["sam-configured"].status == "dependency_unavailable"
|
||||
|
||||
|
||||
def test_segmentation_models_require_runtime_provenance_sidecars(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"unmanifested yolo segmentation weights")
|
||||
(tmp_path / "sam.pt").write_bytes(b"unmanifested sam weights")
|
||||
settings = _settings(tmp_path)
|
||||
|
||||
models = {
|
||||
model.model_id: model
|
||||
for model in ModelRegistryService.list_segmentation_model_capabilities(
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=AvailableSegAdapter,
|
||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||
)
|
||||
}
|
||||
|
||||
assert models["yolo-seg-configured"].configured is False
|
||||
assert models["yolo-seg-configured"].status == "contract_incomplete"
|
||||
assert models["sam-configured"].configured is False
|
||||
assert models["sam-configured"].status == "contract_incomplete"
|
||||
|
||||
|
||||
def test_segmentation_models_report_configured_with_local_weights(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"weights")
|
||||
(tmp_path / "sam.pt").write_bytes(b"weights")
|
||||
settings = _settings(tmp_path)
|
||||
_write_configured_model_sidecars(tmp_path, settings)
|
||||
|
||||
models = {
|
||||
model.model_id: model
|
||||
for model in ModelRegistryService.list_segmentation_model_capabilities(
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=AvailableSegAdapter,
|
||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||
)
|
||||
}
|
||||
|
||||
assert models["yolo-seg-configured"].configured is True
|
||||
assert models["yolo-seg-configured"].status == "configured"
|
||||
assert models["sam-configured"].configured is True
|
||||
assert models["sam-configured"].status == "configured"
|
||||
|
||||
|
||||
def test_segmentation_dependency_check_uses_real_imports_not_find_spec() -> None:
|
||||
source = (ROOT / "backend" / "app" / "services" / "segmentation_adapter.py").read_text(encoding="utf-8")
|
||||
|
||||
assert 'find_spec("ultralytics")' not in source
|
||||
assert "import ultralytics" in source
|
||||
assert "import torch" in source
|
||||
|
||||
|
||||
def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"weights")
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
SegmentationService.run_segmentation(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-seg-configured",
|
||||
confidence_threshold=0.5,
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=AvailableSegAdapter,
|
||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||
)
|
||||
|
||||
assert getattr(exc_info.value, "code", None) == "SEGMENTATION_TILE_MANIFEST_REQUIRED"
|
||||
|
||||
|
||||
def test_configured_segmentation_fails_before_adapter_load_without_sidecar(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"unmanifested weights")
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
NeverLoadSegAdapter.load_calls = 0
|
||||
|
||||
response = SegmentationService.run_segmentation(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-seg-configured",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(_manifest(tmp_path)),
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=NeverLoadSegAdapter,
|
||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||
)
|
||||
|
||||
assert response.status == "failed"
|
||||
assert response.error_code == "SEGMENTATION_MODEL_UNAVAILABLE"
|
||||
assert NeverLoadSegAdapter.load_calls == 0
|
||||
|
||||
|
||||
def test_configured_segmentation_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"structurally valid but unbound weights")
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
# The sidecar passes catalog validation but its source registry/snapshot
|
||||
# was never registered in this production-session fixture.
|
||||
_write_configured_model_sidecars(tmp_path, settings, include_sam=False)
|
||||
NeverLoadSegAdapter.load_calls = 0
|
||||
|
||||
response = SegmentationService.run_segmentation(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-seg-configured",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(
|
||||
_manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
|
||||
),
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=NeverLoadSegAdapter,
|
||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||
)
|
||||
|
||||
assert response.status == "failed"
|
||||
assert response.error_code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
|
||||
assert NeverLoadSegAdapter.load_calls == 0
|
||||
|
||||
|
||||
def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"weights")
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
|
||||
manifest_path = _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
|
||||
|
||||
response = SegmentationService.run_segmentation(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-seg-configured",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(manifest_path),
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=AvailableSegAdapter,
|
||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||
)
|
||||
|
||||
assert response.status == "success"
|
||||
assert response.segmentation_count == 1
|
||||
persisted = [item for item in db.added if isinstance(item, Segmentation)]
|
||||
assert len(persisted) == 1
|
||||
segmentation = persisted[0]
|
||||
assert segmentation.class_name == "building"
|
||||
assert segmentation.confidence == pytest.approx(0.91)
|
||||
geometry = to_shape(segmentation.geometry)
|
||||
assert geometry.geom_type == "MultiPolygon"
|
||||
min_x, min_y, max_x, max_y = geometry.bounds
|
||||
assert 4.0 <= min_x <= 5.0
|
||||
assert 51.0 <= min_y <= 52.0
|
||||
assert max_x <= 5.0
|
||||
assert max_y <= 52.0
|
||||
assert segmentation.area_m2 is not None and segmentation.area_m2 > 0
|
||||
assert segmentation.provenance_json["inference"] == "local"
|
||||
assert segmentation.provenance_json["model_id"] == "yolo-seg-configured"
|
||||
assert segmentation.provenance_json["runtime_model_provenance"]["data_contract_key"] == "geointel.model.pytorch"
|
||||
|
||||
|
||||
def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None:
|
||||
(tmp_path / "sam.pt").write_bytes(b"weights")
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
_write_configured_model_sidecars(tmp_path, settings, include_yolo=False, db=db)
|
||||
manifest_path = _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
|
||||
|
||||
response = SegmentationService.run_segmentation(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="sam-configured",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(manifest_path),
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=AvailableSegAdapter,
|
||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||
)
|
||||
|
||||
assert response.status == "success"
|
||||
assert response.segmentation_count == 1
|
||||
persisted = [item for item in db.added if isinstance(item, Segmentation)]
|
||||
assert persisted[0].class_name == "segment"
|
||||
assert persisted[0].confidence is None
|
||||
|
||||
|
||||
def test_unconfigured_segmentation_run_fails_closed(tmp_path: Path) -> None:
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path, yolo_seg_enabled=False)
|
||||
manifest_path = _manifest(tmp_path)
|
||||
|
||||
response = SegmentationService.run_segmentation(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-seg-configured",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(manifest_path),
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=AvailableSegAdapter,
|
||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||
)
|
||||
|
||||
assert response.status == "failed"
|
||||
assert response.error_code == "SEGMENTATION_MODEL_UNAVAILABLE"
|
||||
assert not [item for item in db.added if isinstance(item, Segmentation)]
|
||||
|
||||
|
||||
def test_pixel_points_to_epsg4326_polygon_uses_tile_transform() -> None:
|
||||
tile = {
|
||||
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
|
||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||
"pixel_window": [0, 0, 100, 100],
|
||||
}
|
||||
|
||||
polygon = pixel_points_to_epsg4326_polygon(
|
||||
points=[[0.0, 0.0], [100.0, 0.0], [100.0, 100.0], [0.0, 100.0]],
|
||||
tile=tile,
|
||||
crs="EPSG:4326",
|
||||
)
|
||||
|
||||
min_x, min_y, max_x, max_y = polygon.bounds
|
||||
assert min_x == pytest.approx(4.0)
|
||||
assert max_x == pytest.approx(5.0)
|
||||
assert min_y == pytest.approx(51.0)
|
||||
assert max_y == pytest.approx(52.0)
|
||||
|
||||
|
||||
def test_pixel_points_to_epsg4326_polygon_rejects_degenerate_input() -> None:
|
||||
tile = {"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01]}
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
pixel_points_to_epsg4326_polygon(points=[[0.0, 0.0], [1.0, 1.0]], tile=tile)
|
||||
|
||||
assert getattr(exc_info.value, "code", None) == "SEGMENTATION_INVALID_MASK"
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Segmentation must honour the same post-processing configuration as detection.
|
||||
|
||||
Segmentation reuses the detection suppressor but passed only the IoU threshold,
|
||||
so it silently fell back to the hardcoded containment constant while detection
|
||||
read a configured one. A deployment tuning containment for a promoted model
|
||||
changed detection behaviour and left segmentation on the old value.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.config import Settings
|
||||
|
||||
|
||||
def test_segmentation_has_its_own_containment_setting() -> None:
|
||||
settings = Settings(_env_file=None)
|
||||
|
||||
assert settings.segmentation_containment_nms_threshold == pytest.approx(0.85)
|
||||
|
||||
|
||||
def test_the_setting_is_independent_of_the_detection_one() -> None:
|
||||
"""Masks and boxes overlap differently; one value need not fit both."""
|
||||
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
yolo_containment_nms_threshold=0.7,
|
||||
segmentation_containment_nms_threshold=0.95,
|
||||
)
|
||||
|
||||
assert settings.yolo_containment_nms_threshold == pytest.approx(0.7)
|
||||
assert settings.segmentation_containment_nms_threshold == pytest.approx(0.95)
|
||||
|
||||
|
||||
def test_the_configured_value_reaches_the_suppressor() -> None:
|
||||
from app.services.detection_service import DetectionService
|
||||
|
||||
def candidate(name: str, geometry, confidence: float):
|
||||
return {
|
||||
"class_name": "building",
|
||||
"confidence": confidence,
|
||||
"geometry": geometry,
|
||||
"bbox": [0.0, 0.0, 1.0, 1.0],
|
||||
"source_tile_path": f"/tiles/{name}.tif",
|
||||
"properties": {"name": name},
|
||||
}
|
||||
|
||||
outer = candidate("outer", box(0, 0, 10, 10), 0.9)
|
||||
# Containment 0.9, IoU 0.09: only the containment rule can act on this pair.
|
||||
mostly_nested = candidate("mostly", box(8.2, 1, 10.2, 6), 0.5)
|
||||
|
||||
strict = DetectionService._suppress_duplicate_candidates(
|
||||
[outer, mostly_nested], iou_threshold=0.5, containment_threshold=0.95
|
||||
)
|
||||
loose = DetectionService._suppress_duplicate_candidates(
|
||||
[outer, mostly_nested], iou_threshold=0.5, containment_threshold=0.7
|
||||
)
|
||||
|
||||
assert len(strict) == 2
|
||||
assert len(loose) == 1
|
||||
@@ -0,0 +1,174 @@
|
||||
"""Segmentation QA must score against the area it actually inferred.
|
||||
|
||||
Detection QA already clips both populations to the union of the persisted
|
||||
inference tiles. Segmentation QA compared candidates against every reference
|
||||
feature in the dataset, so every building outside the inferred tiles counted
|
||||
as a false negative and recall collapsed for no modelling reason.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from geoalchemy2.shape import from_shape
|
||||
from shapely.geometry import MultiPolygon, box
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import AnalysisRun, Dataset, Segmentation, VectorFeature
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
|
||||
from tests.test_sprint9_segmentation_foundation import ( # noqa: F401
|
||||
FakeSession,
|
||||
_authoritative_reference,
|
||||
)
|
||||
|
||||
|
||||
def _manifest(tmp_path: Path, dataset_id, bounds: list[float]) -> str:
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"source_dataset_id": str(dataset_id),
|
||||
"crs": "EPSG:4326",
|
||||
"tiles": [{"path": "tile_0000.tif", "bounds": bounds, "crs": "EPSG:4326"}],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return str(manifest_path)
|
||||
|
||||
|
||||
def _segmentation(project_id, dataset_id, analysis_run_id, geom):
|
||||
return Segmentation(
|
||||
id=uuid4(),
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
analysis_run_id=analysis_run_id,
|
||||
job_id=uuid4(),
|
||||
model_name="fixture-segmenter",
|
||||
model_version="fixture-v1",
|
||||
class_name="building",
|
||||
confidence=0.9,
|
||||
geometry=from_shape(MultiPolygon([geom]), srid=4326),
|
||||
)
|
||||
|
||||
|
||||
def _reference(dataset_id, geom) -> VectorFeature:
|
||||
return VectorFeature(
|
||||
id=uuid4(),
|
||||
dataset_id=dataset_id,
|
||||
feature_class="building",
|
||||
geometry=from_shape(geom, srid=4326),
|
||||
)
|
||||
|
||||
|
||||
def _session(tmp_path: Path, *, with_manifest: bool):
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
reference_dataset_id = uuid4()
|
||||
analysis_run_id = uuid4()
|
||||
|
||||
parameters = {}
|
||||
if with_manifest:
|
||||
parameters = {"tile_manifest_path": _manifest(tmp_path, dataset_id, [0.0, 0.0, 1.0, 1.0])}
|
||||
|
||||
reference_dataset = _authoritative_reference(
|
||||
Dataset(
|
||||
id=reference_dataset_id,
|
||||
project_id=project_id,
|
||||
name="reference.geojson",
|
||||
dataset_type="vector",
|
||||
source="test",
|
||||
dataset_role="reference",
|
||||
)
|
||||
)
|
||||
db = FakeSession(
|
||||
objects={
|
||||
(AnalysisRun, analysis_run_id): AnalysisRun(
|
||||
id=analysis_run_id,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
analysis_type="segmentation",
|
||||
status="success",
|
||||
parameters_json=parameters,
|
||||
),
|
||||
(Dataset, dataset_id): Dataset(
|
||||
id=dataset_id, project_id=project_id, name="fixture.tif", dataset_type="raster", source="test"
|
||||
),
|
||||
(Dataset, reference_dataset_id): reference_dataset,
|
||||
},
|
||||
query_rows={
|
||||
Segmentation: [_segmentation(project_id, dataset_id, analysis_run_id, box(0.1, 0.1, 0.2, 0.2))],
|
||||
VectorFeature: [
|
||||
# Inside the inferred tile: a genuine match.
|
||||
_reference(reference_dataset_id, box(0.1, 0.1, 0.2, 0.2)),
|
||||
# Far outside it: never looked at by the model.
|
||||
_reference(reference_dataset_id, box(8.0, 8.0, 8.1, 8.1)),
|
||||
_reference(reference_dataset_id, box(9.0, 9.0, 9.1, 9.1)),
|
||||
],
|
||||
},
|
||||
)
|
||||
return db, analysis_run_id, reference_dataset_id
|
||||
|
||||
|
||||
def test_segmentation_qa_scores_only_inside_persisted_tile_coverage(tmp_path: Path, monkeypatch) -> None:
|
||||
# A manifest written into tmp_path is only a governed artifact if
|
||||
# tmp_path is the storage root.
|
||||
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
|
||||
db, analysis_run_id, reference_dataset_id = _session(tmp_path, with_manifest=True)
|
||||
|
||||
result = SegmentationService.compare_segmentations_with_reference(
|
||||
db=db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
reference_dataset_id=reference_dataset_id,
|
||||
iou_threshold=0.5,
|
||||
)
|
||||
|
||||
assert result["matches"] == 1
|
||||
assert result["false_negatives"] == 0
|
||||
assert result["recall"] == 1.0
|
||||
assert result["coverage"]["applied"] is True
|
||||
assert result["coverage"]["reference_raw_count"] == 3
|
||||
assert result["coverage"]["reference_evaluated_count"] == 1
|
||||
assert result["coverage"]["reference_excluded_outside_count"] == 2
|
||||
assert any("tile" in warning for warning in result["warnings"])
|
||||
|
||||
|
||||
def test_segmentation_qa_without_manifest_reports_unbounded_coverage(tmp_path: Path) -> None:
|
||||
db, analysis_run_id, reference_dataset_id = _session(tmp_path, with_manifest=False)
|
||||
|
||||
result = SegmentationService.compare_segmentations_with_reference(
|
||||
db=db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
reference_dataset_id=reference_dataset_id,
|
||||
iou_threshold=0.5,
|
||||
)
|
||||
|
||||
# Unchanged behaviour, but the response now says the score was not bounded
|
||||
# by an inference footprint so the recall can be read correctly.
|
||||
assert result["false_negatives"] == 2
|
||||
assert result["coverage"]["applied"] is False
|
||||
assert result["coverage"]["mode"] == "unbounded_no_manifest"
|
||||
|
||||
|
||||
def test_segmentation_qa_rejects_reference_entirely_outside_coverage(tmp_path: Path, monkeypatch) -> None:
|
||||
# A manifest written into tmp_path is only a governed artifact if
|
||||
# tmp_path is the storage root.
|
||||
monkeypatch.setenv("STORAGE_ROOT", str(tmp_path))
|
||||
db, analysis_run_id, reference_dataset_id = _session(tmp_path, with_manifest=True)
|
||||
db.query_rows[VectorFeature] = [
|
||||
_reference(reference_dataset_id, box(8.0, 8.0, 8.1, 8.1)),
|
||||
]
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
SegmentationService.compare_segmentations_with_reference(
|
||||
db=db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
reference_dataset_id=reference_dataset_id,
|
||||
iou_threshold=0.5,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "REFERENCE_FEATURES_OUTSIDE_COVERAGE"
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Regression coverage for bounded, stable segmentation result listings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.api.routes import segmentation as segmentation_routes
|
||||
from app.db.session import get_db
|
||||
from app.schemas.segmentation import SegmentationListResponse
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
|
||||
|
||||
RUN_ID = UUID("00000000-0000-0000-0000-000000000101")
|
||||
DATASET_ID = UUID("00000000-0000-0000-0000-000000000102")
|
||||
PROJECT_ID = UUID("00000000-0000-0000-0000-000000000103")
|
||||
|
||||
|
||||
def _segmentation(index: int) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
id=UUID(int=index + 1),
|
||||
project_id=PROJECT_ID,
|
||||
dataset_id=DATASET_ID,
|
||||
analysis_run_id=RUN_ID,
|
||||
job_id=None,
|
||||
model_name="segmentation-test-model",
|
||||
model_version="1",
|
||||
class_name="building",
|
||||
confidence=0.99 - index / 100,
|
||||
bbox_json=None,
|
||||
area_m2=float(index + 1),
|
||||
mask_path=None,
|
||||
source_tile_path=None,
|
||||
tile_index=index,
|
||||
properties_json={},
|
||||
provenance_json={},
|
||||
created_at=datetime(2026, 8, 23, tzinfo=UTC),
|
||||
)
|
||||
|
||||
|
||||
class _Session:
|
||||
def get(self, _model, identifier):
|
||||
if identifier == RUN_ID:
|
||||
return SimpleNamespace(analysis_type="segmentation")
|
||||
return None
|
||||
|
||||
|
||||
def test_service_returns_one_stable_page_with_complete_metadata(monkeypatch) -> None:
|
||||
rows = [_segmentation(index) for index in range(5)]
|
||||
monkeypatch.setattr(
|
||||
SegmentationService,
|
||||
"_query_segmentation_rows",
|
||||
staticmethod(lambda _db, **_filters: rows),
|
||||
)
|
||||
|
||||
result = SegmentationService.list_segmentations(
|
||||
_Session(),
|
||||
analysis_run_id=RUN_ID,
|
||||
dataset_id=DATASET_ID,
|
||||
limit=2,
|
||||
offset=1,
|
||||
)
|
||||
|
||||
assert [item.id for item in result.items] == [rows[1].id, rows[2].id]
|
||||
assert result.total == 5
|
||||
assert result.limit == 2
|
||||
assert result.offset == 1
|
||||
assert result.truncated is True
|
||||
|
||||
|
||||
def test_service_pages_cover_the_stably_ordered_population_once(monkeypatch) -> None:
|
||||
rows = [_segmentation(index) for index in range(5)]
|
||||
monkeypatch.setattr(
|
||||
SegmentationService,
|
||||
"_query_segmentation_rows",
|
||||
staticmethod(lambda _db, **_filters: rows),
|
||||
)
|
||||
|
||||
seen = []
|
||||
for offset in (0, 2, 4):
|
||||
result = SegmentationService.list_segmentations(
|
||||
_Session(),
|
||||
dataset_id=DATASET_ID,
|
||||
limit=2,
|
||||
offset=offset,
|
||||
)
|
||||
seen.extend(item.id for item in result.items)
|
||||
assert result.total == len(rows)
|
||||
assert result.offset == offset
|
||||
|
||||
assert seen == [row.id for row in rows]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("path", "expected_run_id", "expected_dataset_id"),
|
||||
[
|
||||
(f"/api/v1/segmentation/runs/{RUN_ID}/segmentations", RUN_ID, None),
|
||||
(f"/api/v1/segmentation/datasets/{DATASET_ID}/segmentations", None, DATASET_ID),
|
||||
],
|
||||
)
|
||||
def test_both_listing_routes_forward_the_page_window_and_return_it(
|
||||
monkeypatch,
|
||||
path: str,
|
||||
expected_run_id: UUID | None,
|
||||
expected_dataset_id: UUID | None,
|
||||
) -> None:
|
||||
calls: list[dict] = []
|
||||
|
||||
def _list(_db, analysis_run_id=None, **parameters):
|
||||
calls.append({"analysis_run_id": analysis_run_id, **parameters})
|
||||
return SegmentationListResponse(
|
||||
items=[],
|
||||
total=9,
|
||||
limit=2,
|
||||
offset=4,
|
||||
truncated=True,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(SegmentationService, "list_segmentations", staticmethod(_list))
|
||||
app = FastAPI()
|
||||
app.include_router(segmentation_routes.router, prefix="/api/v1")
|
||||
app.dependency_overrides[get_db] = lambda: object()
|
||||
|
||||
response = TestClient(app).get(
|
||||
path,
|
||||
params={"limit": 2, "offset": 4, "class_name": "building", "min_confidence": 0.5},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["data"] == {
|
||||
"items": [],
|
||||
"total": 9,
|
||||
"limit": 2,
|
||||
"offset": 4,
|
||||
"truncated": True,
|
||||
}
|
||||
assert calls == [
|
||||
{
|
||||
"analysis_run_id": expected_run_id,
|
||||
"limit": 2,
|
||||
"offset": 4,
|
||||
"dataset_id": expected_dataset_id,
|
||||
"class_name": "building",
|
||||
"min_confidence": 0.5,
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,103 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.api.routes.selection_partitions import select_vector_partitions
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset
|
||||
from app.schemas.selection_partitions import VectorPartitionSelectionRequest
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
class DatasetQuery:
|
||||
def __init__(self, datasets):
|
||||
self.datasets = datasets
|
||||
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return self.datasets
|
||||
|
||||
|
||||
class DatasetSession:
|
||||
def __init__(self, datasets):
|
||||
self.datasets = datasets
|
||||
|
||||
def query(self, model):
|
||||
assert model is Dataset
|
||||
return DatasetQuery(self.datasets)
|
||||
|
||||
|
||||
def make_dataset(project_id, dataset_id, *, source_name="grb", product_key="buildings"):
|
||||
return Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name=f"{source_name}-{product_key}",
|
||||
dataset_type="vector",
|
||||
source="official",
|
||||
dataset_role="reference",
|
||||
source_name=source_name,
|
||||
reference_layer_name=product_key,
|
||||
source_metadata={"product_key": product_key, "theme": product_key},
|
||||
provenance_metadata={},
|
||||
metadata_json={},
|
||||
status="ready",
|
||||
)
|
||||
|
||||
|
||||
def test_vector_partition_route_combines_one_governed_product(monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
dataset_ids = [uuid4(), uuid4()]
|
||||
db = DatasetSession([make_dataset(project_id, dataset_id) for dataset_id in dataset_ids])
|
||||
captured = {}
|
||||
|
||||
def select_features(_db, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return {
|
||||
"selection_bbox": kwargs["bbox"],
|
||||
"feature_count": 1,
|
||||
"total_feature_count": 3,
|
||||
"limit": kwargs["limit"],
|
||||
"truncated": False,
|
||||
"geojson": {"type": "FeatureCollection", "features": []},
|
||||
"summary": None,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(VectorFeatureService, "select_features_by_bbox", select_features)
|
||||
payload = VectorPartitionSelectionRequest(
|
||||
dataset_ids=dataset_ids,
|
||||
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.3, "max_y": 51.2},
|
||||
)
|
||||
response = select_vector_partitions(project_id, payload, db)
|
||||
|
||||
assert captured["dataset_ids"] == dataset_ids
|
||||
assert captured["deduplicate_source_features"] is True
|
||||
assert response["data"]["partition_count"] == 2
|
||||
assert response["data"]["dataset_ids"] == dataset_ids
|
||||
|
||||
|
||||
def test_vector_partition_request_has_a_bounded_fan_out() -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
VectorPartitionSelectionRequest(
|
||||
dataset_ids=[uuid4() for _ in range(4097)],
|
||||
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.3, "max_y": 51.2},
|
||||
)
|
||||
|
||||
|
||||
def test_vector_partition_route_rejects_mixed_source_products() -> None:
|
||||
project_id = uuid4()
|
||||
datasets = [
|
||||
make_dataset(project_id, uuid4(), source_name="grb", product_key="buildings"),
|
||||
make_dataset(project_id, uuid4(), source_name="spw_picc", product_key="picc_buildings"),
|
||||
]
|
||||
payload = VectorPartitionSelectionRequest(
|
||||
dataset_ids=[dataset.id for dataset in datasets],
|
||||
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.3, "max_y": 51.2},
|
||||
)
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
select_vector_partitions(project_id, payload, DatasetSession(datasets))
|
||||
assert getattr(exc_info.value, "code", None) == "VECTOR_PARTITION_SOURCE_MISMATCH"
|
||||
@@ -0,0 +1,121 @@
|
||||
"""A selection finer than the source raster must answer, not return zero.
|
||||
|
||||
End-to-end counterpart to ``test_raster_cell_selection``: the analysis reads a
|
||||
real GeoTIFF, so it proves the fallback survives the clip/mask path the service
|
||||
actually uses rather than only the helper in isolation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
np = pytest.importorskip("numpy")
|
||||
rasterio = pytest.importorskip("rasterio")
|
||||
|
||||
from pyproj import Transformer # noqa: E402 - optional rasterio gate precedes geospatial imports
|
||||
from rasterio.transform import from_origin # noqa: E402 - optional rasterio gate precedes geospatial imports
|
||||
|
||||
from app.core.config import Settings # noqa: E402 - optional rasterio gate precedes app imports
|
||||
from app.models import Dataset # noqa: E402 - optional rasterio gate precedes app imports
|
||||
from app.schemas.flood_hazard import FloodHazardSelectionRequest # noqa: E402 - optional rasterio gate precedes app imports
|
||||
from app.services.flood_hazard_acquisition_service import ( # noqa: E402 - optional rasterio gate precedes app imports
|
||||
FloodHazardAcquisitionService,
|
||||
)
|
||||
from app.services.flood_hazard_analysis_service import ( # noqa: E402 - optional rasterio gate precedes app imports
|
||||
FloodHazardAnalysisService,
|
||||
)
|
||||
|
||||
|
||||
TO_4326 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
||||
PRODUCT_KEY = "fluviaal_current_t100"
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, objects):
|
||||
self.objects = objects
|
||||
|
||||
def get(self, model, item_id):
|
||||
return self.objects.get((model, item_id))
|
||||
|
||||
|
||||
def _write_raster(path: Path, *, resolution: float, depth: float) -> None:
|
||||
values = np.full((4, 4), depth, dtype="float32")
|
||||
with rasterio.open(
|
||||
path,
|
||||
"w",
|
||||
driver="GTiff",
|
||||
width=4,
|
||||
height=4,
|
||||
count=1,
|
||||
dtype="float32",
|
||||
crs="EPSG:31370",
|
||||
transform=from_origin(200_000, 210_000, resolution, resolution),
|
||||
nodata=-9999.0,
|
||||
) as output:
|
||||
output.write(values, 1)
|
||||
|
||||
|
||||
def _dataset(project_id, dataset_id, path: Path) -> Dataset:
|
||||
return Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="vmm-flood.tif",
|
||||
dataset_type="raster",
|
||||
source="vmm",
|
||||
source_name=FloodHazardAcquisitionService.PROVIDER,
|
||||
status="ready",
|
||||
storage_path=str(path),
|
||||
source_metadata={"product_key": PRODUCT_KEY, "normalized_value_unit": "m"},
|
||||
)
|
||||
|
||||
|
||||
def _bbox_for(min_x: float, min_y: float, max_x: float, max_y: float) -> dict:
|
||||
left, bottom = TO_4326.transform(min_x, min_y)
|
||||
right, top = TO_4326.transform(max_x, max_y)
|
||||
return {"min_x": left, "min_y": bottom, "max_x": right, "max_y": top, "crs": "EPSG:4326"}
|
||||
|
||||
|
||||
def _analyze(tmp_path: Path, bbox: dict, *, resolution: float = 100.0) -> dict:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
path = tmp_path / "flood.tif"
|
||||
_write_raster(path, resolution=resolution, depth=2.0)
|
||||
dataset = _dataset(project_id, dataset_id, path)
|
||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||
|
||||
return FloodHazardAnalysisService.analyze(
|
||||
db,
|
||||
project_id,
|
||||
dataset_id,
|
||||
FloodHazardSelectionRequest(bbox=bbox),
|
||||
settings=Settings(_env_file=None),
|
||||
)
|
||||
|
||||
|
||||
def test_a_selection_smaller_than_one_cell_reports_the_cell_it_touches(tmp_path: Path) -> None:
|
||||
# A 40 x 30 m rectangle wholly inside one 100 m cell: no cell centre falls
|
||||
# inside it, so the centre rule alone would report an empty selection.
|
||||
result = _analyze(tmp_path, _bbox_for(200_010, 209_960, 200_050, 209_990))
|
||||
|
||||
assert result["inundated_cell_count"] == 1
|
||||
assert result["inundated_fraction"] == pytest.approx(1.0)
|
||||
assert "kleiner dan één rastercel" in result["coverage_warning"]
|
||||
|
||||
|
||||
def test_a_normal_selection_is_unaffected(tmp_path: Path) -> None:
|
||||
result = _analyze(tmp_path, _bbox_for(200_000, 209_700, 200_300, 210_000))
|
||||
|
||||
assert result["inundated_cell_count"] >= 9
|
||||
assert result["coverage_warning"] is None
|
||||
|
||||
|
||||
def test_the_reported_area_matches_the_cells_that_were_analysed(tmp_path: Path) -> None:
|
||||
result = _analyze(tmp_path, _bbox_for(200_010, 209_960, 200_050, 209_990))
|
||||
metrics = {item["metric_key"]: item["metric_value"] for item in result["summary"]["metrics"]}
|
||||
|
||||
# One 100 x 100 m cell, not the 0.12 ha that was drawn.
|
||||
assert metrics["modelled_inundated_area_ha"] == pytest.approx(1.0)
|
||||
assert metrics["selection_area_ha"] == pytest.approx(1.0)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user