From b76cd1837bd7e7820081df9d0b8d7a498ede047d Mon Sep 17 00:00:00 2001 From: Jens Date: Sun, 9 Aug 2026 10:54:52 +0200 Subject: [PATCH] fix(ai): bind model scope to immutable geometry --- .env.example | 5 + backend/app/core/config.py | 11 ++ backend/app/services/detection_service.py | 25 ++- .../model_validation_scope_service.py | 142 ++++++++++++++++++ ...nt122_model_asset_activation_guardrails.py | 4 +- ...t_sprint155_detection_operator_profiles.py | 17 ++- .../test_sprint193_end_user_workbench.py | 2 +- .../tests/test_sprint8b_yolo_foundation.py | 61 +++++++- deploy/unraid/Dockerfile.all-in-one | 1 + deploy/unraid/geointel-unraid-template.xml | 6 +- deploy/unraid/geointel.env.example | 3 + deploy/unraid/run-dockerman-container.sh | 4 + docs/AI_PIPELINES.md | 28 +++- docs/CODEX_EXECUTION_LOG.md | 28 ++++ docs/TODO.md | 6 +- .../src/components/detection/DetectionLab.tsx | 11 +- .../detection/DetectionModelManagement.tsx | 16 +- .../components/detection/detectionProfiles.ts | 30 ++-- .../build_model_validation_scope_manifest.py | 89 +++++++++++ 19 files changed, 431 insertions(+), 58 deletions(-) create mode 100644 backend/app/services/model_validation_scope_service.py create mode 100644 scripts/build_model_validation_scope_manifest.py diff --git a/.env.example b/.env.example index 4193deb7..85a0dc4d 100644 --- a/.env.example +++ b/.env.example @@ -118,6 +118,11 @@ YOLO_MODEL_DISPLAY_NAME=Configured YOLO detector YOLO_MODEL_VERSION= YOLO_MODEL_CLASSES=building YOLO_ENFORCE_VALIDATION_SCOPE=false +# Required when scope enforcement is enabled. The manifest is bound to exact +# model bytes and contains the allowed EPSG:4326 validation geometry. +YOLO_VALIDATION_SCOPE_MANIFEST_PATH= +YOLO_VALIDATION_SCOPE_MANIFEST_SHA256= +# Deprecated display metadata; never used as an inference authorization gate. YOLO_VALIDATED_AREA_NAMES=Mol,Kempen YOLO_CONFIG_DIR=./storage/ultralytics YOLO_DEVICE=cpu diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 3128c37e..0f716021 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -359,6 +359,17 @@ class Settings(BaseSettings): yolo_model_version: str | None = Field(default=None, validation_alias="YOLO_MODEL_VERSION") yolo_model_classes: str = Field(default="building", validation_alias="YOLO_MODEL_CLASSES") yolo_enforce_validation_scope: bool = Field(default=False, validation_alias="YOLO_ENFORCE_VALIDATION_SCOPE") + yolo_validation_scope_manifest_path: str | None = Field( + default=None, + validation_alias="YOLO_VALIDATION_SCOPE_MANIFEST_PATH", + ) + yolo_validation_scope_manifest_sha256: str | None = Field( + default=None, + validation_alias="YOLO_VALIDATION_SCOPE_MANIFEST_SHA256", + ) + # Deprecated compatibility field. Mutable Area names are never an + # inference authorization boundary; deployments must use the immutable + # checksum-bound scope manifest above. yolo_validated_area_names: str = Field(default="Mol,Kempen", validation_alias="YOLO_VALIDATED_AREA_NAMES") yolo_device: str = Field(default="cpu", validation_alias="YOLO_DEVICE") yolo_require_cuda: bool = Field(default=False, validation_alias="YOLO_REQUIRE_CUDA") diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py index ea5d1c4b..329eef3c 100644 --- a/backend/app/services/detection_service.py +++ b/backend/app/services/detection_service.py @@ -22,6 +22,7 @@ from app.services.detection_qa_service import DetectionQaService from app.services.dataset_consumption_gate_service import DatasetConsumptionGate from app.services.model_asset_catalog_service import ModelAssetCatalogService from app.services.model_registry_service import ModelRegistryService +from app.services.model_validation_scope_service import ModelValidationScopeService from app.services.qa_service import QaService from app.services.quality_service import QualityService from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService @@ -235,16 +236,30 @@ class DetectionService: @staticmethod def _validate_model_area_scope(db, dataset: Dataset, settings: Settings) -> None: - allowed_names = [value.strip().casefold() for value in settings.yolo_validated_area_names.split(",") if value.strip()] area = db.get(Area, dataset.area_id) if dataset.area_id else None - area_name = area.name.strip() if area is not None else "" - if not area_name or not any(token in area_name.casefold() for token in allowed_names): + if area is None or area.geometry is None: raise AppError( code="DETECTION_VALIDATION_SCOPE_UNAVAILABLE", - message="Configured YOLO inference is not validated for this Dataset area.", - details={"dataset_id": str(dataset.id), "dataset_area": area_name or None, "validated_area_names": allowed_names}, + message="Configured YOLO inference requires a persisted Dataset area geometry.", + details={"dataset_id": str(dataset.id)}, status_code=422, ) + try: + area_geometry = to_shape(area.geometry) + except Exception as exc: + raise AppError( + code="DETECTION_VALIDATION_SCOPE_UNAVAILABLE", + message="The persisted Dataset area geometry cannot be validated for model inference.", + details={"dataset_id": str(dataset.id), "error_type": type(exc).__name__}, + status_code=422, + ) from exc + ModelValidationScopeService.assert_area_covered( + area_geometry=area_geometry, + manifest_path=settings.yolo_validation_scope_manifest_path, + expected_manifest_sha256=settings.yolo_validation_scope_manifest_sha256, + model_id=settings.yolo_model_id, + model_path=settings.yolo_model_path, + ) @staticmethod def _fail_run_after_exception(db, analysis_run: AnalysisRun, job: Job, exc: Exception, fallback_code: str) -> None: try: diff --git a/backend/app/services/model_validation_scope_service.py b/backend/app/services/model_validation_scope_service.py new file mode 100644 index 00000000..d1cbb0f4 --- /dev/null +++ b/backend/app/services/model_validation_scope_service.py @@ -0,0 +1,142 @@ +"""Checksum-bound geographic validation scope for production model inference.""" + +from __future__ import annotations + +from hashlib import sha256 +import json +from pathlib import Path +import re +from typing import Any + +from shapely.geometry import shape +from shapely.geometry.base import BaseGeometry + +from app.core.errors import AppError + + +_SHA256 = re.compile(r"^[0-9a-f]{64}$") + + +class ModelValidationScopeService: + """Load an immutable, model-bound AOI and prove that an input is covered.""" + + SCHEMA_VERSION = "geointel.model-validation-scope/v1" + + @classmethod + def assert_area_covered( + cls, + *, + area_geometry: BaseGeometry, + manifest_path: str | None, + expected_manifest_sha256: str | None, + model_id: str, + model_path: str | None, + ) -> dict[str, str]: + path = Path(manifest_path).expanduser() if manifest_path else None + expected_checksum = (expected_manifest_sha256 or "").strip().lower() + if path is None or not expected_checksum: + cls._raise( + "DETECTION_VALIDATION_SCOPE_NOT_CONFIGURED", + "Configured YOLO inference requires a checksum-bound geographic validation-scope manifest.", + ) + if not _SHA256.fullmatch(expected_checksum): + cls._raise( + "DETECTION_VALIDATION_SCOPE_INVALID", + "The configured validation-scope checksum must be a lowercase SHA-256 digest.", + manifest_path=str(path), + ) + try: + raw_manifest = path.read_bytes() + payload = json.loads(raw_manifest.decode("utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + cls._raise( + "DETECTION_VALIDATION_SCOPE_INVALID", + "The configured validation-scope manifest is missing or unreadable.", + manifest_path=str(path), + error_type=type(exc).__name__, + ) + observed_manifest_sha256 = sha256(raw_manifest).hexdigest() + if observed_manifest_sha256 != expected_checksum: + cls._raise( + "DETECTION_VALIDATION_SCOPE_CHECKSUM_MISMATCH", + "The validation-scope manifest does not match its configured checksum.", + manifest_path=str(path), + expected=expected_checksum, + observed=observed_manifest_sha256, + ) + if not isinstance(payload, dict) or payload.get("schema_version") != cls.SCHEMA_VERSION: + cls._raise( + "DETECTION_VALIDATION_SCOPE_INVALID", + "The validation-scope manifest has an unsupported schema.", + manifest_path=str(path), + ) + if payload.get("model_id") != model_id: + cls._raise( + "DETECTION_VALIDATION_SCOPE_MODEL_MISMATCH", + "The validation scope is not bound to the selected model identity.", + expected_model_id=model_id, + observed_model_id=payload.get("model_id"), + ) + configured_model_path = Path(model_path).expanduser() if model_path else None + if configured_model_path is None or not configured_model_path.is_file(): + cls._raise( + "DETECTION_VALIDATION_SCOPE_MODEL_UNAVAILABLE", + "The model bytes bound by the validation scope are unavailable.", + ) + declared_model_sha256 = str(payload.get("model_sha256") or "").strip().lower() + observed_model_sha256 = cls._file_sha256(configured_model_path) + if not _SHA256.fullmatch(declared_model_sha256) or declared_model_sha256 != observed_model_sha256: + cls._raise( + "DETECTION_VALIDATION_SCOPE_MODEL_MISMATCH", + "The validation scope is not bound to the exact selected model bytes.", + expected=declared_model_sha256 or None, + observed=observed_model_sha256, + ) + if payload.get("crs") != "EPSG:4326": + cls._raise( + "DETECTION_VALIDATION_SCOPE_INVALID", + "The validation-scope geometry must explicitly use EPSG:4326.", + observed_crs=payload.get("crs"), + ) + try: + scope_geometry = shape(payload.get("geometry")) + except Exception as exc: + cls._raise( + "DETECTION_VALIDATION_SCOPE_INVALID", + "The validation-scope geometry is not valid GeoJSON.", + error_type=type(exc).__name__, + ) + if ( + scope_geometry.is_empty + or not scope_geometry.is_valid + or scope_geometry.geom_type not in {"Polygon", "MultiPolygon"} + ): + cls._raise( + "DETECTION_VALIDATION_SCOPE_INVALID", + "The validation scope must be a non-empty valid Polygon or MultiPolygon.", + geometry_type=scope_geometry.geom_type, + ) + if area_geometry.is_empty or not area_geometry.is_valid or not scope_geometry.covers(area_geometry): + cls._raise( + "DETECTION_VALIDATION_SCOPE_UNAVAILABLE", + "Configured YOLO inference is not validated for the complete Dataset area.", + scope_key=payload.get("scope_key"), + ) + return { + "scope_key": str(payload.get("scope_key") or "unspecified"), + "manifest_path": str(path.resolve()), + "manifest_sha256": observed_manifest_sha256, + "model_sha256": observed_model_sha256, + } + + @staticmethod + def _file_sha256(path: Path) -> str: + digest = sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + @staticmethod + def _raise(code: str, message: str, **details: Any) -> None: + raise AppError(code=code, message=message, details=details, status_code=422) diff --git a/backend/tests/test_sprint122_model_asset_activation_guardrails.py b/backend/tests/test_sprint122_model_asset_activation_guardrails.py index 28884bb5..3e06bd05 100644 --- a/backend/tests/test_sprint122_model_asset_activation_guardrails.py +++ b/backend/tests/test_sprint122_model_asset_activation_guardrails.py @@ -25,9 +25,9 @@ def test_detection_lab_explains_explicit_model_asset_and_threshold_selection() - assert "Lokaal modelbestand" in lab assert "GeoIntel kiest automatisch het actieve lokale model" in lab - assert "Gevalideerde YOLO-profielen" in lab + assert "Historische YOLO-controleprofielen" in lab assert "DETECTION_OPERATOR_PROFILES" in lab - assert "kandidaat, extra controle vereist" in lab + assert "historisch, geen releasebewijs" in lab assert "will_download_models" in lab diff --git a/backend/tests/test_sprint155_detection_operator_profiles.py b/backend/tests/test_sprint155_detection_operator_profiles.py index 476a0cb6..260a3194 100644 --- a/backend/tests/test_sprint155_detection_operator_profiles.py +++ b/backend/tests/test_sprint155_detection_operator_profiles.py @@ -4,7 +4,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -def test_detection_operator_profiles_define_explicit_yolo_candidates_and_promoted_profile() -> None: +def test_detection_operator_profiles_define_explicit_historical_yolo_controls_without_promotion_claim() -> None: profiles = ROOT / "frontend" / "src" / "components" / "detection" / "detectionProfiles.ts" source = profiles.read_text(encoding="utf-8") @@ -17,16 +17,17 @@ def test_detection_operator_profiles_define_explicit_yolo_candidates_and_promote assert "conservative-review" in source assert "confidenceThreshold: 0.15" in source assert "confidenceThreshold: 0.35" in source - assert "defaultApproved: true" in source - assert "promotionRecommendation: 'promote_candidate'" in source + assert "defaultApproved" not in source + assert "promotionRecommendation" not in source + assert "independentTestProven: false" in source assert "positiveSampleCount: 7" in source assert "precision: 0.6140895327792112" in source assert "recall: 0.6062221049337548" in source assert "f1: 0.6068607646002744" in source assert "f1: 0.5432865390636915" in source assert "maxBackgroundDetections: 0" in source - assert "lege-achtergrondtest is geslaagd" in source - assert "Postel blijft met 47,5% F1" in source + assert "Slechts drie pure-achtergrondbeelden" in source + assert "ruimtelijke onafhankelijkheid niet bewezen" in source assert "controlekandidaat en niet als grondwaarheid" in source @@ -39,11 +40,11 @@ def test_detection_lab_surfaces_profiles_as_deliberate_operator_actions() -> Non ) assert "DETECTION_OPERATOR_PROFILES" in lab - assert "Gevalideerde YOLO-profielen" in lab + assert "Historische YOLO-controleprofielen" in lab assert "profile.displayName" in lab assert "profile.confidenceThreshold" in lab - assert "kandidaat, extra controle vereist" in lab - assert "standaardprofiel" in lab + assert "historisch, geen releasebewijs" in lab + assert "historische F1" in lab assert "Profiel gebruiken" in lab assert "onApplyOperatorProfile(profile)" in lab assert "Recommended starting threshold: 0.25" not in lab diff --git a/backend/tests/test_sprint193_end_user_workbench.py b/backend/tests/test_sprint193_end_user_workbench.py index b6c70da0..ee37c816 100644 --- a/backend/tests/test_sprint193_end_user_workbench.py +++ b/backend/tests/test_sprint193_end_user_workbench.py @@ -72,7 +72,7 @@ def test_visible_ai_and_quality_labels_are_end_user_facing() -> None: providers = read("frontend/src/components/providers/ProviderPanel.tsx") assert "Aanbevolen controleprofiel kleine gebouwen" in profiles - assert "Postel blijft met 47,5% F1" in profiles + assert "Slechts drie pure-achtergrondbeelden" in profiles assert "controlekandidaat en niet als grondwaarheid" in profiles assert "qualityStatusLabel" in quality assert "nog niet uitgevoerd" in quality diff --git a/backend/tests/test_sprint8b_yolo_foundation.py b/backend/tests/test_sprint8b_yolo_foundation.py index a42eb6f0..eafa1b23 100644 --- a/backend/tests/test_sprint8b_yolo_foundation.py +++ b/backend/tests/test_sprint8b_yolo_foundation.py @@ -8,6 +8,8 @@ from types import SimpleNamespace from uuid import uuid4 import pytest +from geoalchemy2.shape import from_shape +from shapely.geometry import box, mapping from app.core.config import Settings from app.core.errors import AppError @@ -15,6 +17,7 @@ from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, Sour from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon from app.services.detection_service import DetectionService from app.services.model_registry_service import ModelRegistryService +from app.services.model_validation_scope_service import ModelValidationScopeService from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService from app.services.yolo_adapter import YoloDetectionAdapter @@ -207,6 +210,28 @@ def _settings(tmp_path: Path, **overrides) -> Settings: return Settings(**values) +def _scope_settings(tmp_path: Path, scope_geometry=None, **overrides) -> Settings: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"scope-bound-model") + payload = { + "schema_version": ModelValidationScopeService.SCHEMA_VERSION, + "model_id": "yolo-configured", + "model_sha256": sha256(model_path.read_bytes()).hexdigest(), + "scope_key": "mol-kempen-test", + "crs": "EPSG:4326", + "geometry": mapping(scope_geometry or box(4.0, 50.8, 5.5, 52.0)), + } + manifest_path = tmp_path / "model-validation-scope.json" + manifest_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") + values = { + "yolo_model_path": str(model_path), + "yolo_validation_scope_manifest_path": str(manifest_path), + "yolo_validation_scope_manifest_sha256": sha256(manifest_path.read_bytes()).hexdigest(), + } + values.update(overrides) + return _settings(tmp_path, **values) + + def _write_model_sidecar( model_path: Path, settings: Settings, @@ -399,21 +424,49 @@ def test_yolo_runtime_rejects_cpu_device_when_cuda_is_required(tmp_path: Path, m def test_yolo_validation_scope_requires_persisted_validated_area(tmp_path: Path) -> None: dataset = Dataset(id=uuid4(), project_id=uuid4(), name="image.tif", dataset_type="raster", source="test", area_id=uuid4()) - wrong_area = Area(id=dataset.area_id, project_id=dataset.project_id, name="Brussels", geometry="MULTIPOLYGON EMPTY") + wrong_area = Area( + id=dataset.area_id, + project_id=dataset.project_id, + name="Mol validation bypass", + geometry=from_shape(box(-74.1, 40.6, -73.8, 40.9), srid=4326), + ) db = FakeSession(objects={(Area, dataset.area_id): wrong_area}) with pytest.raises(AppError) as exc_info: - DetectionService._validate_model_area_scope(db, dataset, _settings(tmp_path, yolo_validated_area_names="Mol,Kempen")) + DetectionService._validate_model_area_scope(db, dataset, _scope_settings(tmp_path)) assert exc_info.value.code == "DETECTION_VALIDATION_SCOPE_UNAVAILABLE" def test_yolo_validation_scope_accepts_bound_mol_area(tmp_path: Path) -> None: dataset = Dataset(id=uuid4(), project_id=uuid4(), name="image.tif", dataset_type="raster", source="test", area_id=uuid4()) - area = Area(id=dataset.area_id, project_id=dataset.project_id, name="Gemeente Mol", geometry="MULTIPOLYGON EMPTY") + area = Area( + id=dataset.area_id, + project_id=dataset.project_id, + name="Een wijzigbare weergavenaam", + geometry=from_shape(box(5.0, 51.1, 5.2, 51.3), srid=4326), + ) db = FakeSession(objects={(Area, dataset.area_id): area}) - DetectionService._validate_model_area_scope(db, dataset, _settings(tmp_path, yolo_validated_area_names="Mol,Kempen")) + DetectionService._validate_model_area_scope(db, dataset, _scope_settings(tmp_path)) + + +def test_yolo_validation_scope_rejects_tampered_manifest(tmp_path: Path) -> None: + dataset = Dataset(id=uuid4(), project_id=uuid4(), name="image.tif", dataset_type="raster", source="test", area_id=uuid4()) + area = Area( + id=dataset.area_id, + project_id=dataset.project_id, + name="Gemeente Mol", + geometry=from_shape(box(5.0, 51.1, 5.2, 51.3), srid=4326), + ) + settings = _scope_settings(tmp_path) + Path(settings.yolo_validation_scope_manifest_path).write_text("{}", encoding="utf-8") + db = FakeSession(objects={(Area, dataset.area_id): area}) + + with pytest.raises(AppError) as exc_info: + DetectionService._validate_model_area_scope(db, dataset, settings) + + assert exc_info.value.code == "DETECTION_VALIDATION_SCOPE_CHECKSUM_MISMATCH" def test_yolo_run_requires_tile_manifest_path(tmp_path: Path) -> None: diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 52d83233..8d41da6c 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -113,6 +113,7 @@ COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_bu COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py COPY scripts/provision_geographic_scope.py /app/scripts/provision_geographic_scope.py +COPY scripts/build_model_validation_scope_manifest.py /app/scripts/build_model_validation_scope_manifest.py COPY scripts/provision_belgium_north_sea_scope.py /app/scripts/provision_belgium_north_sea_scope.py COPY scripts/provision_release_golden_areas.py /app/scripts/provision_release_golden_areas.py COPY scripts/provision_regional_grb_buildings.py /app/scripts/provision_regional_grb_buildings.py diff --git a/deploy/unraid/geointel-unraid-template.xml b/deploy/unraid/geointel-unraid-template.xml index ae02ca04..6aae8de3 100644 --- a/deploy/unraid/geointel-unraid-template.xml +++ b/deploy/unraid/geointel-unraid-template.xml @@ -137,8 +137,10 @@ cuda:0 true building - true - Mol,Kempen + true + /app/storage/operator-data/model-validation-scopes/active-building-model.json + + Mol,Kempen 640 100 1000 diff --git a/deploy/unraid/geointel.env.example b/deploy/unraid/geointel.env.example index ab0f1bf0..f9867917 100644 --- a/deploy/unraid/geointel.env.example +++ b/deploy/unraid/geointel.env.example @@ -166,6 +166,9 @@ YOLO_DEVICE=cuda:0 YOLO_REQUIRE_CUDA=true YOLO_MODEL_CLASSES=building YOLO_ENFORCE_VALIDATION_SCOPE=true +YOLO_VALIDATION_SCOPE_MANIFEST_PATH=/app/storage/operator-data/model-validation-scopes/active-building-model.json +YOLO_VALIDATION_SCOPE_MANIFEST_SHA256= +# Deprecated display metadata; never used as an inference authorization gate. YOLO_VALIDATED_AREA_NAMES=Mol,Kempen YOLO_IMAGE_SIZE=640 YOLO_MAX_TILES=100 diff --git a/deploy/unraid/run-dockerman-container.sh b/deploy/unraid/run-dockerman-container.sh index 034bb985..1563dabd 100644 --- a/deploy/unraid/run-dockerman-container.sh +++ b/deploy/unraid/run-dockerman-container.sh @@ -136,6 +136,8 @@ YOLO_MODEL_DISPLAY_NAME="${YOLO_MODEL_DISPLAY_NAME:-Configured YOLO detector}" YOLO_MODEL_VERSION="${YOLO_MODEL_VERSION:-}" YOLO_MODEL_CLASSES="${YOLO_MODEL_CLASSES:-building}" YOLO_ENFORCE_VALIDATION_SCOPE="${YOLO_ENFORCE_VALIDATION_SCOPE:-true}" +YOLO_VALIDATION_SCOPE_MANIFEST_PATH="${YOLO_VALIDATION_SCOPE_MANIFEST_PATH:-}" +YOLO_VALIDATION_SCOPE_MANIFEST_SHA256="${YOLO_VALIDATION_SCOPE_MANIFEST_SHA256:-}" YOLO_VALIDATED_AREA_NAMES="${YOLO_VALIDATED_AREA_NAMES:-Mol,Kempen}" YOLO_CONFIG_DIR="${YOLO_CONFIG_DIR:-/app/storage/ultralytics}" YOLO_DEVICE="${YOLO_DEVICE:-cuda:0}" @@ -413,6 +415,8 @@ docker run -d \ -e YOLO_MODEL_VERSION="$YOLO_MODEL_VERSION" \ -e YOLO_MODEL_CLASSES="$YOLO_MODEL_CLASSES" \ -e YOLO_ENFORCE_VALIDATION_SCOPE="$YOLO_ENFORCE_VALIDATION_SCOPE" \ + -e YOLO_VALIDATION_SCOPE_MANIFEST_PATH="$YOLO_VALIDATION_SCOPE_MANIFEST_PATH" \ + -e YOLO_VALIDATION_SCOPE_MANIFEST_SHA256="$YOLO_VALIDATION_SCOPE_MANIFEST_SHA256" \ -e YOLO_VALIDATED_AREA_NAMES="$YOLO_VALIDATED_AREA_NAMES" \ -e YOLO_CONFIG_DIR="$YOLO_CONFIG_DIR" \ -e YOLO_DEVICE="$YOLO_DEVICE" \ diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 8a4dac3a..1aac8f68 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -121,8 +121,13 @@ Environment variables: - `YOLO_REQUIRE_CUDA` (set to `true` on the production server; inference then fails closed when CUDA is unavailable or `YOLO_DEVICE` selects CPU) - `YOLO_MODEL_CLASSES` (the active promoted detector is `building` only) -- `YOLO_ENFORCE_VALIDATION_SCOPE` and `YOLO_VALIDATED_AREA_NAMES` (production - rejects inference when the raster is not bound to a persisted validated Area) +- `YOLO_ENFORCE_VALIDATION_SCOPE` (keep `true` in production) +- `YOLO_VALIDATION_SCOPE_MANIFEST_PATH` and + `YOLO_VALIDATION_SCOPE_MANIFEST_SHA256` (production accepts inference only + when the exact active model bytes match the manifest and the complete + persisted Dataset AOI is covered by its valid EPSG:4326 geometry) +- `YOLO_VALIDATED_AREA_NAMES` is deprecated display metadata and never grants + inference access - `YOLO_IMAGE_SIZE` - `YOLO_MAX_TILES` - `YOLO_MAX_DETECTIONS` @@ -136,6 +141,25 @@ the upstream default would cap recall before QA/QC begins. Operators may lower the value for small rasters or raise it for dense urban tiles after reviewing runtime and false-positive behavior. +Create a new immutable scope artifact whenever either the model bytes or the +governed validation boundary changes: + +```bash +python /app/scripts/build_model_validation_scope_manifest.py \ + --model /app/models/active-building.pt \ + --model-id yolo-configured \ + --scope-geojson /app/storage/operator-data/geographic-scopes/kempen-transport-region/kempen_transport_region_boundary_YYYY-MM-DD.geojson \ + --scope-key kempen-transport-region \ + --authority "Digitaal Vlaanderen VRBG/Refgem" \ + --snapshot-date YYYY-MM-DD \ + --output /app/storage/operator-data/model-validation-scopes/active-building-model.json +``` + +The command refuses to overwrite an existing manifest and prints the checksum +for `YOLO_VALIDATION_SCOPE_MANIFEST_SHA256`. Area names are intentionally not +part of this decision: they are mutable presentation text, not accuracy or +authorization evidence. + After YOLO boxes are georeferenced, configured-YOLO runs apply a GeoIntel cross-tile duplicate suppression pass before persistence. Candidates are grouped by canonical class and sorted by confidence; lower-confidence same-class diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 826520e4..d8481d3c 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -12444,3 +12444,31 @@ Open: `not_evaluable` without governed evidence. Phase 4 remains **in progress**, Phase 5 remains **not ready**, and promotion/training feedback from protected data is not authorized. + +## 2026-08-09 - Model accuracy boundary and truthful evidence labels + +### Changed + +- Replaced the configured-YOLO Area-name substring gate with a fail-closed, + checksum-bound validation-scope manifest. The exact model SHA-256 must match + and the immutable EPSG:4326 scope geometry must cover the complete persisted + Dataset AOI. Renaming an Area can no longer widen model applicability. +- Added an operator tool that builds the model-bound manifest from exact model + bytes and governed scope GeoJSON without overwriting prior evidence. +- Corrected the detection UI: legacy Mol/Kempen profile scores are now labelled + historical calibration context, spatial independence is explicitly unproven, + and no profile is marked approved or promotion-ready. + +### Verified + +- Targeted backend/UI contract selection: 34 passed. +- `git diff --check`: passed before the documentation update; final check is + part of the handoff verification. + +### Remaining limitations + +- The current active model is still not nationally validated. A new governed + corpus, independent spatial split, representative human review and immutable + product benchmark remain required before any production-accuracy claim. +- Every deployed model asset needs its own generated scope manifest and exact + configured manifest checksum before enforced inference is available. diff --git a/docs/TODO.md b/docs/TODO.md index 8b980b8b..12412c4f 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -1115,8 +1115,10 @@ This file now starts with the current implementation status. Older preparation/b 112 Ruff findings and add real frontend lint. - [ ] P2-02: fix CRS ingest, metre buffering and Area geometry/CRS updates; auditably quarantine or repair the four legacy Geel detections. -- [ ] P2-03: isolate coverage by source/theme/layer/zone, make official source - identity server-attested and replace mutable-name legal/model scope checks. +- [ ] P2-03: isolate coverage by source/theme/layer/zone and make official + source identity server-attested. The mutable-name YOLO scope bypass is fixed + with a model/checksum-bound geometry manifest; equivalent legal-scope checks + still require the same review. - [ ] P2-04: make derived persistence transactional, require complete RunManifest hashes and expose every fallback/persistence failure. - [ ] P2-05: remove every protected-test feedback path, introduce a test vault diff --git a/frontend/src/components/detection/DetectionLab.tsx b/frontend/src/components/detection/DetectionLab.tsx index b5e043e2..b135855d 100644 --- a/frontend/src/components/detection/DetectionLab.tsx +++ b/frontend/src/components/detection/DetectionLab.tsx @@ -21,11 +21,8 @@ const DETECTION_PAGE_SIZE_OPTIONS = [25, 50, 100] as const const DEFAULT_DETECTION_PAGE_SIZE = 50 function detectionQualityInterpretation(f1: number | null | undefined): string { - if (typeof f1 !== 'number' || !Number.isFinite(f1)) return 'Nog geen gevalideerde kwaliteitsmeting.' - if (f1 >= 0.85) return 'Sterk resultaat; steekproefcontrole blijft vereist.' - if (f1 >= 0.70) return 'Bruikbaar met gerichte handmatige controle.' - if (f1 >= 0.50) return 'Verkennend resultaat; beoordeel fouten voor operationeel gebruik.' - return 'Onvoldoende betrouwbaar voor operationeel gebruik.' + if (typeof f1 !== 'number' || !Number.isFinite(f1)) return 'Nog geen onafhankelijke kwaliteitsmeting beschikbaar.' + return `Historische F1 ${f1.toFixed(3)} is alleen kalibratiecontext. Een actuele, ruimtelijk onafhankelijke QA-run bepaalt of dit resultaat lokaal bruikbaar is.` } function detectionStatusLabel(status: string): string { @@ -311,8 +308,8 @@ export function DetectionLab({

{yoloRuntimeReady ? `${yoloPreflight?.runtime.cuda_available ? 'GPU' : 'CPU'} ยท lokaal model gevonden` : 'Controleer de modelconfiguratie onder beheer.'}

- Validatiescope - {selectedOperatorProfile ? `F1 ${selectedOperatorProfile.f1.toFixed(3)}` : 'Nog niet gekoppeld'} + Bewijsstatus + {selectedOperatorProfile ? `Historische F1 ${selectedOperatorProfile.f1.toFixed(3)}` : 'Nog niet gekoppeld'}

{selectedOperatorProfile ? selectedOperatorProfile.validationScope : 'Kies een modelprofiel met gedocumenteerd evaluatiebewijs.'}

0 ? 'ai-user-summary-card ai-user-summary-card-ready' : 'ai-user-summary-card'}> diff --git a/frontend/src/components/detection/DetectionModelManagement.tsx b/frontend/src/components/detection/DetectionModelManagement.tsx index d7c15d85..6e1c726a 100644 --- a/frontend/src/components/detection/DetectionModelManagement.tsx +++ b/frontend/src/components/detection/DetectionModelManagement.tsx @@ -141,7 +141,7 @@ export function DetectionModelManagement({ {selectedModelAsset ? 'model gekozen' : 'geen model gekozen'}
-
+
{DETECTION_OPERATOR_PROFILES.map((profile) => { const profileAsset = modelAssets.find((asset) => asset.model_asset_id === profile.modelAssetId) const profileSelected = @@ -154,18 +154,18 @@ export function DetectionModelManagement({ >
{profile.displayName} - - {profile.defaultApproved ? 'standaardprofiel' : 'kandidaat, extra controle vereist'} + + {profile.independentTestProven ? 'onafhankelijk getoetst' : 'historisch, geen releasebewijs'}

{profile.description}

drempel {profile.confidenceThreshold.toFixed(2)} - precisie {profile.precision.toFixed(3)} - herkenningsgraad {profile.recall.toFixed(3)} - F1 {profile.f1.toFixed(3)} - testgebieden {profile.positiveSampleCount} - max. achtergrondfouten {profile.maxBackgroundDetections} + historische precisie {profile.precision.toFixed(3)} + historische herkenningsgraad {profile.recall.toFixed(3)} + historische F1 {profile.f1.toFixed(3)} + positieve controles {profile.positiveSampleCount} + gemeten achtergrondfouten {profile.maxBackgroundDetections}

{profile.limitationMessage}