fix(results): gate operational AI exports on QA
This commit is contained in:
@@ -40,7 +40,12 @@ def export_geojson(payload: GeoJsonExportRequest, db: Session = Depends(get_db))
|
||||
)
|
||||
if payload.export_kind == "detection_run" and payload.analysis_run_id is not None:
|
||||
return envelope(
|
||||
ExportService.export_detection_run_geojson(db, payload.analysis_run_id, payload.name).model_dump(mode="json")
|
||||
ExportService.export_detection_run_geojson(
|
||||
db,
|
||||
payload.analysis_run_id,
|
||||
payload.name,
|
||||
intended_use=payload.intended_use,
|
||||
).model_dump(mode="json")
|
||||
)
|
||||
if payload.export_kind == "segmentation_run" and payload.analysis_run_id is not None:
|
||||
return envelope(
|
||||
|
||||
@@ -10,6 +10,7 @@ from app.schemas.operations import VectorSelectionBBox
|
||||
|
||||
|
||||
ExportKind = Literal["dataset", "detection_run", "segmentation_run", "vector_selection"]
|
||||
DetectionExportIntendedUse = Literal["review", "operational"]
|
||||
MapResultMode = Literal["current", "evolution"]
|
||||
|
||||
|
||||
@@ -21,6 +22,7 @@ class GeoJsonExportRequest(BaseModel):
|
||||
name: str | None = None
|
||||
bbox: VectorSelectionBBox | None = None
|
||||
limit: int = 250
|
||||
intended_use: DetectionExportIntendedUse = "review"
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_target(self) -> "GeoJsonExportRequest":
|
||||
@@ -33,6 +35,8 @@ class GeoJsonExportRequest(BaseModel):
|
||||
raise ValueError("bbox is required for vector selection GeoJSON exports")
|
||||
if self.export_kind in {"detection_run", "segmentation_run"} and self.analysis_run_id is None:
|
||||
raise ValueError("analysis_run_id is required for run GeoJSON exports")
|
||||
if self.intended_use == "operational" and self.export_kind != "detection_run":
|
||||
raise ValueError("operational intended_use is supported only for detection run exports")
|
||||
return self
|
||||
|
||||
|
||||
|
||||
@@ -35,6 +35,85 @@ from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
class ExportService:
|
||||
@staticmethod
|
||||
def _detection_export_trust(db: Session, run: AnalysisRun) -> dict[str, Any]:
|
||||
"""Classify persisted AI output without turning confidence into truth."""
|
||||
|
||||
checks = (
|
||||
db.query(QualityCheck)
|
||||
.filter(
|
||||
QualityCheck.analysis_run_id == run.id,
|
||||
QualityCheck.check_type == "detections_vs_reference",
|
||||
)
|
||||
.order_by(QualityCheck.created_at.desc())
|
||||
.all()
|
||||
)
|
||||
quality_check = checks[0] if checks else None
|
||||
reasons: list[str] = []
|
||||
reference_dataset = None
|
||||
if quality_check is None:
|
||||
reasons.append("authoritative_qa_missing")
|
||||
else:
|
||||
findings = quality_check.findings_json if isinstance(quality_check.findings_json, dict) else {}
|
||||
coverage = findings.get("coverage") if isinstance(findings.get("coverage"), dict) else {}
|
||||
temporal = findings.get("temporal_compatibility") if isinstance(findings.get("temporal_compatibility"), dict) else {}
|
||||
warnings = findings.get("warnings") if isinstance(findings.get("warnings"), list) else []
|
||||
if quality_check.status != "ok":
|
||||
reasons.append("quality_check_not_ok")
|
||||
if findings.get("unsupported_geometry") is True:
|
||||
reasons.append("unsupported_geometry")
|
||||
false_positives = findings.get("false_positives")
|
||||
false_negatives = findings.get("false_negatives")
|
||||
if (
|
||||
isinstance(false_positives, bool)
|
||||
or not isinstance(false_positives, (int, float))
|
||||
or false_positives != 0
|
||||
):
|
||||
reasons.append("false_positives_present")
|
||||
if (
|
||||
isinstance(false_negatives, bool)
|
||||
or not isinstance(false_negatives, (int, float))
|
||||
or false_negatives != 0
|
||||
):
|
||||
reasons.append("false_negatives_present")
|
||||
if warnings:
|
||||
reasons.append("quality_warnings_present")
|
||||
if coverage.get("applied") is not True:
|
||||
reasons.append("inference_coverage_not_proven")
|
||||
if temporal.get("status") != "compatible":
|
||||
reasons.append("temporal_compatibility_not_proven")
|
||||
reference_dataset = db.get(Dataset, quality_check.reference_dataset_id)
|
||||
if reference_dataset is None:
|
||||
reasons.append("reference_dataset_missing")
|
||||
else:
|
||||
try:
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
reference_dataset,
|
||||
purpose="reference_validation",
|
||||
reference_task="building_validation",
|
||||
)
|
||||
except AppError:
|
||||
reasons.append("reference_not_authoritative_for_buildings")
|
||||
|
||||
operational_use_allowed = not reasons
|
||||
return {
|
||||
"schema_version": "geointel.result-trust/v1",
|
||||
"classification": (
|
||||
"authoritative_reference_checked_ai_output"
|
||||
if operational_use_allowed
|
||||
else "unverified_ai_review_output"
|
||||
),
|
||||
"authoritative": False,
|
||||
"operational_use_allowed": operational_use_allowed,
|
||||
"operator_review_required": True,
|
||||
"quality_check_id": str(quality_check.id) if quality_check else None,
|
||||
"reference_dataset_id": str(reference_dataset.id) if reference_dataset else None,
|
||||
"blocking_reasons": sorted(set(reasons)),
|
||||
"limitation": (
|
||||
"AI output is not ground truth. Operational use is bounded to the exact source, AOI, model and reference QA evidence."
|
||||
),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _assert_run_source_dataset_exportable(db: Session, run: AnalysisRun) -> Dataset:
|
||||
"""Block an output export when its persisted source dataset is unsafe."""
|
||||
@@ -429,13 +508,33 @@ class ExportService:
|
||||
return ExportService._create_response(export)
|
||||
|
||||
@staticmethod
|
||||
def export_detection_run_geojson(db: Session, analysis_run_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse:
|
||||
def export_detection_run_geojson(
|
||||
db: Session,
|
||||
analysis_run_id: uuid.UUID,
|
||||
name: str | None = None,
|
||||
*,
|
||||
intended_use: str = "review",
|
||||
) -> ExportCreateResponse:
|
||||
run = db.get(AnalysisRun, analysis_run_id)
|
||||
if not run or run.analysis_type != "detection":
|
||||
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
|
||||
ExportService._assert_run_source_dataset_exportable(db, run)
|
||||
|
||||
trust = ExportService._detection_export_trust(db, run)
|
||||
if intended_use == "operational" and not trust["operational_use_allowed"]:
|
||||
raise AppError(
|
||||
code="DETECTION_OPERATIONAL_EXPORT_BLOCKED",
|
||||
message="Operational detection export requires complete authoritative QA with no remaining errors or warnings.",
|
||||
details=trust,
|
||||
status_code=409,
|
||||
)
|
||||
feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id)
|
||||
feature_collection["geointel_result"] = trust
|
||||
for feature in feature_collection.get("features", []):
|
||||
properties = feature.get("properties") if isinstance(feature, dict) else None
|
||||
if isinstance(properties, dict):
|
||||
properties["result_classification"] = trust["classification"]
|
||||
properties["authoritative"] = False
|
||||
filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson")
|
||||
export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename)
|
||||
metadata = {
|
||||
@@ -444,6 +543,8 @@ class ExportService:
|
||||
"project_id": str(run.project_id),
|
||||
"dataset_id": str(run.dataset_id) if run.dataset_id else None,
|
||||
"feature_count": len(feature_collection.get("features", [])),
|
||||
"intended_use": intended_use,
|
||||
"result_trust": trust,
|
||||
}
|
||||
export = ExportService._write_json_export(
|
||||
db,
|
||||
|
||||
@@ -4,11 +4,12 @@ import json
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, Export, Project, QualityCheck, SourceRegistry, SourceSnapshot
|
||||
from app.models import AnalysisRun, Area, Dataset, Export, Project, QualityCheck, SourceRegistry, SourceSnapshot
|
||||
from app.schemas.export import ExportCreateResponse
|
||||
from app.services.export_service import ExportService
|
||||
from app.services.storage_service import StorageService
|
||||
@@ -110,6 +111,143 @@ def _govern_fixture_dataset(dataset: Dataset) -> Dataset:
|
||||
return dataset
|
||||
|
||||
|
||||
def _authoritative_building_reference(dataset: Dataset) -> Dataset:
|
||||
dataset = _govern_fixture_dataset(dataset)
|
||||
dataset.dataset_role = "reference"
|
||||
dataset.source_registry.usage_policy_json = {
|
||||
"ground_truth_allowed": True,
|
||||
"validation_authority": {"building_validation": "primary"},
|
||||
}
|
||||
return dataset
|
||||
|
||||
|
||||
def test_detection_export_is_machine_labelled_as_unverified_review_output(tmp_path, monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
run_id = uuid4()
|
||||
source_dataset = _govern_fixture_dataset(
|
||||
Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="ortho.tif",
|
||||
dataset_type="raster",
|
||||
source="fixture",
|
||||
status="ready",
|
||||
)
|
||||
)
|
||||
run = AnalysisRun(
|
||||
id=run_id,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
analysis_type="detection",
|
||||
status="success",
|
||||
model_name="yolo-configured",
|
||||
)
|
||||
export_path = tmp_path / "detections-review.geojson"
|
||||
db = FakeSession({(Dataset, dataset_id): source_dataset, (AnalysisRun, run_id): run})
|
||||
monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path))
|
||||
|
||||
response = ExportService.export_detection_run_geojson(db, run_id, intended_use="review")
|
||||
|
||||
content = json.loads(export_path.read_text(encoding="utf-8"))
|
||||
trust = content["geointel_result"]
|
||||
assert response.metadata_json["intended_use"] == "review"
|
||||
assert trust["classification"] == "unverified_ai_review_output"
|
||||
assert trust["authoritative"] is False
|
||||
assert trust["operational_use_allowed"] is False
|
||||
assert trust["blocking_reasons"] == ["authoritative_qa_missing"]
|
||||
|
||||
|
||||
def test_detection_operational_export_fails_closed_without_authoritative_qa(tmp_path, monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
run_id = uuid4()
|
||||
source_dataset = _govern_fixture_dataset(
|
||||
Dataset(id=dataset_id, project_id=project_id, name="ortho.tif", dataset_type="raster", source="fixture", status="ready")
|
||||
)
|
||||
run = AnalysisRun(
|
||||
id=run_id,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
analysis_type="detection",
|
||||
status="success",
|
||||
model_name="yolo-configured",
|
||||
)
|
||||
db = FakeSession({(Dataset, dataset_id): source_dataset, (AnalysisRun, run_id): run})
|
||||
monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(tmp_path / "blocked.geojson"))
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
ExportService.export_detection_run_geojson(db, run_id, intended_use="operational")
|
||||
|
||||
assert exc_info.value.code == "DETECTION_OPERATIONAL_EXPORT_BLOCKED"
|
||||
|
||||
|
||||
def test_detection_operational_export_requires_zero_error_authoritative_qa(tmp_path, monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
reference_id = uuid4()
|
||||
run_id = uuid4()
|
||||
check_id = uuid4()
|
||||
source_dataset = _govern_fixture_dataset(
|
||||
Dataset(id=dataset_id, project_id=project_id, name="ortho.tif", dataset_type="raster", source="fixture", status="ready")
|
||||
)
|
||||
reference = _authoritative_building_reference(
|
||||
Dataset(
|
||||
id=reference_id,
|
||||
project_id=project_id,
|
||||
name="grb.geojson",
|
||||
dataset_type="vector",
|
||||
source="fixture",
|
||||
dataset_role="reference",
|
||||
status="ready",
|
||||
)
|
||||
)
|
||||
run = AnalysisRun(
|
||||
id=run_id,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
analysis_type="detection",
|
||||
status="success",
|
||||
model_name="yolo-configured",
|
||||
)
|
||||
check = QualityCheck(
|
||||
id=check_id,
|
||||
project_id=project_id,
|
||||
analysis_run_id=run_id,
|
||||
candidate_dataset_id=dataset_id,
|
||||
reference_dataset_id=reference_id,
|
||||
check_type="detections_vs_reference",
|
||||
status="ok",
|
||||
findings_json={
|
||||
"false_positives": 0,
|
||||
"false_negatives": 0,
|
||||
"warnings": [],
|
||||
"unsupported_geometry": False,
|
||||
"coverage": {"applied": True},
|
||||
"temporal_compatibility": {"status": "compatible"},
|
||||
},
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
export_path = tmp_path / "detections-operational.geojson"
|
||||
db = FakeSession(
|
||||
{
|
||||
(Dataset, dataset_id): source_dataset,
|
||||
(Dataset, reference_id): reference,
|
||||
(AnalysisRun, run_id): run,
|
||||
(QualityCheck, check_id): check,
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path))
|
||||
|
||||
response = ExportService.export_detection_run_geojson(db, run_id, intended_use="operational")
|
||||
|
||||
trust = json.loads(export_path.read_text(encoding="utf-8"))["geointel_result"]
|
||||
assert response.metadata_json["intended_use"] == "operational"
|
||||
assert trust["operational_use_allowed"] is True
|
||||
assert trust["quality_check_id"] == str(check_id)
|
||||
assert trust["reference_dataset_id"] == str(reference_id)
|
||||
|
||||
|
||||
def test_dataset_geojson_export_persists_export_and_writes_artifact(tmp_path, monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
|
||||
+28
-1
@@ -2220,10 +2220,34 @@ Detection run export request:
|
||||
{
|
||||
"export_kind": "detection_run",
|
||||
"analysis_run_id": "uuid",
|
||||
"intended_use": "review",
|
||||
"name": "optional-basename"
|
||||
}
|
||||
```
|
||||
|
||||
`intended_use` is `review` by default. Every detection FeatureCollection and
|
||||
export record contains a machine-readable `geointel_result` / `result_trust`
|
||||
contract. Raw model output remains `authoritative: false` and is classified as
|
||||
`unverified_ai_review_output` when authoritative QA is absent or incomplete.
|
||||
Feature properties repeat the classification so it survives GIS workflows
|
||||
that discard collection-level metadata.
|
||||
|
||||
`intended_use: operational` fails closed with
|
||||
`DETECTION_OPERATIONAL_EXPORT_BLOCKED` unless the same persisted run has a
|
||||
completed detection-versus-reference check that proves all of the following:
|
||||
|
||||
- the reference Dataset is authoritative and specifically approved as primary
|
||||
authority for building validation;
|
||||
- inference coverage was derived from the persisted tile-manifest union;
|
||||
- imagery/reference time compatibility is proven;
|
||||
- geometry is supported and no QA warnings remain;
|
||||
- false-positive and false-negative counts are both explicitly present and
|
||||
zero.
|
||||
|
||||
Even a passing operational export remains AI-derived and therefore retains
|
||||
`authoritative: false`, the exact quality-check/reference identifiers and an
|
||||
operator-review limitation. Model confidence by itself never unlocks export.
|
||||
|
||||
Segmentation run export request:
|
||||
|
||||
```json
|
||||
@@ -2251,7 +2275,10 @@ Response persists an `exports` row and writes a deterministic JSON artifact:
|
||||
|
||||
Vector dataset exports use the stored dataset GeoJSON. Detection and
|
||||
segmentation exports use persisted first-class geometry records and the
|
||||
existing Detection/Segmentation GeoJSON conversion services. Vector selection
|
||||
existing Detection/Segmentation GeoJSON conversion services. Detection export
|
||||
in the frontend is deliberately labelled as a control layer and requests
|
||||
`intended_use: review`; it cannot silently produce an operationally approved
|
||||
artifact. Vector selection
|
||||
exports query persisted PostGIS `vector_features` with the supplied EPSG:4326
|
||||
bbox, write the selected FeatureCollection as a `vector_selection_geojson`
|
||||
artifact, and persist bbox/feature-count metadata in the export record. When
|
||||
|
||||
@@ -12472,3 +12472,29 @@ Open:
|
||||
product benchmark remain required before any production-accuracy claim.
|
||||
- Every deployed model asset needs its own generated scope manifest and exact
|
||||
configured manifest checksum before enforced inference is available.
|
||||
|
||||
## 2026-08-09 - Detection result trust and operational export gate
|
||||
|
||||
### Changed
|
||||
|
||||
- Detection GeoJSON is now explicitly a review/control layer by default. The
|
||||
collection, every feature and the persisted Export metadata carry a stable
|
||||
result-trust classification; AI confidence is never represented as truth.
|
||||
- Added an explicit `operational` export intent that fails closed unless a
|
||||
persisted authoritative building-reference QA proves exact tile coverage,
|
||||
compatible time, supported geometry, zero warnings, zero false positives
|
||||
and zero false negatives.
|
||||
- Updated the Downloads copy so users no longer see an unqualified
|
||||
"Gebouwdetecties" export label.
|
||||
|
||||
### Verified
|
||||
|
||||
- Focused export and detection-QA suite: 24 passed.
|
||||
- Scoped Ruff check: passed.
|
||||
- Frontend TypeScript check: passed.
|
||||
|
||||
### Remaining limitations
|
||||
|
||||
- The active model remains review-required and Kempen-scoped. The export gate
|
||||
prevents overstated results; it does not replace the missing independent,
|
||||
human-reviewed national accuracy corpus or improve model weights by itself.
|
||||
|
||||
+3
-1
@@ -1120,7 +1120,9 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
with a model/checksum-bound geometry manifest; equivalent legal-scope checks
|
||||
still require the same review.
|
||||
- [ ] P2-04: make derived persistence transactional, require complete
|
||||
RunManifest hashes and expose every fallback/persistence failure.
|
||||
RunManifest hashes and expose every fallback/persistence failure. Detection
|
||||
exports now carry a machine-readable trust contract and fail closed for
|
||||
operational use without zero-error authoritative QA.
|
||||
- [ ] P2-05: remove every protected-test feedback path, introduce a test vault
|
||||
and make the sampler reject protected IDs, paths and assessment fields.
|
||||
- [ ] P2-06: complete representative human V56 review; add independent AOIs,
|
||||
|
||||
@@ -49,7 +49,7 @@ function formatExportType(value: string): string {
|
||||
map_analysis_json: 'Gebiedsanalyse (JSON)',
|
||||
map_evolution_json: 'Historische vergelijking (JSON)',
|
||||
dataset_geojson: 'Kaartlaag (GeoJSON)',
|
||||
detection_geojson: 'Gebouwdetecties (GeoJSON)',
|
||||
detection_geojson: 'Gebouwdetecties - controlelaag (GeoJSON)',
|
||||
segmentation_geojson: 'Segmentaties (GeoJSON)',
|
||||
project_metadata_json: 'Werkruimtedata (JSON)',
|
||||
project_report_html: 'Projectrapport (HTML)',
|
||||
@@ -155,7 +155,7 @@ export function ExportCenter({
|
||||
},
|
||||
{
|
||||
key: 'detection_geojson',
|
||||
label: 'Herkende gebouwen (GeoJSON)',
|
||||
label: 'Herkende gebouwen - controlelaag (GeoJSON)',
|
||||
detail: 'Bewaarde geometrieën uit beeldanalyse',
|
||||
item: getLatestExportByType(exports, 'detection_geojson'),
|
||||
},
|
||||
@@ -337,7 +337,7 @@ export function ExportCenter({
|
||||
</div>
|
||||
{selectedDetectionRunId ? <div className="handoff-action-card">
|
||||
<span>Gebouwanalyse</span>
|
||||
<strong>Herkende gebouwen (GeoJSON)</strong>
|
||||
<strong>Herkende gebouwen - controlelaag (GeoJSON)</strong>
|
||||
<p>Bewaar de persistente detectiegeometrie van de geselecteerde analyserun.</p>
|
||||
<button type="button" className="secondary-action" onClick={onExportDetectionRun} disabled={!selectedDetectionRunId || exporting}>
|
||||
Gebouwanalyse bewaren
|
||||
|
||||
@@ -84,6 +84,7 @@ export function useExportWorkflow({
|
||||
const response = await exportsApi.exportGeojson(selectedProjectId, {
|
||||
analysis_run_id: selectedDetectionRunId,
|
||||
export_kind: 'detection_run',
|
||||
intended_use: 'review',
|
||||
})
|
||||
setLatestExport(response)
|
||||
await loadExports(selectedProjectId)
|
||||
|
||||
@@ -21,6 +21,7 @@ export const exportsApi = {
|
||||
name?: string
|
||||
bbox?: VectorSelectionBBox
|
||||
limit?: number
|
||||
intended_use?: 'review' | 'operational'
|
||||
}
|
||||
| string,
|
||||
): Promise<ExportCreateResponse> => {
|
||||
|
||||
Reference in New Issue
Block a user