From 9146981802149d09fe0228b28b99de5ae9e88a31 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 18 Jul 2026 00:53:00 +0200 Subject: [PATCH] Complete temporal detection safety gate --- CHANGELOG.md | 15 ++ backend/app/core/request_context.py | 18 +++ backend/app/main.py | 26 ++- backend/app/services/detection_service.py | 37 +++++ .../temporal_compatibility_service.py | 153 ++++++++++++++++++ .../test_rc_detection_temporal_safety.py | 137 ++++++++++++++++ .../tests/test_rc_runtime_observability.py | 31 ++++ .../test_sprint8_detection_foundation.py | 19 +++ ...est_sprint8c_detection_visualization_qa.py | 68 +++++++- docs/CODEX_EXECUTION_LOG.md | 54 +++++++ docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md | 24 ++- docs/ROLLBACK_AND_RECOVERY.md | 7 + docs/TODO.md | 6 +- frontend/src/hooks/useDetectionWorkflow.ts | 14 +- scripts/README.md | 19 +++ scripts/runtime_state_report.py | 84 ++++++++++ 16 files changed, 689 insertions(+), 23 deletions(-) create mode 100644 backend/app/core/request_context.py create mode 100644 backend/app/services/temporal_compatibility_service.py create mode 100644 backend/tests/test_rc_detection_temporal_safety.py create mode 100644 backend/tests/test_rc_runtime_observability.py create mode 100644 scripts/runtime_state_report.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d9a20f7..d7e7133d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,21 @@ - Made system capabilities report real PostGIS and configured local YOLO state, added request IDs and exception logging, reduced SQL engine logging, and terminalized impossible orphaned work after all-in-one restarts. +- Added non-logging PostgreSQL credential rotation with atomic `.env` update, + role update, managed-container restart and health verification. +- Completed a current secure Tower backup and isolated restore drill with + PostGIS, Alembic and critical table-count reconciliation. +- Removed every implicit first-raster fallback from Detection Lab; raster + selection is now an explicit operator decision in normal runs, guided runs + and calibration. +- Added fail-closed detection source and QA temporal compatibility checks. + Historical orthophotos marked unsupported cannot run through the current + model, and historical QA requires an overlapping reference validity period. +- Persisted the temporal compatibility decision in existing QualityCheck + parameters/findings and added correlated request, job, analysis-run and + quality-check logging. +- Added a read-only stale-runtime report plus an explicitly confirmed + reconciliation mode. ## Sprint 240 Operational forest, agriculture, nature and soil themes (2026-07-17) diff --git a/backend/app/core/request_context.py b/backend/app/core/request_context.py new file mode 100644 index 00000000..20546aae --- /dev/null +++ b/backend/app/core/request_context.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from contextvars import ContextVar, Token + + +_request_id: ContextVar[str] = ContextVar("geointel_request_id", default="-") + + +def get_request_id() -> str: + return _request_id.get() + + +def set_request_id(value: str) -> Token: + return _request_id.set(value) + + +def reset_request_id(token: Token) -> None: + _request_id.reset(token) diff --git a/backend/app/main.py b/backend/app/main.py index d5902ecb..a977a167 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,6 +1,8 @@ from __future__ import annotations import logging +import re +import time import uuid from contextlib import asynccontextmanager @@ -13,11 +15,13 @@ from app.api.routes import analysis, areas, assistant, datasets, demo, detection from app.core.config import get_settings from app.core.errors import AppError from app.core.logging import configure_logging +from app.core.request_context import reset_request_id, set_request_id from app.db.session import SessionLocal from app.services.runtime_reconciliation_service import RuntimeReconciliationService logger = logging.getLogger("geointel") +SAFE_REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") def _to_error_payload( @@ -91,11 +95,25 @@ def create_app() -> FastAPI: @app.middleware("http") async def request_identity(request: Request, call_next): - request_id = request.headers.get("x-request-id") or str(uuid.uuid4()) + supplied_request_id = request.headers.get("x-request-id", "") + request_id = supplied_request_id if SAFE_REQUEST_ID.fullmatch(supplied_request_id) else str(uuid.uuid4()) request.state.request_id = request_id - response = await call_next(request) - response.headers["x-request-id"] = request_id - return response + token = set_request_id(request_id) + started_at = time.perf_counter() + try: + response = await call_next(request) + response.headers["x-request-id"] = request_id + logger.info( + "request_complete request_id=%s method=%s path=%s status=%s duration_ms=%.1f", + request_id, + request.method, + request.url.path, + response.status_code, + (time.perf_counter() - started_at) * 1000, + ) + return response + finally: + reset_request_id(token) @app.exception_handler(AppError) async def app_error(request: Request, exc: AppError): # noqa: ARG001 diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py index 7c21926b..5ebc8fd9 100644 --- a/backend/app/services/detection_service.py +++ b/backend/app/services/detection_service.py @@ -2,6 +2,7 @@ from __future__ import annotations import uuid import json +import logging from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -13,6 +14,7 @@ from sqlalchemy import func from app.core.config import Settings, get_settings from app.core.errors import AppError +from app.core.request_context import get_request_id from app.models import AnalysisRun, Dataset, Detection, Job, Project, VectorFeature from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon @@ -21,9 +23,13 @@ from app.services.model_asset_catalog_service import ModelAssetCatalogService from app.services.model_registry_service import ModelRegistryService from app.services.qa_service import QaService from app.services.quality_service import QualityService +from app.services.temporal_compatibility_service import TemporalCompatibilityService from app.services.yolo_adapter import YoloDetectionAdapter +logger = logging.getLogger("geointel.detection") + + class DetectionService: @staticmethod def _now() -> datetime: @@ -58,6 +64,7 @@ class DetectionService: details={"dataset_type": dataset.dataset_type}, status_code=400, ) + TemporalCompatibilityService.ensure_detection_source_supported(dataset) selected_model_asset = None if model_id == resolved_settings.yolo_model_id and model_asset_id: @@ -96,6 +103,15 @@ class DetectionService: } job = DetectionService._create_job(db, project_id, dataset_id, run_parameters) analysis_run = DetectionService._create_analysis_run(db, project_id, dataset_id, job.id, model, run_parameters) + logger.info( + "detection_started request_id=%s project_id=%s dataset_id=%s job_id=%s analysis_run_id=%s model_id=%s", + get_request_id(), + project_id, + dataset_id, + job.id, + analysis_run.id, + model.model_id, + ) if not model.configured: message = model.limitation_message @@ -281,6 +297,13 @@ class DetectionService: raise AppError(code="INVALID_DATASET_SCOPE", message="Reference dataset does not belong to detection project", status_code=400) if reference_dataset.dataset_type not in {"vector", "geojson"}: raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400) + candidate_dataset = db.get(Dataset, run.dataset_id) + if not candidate_dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Detection source dataset not found", status_code=404) + temporal_compatibility = TemporalCompatibilityService.assess_detection_qa( + candidate_dataset, + reference_dataset, + ) detections = DetectionService._query_detection_rows( db, @@ -421,6 +444,7 @@ class DetectionService: "class_name": class_name, "min_confidence": min_confidence, "coverage_policy": coverage_summary["mode"], + "temporal_compatibility": temporal_compatibility, }, findings={ "matches": evidence.matches, @@ -429,6 +453,7 @@ class DetectionService: "warnings": coverage_warnings + evidence.warnings, "unsupported_geometry": evidence.unsupported, "coverage": coverage_summary, + "temporal_compatibility": temporal_compatibility, "box_to_footprint_diagnostics": box_to_footprint_diagnostics, "match_evidence": evidence.match_evidence, "false_positive_evidence": evidence.false_positive_evidence, @@ -443,6 +468,17 @@ class DetectionService: "false_negative_count": evidence.false_negatives, }, ) + logger.info( + "detection_qa_completed request_id=%s job_id=%s analysis_run_id=%s quality_check_id=%s " + "candidate_dataset_id=%s reference_dataset_id=%s status=%s", + get_request_id(), + run.job_id, + analysis_run_id, + quality_check.id, + run.dataset_id, + reference_dataset_id, + status, + ) return { "status": status, "quality_check_id": str(quality_check.id), @@ -462,6 +498,7 @@ class DetectionService: "iou_threshold": iou_threshold, "warnings": coverage_warnings + evidence.warnings, "coverage": coverage_summary, + "temporal_compatibility": temporal_compatibility, "box_to_footprint_diagnostics": box_to_footprint_diagnostics, "match_evidence": evidence.match_evidence, "false_positive_evidence": evidence.false_positive_evidence, diff --git a/backend/app/services/temporal_compatibility_service.py b/backend/app/services/temporal_compatibility_service.py new file mode 100644 index 00000000..6a628302 --- /dev/null +++ b/backend/app/services/temporal_compatibility_service.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any + +from app.core.errors import AppError +from app.models import Dataset + + +@dataclass(frozen=True) +class TemporalInterval: + start: datetime | None + end: datetime | None + granularity: str | None + + @property + def bounded(self) -> bool: + return self.start is not None and self.end is not None + + def as_dict(self) -> dict[str, Any]: + return { + "start": self.start.isoformat() if self.start else None, + "end": self.end.isoformat() if self.end else None, + "granularity": self.granularity, + } + + +class TemporalCompatibilityService: + @staticmethod + def ensure_detection_source_supported(dataset: Dataset) -> None: + metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {} + if metadata.get("supports_detection") is False: + raise AppError( + code="DETECTION_SOURCE_TEMPORALLY_UNSUPPORTED", + message="The selected raster edition is not approved for the configured detection model", + details={ + "dataset_id": str(dataset.id), + "source_name": dataset.source_name, + "product_key": metadata.get("product_key"), + "observed_at": TemporalCompatibilityService._iso(dataset.observed_at), + "valid_from": TemporalCompatibilityService._iso(dataset.valid_from), + "valid_to": TemporalCompatibilityService._iso(dataset.valid_to), + }, + status_code=422, + ) + + @staticmethod + def assess_detection_qa(candidate: Dataset, reference: Dataset) -> dict[str, Any]: + candidate_interval = TemporalCompatibilityService._interval(candidate) + reference_interval = TemporalCompatibilityService._interval(reference) + candidate_historical = TemporalCompatibilityService._is_historical_detection_source(candidate) + + if candidate_historical: + if not reference_interval.bounded: + TemporalCompatibilityService._raise_mismatch( + candidate, + reference, + candidate_interval, + reference_interval, + "Historical imagery requires a reference dataset with an explicit compatible validity period.", + ) + if not TemporalCompatibilityService._overlaps(candidate_interval, reference_interval): + TemporalCompatibilityService._raise_mismatch( + candidate, + reference, + candidate_interval, + reference_interval, + "The historical imagery and reference dataset validity periods do not overlap.", + ) + + if ( + candidate_interval.bounded + and reference_interval.bounded + and not TemporalCompatibilityService._overlaps(candidate_interval, reference_interval) + ): + TemporalCompatibilityService._raise_mismatch( + candidate, + reference, + candidate_interval, + reference_interval, + "The candidate and reference dataset validity periods do not overlap.", + ) + + return { + "status": "compatible", + "policy": "explicit_interval_overlap_for_historical_sources", + "candidate_dataset_id": str(candidate.id), + "reference_dataset_id": str(reference.id), + "candidate_historical": candidate_historical, + "candidate_interval": candidate_interval.as_dict(), + "reference_interval": reference_interval.as_dict(), + } + + @staticmethod + def _is_historical_detection_source(dataset: Dataset) -> bool: + metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {} + if metadata.get("supports_detection") is False: + return True + product_key = str(metadata.get("product_key") or "").strip().lower() + return dataset.source_name == "digitaal_vlaanderen_orthophoto" and product_key not in {"", "most_recent"} + + @staticmethod + def _interval(dataset: Dataset) -> TemporalInterval: + start = TemporalCompatibilityService._utc(dataset.valid_from or dataset.observed_at) + end = TemporalCompatibilityService._utc(dataset.valid_to) + granularity = dataset.temporal_granularity + + if start is not None and end is None and granularity == "year": + end = datetime(start.year, 12, 31, 23, 59, 59, tzinfo=UTC) + elif start is not None and end is None and granularity == "day": + end = start.replace(hour=23, minute=59, second=59, microsecond=999999) + + return TemporalInterval(start=start, end=end, granularity=granularity) + + @staticmethod + def _overlaps(left: TemporalInterval, right: TemporalInterval) -> bool: + if not left.bounded or not right.bounded: + return True + return left.start <= right.end and right.start <= left.end + + @staticmethod + def _raise_mismatch( + candidate: Dataset, + reference: Dataset, + candidate_interval: TemporalInterval, + reference_interval: TemporalInterval, + message: str, + ) -> None: + raise AppError( + code="DETECTION_QA_TEMPORAL_MISMATCH", + message=message, + details={ + "candidate_dataset_id": str(candidate.id), + "reference_dataset_id": str(reference.id), + "candidate_interval": candidate_interval.as_dict(), + "reference_interval": reference_interval.as_dict(), + }, + status_code=422, + ) + + @staticmethod + def _utc(value: datetime | None) -> datetime | None: + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + @staticmethod + def _iso(value: datetime | None) -> str | None: + normalized = TemporalCompatibilityService._utc(value) + return normalized.isoformat() if normalized else None diff --git a/backend/tests/test_rc_detection_temporal_safety.py b/backend/tests/test_rc_detection_temporal_safety.py new file mode 100644 index 00000000..e5b4c6e0 --- /dev/null +++ b/backend/tests/test_rc_detection_temporal_safety.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from uuid import uuid4 + +import pytest + +from app.core.errors import AppError +from app.models import Dataset +from app.services.temporal_compatibility_service import TemporalCompatibilityService + + +ROOT = Path(__file__).parents[2] + + +def dataset( + *, + dataset_type: str, + source_name: str, + observed_at: datetime | None = None, + valid_from: datetime | None = None, + valid_to: datetime | None = None, + temporal_granularity: str | None = None, + source_metadata: dict | None = None, +) -> Dataset: + return Dataset( + id=uuid4(), + project_id=uuid4(), + name="temporal-source", + dataset_type=dataset_type, + source=source_name, + source_name=source_name, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + temporal_granularity=temporal_granularity, + source_metadata=source_metadata, + ) + + +def test_historical_orthophoto_is_rejected_for_detection() -> None: + historical = dataset( + dataset_type="raster", + source_name="digitaal_vlaanderen_orthophoto", + observed_at=datetime(2020, 1, 1, tzinfo=UTC), + valid_from=datetime(2020, 1, 1, tzinfo=UTC), + valid_to=datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC), + temporal_granularity="year", + source_metadata={"product_key": "2020", "supports_detection": False}, + ) + + with pytest.raises(AppError) as exc_info: + TemporalCompatibilityService.ensure_detection_source_supported(historical) + + assert exc_info.value.code == "DETECTION_SOURCE_TEMPORALLY_UNSUPPORTED" + assert exc_info.value.status_code == 422 + + +def test_historical_detection_qa_rejects_current_reference() -> None: + historical = dataset( + dataset_type="raster", + source_name="digitaal_vlaanderen_orthophoto", + valid_from=datetime(2020, 1, 1, tzinfo=UTC), + valid_to=datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC), + temporal_granularity="year", + source_metadata={"product_key": "2020", "supports_detection": False}, + ) + current_reference = dataset( + dataset_type="vector", + source_name="grb", + valid_from=datetime(2026, 7, 1, tzinfo=UTC), + valid_to=datetime(2026, 7, 31, 23, 59, 59, tzinfo=UTC), + temporal_granularity="month", + ) + + with pytest.raises(AppError) as exc_info: + TemporalCompatibilityService.assess_detection_qa(historical, current_reference) + + assert exc_info.value.code == "DETECTION_QA_TEMPORAL_MISMATCH" + assert exc_info.value.status_code == 422 + + +def test_historical_detection_qa_accepts_overlapping_reference_edition() -> None: + historical = dataset( + dataset_type="raster", + source_name="digitaal_vlaanderen_orthophoto", + valid_from=datetime(2020, 1, 1, tzinfo=UTC), + valid_to=datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC), + temporal_granularity="year", + source_metadata={"product_key": "2020", "supports_detection": False}, + ) + historical_reference = dataset( + dataset_type="vector", + source_name="manual", + valid_from=datetime(2020, 6, 1, tzinfo=UTC), + valid_to=datetime(2020, 6, 30, 23, 59, 59, tzinfo=UTC), + temporal_granularity="month", + ) + + result = TemporalCompatibilityService.assess_detection_qa(historical, historical_reference) + + assert result["status"] == "compatible" + assert result["candidate_historical"] is True + assert result["candidate_interval"]["start"].startswith("2020-01-01") + assert result["reference_interval"]["start"].startswith("2020-06-01") + + +def test_current_source_with_unbounded_current_reference_remains_supported() -> None: + current = dataset( + dataset_type="raster", + source_name="digitaal_vlaanderen_orthophoto", + observed_at=datetime(2026, 7, 17, tzinfo=UTC), + valid_from=datetime(2026, 7, 17, tzinfo=UTC), + temporal_granularity="snapshot", + source_metadata={"product_key": "most_recent", "supports_detection": True}, + ) + current_reference = dataset( + dataset_type="vector", + source_name="grb", + observed_at=datetime(2026, 7, 16, tzinfo=UTC), + temporal_granularity="snapshot", + ) + + TemporalCompatibilityService.ensure_detection_source_supported(current) + result = TemporalCompatibilityService.assess_detection_qa(current, current_reference) + + assert result["status"] == "compatible" + assert result["candidate_historical"] is False + + +def test_detection_frontend_has_no_implicit_first_raster_fallback() -> None: + source = (ROOT / "frontend" / "src" / "hooks" / "useDetectionWorkflow.ts").read_text(encoding="utf-8") + + assert "rasterDatasets[0]" not in source + assert "setSelectedDetectionDatasetId(rasterDatasets" not in source + assert "const datasetId = selectedDetectionDatasetId" in source diff --git a/backend/tests/test_rc_runtime_observability.py b/backend/tests/test_rc_runtime_observability.py new file mode 100644 index 00000000..a3e5004b --- /dev/null +++ b/backend/tests/test_rc_runtime_observability.py @@ -0,0 +1,31 @@ +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.main import app + + +ROOT = Path(__file__).parents[2] + + +def test_valid_request_id_is_returned() -> None: + response = TestClient(app).get("/health/live", headers={"x-request-id": "rc3-check.123"}) + + assert response.status_code == 200 + assert response.headers["x-request-id"] == "rc3-check.123" + + +def test_unsafe_request_id_is_replaced() -> None: + response = TestClient(app).get("/health/live", headers={"x-request-id": "unsafe request/id"}) + + assert response.status_code == 200 + assert response.headers["x-request-id"] != "unsafe request/id" + assert " " not in response.headers["x-request-id"] + + +def test_runtime_report_is_read_only_by_default_and_requires_confirmation() -> None: + source = (ROOT / "scripts" / "runtime_state_report.py").read_text(encoding="utf-8") + + assert '"mode": "read_only"' in source + assert "if args.reconcile and args.confirm != RECONCILE_CONFIRMATION" in source + assert "RuntimeReconciliationService.reconcile(db)" in source diff --git a/backend/tests/test_sprint8_detection_foundation.py b/backend/tests/test_sprint8_detection_foundation.py index 15fe3180..4e144904 100644 --- a/backend/tests/test_sprint8_detection_foundation.py +++ b/backend/tests/test_sprint8_detection_foundation.py @@ -105,6 +105,25 @@ def test_non_raster_dataset_request_is_rejected() -> None: assert getattr(exc_info.value, "code", None) == "INVALID_DATASET_TYPE" +def test_historical_raster_marked_unsupported_is_rejected_before_run_creation() -> None: + db, project_id, dataset_id = _project_and_dataset() + source = db.get(Dataset, dataset_id) + source.source_name = "digitaal_vlaanderen_orthophoto" + source.source_metadata = {"product_key": "2020", "supports_detection": False} + + with pytest.raises(Exception) as exc_info: + DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-placeholder", + confidence_threshold=0.5, + ) + + assert getattr(exc_info.value, "code", None) == "DETECTION_SOURCE_TEMPORALLY_UNSUPPORTED" + assert db.added == [] + + def test_fixture_detector_persists_detections_only_with_explicit_fixture_mode() -> None: db, project_id, dataset_id = _project_and_dataset() diff --git a/backend/tests/test_sprint8c_detection_visualization_qa.py b/backend/tests/test_sprint8c_detection_visualization_qa.py index 19b44e82..7022395a 100644 --- a/backend/tests/test_sprint8c_detection_visualization_qa.py +++ b/backend/tests/test_sprint8c_detection_visualization_qa.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from datetime import UTC, datetime from uuid import uuid4 import pytest @@ -11,7 +12,7 @@ from shapely.geometry import Polygon, box from app.core.errors import AppError from app.main import app from app.db.session import get_db -from app.models import AnalysisRun, Dataset, Detection, Job, Metric, Project, QualityCheck, VectorFeature +from app.models import AnalysisRun, Dataset, Detection, Metric, QualityCheck, VectorFeature from app.services.detection_service import DetectionService @@ -89,6 +90,17 @@ def _detection(project_id, dataset_id, analysis_run_id, class_name="building", c ) +def _source_dataset(project_id, dataset_id): + return Dataset( + id=dataset_id, + project_id=project_id, + name="source.tif", + dataset_type="raster", + source="manual", + source_name="manual", + ) + + def test_detection_geojson_feature_collection_shape() -> None: project_id = uuid4() dataset_id = uuid4() @@ -206,6 +218,7 @@ def test_detection_qa_persists_quality_check_and_metrics() -> None: db = FakeSession( objects={ (AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="detection", status="success", parameters_json={}), + (Dataset, dataset_id): _source_dataset(project_id, dataset_id), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={Detection: [detection], VectorFeature: [reference_feature]}, @@ -227,6 +240,8 @@ def test_detection_qa_persists_quality_check_and_metrics() -> None: assert result["quality_check_id"] == str(quality_checks[0].id) assert quality_checks[0].analysis_run_id == analysis_run_id assert quality_checks[0].reference_dataset_id == reference_dataset_id + assert quality_checks[0].parameters_json["temporal_compatibility"]["status"] == "compatible" + assert quality_checks[0].findings_json["temporal_compatibility"]["status"] == "compatible" assert [metric.metric_key for metric in metrics] == [ "precision", "recall", @@ -237,6 +252,53 @@ def test_detection_qa_persists_quality_check_and_metrics() -> None: ] +def test_detection_qa_rejects_non_overlapping_historical_reference_editions() -> None: + project_id = uuid4() + dataset_id = uuid4() + reference_dataset_id = uuid4() + analysis_run_id = uuid4() + source_dataset = _source_dataset(project_id, dataset_id) + source_dataset.source_name = "digitaal_vlaanderen_orthophoto" + source_dataset.source_metadata = {"product_key": "2020", "supports_detection": False} + source_dataset.valid_from = datetime(2020, 1, 1, tzinfo=UTC) + source_dataset.valid_to = datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC) + reference_dataset = Dataset( + id=reference_dataset_id, + project_id=project_id, + name="current-grb.geojson", + dataset_type="vector", + source="grb", + source_name="grb", + dataset_role="reference", + valid_from=datetime(2026, 7, 1, tzinfo=UTC), + valid_to=datetime(2026, 7, 31, 23, 59, 59, tzinfo=UTC), + ) + db = FakeSession( + objects={ + (AnalysisRun, analysis_run_id): AnalysisRun( + id=analysis_run_id, + project_id=project_id, + dataset_id=dataset_id, + analysis_type="detection", + status="success", + parameters_json={}, + ), + (Dataset, dataset_id): source_dataset, + (Dataset, reference_dataset_id): reference_dataset, + }, + ) + + with pytest.raises(AppError) as exc_info: + DetectionService.compare_detections_with_reference( + db=db, + analysis_run_id=analysis_run_id, + reference_dataset_id=reference_dataset_id, + ) + + assert exc_info.value.code == "DETECTION_QA_TEMPORAL_MISMATCH" + assert db.added == [] + + def test_detection_qa_no_match_case_persists_zero_scores() -> None: project_id = uuid4() dataset_id = uuid4() @@ -260,6 +322,7 @@ def test_detection_qa_no_match_case_persists_zero_scores() -> None: db = FakeSession( objects={ (AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="detection", status="success", parameters_json={}), + (Dataset, dataset_id): _source_dataset(project_id, dataset_id), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={Detection: [detection], VectorFeature: [reference_feature]}, @@ -311,6 +374,7 @@ def test_configured_yolo_qa_requires_persisted_tile_manifest_provenance() -> Non model_name="yolo-configured", parameters_json={"model_id": "yolo-configured"}, ), + (Dataset, dataset_id): _source_dataset(project_id, dataset_id), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={Detection: [detection], VectorFeature: [reference_feature]}, @@ -391,6 +455,7 @@ def test_detection_qa_excludes_references_outside_persisted_tile_coverage(tmp_pa "tile_manifest_path": str(manifest_path), }, ), + (Dataset, dataset_id): _source_dataset(project_id, dataset_id), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={Detection: [detection], VectorFeature: [inside_reference, outside_reference]}, @@ -452,6 +517,7 @@ def test_detection_qa_reports_box_to_footprint_diagnostic_without_changing_stric "tile_manifest_path": str(manifest_path), }, ), + (Dataset, dataset_id): _source_dataset(project_id, dataset_id), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={Detection: [detection], VectorFeature: [reference_feature]}, diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 8b0b4344..14d28e9b 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -36,6 +36,27 @@ - Local Docker CLI is unavailable on the Windows Codex host. Docker config, image build, live health truthfulness, backup and restore are therefore scheduled on the Docker-enabled Tower after push. +- Pushed and deployed commit `9c402e0`; subsequently pushed the cwd-independent + backup fix `38a3bd0` and secret rotation command `fc42ea9`. +- Tower Docker config, all-in-one build, live migration smoke and browser + runtime smoke passed. `/health/live` and `/health/ready` report `ok`, + PostGIS 3.6, Alembic `202607160001`, writable storage and configured YOLO. +- Startup reconciliation changed five impossible orphaned jobs and two + analysis runs to terminal `failed` state with `PROCESS_INTERRUPTED`; no + `running` rows remain. +- Created the full SHA-256 inventory backup + `rc-belgium-north-sea-38a3bd0`, verified it read-only and restored it in an + isolated temporary database. The first restore attempt exposed stale glibc + collation metadata in `template1`/`postgres`; those empty system databases + were reindexed and refreshed after the backup, then restore passed. +- Rotated the production PostgreSQL password with a generated 256-bit value + without logging it, atomically updated the Tower `.env` and recreated the + healthy container. +- Created the current secure backup + `rc-belgium-north-sea-fc42ea9-secure` (1.4 GiB), verified all checksums and + repeated isolated restore. PostGIS/Alembic and all retained critical table + counts match; the temporary database was removed. +- RC-1 and RC-2 are complete. RC-3 is active. ## Sprint 229 - Governed ALZ definitive release promotion (2026-07-17) @@ -10340,3 +10361,36 @@ Validation: TypeScript typecheck and the production Vite build. - Tower deployment and browser acceptance results are recorded after the rebuilt all-in-one runtime is verified. +## 2026-07-17 - Belgium/North Sea RC-3 local gate + +- Removed automatic selection and every `rasterDatasets[0]` execution fallback + from Detection Lab. Normal detection, guided preparation and calibration now + require an explicit raster selection. +- Added `TemporalCompatibilityService`. Official raster editions carrying + `supports_detection=false` fail before Job/AnalysisRun creation with + `DETECTION_SOURCE_TEMPORALLY_UNSUPPORTED`. +- Detection QA now loads the persisted source Dataset and rejects historical + candidate/reference periods that are absent or non-overlapping with + `DETECTION_QA_TEMPORAL_MISMATCH`. +- Compatible temporal evidence is persisted under + `parameters_json.temporal_compatibility` and + `findings_json.temporal_compatibility` on the existing QualityCheck. +- Added sanitized request IDs, request-duration logs and correlated detection + start/QA completion logs. +- Added `scripts/runtime_state_report.py`; default mode is read-only and + mutation requires `--reconcile --confirm reconcile-interrupted-runtime`. + +Validation: + +- `python -m compileall backend/app`: passed. +- `python -m pytest backend`: 974 passed. +- `npm run typecheck`: passed. +- `npm run build`: passed. +- `bash scripts/run_readiness_check.sh`: passed, including 122-route API audit, + 974 tests, Alembic head, typecheck and build. +- `python -m alembic heads`: one head, `202607160001`. +- `python -m alembic upgrade head --sql`: passed. +- backup/restore/live-smoke shell syntax checks: passed. +- Local `docker compose config` was unavailable because this Windows + workstation has no Docker CLI; Tower validation remains mandatory before + RC-3 completion. diff --git a/docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md b/docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md index 3cd80e78..2de9d2cf 100644 --- a/docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md +++ b/docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md @@ -144,9 +144,9 @@ editions and licences must still pass source-specific probes before activation. | Phase | State | Purpose | | --- | --- | --- | | RC-0 | complete | freeze scope and produce release evidence baseline | -| RC-1 | in progress | backup, restore and data safety | -| RC-2 | verification pending | health, capabilities and stale-runtime correctness | -| RC-3 | pending | temporal detection/QA correctness and observability | +| RC-1 | complete | backup, restore and data safety | +| RC-2 | complete | health, capabilities and stale-runtime correctness | +| RC-3 | in progress | temporal detection/QA correctness and observability | | RC-4 | pending | national/maritime scope and provider coverage contracts | | RC-5 | pending | deployment, secrets, configuration, fresh install and rollback | | RC-6 | pending | complete CI, dependency and supply-chain gates | @@ -190,6 +190,12 @@ Runtime evidence is stored outside Git under `storage/release-evidence/`. ## RC-1 - Backup, restore and data safety +**State: complete.** Tower retains the checksum-verified 1.4 GiB backup +`rc-belgium-north-sea-fc42ea9-secure`. Its manifest records a non-default +database password. An isolated restore reached PostGIS 3.6 and Alembic +`202607160001`, reconciled every retained critical table count and removed the +temporary database. + ### Work - Add a safe production backup command for PostgreSQL custom-format dumps. @@ -219,8 +225,11 @@ Runtime evidence is stored outside Git under `storage/release-evidence/`. ## RC-2 - Truthful runtime state -**State: implementation complete, live verification pending.** Local compile, -963 backend tests, frontend typecheck/build and the full readiness gate pass. +**State: complete.** Local compile, 963 backend tests, frontend +typecheck/build and the full readiness gate pass. Tower reports green +DB/PostGIS/migration/storage readiness and configured YOLO capability. Startup +reconciled five orphaned jobs and two orphaned analysis runs; zero remain +`running`. ### Work @@ -253,6 +262,11 @@ Runtime evidence is stored outside Git under `storage/release-evidence/`. ## RC-3 - Temporal QA correctness and observability +**State: implementation complete; live deployment acceptance pending.** The +local release gate passes backend compilation, all 974 backend tests, frontend +typecheck/build, the integrated readiness script, one Alembic head and complete +offline migration SQL generation. + ### Work - Stop Detection Lab from silently selecting the first raster. diff --git a/docs/ROLLBACK_AND_RECOVERY.md b/docs/ROLLBACK_AND_RECOVERY.md index 4456f27e..4b158018 100644 --- a/docs/ROLLBACK_AND_RECOVERY.md +++ b/docs/ROLLBACK_AND_RECOVERY.md @@ -32,3 +32,10 @@ Storage and model files are inventoried rather than copied into the database dump. Release backups must therefore be paired with the persistent storage volume backup policy. Use `--inventory-mode sha256` for final release evidence. + +An old persistent volume can also retain glibc collation metadata for the +empty `postgres` and `template1` system databases. If `createdb` fails for that +reason, first complete and verify the production backup, then reindex each +affected system database and run `ALTER DATABASE REFRESH COLLATION +VERSION`. Never refresh before reindexing, and never treat this maintenance as +a substitute for the isolated restore proof. diff --git a/docs/TODO.md b/docs/TODO.md index 8986db82..e09cc7ca 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -10,9 +10,9 @@ maritieme zones. - [x] RC-0: Belgische en maritieme scopefreeze vastleggen. - [x] RC-0: autonome roadmap zonder afzonderlijke RC-12 vastleggen. - [x] RC-0: deterministisch release-evidence manifest implementeren en draaien. -- [ ] RC-1: actuele productiebackup, verificatie en geisoleerde restore bewijzen. -- [ ] RC-2: liveness/readiness/capabilities fail-closed en waarheidsgetrouw maken. -- [ ] RC-2: verweesde jobs en analysis runs na een procesherstart verzoenen. +- [x] RC-1: actuele productiebackup, verificatie en geisoleerde restore bewijzen. +- [x] RC-2: liveness/readiness/capabilities fail-closed en waarheidsgetrouw maken. +- [x] RC-2: verweesde jobs en analysis runs na een procesherstart verzoenen. - [ ] RC-3: expliciete rasterkeuze en temporeel compatibele detectie-QA afdwingen. - [ ] RC-4: nationale basisdekking, Wallonie, Brussel en Belgische Noordzee via beheerde providers en golden areas operationaliseren. diff --git a/frontend/src/hooks/useDetectionWorkflow.ts b/frontend/src/hooks/useDetectionWorkflow.ts index cae25236..2f66cd83 100644 --- a/frontend/src/hooks/useDetectionWorkflow.ts +++ b/frontend/src/hooks/useDetectionWorkflow.ts @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react' +import { useState } from 'react' import { datasetsApi, detectionApi } from '../services/api' import type { DatasetCreateResponse, @@ -126,12 +126,6 @@ export function useDetectionWorkflow({ const [detectionCalibrationError, setDetectionCalibrationError] = useState(null) const [detectionWorkflowStage, setDetectionWorkflowStage] = useState('idle') - useEffect(() => { - if (!selectedDetectionDatasetId && rasterDatasets.length > 0) { - setSelectedDetectionDatasetId(rasterDatasets[0].id) - } - }, [rasterDatasets, selectedDetectionDatasetId]) - const loadDetectionModels = async () => { setLoadingDetectionModels(true) setDetectionModelError(null) @@ -259,7 +253,7 @@ export function useDetectionWorkflow({ setDetectionRunError('Select a project first') return } - const datasetId = selectedDetectionDatasetId || rasterDatasets[0]?.id + const datasetId = selectedDetectionDatasetId if (!datasetId) { setDetectionRunError('Select a raster dataset') return @@ -317,7 +311,7 @@ export function useDetectionWorkflow({ setDetectionRunError('De regionale werkruimte is nog niet geladen') return null } - const datasetId = datasetIdOverride || selectedDetectionDatasetId || rasterDatasets[0]?.id + const datasetId = datasetIdOverride || selectedDetectionDatasetId if (!datasetId) { setDetectionRunError('Kies of voeg eerst een gegeorefereerd luchtbeeld toe') return null @@ -446,7 +440,7 @@ export function useDetectionWorkflow({ setDetectionCalibrationError('Select a project before calibration') return } - const datasetId = selectedDetectionDatasetId || rasterDatasets[0]?.id + const datasetId = selectedDetectionDatasetId if (!datasetId) { setDetectionCalibrationError('Select a raster dataset before calibration') return diff --git a/scripts/README.md b/scripts/README.md index 03f97414..c5248b8f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,6 +4,25 @@ Setup-, import-, demo- en maintenance-scripts voor GeoIntel. ## Runtime verification +Inspect interrupted runtime state without changing it: + +```bash +python scripts/runtime_state_report.py +``` + +Only after reviewing that report, explicitly reconcile records that can no +longer be running: + +```bash +python scripts/runtime_state_report.py \ + --reconcile \ + --confirm reconcile-interrupted-runtime +``` + +The default mode is read-only. Reconciliation only marks currently `running` +jobs and analysis runs as failed with `PROCESS_INTERRUPTED`; it does not delete +jobs, results, datasets or artifacts. + Audit the active backend route surface against `docs/API_CONTRACTS.md`: ```bash diff --git a/scripts/runtime_state_report.py b/scripts/runtime_state_report.py new file mode 100644 index 00000000..ff079c53 --- /dev/null +++ b/scripts/runtime_state_report.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Report stale runtime records and optionally reconcile them explicitly.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +BACKEND = ROOT / "backend" +sys.path.insert(0, str(BACKEND)) + +from app.db.session import SessionLocal # noqa: E402 +from app.models import AnalysisRun, Job # noqa: E402 +from app.services.runtime_reconciliation_service import RuntimeReconciliationService # noqa: E402 + + +RECONCILE_CONFIRMATION = "reconcile-interrupted-runtime" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--reconcile", + action="store_true", + help="Mark running jobs and analysis runs as failed with PROCESS_INTERRUPTED.", + ) + parser.add_argument( + "--confirm", + default="", + help=f"Required with --reconcile: {RECONCILE_CONFIRMATION}", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.reconcile and args.confirm != RECONCILE_CONFIRMATION: + print( + json.dumps( + { + "status": "refused", + "reason": "Explicit reconciliation confirmation is missing.", + "required_confirmation": RECONCILE_CONFIRMATION, + }, + indent=2, + ) + ) + return 2 + + with SessionLocal() as db: + running_jobs = db.query(Job).filter(Job.status == "running").order_by(Job.created_at).all() + running_runs = ( + db.query(AnalysisRun) + .filter(AnalysisRun.status == "running") + .order_by(AnalysisRun.created_at) + .all() + ) + result = { + "status": "ok", + "mode": "read_only", + "running_job_count": len(running_jobs), + "running_analysis_run_count": len(running_runs), + "running_job_ids": [str(item.id) for item in running_jobs], + "running_analysis_run_ids": [str(item.id) for item in running_runs], + } + if args.reconcile: + reconciled = RuntimeReconciliationService.reconcile(db) + result.update( + { + "mode": "reconciled", + "interrupted_jobs": reconciled.interrupted_jobs, + "interrupted_analysis_runs": reconciled.interrupted_analysis_runs, + } + ) + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())