Add governed GRB refresh workflow

backend/README.md

@@ -1469,3 +1469,41 @@ Runtime controls are `SOURCE_CATALOG_PROBE_ENABLED`,
 `SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS`. Metadata links remain restricted to
 the official HTTPS CSW path on `metadata.vlaanderen.be` even when an operator
 overrides the capabilities endpoint.
+
+## Governed regional GRB refresh
+
+`GET /api/v1/projects/{project_id}/datasets/grb-refresh-plan` combines the
+explicit official GRB edition probe with the four existing regional snapshot
+series. It is read-only: it reports local feature/storage impact and whether
+buildings, roads, water and parcels are current, updateable or require review.
+
+Regional refreshes use a two-phase operator flow inside the all-in-one
+container. First stage and validate every source partition without touching
+PostGIS:
+
+```bash
+docker exec geointel python /app/scripts/manage_grb_refresh.py stage \
+  --project-id 82a85913-c522-45d7-84a1-02b393d89e55 \
+  --api-url http://127.0.0.1:8000/api/v1 \
+  --confirm-edition 2026-07-15 \
+  --layers buildings roads water parcels
+```
+
+The JSON result gives `plan_path`, exact feature deltas, artifact sizes and
+`plan_sha256`. Review that evidence, then apply those exact staged bytes:
+
+```bash
+docker exec geointel python /app/scripts/manage_grb_refresh.py apply \
+  --project-id 82a85913-c522-45d7-84a1-02b393d89e55 \
+  --api-url http://127.0.0.1:8000/api/v1 \
+  --confirm-edition 2026-07-15 \
+  --confirm-plan-sha256 SHA256_FROM_STAGE
+```
+
+Both commands fail when project/scope, official edition, manifests, partition
+count or any checksum differs. Interrupted staging is safely resumable because
+the existing municipal manifests are reused. Apply imports through
+DatasetService/VectorFeatureService, creates new temporal Datasets and retains
+all previous snapshots. Do not add `--force` to this coordinator; a source
+refetch remains a separate deliberate recovery action in the lower-level
+operators.

backend/app/api/routes/datasets.py

@@ -46,6 +46,7 @@ from app.services.vector_feature_service import VectorFeatureService
 from app.services.dataset_service import DatasetService
 from app.services.source_freshness_service import SourceFreshnessService
 from app.services.source_catalog_probe_service import SourceCatalogProbeService
+from app.services.grb_refresh_plan_service import GrbRefreshPlanService
 from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
 from app.services.dhmv_acquisition_service import DhmvAcquisitionService
 from app.services.terrain_analysis_service import TerrainAnalysisService
@@ -266,6 +267,22 @@ def probe_dataset_source_catalogs(
     return envelope(report.model_dump())
 
 
+@router.get("/datasets/grb-refresh-plan", response_model=dict)
+def plan_grb_dataset_refresh(
+    project_id: UUID,
+    scope: str = Query(default=GrbRefreshPlanService.SCOPE),
+    refresh_catalog: bool = Query(default=False),
+    db: Session = Depends(get_db),
+):
+    report = GrbRefreshPlanService.build(
+        db,
+        project_id,
+        scope=scope,
+        refresh_catalog=refresh_catalog,
+    )
+    return envelope(report.model_dump())
+
+
 @router.get("/datasets/{dataset_id}", response_model=dict)
 def get_dataset(
     project_id: UUID,

backend/app/schemas/__init__.py

@@ -16,6 +16,7 @@ from .source_catalog import (
     SourceCatalogProbeReport,
     SourceCatalogProbeSummary,
 )
+from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
 from .detection import (
     DetectionListResponse,
     DetectionModelCapability,
@@ -149,6 +150,9 @@ __all__ = [
     "SourceCatalogProbeItem",
     "SourceCatalogProbeReport",
     "SourceCatalogProbeSummary",
+    "GrbRefreshLayerPlan",
+    "GrbRefreshPlan",
+    "GrbRefreshPlanSummary",
     "DetectionListResponse",
     "DetectionModelCapability",
     "DetectionModelsResponse",

backend/app/schemas/grb_refresh.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from datetime import date, datetime
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel
+
+
+GrbRefreshLayerStatus = Literal[
+    "current",
+    "update_available",
+    "not_loaded",
+    "review_required",
+    "remote_unavailable",
+]
+
+
+class GrbRefreshLayerPlan(BaseModel):
+    theme: Literal["buildings", "roads", "water", "parcels"]
+    display_name: str
+    collections: list[str]
+    temporal_series_key: str
+    status: GrbRefreshLayerStatus
+    local_dataset_id: UUID | None = None
+    local_source_version: str | None = None
+    local_observed_at: datetime | None = None
+    local_imported_at: datetime | None = None
+    local_feature_count: int | None = None
+    local_size_bytes: int | None = None
+    retained_after_refresh: bool = True
+    action_message: str
+
+
+class GrbRefreshPlanSummary(BaseModel):
+    layer_count: int
+    current_count: int
+    update_available_count: int
+    not_loaded_count: int
+    review_required_count: int
+    remote_unavailable_count: int
+    new_dataset_count_if_applied: int
+    retained_dataset_count: int
+    current_feature_count: int
+    current_size_bytes: int
+
+
+class GrbRefreshPlan(BaseModel):
+    project_id: UUID
+    scope: str
+    generated_at: datetime
+    remote_status: str
+    remote_version: str | None = None
+    remote_edition_date: date | None = None
+    catalog_checked_at: datetime | None = None
+    summary: GrbRefreshPlanSummary
+    layers: list[GrbRefreshLayerPlan]
+    execution_mode: Literal["operator_stage_then_apply"] = "operator_stage_then_apply"
+    staging_required: bool = True
+    automatic_import: bool = False
+    destructive_replacement: bool = False
+    message: str
+    limitations: list[str]

backend/app/services/grb_refresh_plan_service.py

@@ -0,0 +1,186 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from datetime import date, datetime, timezone
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.core.errors import AppError
+from app.models import Dataset, Project
+from app.schemas.grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
+from app.services.source_catalog_probe_service import SourceCatalogProbeService
+
+
+@dataclass(frozen=True)
+class _LayerDefinition:
+    theme: str
+    display_name: str
+    collections: tuple[str, ...]
+
+    @property
+    def series_key(self) -> str:
+        return f"grb:{self.theme}:kempen-transport-region"
+
+
+class GrbRefreshPlanService:
+    SCOPE = "kempen-transport-region"
+    LAYERS = (
+        _LayerDefinition("buildings", "Gebouwen", ("GBG",)),
+        _LayerDefinition("roads", "Wegen", ("Wegsegment",)),
+        _LayerDefinition("water", "Water", ("WTZ", "WLAS", "WGR")),
+        _LayerDefinition("parcels", "Percelen", ("ADP",)),
+    )
+    _EDITION_DATE = re.compile(r"(?<!\d)(20\d{2}-\d{2}-\d{2})(?!\d)")
+
+    @classmethod
+    def _parse_edition_date(cls, value: str | None) -> date | None:
+        match = cls._EDITION_DATE.search(value or "")
+        if not match:
+            return None
+        try:
+            return date.fromisoformat(match.group(1))
+        except ValueError:
+            return None
+
+    @staticmethod
+    def _dataset_feature_count(dataset: Dataset) -> int | None:
+        metadata = dataset.metadata_json if isinstance(dataset.metadata_json, dict) else {}
+        value = metadata.get("feature_count")
+        try:
+            return int(value) if value is not None else None
+        except (TypeError, ValueError):
+            return None
+
+    @staticmethod
+    def _latest_dataset(rows: list[Dataset], series_key: str) -> Dataset | None:
+        candidates = [
+            row
+            for row in rows
+            if row.temporal_series_key == series_key and row.status == "ready"
+        ]
+        if not candidates:
+            return None
+        minimum = datetime.min.replace(tzinfo=timezone.utc)
+        return max(candidates, key=lambda row: (row.observed_at or row.imported_at or row.created_at or minimum, str(row.id)))
+
+    @classmethod
+    def build(
+        cls,
+        db: Session,
+        project_id: UUID,
+        *,
+        scope: str = SCOPE,
+        refresh_catalog: bool = False,
+        now: datetime | None = None,
+    ) -> GrbRefreshPlan:
+        if scope != cls.SCOPE:
+            raise AppError(
+                code="GRB_REFRESH_SCOPE_UNSUPPORTED",
+                message=f"Only the governed scope '{cls.SCOPE}' is supported",
+                status_code=400,
+            )
+        if not db.get(Project, project_id):
+            raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
+
+        generated_at = now or datetime.now(timezone.utc)
+        catalog = SourceCatalogProbeService.audit_project(db, project_id, force=refresh_catalog)
+        grb_probe = next((item for item in catalog.items if item.source_name == "grb"), None)
+        remote_version = grb_probe.remote_version if grb_probe else None
+        remote_edition_date = cls._parse_edition_date(remote_version)
+        remote_available = bool(grb_probe and grb_probe.status == "available" and grb_probe.reachable)
+        rows = (
+            db.query(Dataset)
+            .filter(Dataset.project_id == project_id, Dataset.source_name == "grb")
+            .all()
+        )
+
+        layer_plans: list[GrbRefreshLayerPlan] = []
+        for definition in cls.LAYERS:
+            local = cls._latest_dataset(rows, definition.series_key)
+            local_date = cls._parse_edition_date(local.source_version if local else None)
+            if not remote_available:
+                status = "remote_unavailable"
+                action = "De officiële catalogus is niet bereikbaar; er wordt geen vernieuwingsbeslissing genomen."
+            elif remote_edition_date is None:
+                status = "review_required"
+                action = "De officiële editie bevat geen herkenbare datum en vereist menselijke beoordeling."
+            elif local is None:
+                status = "not_loaded"
+                action = "Deze laag kan als nieuwe, afzonderlijke GRB-snapshot worden voorbereid."
+            elif local_date is None:
+                status = "review_required"
+                action = "De lokale editie is niet veilig datumvergelijkbaar; controleer de provenance vóór staging."
+            elif local_date == remote_edition_date:
+                status = "current"
+                action = "De lokale snapshot gebruikt dezelfde officiële editie; geen import nodig."
+            elif local_date < remote_edition_date:
+                status = "update_available"
+                action = "Stage eerst alle bronartifacts en controleer exacte aantallen en checksums vóór import."
+            else:
+                status = "review_required"
+                action = "De lokale editie lijkt nieuwer dan de catalogus; automatische terugval is verboden."
+
+            layer_plans.append(
+                GrbRefreshLayerPlan(
+                    theme=definition.theme,
+                    display_name=definition.display_name,
+                    collections=list(definition.collections),
+                    temporal_series_key=definition.series_key,
+                    status=status,
+                    local_dataset_id=local.id if local else None,
+                    local_source_version=local.source_version if local else None,
+                    local_observed_at=local.observed_at if local else None,
+                    local_imported_at=local.imported_at if local else None,
+                    local_feature_count=cls._dataset_feature_count(local) if local else None,
+                    local_size_bytes=local.size_bytes if local else None,
+                    action_message=action,
+                )
+            )
+
+        counts = {status: sum(item.status == status for item in layer_plans) for status in (
+            "current", "update_available", "not_loaded", "review_required", "remote_unavailable"
+        )}
+        actionable = counts["update_available"] + counts["not_loaded"]
+        summary = GrbRefreshPlanSummary(
+            layer_count=len(layer_plans),
+            current_count=counts["current"],
+            update_available_count=counts["update_available"],
+            not_loaded_count=counts["not_loaded"],
+            review_required_count=counts["review_required"],
+            remote_unavailable_count=counts["remote_unavailable"],
+            new_dataset_count_if_applied=actionable,
+            retained_dataset_count=sum(item.local_dataset_id is not None for item in layer_plans),
+            current_feature_count=sum(item.local_feature_count or 0 for item in layer_plans),
+            current_size_bytes=sum(item.local_size_bytes or 0 for item in layer_plans),
+        )
+        if actionable:
+            message = (
+                f"{actionable} GRB-laag{' is' if actionable == 1 else 'en zijn'} voorbereidbaar voor editie "
+                f"{remote_edition_date.isoformat() if remote_edition_date else remote_version}. "
+                "Staging berekent eerst de exacte impact; import vereist daarna de plan-checksum."
+            )
+        elif counts["current"] == len(layer_plans):
+            message = "Alle beheerde regionale GRB-lagen gebruiken de officiële cataloguseditie."
+        else:
+            message = "Er is menselijke beoordeling nodig voordat een GRB-staging kan starten."
+
+        return GrbRefreshPlan(
+            project_id=project_id,
+            scope=scope,
+            generated_at=generated_at,
+            remote_status=grb_probe.status if grb_probe else "unavailable",
+            remote_version=remote_version,
+            remote_edition_date=remote_edition_date,
+            catalog_checked_at=grb_probe.checked_at if grb_probe else None,
+            summary=summary,
+            layers=layer_plans,
+            message=message,
+            limitations=[
+                "Dit endpoint is read-only en start geen download, import of databasejob.",
+                "Staging bewaart bronartifacts buiten PostGIS; apply vereist de exacte staged plan-checksum.",
+                "Een refresh maakt nieuwe immutable Datasets en verwijdert of overschrijft oude snapshots niet.",
+                "Exacte feature- en opslagverschillen zijn pas bekend nadat alle regionale partitions staged en gevalideerd zijn.",
+            ],
+        )

backend/tests/test_sprint223_governed_grb_refresh.py

@@ -0,0 +1,250 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from types import SimpleNamespace
+import uuid
+
+import pytest
+
+from app.core.errors import AppError
+from app.models import Dataset
+from app.services.grb_refresh_plan_service import GrbRefreshPlanService
+
+
+NOW = datetime(2026, 7, 16, 16, 0, tzinfo=timezone.utc)
+PROJECT_ID = uuid.uuid4()
+
+
+class _Query:
+    def __init__(self, rows: list[Dataset]) -> None:
+        self.rows = rows
+
+    def filter(self, *_args):
+        return self
+
+    def all(self) -> list[Dataset]:
+        return self.rows
+
+
+class _Db:
+    def __init__(self, rows: list[Dataset]) -> None:
+        self.rows = rows
+
+    def get(self, _model, identifier):
+        return SimpleNamespace(id=identifier)
+
+    def query(self, _model):
+        return _Query(self.rows)
+
+
+def _dataset(theme: str, version: str = "2026-07-14", count: int = 100) -> Dataset:
+    return Dataset(
+        id=uuid.uuid4(),
+        project_id=PROJECT_ID,
+        name=f"grb_{theme}.geojson",
+        dataset_type="vector",
+        source="operator_official_import",
+        source_name="grb",
+        dataset_role="reference",
+        reference_layer_name=theme,
+        source_version=version,
+        temporal_series_key=f"grb:{theme}:kempen-transport-region",
+        observed_at=datetime.fromisoformat(f"{version}T00:00:00+00:00"),
+        imported_at=NOW,
+        metadata_json={"feature_count": count},
+        size_bytes=1000,
+        status="ready",
+    )
+
+
+def _catalog(status: str = "available", version: str | None = "Toestand 2026-07-15"):
+    return SimpleNamespace(
+        items=[
+            SimpleNamespace(
+                source_name="grb",
+                status=status,
+                reachable=status == "available",
+                remote_version=version,
+                checked_at=NOW,
+            )
+        ]
+    )
+
+
+def test_refresh_plan_marks_all_older_immutable_snapshots_as_update_available(monkeypatch) -> None:
+    rows = [_dataset(theme, count=(index + 1) * 100) for index, theme in enumerate(("buildings", "roads", "water", "parcels"))]
+    monkeypatch.setattr(
+        "app.services.grb_refresh_plan_service.SourceCatalogProbeService.audit_project",
+        lambda *_args, **_kwargs: _catalog(),
+    )
+
+    plan = GrbRefreshPlanService.build(_Db(rows), PROJECT_ID, now=NOW)
+
+    assert plan.remote_edition_date.isoformat() == "2026-07-15"
+    assert plan.summary.update_available_count == 4
+    assert plan.summary.new_dataset_count_if_applied == 4
+    assert plan.summary.retained_dataset_count == 4
+    assert plan.summary.current_feature_count == 1000
+    assert plan.summary.current_size_bytes == 4000
+    assert all(item.status == "update_available" for item in plan.layers)
+    assert all(item.retained_after_refresh for item in plan.layers)
+    assert plan.automatic_import is False
+    assert plan.destructive_replacement is False
+
+
+@pytest.mark.parametrize(
+    ("rows", "catalog", "expected"),
+    [
+        ([_dataset("buildings", "2026-07-15")], _catalog(), "current"),
+        ([], _catalog(), "not_loaded"),
+        ([_dataset("buildings")], _catalog("unavailable"), "remote_unavailable"),
+        ([_dataset("buildings")], _catalog("available", "Onbekende toestand"), "review_required"),
+    ],
+)
+def test_refresh_plan_status_matrix(monkeypatch, rows, catalog, expected) -> None:
+    monkeypatch.setattr(
+        "app.services.grb_refresh_plan_service.SourceCatalogProbeService.audit_project",
+        lambda *_args, **_kwargs: catalog,
+    )
+    plan = GrbRefreshPlanService.build(_Db(rows), PROJECT_ID, now=NOW)
+    buildings = next(item for item in plan.layers if item.theme == "buildings")
+    assert buildings.status == expected
+
+
+def test_refresh_plan_rejects_an_unapproved_scope(monkeypatch) -> None:
+    monkeypatch.setattr(
+        "app.services.grb_refresh_plan_service.SourceCatalogProbeService.audit_project",
+        lambda *_args, **_kwargs: _catalog(),
+    )
+    with pytest.raises(AppError) as raised:
+        GrbRefreshPlanService.build(_Db([]), PROJECT_ID, scope="mol", now=NOW)
+    assert raised.value.code == "GRB_REFRESH_SCOPE_UNSUPPORTED"
+
+
+def test_refresh_plan_route_uses_the_canonical_envelope(monkeypatch) -> None:
+    from app.api.routes import datasets as dataset_routes
+
+    monkeypatch.setattr(
+        "app.services.grb_refresh_plan_service.SourceCatalogProbeService.audit_project",
+        lambda *_args, **_kwargs: _catalog(),
+    )
+    response = dataset_routes.plan_grb_dataset_refresh(
+        project_id=PROJECT_ID,
+        scope="kempen-transport-region",
+        refresh_catalog=False,
+        db=_Db([]),
+    )
+    assert list(response) == ["data"]
+    assert response["data"]["project_id"] == PROJECT_ID
+    assert response["data"]["automatic_import"] is False
+
+
+def _load_operator_module():
+    root = Path(__file__).resolve().parents[2]
+    scripts = root / "scripts"
+    if str(scripts) not in sys.path:
+        sys.path.insert(0, str(scripts))
+    spec = importlib.util.spec_from_file_location("manage_grb_refresh_s223", scripts / "manage_grb_refresh.py")
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_operator_requires_exact_edition_and_plan_hash() -> None:
+    module = _load_operator_module()
+    assert module.require_confirmed_edition({"remote_edition_date": "2026-07-15"}, "2026-07-15") == "2026-07-15"
+    with pytest.raises(RuntimeError, match="confirm-edition"):
+        module.require_confirmed_edition({"remote_edition_date": "2026-07-15"}, "2026-07-14")
+
+    payload = {"status": "staged", "layers": []}
+    first = module.canonical_plan_sha256(payload)
+    assert first == module.canonical_plan_sha256({**payload, "plan_sha256": first})
+    assert first != module.canonical_plan_sha256({"status": "staged", "layers": [{"theme": "roads"}]})
+
+
+def test_operator_validates_every_staged_artifact_and_partition(tmp_path) -> None:
+    module = _load_operator_module()
+    manifest_dir = tmp_path / "kempen-transport-region" / "buildings" / "2026-07-15"
+    partition_dir = manifest_dir / "partitions"
+    partition_dir.mkdir(parents=True)
+    artifact = manifest_dir / "buildings.geojson"
+    partition = partition_dir / "13025_mol.geojson"
+    artifact.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
+    partition.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
+    manifest = {
+        "status": "complete",
+        "scope": "kempen-transport-region",
+        "theme": "buildings",
+        "observed_at": "2026-07-15",
+        "reference_truncated": False,
+        "member_count": 1,
+        "feature_count": 1,
+        "artifact_filename": artifact.name,
+        "artifact_sha256": module.sha256_file(artifact),
+        "artifact_size_bytes": artifact.stat().st_size,
+        "partitions": [{"filename": partition.name, "sha256": module.sha256_file(partition)}],
+    }
+    path = manifest_dir / "regional_buildings_manifest.json"
+    path.write_text(json.dumps(manifest), encoding="utf-8")
+
+    assert module.validate_manifest(
+        path,
+        output_root=tmp_path,
+        scope="kempen-transport-region",
+        theme="buildings",
+        edition="2026-07-15",
+    )["feature_count"] == 1
+
+    partition.write_text("changed", encoding="utf-8")
+    with pytest.raises(RuntimeError, match="partition checksum"):
+        module.validate_manifest(
+            path,
+            output_root=tmp_path,
+            scope="kempen-transport-region",
+            theme="buildings",
+            edition="2026-07-15",
+        )
+
+
+def test_operator_builds_only_allowlisted_local_subprocess_commands(tmp_path) -> None:
+    module = _load_operator_module()
+    args = SimpleNamespace(
+        scope="kempen-transport-region",
+        api_url="http://127.0.0.1:8000/api/v1",
+        output_root=tmp_path,
+        request_timeout=180,
+        api_timeout=180,
+        batch_size=1000,
+        page_limit=1000,
+        max_features_per_member=100000,
+        max_total_features=1500000,
+    )
+    commands = module.build_operator_commands(
+        args,
+        ["buildings", "roads", "water", "parcels"],
+        "2026-07-15",
+        fetch_only=True,
+    )
+    flattened = [argument for _label, command in commands for argument in command]
+    assert len(commands) == 2
+    assert "--fetch-only" in flattened
+    assert "provision_regional_grb_buildings.py" in " ".join(flattened)
+    assert "provision_regional_grb_context.py" in " ".join(flattened)
+    assert "--force" not in flattened
+
+
+def test_refresh_api_and_frontend_remain_explicit_only() -> None:
+    root = Path(__file__).resolve().parents[2]
+    service = (root / "backend" / "app" / "services" / "grb_refresh_plan_service.py").read_text(encoding="utf-8")
+    hook = (root / "frontend" / "src" / "hooks" / "useSourceFreshness.ts").read_text(encoding="utf-8")
+    operator = (root / "scripts" / "manage_grb_refresh.py").read_text(encoding="utf-8")
+    assert "DatasetService" not in service
+    assert "VectorFeatureService" not in service
+    assert "void probeCatalogs(" not in hook
+    assert 'choices=("plan", "stage", "apply")' in operator
+    assert "--confirm-plan-sha256" in operator