Add governed Statbel edition probe

backend/app/services/source_catalog_probe_service.py

@@ -24,6 +24,11 @@ from app.schemas.source_catalog import (
     SourceCatalogProbeReport,
     SourceCatalogProbeSummary,
 )
+from app.services.statbel_catalog_probe import (
+    StatbelCatalogError,
+    parse_statbel_population_catalog,
+    validate_statbel_catalog_url,
+)
 
 
 _GMD = "http://www.isotc211.org/2005/gmd"
@@ -44,6 +49,7 @@ _ALZ_SNAPSHOT = re.compile(
     re.IGNORECASE,
 )
 _ALZ_EDITION = re.compile(r"^(20\d{2})-(?:definitive|v3)$", re.IGNORECASE)
+_YEAR_EDITION = re.compile(r"^20\d{2}$")
 
 
 class CatalogProbeFailure(RuntimeError):
@@ -159,12 +165,19 @@ def _with_capabilities_query(base_url: str, service_type: str) -> str:
     return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, query, ""))
 
 
-def _bounded_fetch(url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> FetchResult:
+def _bounded_fetch(
+    url: str,
+    settings: Settings,
+    opener: Callable[..., Any] | None = None,
+    *,
+    max_response_mb: int | None = None,
+    accept: str = "application/xml,text/xml,text/html,application/json",
+) -> FetchResult:
     request = Request(
         url,
-        headers={"Accept": "application/xml,text/xml,text/html,application/json", "User-Agent": "GeoIntel/0.1 source-catalog-probe"},
+        headers={"Accept": accept, "User-Agent": "GeoIntel/0.1 source-catalog-probe"},
     )
-    max_bytes = settings.source_catalog_probe_max_response_mb * 1024 * 1024
+    max_bytes = (max_response_mb or settings.source_catalog_probe_max_response_mb) * 1024 * 1024
     try:
         with (opener or urlopen)(request, timeout=settings.source_catalog_probe_timeout_seconds) as response:
             content_length = _header(response.headers, "Content-Length")
@@ -509,6 +522,56 @@ def _probe_alz_remote(
     )
 
 
+def _probe_statbel_remote(
+    contract: ProbeContract,
+    settings: Settings,
+    *,
+    opener: Callable[..., Any] | None,
+    now: datetime,
+) -> RemoteProbe:
+    try:
+        catalog_url = validate_statbel_catalog_url(contract.endpoint_url)
+        response = _bounded_fetch(
+            catalog_url,
+            settings,
+            opener,
+            max_response_mb=settings.source_catalog_statbel_max_response_mb,
+            accept="text/turtle,application/x-turtle,application/octet-stream,text/plain",
+        )
+        validate_statbel_catalog_url(response.final_url)
+        content_type = response.content_type.lower()
+        if content_type and not any(token in content_type for token in ("turtle", "octet-stream", "text/plain")):
+            raise StatbelCatalogError(
+                "CATALOG_STATBEL_INVALID_CONTENT_TYPE",
+                "De officiële Statbel DCAT-catalogus heeft geen ondersteund Turtle-contenttype.",
+            )
+        release = parse_statbel_population_catalog(response.content)
+    except StatbelCatalogError as exc:
+        raise CatalogProbeFailure(exc.code, exc.message) from exc
+
+    layout = "nieuwe REDEGEO-sectorindeling" if release.current_distribution_variant == "new" else "actuele sectorindeling"
+    message = f"De officiële Statbel DCAT-catalogus bevestigt bevolkingseditie {release.version} met de {layout}."
+    if release.legacy_distribution_available:
+        message += " De oude 2025-indeling is alleen overgangsevidentie en wordt niet als actuele GeoIntel-editie gebruikt."
+    return RemoteProbe(
+        status="available",
+        reachable=True,
+        checked_at=now,
+        expected_layers=contract.expected_layers,
+        matched_layers=contract.expected_layers,
+        advertised_layer_count=release.distribution_count,
+        metadata_url=release.landing_page,
+        metadata_identifier=release.identifier,
+        remote_title=f"Bevolking per statistische sector {release.version} ({layout})",
+        remote_version=release.version,
+        remote_modified_at=release.catalog_modified_at,
+        capabilities_sha256=sha256(response.content).hexdigest(),
+        capabilities_etag=response.etag,
+        capabilities_last_modified_at=response.last_modified_at,
+        message=message,
+    )
+
+
 def _probe_remote(
     contract: ProbeContract,
     settings: Settings,
@@ -519,6 +582,8 @@ def _probe_remote(
     try:
         if contract.service_type == "HTML":
             return _probe_alz_remote(contract, settings, opener=opener, now=now)
+        if contract.service_type == "DCAT":
+            return _probe_statbel_remote(contract, settings, opener=opener, now=now)
         capabilities = _bounded_fetch(_validate_capabilities_url(contract.endpoint_url), settings, opener)
         _validate_capabilities_url(capabilities.final_url)
         content_type = capabilities.content_type.lower()
@@ -635,6 +700,10 @@ def _latest_local_version(source_name: str, datasets: list[Dataset]) -> str | No
                 ),
             )
             return latest.source_version
+    elif source_name == "statbel":
+        annual = [item for item in candidates if _YEAR_EDITION.fullmatch((item.source_version or "").strip())]
+        if annual:
+            return max(annual, key=lambda item: int((item.source_version or "0").strip())).source_version
     if not candidates:
         return None
     latest = max(
@@ -660,6 +729,8 @@ def _normalized_version(source_name: str, version: str | None) -> str | None:
     if source_name == _ALZ_SOURCE_NAME:
         match = _ALZ_EDITION.fullmatch(value)
         return f"{match.group(1)}-v3" if match else None
+    if source_name == "statbel" and _YEAR_EDITION.fullmatch(value):
+        return value
     return None
 
 
@@ -711,6 +782,13 @@ class SourceCatalogProbeService:
                 endpoint_url=_with_capabilities_query(active_settings.orthophoto_wms_url, "WMS"),
                 expected_layers=(active_settings.orthophoto_wms_layer, "Vliegdagcontour"),
             ),
+            ProbeContract(
+                source_name="statbel",
+                display_name="Bevolking per statistische sector (Statbel)",
+                service_type="DCAT",
+                endpoint_url=active_settings.source_catalog_statbel_dcat_url,
+                expected_layers=("population_txt_current", "landing_page", "cc_by_4_0"),
+            ),
             ProbeContract(
                 source_name=_ALZ_SOURCE_NAME,
                 display_name="Landbouwgebruikspercelen (ALZ)",
@@ -789,8 +867,9 @@ class SourceCatalogProbeService:
             summary=summary,
             items=items,
             limitations=[
-                "Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities, gekoppelde ISO 19139 metadata en de officiële ALZ-publicatiepagina.",
+                "Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities, gekoppelde ISO 19139 metadata, de officiële Statbel DCAT-catalogus en de officiële ALZ-publicatiepagina.",
                 "Er worden geen features, rasters of modelbestanden opgehaald en geen datasets aangemaakt of overschreven.",
+                "Statbel distributielinks worden alleen als release-evidentie gevalideerd; de oude en nieuwe 2025-sectorindeling blijven semantisch gescheiden.",
                 "Voor ALZ is alleen de nieuwste definitieve v3-editie vergelijkbaar; voorlopige v1/v2-snapshots zijn uitsluitend informatief.",
                 "Een versieverschil is controlesignaal, geen bewijs dat een lokale dataset onbruikbaar is en geen automatische importopdracht.",
             ],