Add governed Statbel edition probe

backend/tests/test_sprint222_source_catalog_probes.py

@@ -22,6 +22,7 @@ ORTHO_METADATA_URL = (
     "id=f5304d6d-0dd4-43fd-a726-427af31e8d61&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd"
 )
 ALZ_RELEASE_URL = "https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen"
+STATBEL_DCAT_URL = "https://doc.statbel.be/publications/DCAT/DCAT_opendata_datasets.ttl"
 
 
 def _wfs_capabilities(metadata_url: str = GRB_METADATA_URL) -> bytes:
@@ -95,6 +96,30 @@ def _alz_release_page(*, include_snapshot: bool = True, download_host: str = "ww
     """.encode()
 
 
+def _statbel_dcat() -> bytes:
+    return b"""
+    @prefix dcat: <http://www.w3.org/ns/dcat#> .
+    @prefix dct: <http://purl.org/dc/terms/> .
+    @prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+    <http://data.gov.be/catalog/statbelopen> a dcat:Catalog ;
+      dct:modified "2026-07-07"^^xsd:date .
+
+    <https://statbel.fgov.be/node/6475#id> a dcat:Dataset ;
+      dct:title "Bevolking per statistische sector"@nl ;
+      dct:alternative "Bevolking per statistische sector [Periode: 2025]"@nl ;
+      dct:identifier "NodeID6475" ;
+      dct:license <https://creativecommons.org/licenses/by/4.0/> ;
+      dct:temporal [ dcat:startDate "2025-01-01"^^xsd:date ] ;
+      dcat:landingPage <https://statbel.fgov.be/nl/open-data/bevolking-statistische-sector-12> ;
+      dcat:distribution
+        <https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2025_NEW.zip#distribution6475>,
+        <https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2025_NEW.xlsx#distribution6475>,
+        <https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2025_OLD.zip#distribution6475>,
+        <https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2025_OLD.xlsx#distribution6475> .
+    """
+
+
 class _Response:
     def __init__(self, content: bytes, content_type: str = "text/xml", *, content_length: int | None = None) -> None:
         self.content = content
@@ -149,7 +174,7 @@ def _dataset(source_name: str, version: str) -> Dataset:
         id=uuid.uuid4(),
         project_id=uuid.uuid4(),
         name=f"{source_name} source",
-        dataset_type="vector" if source_name == "grb" else "raster",
+        dataset_type="vector" if source_name in {"grb", "statbel", "agentschap_landbouw_zeevisserij_agricultural_parcels"} else "raster",
         source=source_name,
         source_name=source_name,
         source_version=version,
@@ -163,6 +188,8 @@ def _settings(**overrides) -> Settings:
         "SOURCE_CATALOG_PROBE_ENABLED": True,
         "SOURCE_CATALOG_GRB_WFS_URL": "https://geo.api.vlaanderen.be/GRB/wfs",
         "SOURCE_CATALOG_ALZ_RELEASE_URL": ALZ_RELEASE_URL,
+        "SOURCE_CATALOG_STATBEL_DCAT_URL": STATBEL_DCAT_URL,
+        "SOURCE_CATALOG_STATBEL_MAX_RESPONSE_MB": 5,
         "SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS": 3,
         "SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB": 1,
         "SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS": 0,
@@ -176,6 +203,8 @@ def _settings(**overrides) -> Settings:
 def _opener(request, timeout):
     assert timeout == 3
     url = request.full_url
+    if url == STATBEL_DCAT_URL:
+        return _Response(_statbel_dcat(), "application/octet-stream")
     if url == ALZ_RELEASE_URL:
         return _Response(_alz_release_page(), "text/html; charset=utf-8")
     if "metadata.vlaanderen.be" in url:
@@ -195,6 +224,7 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions
             [
                 _dataset("grb", "2026-07-14"),
                 _dataset("digitaal_vlaanderen_orthophoto", "most_recent_at_2026-07-14"),
+                _dataset("statbel", "2025"),
                 _dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2025-definitive"),
             ]
         ),
@@ -205,7 +235,7 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions
     )
     by_source = {item.source_name: item for item in report.items}
 
-    assert report.summary.available_count == 3
+    assert report.summary.available_count == 4
     assert report.summary.different_version_count == 1
     assert by_source["grb"].remote_version == "Toestand 2026-07-15"
     assert by_source["grb"].comparison_status == "different"
@@ -214,6 +244,14 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions
     assert by_source["digitaal_vlaanderen_orthophoto"].remote_version == "2025.04"
     assert by_source["digitaal_vlaanderen_orthophoto"].comparison_status == "not_comparable"
     assert by_source["digitaal_vlaanderen_orthophoto"].remote_published_at.year == 2025
+    statbel = by_source["statbel"]
+    assert statbel.service_type == "DCAT"
+    assert statbel.remote_version == "2025"
+    assert statbel.local_source_version == "2025"
+    assert statbel.comparison_status == "same"
+    assert statbel.metadata_identifier == "NodeID6475"
+    assert statbel.matched_layers == ["population_txt_current", "landing_page", "cc_by_4_0"]
+    assert "REDEGEO" in statbel.message
     alz = by_source["agentschap_landbouw_zeevisserij_agricultural_parcels"]
     assert alz.service_type == "HTML"
     assert alz.remote_version == "2025-v3"
@@ -309,10 +347,10 @@ def test_catalog_probe_cache_is_explicitly_bypassable() -> None:
     second = SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW)
     SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW, force=True)
 
-    assert first_call_count == 5
+    assert first_call_count == 6
     assert all(item.cached is False for item in first.items)
     assert all(item.cached is True for item in second.items)
-    assert len(calls) == 10
+    assert len(calls) == 12
 
 
 def test_catalog_probe_can_be_disabled_without_network_access() -> None:
@@ -327,7 +365,7 @@ def test_catalog_probe_can_be_disabled_without_network_access() -> None:
         now=NOW,
     )
 
-    assert report.summary.disabled_count == 3
+    assert report.summary.disabled_count == 4
     assert all(item.status == "disabled" for item in report.items)
 
 
@@ -350,7 +388,7 @@ def test_catalog_probe_route_returns_canonical_envelope(monkeypatch) -> None:
 
     assert list(response) == ["data"]
     assert response["data"]["project_id"] == project_id
-    assert response["data"]["summary"]["provider_count"] == 3
+    assert response["data"]["summary"]["provider_count"] == 4
 
 
 def test_catalog_probe_remains_explicit_and_never_imports_provider_data() -> None: