Add governed ALZ edition probe

2026-07-16 20:27:07 +02:00
parent 5e897206b2
commit 0a23e3484c
18 changed files with 451 additions and 38 deletions
@@ -21,6 +21,7 @@ ORTHO_METADATA_URL = (
    "https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&service=CSW&"
    "id=f5304d6d-0dd4-43fd-a726-427af31e8d61&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd"
 )
+ALZ_RELEASE_URL = "https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen"


 def _wfs_capabilities(metadata_url: str = GRB_METADATA_URL) -> bytes:
@@ -77,6 +78,23 @@ def _metadata(identifier: str, title: str, edition: str, modified: str, publishe
    """.encode()


+def _alz_release_page(*, include_snapshot: bool = True, download_host: str = "www.landbouwvlaanderen.be") -> bytes:
+    snapshot = (
+        f'<a href="https://{download_host}/bestanden/gis/agpa_2026_2026-06-02_public.zip">'
+        "Landbouwgebruikspercelen 2026 – 1e snapshot (extractie 02-06-2026) - GPKG</a>"
+        if include_snapshot
+        else ""
+    )
+    return f"""
+    <html><body>
+      {snapshot}
+      <p>Definitieve datasets</p>
+      <a href="https://{download_host}/bestanden/gis/agpa_2025_2026-05-13_public.zip">Downloaden</a>
+      <a href="https://{download_host}/bestanden/gis/agpa_2024_2025-03-27_public.zip">Downloaden</a>
+    </body></html>
+    """.encode()
+
+
 class _Response:
    def __init__(self, content: bytes, content_type: str = "text/xml", *, content_length: int | None = None) -> None:
        self.content = content
@@ -97,8 +115,8 @@ class _Response:


 class _RedirectedResponse(_Response):
-    def __init__(self, content: bytes, final_url: str) -> None:
-        super().__init__(content)
+    def __init__(self, content: bytes, final_url: str, content_type: str = "text/xml") -> None:
+        super().__init__(content, content_type)
        self.final_url = final_url

    def geturl(self) -> str:
@@ -144,6 +162,7 @@ def _settings(**overrides) -> Settings:
    values = {
        "SOURCE_CATALOG_PROBE_ENABLED": True,
        "SOURCE_CATALOG_GRB_WFS_URL": "https://geo.api.vlaanderen.be/GRB/wfs",
+        "SOURCE_CATALOG_ALZ_RELEASE_URL": ALZ_RELEASE_URL,
        "SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS": 3,
        "SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB": 1,
        "SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS": 0,
@@ -157,6 +176,8 @@ def _settings(**overrides) -> Settings:
 def _opener(request, timeout):
    assert timeout == 3
    url = request.full_url
+    if url == ALZ_RELEASE_URL:
+        return _Response(_alz_release_page(), "text/html; charset=utf-8")
    if "metadata.vlaanderen.be" in url:
        if "f5304d6d" in url:
            return _Response(_metadata("f5304d6d", "Orthofoto meest recent, 2025.04", "2025.04", "2026-04-27", "2025-12-11"))
@@ -170,7 +191,13 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions
    SourceCatalogProbeService.clear_cache()
    project_id = uuid.uuid4()
    report = SourceCatalogProbeService.audit_project(
-        _Db([_dataset("grb", "2026-07-14"), _dataset("digitaal_vlaanderen_orthophoto", "most_recent_at_2026-07-14")]),
+        _Db(
+            [
+                _dataset("grb", "2026-07-14"),
+                _dataset("digitaal_vlaanderen_orthophoto", "most_recent_at_2026-07-14"),
+                _dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2025-definitive"),
+            ]
+        ),
        project_id,
        settings=_settings(),
        opener=_opener,
@@ -178,7 +205,7 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions
    )
    by_source = {item.source_name: item for item in report.items}

-    assert report.summary.available_count == 2
+    assert report.summary.available_count == 3
    assert report.summary.different_version_count == 1
    assert by_source["grb"].remote_version == "Toestand 2026-07-15"
    assert by_source["grb"].comparison_status == "different"
@@ -187,6 +214,13 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions
    assert by_source["digitaal_vlaanderen_orthophoto"].remote_version == "2025.04"
    assert by_source["digitaal_vlaanderen_orthophoto"].comparison_status == "not_comparable"
    assert by_source["digitaal_vlaanderen_orthophoto"].remote_published_at.year == 2025
+    alz = by_source["agentschap_landbouw_zeevisserij_agricultural_parcels"]
+    assert alz.service_type == "HTML"
+    assert alz.remote_version == "2025-v3"
+    assert alz.comparison_status == "same"
+    assert alz.matched_layers == ["definitive_archive", "current_snapshot"]
+    assert "2026-v1" in alz.message
+    assert "voorlopig" in alz.message
    assert all(item.capabilities_sha256 for item in report.items)


@@ -275,10 +309,10 @@ def test_catalog_probe_cache_is_explicitly_bypassable() -> None:
    second = SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW)
    SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW, force=True)

-    assert first_call_count == 4
+    assert first_call_count == 5
    assert all(item.cached is False for item in first.items)
    assert all(item.cached is True for item in second.items)
-    assert len(calls) == 8
+    assert len(calls) == 10


 def test_catalog_probe_can_be_disabled_without_network_access() -> None:
@@ -293,7 +327,7 @@ def test_catalog_probe_can_be_disabled_without_network_access() -> None:
        now=NOW,
    )

-    assert report.summary.disabled_count == 2
+    assert report.summary.disabled_count == 3
    assert all(item.status == "disabled" for item in report.items)


@@ -316,7 +350,7 @@ def test_catalog_probe_route_returns_canonical_envelope(monkeypatch) -> None:

    assert list(response) == ["data"]
    assert response["data"]["project_id"] == project_id
-    assert response["data"]["summary"]["provider_count"] == 2
+    assert response["data"]["summary"]["provider_count"] == 3


 def test_catalog_probe_remains_explicit_and_never_imports_provider_data() -> None:
@@ -330,3 +364,75 @@ def test_catalog_probe_remains_explicit_and_never_imports_provider_data() -> Non
    assert "VectorFeatureService" not in service
    assert "--probe-catalogs" in operator
    assert "/datasets/source-catalog-probes" in operator
+
+
+def test_alz_catalog_probe_rejects_release_page_redirect_outside_allowlist() -> None:
+    def redirected_opener(request, timeout):
+        if request.full_url == ALZ_RELEASE_URL:
+            return _RedirectedResponse(
+                _alz_release_page(),
+                "https://example.com/open-geodata-landbouwgebruikspercelen",
+                "text/html",
+            )
+        return _opener(request, timeout)
+
+    report = SourceCatalogProbeService.audit_project(
+        _Db([]), uuid.uuid4(), settings=_settings(), opener=redirected_opener, now=NOW
+    )
+    alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels")
+
+    assert alz.status == "unavailable"
+    assert alz.error_code == "CATALOG_ALZ_RELEASE_URL_REJECTED"
+
+
+def test_alz_catalog_probe_rejects_untrusted_download_host() -> None:
+    def malicious_opener(request, timeout):
+        if request.full_url == ALZ_RELEASE_URL:
+            return _Response(_alz_release_page(download_host="example.com"), "text/html")
+        return _opener(request, timeout)
+
+    report = SourceCatalogProbeService.audit_project(
+        _Db([]), uuid.uuid4(), settings=_settings(), opener=malicious_opener, now=NOW
+    )
+    alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels")
+
+    assert alz.status == "unavailable"
+    assert alz.error_code == "CATALOG_ALZ_DOWNLOAD_URL_REJECTED"
+
+
+def test_alz_catalog_probe_degrades_without_current_snapshot_but_keeps_definitive_evidence() -> None:
+    def archive_only_opener(request, timeout):
+        if request.full_url == ALZ_RELEASE_URL:
+            return _Response(_alz_release_page(include_snapshot=False), "text/html")
+        return _opener(request, timeout)
+
+    report = SourceCatalogProbeService.audit_project(
+        _Db([_dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2024-definitive")]),
+        uuid.uuid4(),
+        settings=_settings(),
+        opener=archive_only_opener,
+        now=NOW,
+    )
+    alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels")
+
+    assert alz.status == "degraded"
+    assert alz.remote_version == "2025-v3"
+    assert alz.comparison_status == "different"
+    assert alz.matched_layers == ["definitive_archive"]
+    assert alz.missing_layers == ["current_snapshot"]
+    assert alz.error_code == "CATALOG_ALZ_CURRENT_SNAPSHOT_MISSING"
+
+
+def test_alz_catalog_probe_compares_latest_definitive_year_not_latest_import_order() -> None:
+    older = _dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2024-definitive")
+    newer = _dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2025-definitive")
+    older.imported_at = NOW
+    newer.imported_at = NOW.replace(year=2025)
+
+    report = SourceCatalogProbeService.audit_project(
+        _Db([older, newer]), uuid.uuid4(), settings=_settings(), opener=_opener, now=NOW
+    )
+    alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels")
+
+    assert alz.local_source_version == "2025-definitive"
+    assert alz.comparison_status == "same"