Add governed ALZ edition probe

backend/app/core/config.py

@@ -36,6 +36,10 @@ class Settings(BaseSettings):
         default="https://geo.api.vlaanderen.be/GRB/wfs",
         validation_alias="SOURCE_CATALOG_GRB_WFS_URL",
     )
+    source_catalog_alz_release_url: str = Field(
+        default="https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen",
+        validation_alias="SOURCE_CATALOG_ALZ_RELEASE_URL",
+    )
     source_catalog_probe_timeout_seconds: int = Field(
         default=10,
         ge=1,

backend/app/schemas/source_catalog.py

@@ -14,7 +14,7 @@ SourceCatalogComparisonStatus = Literal["same", "different", "not_comparable", "
 class SourceCatalogProbeItem(BaseModel):
     source_name: str
     display_name: str
-    service_type: Literal["WFS", "WMS"]
+    service_type: Literal["WFS", "WMS", "HTML"]
     endpoint_url: str
     status: SourceCatalogProbeStatus
     reachable: bool

backend/app/services/source_catalog_probe_service.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass, replace
 from datetime import datetime, timedelta, timezone
 from email.utils import parsedate_to_datetime
 from hashlib import sha256
+from html.parser import HTMLParser
 import re
 from threading import Lock
 from typing import Any, Callable
@@ -32,6 +33,17 @@ _XLINK = "http://www.w3.org/1999/xlink"
 _METADATA_HOST = "metadata.vlaanderen.be"
 _VERSION_DATE = re.compile(r"^(?:toestand\s+)?(\d{4}-\d{2}-\d{2})$", re.IGNORECASE)
 _ORTHOPHOTO_EDITION = re.compile(r"^\d{4}\.\d{2}$")
+_ALZ_SOURCE_NAME = "agentschap_landbouw_zeevisserij_agricultural_parcels"
+_ALZ_RELEASE_HOST = "landbouwcijfers.vlaanderen.be"
+_ALZ_RELEASE_PATH = "/open-geodata-landbouwgebruikspercelen"
+_ALZ_DOWNLOAD_HOST = "www.landbouwvlaanderen.be"
+_ALZ_DOWNLOAD_PATH = re.compile(r"^/bestanden/gis/agpa_(20\d{2})_(\d{4}-\d{2}-\d{2})_public\.zip$")
+_ALZ_SNAPSHOT = re.compile(
+    r"^Landbouwgebruikspercelen\s+(20\d{2})\s*-\s*(\d+)e\s+snapshot\s*"
+    r"\(extractie\s+(\d{2}-\d{2}-\d{4})\)(?:\s*-\s*GPKG)?$",
+    re.IGNORECASE,
+)
+_ALZ_EDITION = re.compile(r"^(20\d{2})-(?:definitive|v3)$", re.IGNORECASE)
 
 
 class CatalogProbeFailure(RuntimeError):
@@ -88,6 +100,32 @@ class CacheEntry:
     probe: RemoteProbe
 
 
+class _AnchorParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.anchors: list[tuple[str, str]] = []
+        self._href: str | None = None
+        self._text: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag.lower() != "a" or self._href is not None:
+            return
+        href = dict(attrs).get("href")
+        if href:
+            self._href = href.strip()
+            self._text = []
+
+    def handle_data(self, data: str) -> None:
+        if self._href is not None:
+            self._text.append(data)
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag.lower() == "a" and self._href is not None:
+            self.anchors.append((self._href, "".join(self._text)))
+            self._href = None
+            self._text = []
+
+
 _REMOTE_CACHE: dict[str, CacheEntry] = {}
 _CACHE_LOCK = Lock()
 
@@ -124,7 +162,7 @@ def _with_capabilities_query(base_url: str, service_type: str) -> str:
 def _bounded_fetch(url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> FetchResult:
     request = Request(
         url,
-        headers={"Accept": "application/xml,text/xml,application/json", "User-Agent": "GeoIntel/0.1 source-catalog-probe"},
+        headers={"Accept": "application/xml,text/xml,text/html,application/json", "User-Agent": "GeoIntel/0.1 source-catalog-probe"},
     )
     max_bytes = settings.source_catalog_probe_max_response_mb * 1024 * 1024
     try:
@@ -226,6 +264,127 @@ def _validate_capabilities_url(url: str) -> str:
     return url
 
 
+def _validate_alz_release_url(url: str) -> str:
+    parsed = urlsplit(url)
+    if (
+        parsed.scheme != "https"
+        or parsed.hostname != _ALZ_RELEASE_HOST
+        or parsed.port not in {None, 443}
+        or parsed.username
+        or parsed.password
+        or parsed.path.rstrip("/") != _ALZ_RELEASE_PATH
+        or parsed.query
+        or parsed.fragment
+    ):
+        raise CatalogProbeFailure(
+            "CATALOG_ALZ_RELEASE_URL_REJECTED",
+            "De ingestelde ALZ-publicatiepagina valt buiten de toegestane officiële URL.",
+        )
+    return url
+
+
+def _validate_alz_download_url(url: str) -> tuple[int, datetime]:
+    parsed = urlsplit(url)
+    match = _ALZ_DOWNLOAD_PATH.fullmatch(parsed.path)
+    if (
+        parsed.scheme != "https"
+        or parsed.hostname != _ALZ_DOWNLOAD_HOST
+        or parsed.port not in {None, 443}
+        or parsed.username
+        or parsed.password
+        or parsed.query
+        or parsed.fragment
+        or not match
+    ):
+        raise CatalogProbeFailure(
+            "CATALOG_ALZ_DOWNLOAD_URL_REJECTED",
+            "De ALZ-publicatiepagina bevat een datasetlink buiten de toegestane officiële URL-structuur.",
+        )
+    try:
+        published_at = datetime.strptime(match.group(2), "%Y-%m-%d").replace(tzinfo=timezone.utc)
+    except ValueError as exc:
+        raise CatalogProbeFailure(
+            "CATALOG_ALZ_DOWNLOAD_URL_REJECTED",
+            "De ALZ-datasetlink bevat geen geldige publicatiedatum.",
+        ) from exc
+    return int(match.group(1)), published_at
+
+
+def _normalized_html_text(value: str) -> str:
+    return re.sub(r"\s+", " ", value.replace("\xad", "").replace("–", "-").replace("—", "-")).strip()
+
+
+def _parse_alz_release_page(content: bytes) -> dict[str, Any]:
+    try:
+        html = content.decode("utf-8")
+    except UnicodeDecodeError as exc:
+        raise CatalogProbeFailure(
+            "CATALOG_ALZ_INVALID_HTML",
+            "De officiële ALZ-publicatiepagina is niet geldige UTF-8 HTML.",
+        ) from exc
+    parser = _AnchorParser()
+    try:
+        parser.feed(html)
+        parser.close()
+    except Exception as exc:
+        raise CatalogProbeFailure(
+            "CATALOG_ALZ_INVALID_HTML",
+            "De officiële ALZ-publicatiepagina kon niet veilig worden ontleed.",
+        ) from exc
+
+    definitive: list[tuple[int, datetime]] = []
+    snapshots: list[tuple[int, int, datetime]] = []
+    for href, raw_text in parser.anchors:
+        text = _normalized_html_text(raw_text)
+        looks_like_alz_release = (
+            text.casefold() == "downloaden"
+            or text.casefold().startswith("landbouwgebruikspercelen ")
+            or "agpa_" in href.casefold()
+        )
+        if not looks_like_alz_release:
+            continue
+        year, file_date = _validate_alz_download_url(href)
+        snapshot = _ALZ_SNAPSHOT.fullmatch(text)
+        if snapshot:
+            snapshot_year = int(snapshot.group(1))
+            snapshot_number = int(snapshot.group(2))
+            try:
+                extraction_date = datetime.strptime(snapshot.group(3), "%d-%m-%Y").replace(tzinfo=timezone.utc)
+            except ValueError as exc:
+                raise CatalogProbeFailure(
+                    "CATALOG_ALZ_SNAPSHOT_INVALID",
+                    "De actuele ALZ-snapshot bevat geen geldige extractiedatum.",
+                ) from exc
+            if snapshot_year != year or extraction_date != file_date or snapshot_number not in {1, 2, 3}:
+                raise CatalogProbeFailure(
+                    "CATALOG_ALZ_SNAPSHOT_INVALID",
+                    "De actuele ALZ-snapshot is niet consistent met de officiële datasetlink.",
+                )
+            snapshots.append((year, snapshot_number, extraction_date))
+        elif text.casefold() == "downloaden":
+            definitive.append((year, file_date))
+        else:
+            raise CatalogProbeFailure(
+                "CATALOG_ALZ_RELEASE_UNRECOGNIZED",
+                "De ALZ-publicatiepagina bevat een niet-herkende landbouwdatasetpublicatie.",
+            )
+
+    if not definitive:
+        raise CatalogProbeFailure(
+            "CATALOG_ALZ_DEFINITIVE_MISSING",
+            "De officiële ALZ-publicatiepagina bevat geen herkenbare definitieve landbouwperceeleditie.",
+        )
+    latest_definitive = max(definitive, key=lambda item: (item[0], item[1]))
+    latest_snapshot = max(snapshots, key=lambda item: (item[0], item[1], item[2])) if snapshots else None
+    if latest_snapshot and latest_snapshot[1] == 3 and latest_snapshot[0] >= latest_definitive[0]:
+        latest_definitive = (latest_snapshot[0], latest_snapshot[2])
+    return {
+        "definitive": latest_definitive,
+        "definitive_count": len(definitive),
+        "snapshot": latest_snapshot,
+    }
+
+
 def _node_text(node: ElementTree.Element | None) -> str | None:
     if node is None:
         return None
@@ -282,6 +441,74 @@ def _parse_metadata(content: bytes) -> dict[str, Any]:
     }
 
 
+def _probe_alz_remote(
+    contract: ProbeContract,
+    settings: Settings,
+    *,
+    opener: Callable[..., Any] | None,
+    now: datetime,
+) -> RemoteProbe:
+    release_url = _validate_alz_release_url(contract.endpoint_url)
+    response = _bounded_fetch(release_url, settings, opener)
+    _validate_alz_release_url(response.final_url)
+    content_type = response.content_type.lower()
+    if content_type and "html" not in content_type and "text" not in content_type:
+        raise CatalogProbeFailure(
+            "CATALOG_ALZ_INVALID_CONTENT_TYPE",
+            "De officiële ALZ-publicatiepagina is geen HTML-respons.",
+        )
+    release = _parse_alz_release_page(response.content)
+    definitive_year, definitive_date = release["definitive"]
+    snapshot = release["snapshot"]
+    matched = ["definitive_archive"]
+    missing: list[str] = []
+    if snapshot:
+        matched.append("current_snapshot")
+    else:
+        missing.append("current_snapshot")
+
+    definitive_version = f"{definitive_year}-v3"
+    if snapshot:
+        snapshot_year, snapshot_number, snapshot_date = snapshot
+        snapshot_version = f"{snapshot_year}-v{snapshot_number}"
+        if snapshot_number < 3:
+            message = (
+                f"De officiële ALZ-publicatiepagina bevestigt definitieve editie {definitive_version}. "
+                f"De actuele publicatie {snapshot_version} van {snapshot_date.date().isoformat()} is voorlopig "
+                "en wordt niet als historische vervanging aangemerkt."
+            )
+        else:
+            message = f"De officiële ALZ-publicatiepagina bevestigt definitieve editie {definitive_version}."
+        remote_title = f"Landbouwgebruikspercelen {definitive_version}; actuele publicatie {snapshot_version}"
+    else:
+        message = (
+            f"De officiële ALZ-publicatiepagina bevestigt definitieve editie {definitive_version}, "
+            "maar bevat geen herkenbare actuele snapshot."
+        )
+        remote_title = f"Landbouwgebruikspercelen {definitive_version}"
+
+    return RemoteProbe(
+        status="available" if snapshot else "degraded",
+        reachable=True,
+        checked_at=now,
+        expected_layers=contract.expected_layers,
+        matched_layers=tuple(matched),
+        missing_layers=tuple(missing),
+        advertised_layer_count=release["definitive_count"] + (1 if snapshot else 0),
+        metadata_url=response.final_url,
+        metadata_identifier="alz-agricultural-use-parcels",
+        remote_title=remote_title,
+        remote_version=definitive_version,
+        remote_modified_at=response.last_modified_at,
+        remote_published_at=definitive_date,
+        capabilities_sha256=sha256(response.content).hexdigest(),
+        capabilities_etag=response.etag,
+        capabilities_last_modified_at=response.last_modified_at,
+        message=message,
+        error_code="CATALOG_ALZ_CURRENT_SNAPSHOT_MISSING" if not snapshot else None,
+    )
+
+
 def _probe_remote(
     contract: ProbeContract,
     settings: Settings,
@@ -290,6 +517,8 @@ def _probe_remote(
     now: datetime,
 ) -> RemoteProbe:
     try:
+        if contract.service_type == "HTML":
+            return _probe_alz_remote(contract, settings, opener=opener, now=now)
         capabilities = _bounded_fetch(_validate_capabilities_url(contract.endpoint_url), settings, opener)
         _validate_capabilities_url(capabilities.final_url)
         content_type = capabilities.content_type.lower()
@@ -394,6 +623,18 @@ def _latest_local_version(source_name: str, datasets: list[Dataset]) -> str | No
         ]
         if explicit_current:
             candidates = explicit_current
+    elif source_name == _ALZ_SOURCE_NAME:
+        definitive = [item for item in candidates if _ALZ_EDITION.fullmatch((item.source_version or "").strip())]
+        if definitive:
+            latest = max(
+                definitive,
+                key=lambda item: (
+                    int(_ALZ_EDITION.fullmatch((item.source_version or "").strip()).group(1)),
+                    _utc(item.observed_at) if item.observed_at else datetime.min.replace(tzinfo=timezone.utc),
+                    str(item.id),
+                ),
+            )
+            return latest.source_version
     if not candidates:
         return None
     latest = max(
@@ -416,6 +657,9 @@ def _normalized_version(source_name: str, version: str | None) -> str | None:
         return match.group(1) if match else None
     if source_name == "digitaal_vlaanderen_orthophoto" and _ORTHOPHOTO_EDITION.fullmatch(value):
         return value
+    if source_name == _ALZ_SOURCE_NAME:
+        match = _ALZ_EDITION.fullmatch(value)
+        return f"{match.group(1)}-v3" if match else None
     return None
 
 
@@ -467,6 +711,13 @@ class SourceCatalogProbeService:
                 endpoint_url=_with_capabilities_query(active_settings.orthophoto_wms_url, "WMS"),
                 expected_layers=(active_settings.orthophoto_wms_layer, "Vliegdagcontour"),
             ),
+            ProbeContract(
+                source_name=_ALZ_SOURCE_NAME,
+                display_name="Landbouwgebruikspercelen (ALZ)",
+                service_type="HTML",
+                endpoint_url=active_settings.source_catalog_alz_release_url,
+                expected_layers=("definitive_archive", "current_snapshot"),
+            ),
         )
         items: list[SourceCatalogProbeItem] = []
         for contract in contracts:
@@ -538,8 +789,9 @@ class SourceCatalogProbeService:
             summary=summary,
             items=items,
             limitations=[
-                "Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities en gekoppelde ISO 19139 metadata.",
+                "Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities, gekoppelde ISO 19139 metadata en de officiële ALZ-publicatiepagina.",
                 "Er worden geen features, rasters of modelbestanden opgehaald en geen datasets aangemaakt of overschreven.",
+                "Voor ALZ is alleen de nieuwste definitieve v3-editie vergelijkbaar; voorlopige v1/v2-snapshots zijn uitsluitend informatief.",
                 "Een versieverschil is controlesignaal, geen bewijs dat een lokale dataset onbruikbaar is en geen automatische importopdracht.",
             ],
         )