From 0a23e3484ce687a8f000ea5adb6c171f040b7794 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 16 Jul 2026 20:27:07 +0200 Subject: [PATCH] Add governed ALZ edition probe --- .env.example | 1 + CHANGELOG.md | 17 ++ backend/README.md | 20 +- backend/app/core/config.py | 4 + backend/app/schemas/source_catalog.py | 2 +- .../services/source_catalog_probe_service.py | 256 +++++++++++++++++- .../test_sprint222_source_catalog_probes.py | 122 ++++++++- deploy/unraid/geointel-unraid-template.xml | 3 +- deploy/unraid/geointel.env.example | 5 +- deploy/unraid/run-dockerman-container.sh | 2 + docker-compose.yml | 1 + docs/API_CONTRACTS.md | 21 +- docs/DATA_SOURCES.md | 9 +- docs/ENVIRONMENT_SPEC.md | 1 + docs/TODO.md | 7 +- frontend/README.md | 9 +- .../status/SourceFreshnessPanel.tsx | 7 +- frontend/src/types.ts | 2 +- 18 files changed, 451 insertions(+), 38 deletions(-) diff --git a/.env.example b/.env.example index 3875ece0..d5b2a817 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,7 @@ ORTHOPHOTO_MAX_SIDE_M=1024 ORTHOPHOTO_CACHE_TTL_HOURS=24 SOURCE_CATALOG_PROBE_ENABLED=true SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs +SOURCE_CATALOG_ALZ_RELEASE_URL=https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10 SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2 SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900 diff --git a/CHANGELOG.md b/CHANGELOG.md index d3cffe95..f63df283 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ # Changelog +## Sprint 225 Governed ALZ edition probe (2026-07-16) + +- Extended the explicit read-only source catalog audit with the official ALZ + agricultural-use parcel publication page without adding an importer, + scheduler or background provider request. +- Added strict release-page, redirect and archive-link allowlists and bounded + HTML parsing; the probe never downloads the referenced ALZ ZIP archives. +- Normalized only definitive third snapshots as comparable editions, so local + `2025-definitive` evidence matches official `2025-v3` while current + `2026-v1` remains visibly provisional. +- Reused the canonical source-catalog envelope, Status UI and operator command, + and propagated the fixed release URL through Docker and Unraid runtime + configuration. +- Added focused release parsing, version ordering, cache, failure-isolation and + trust-boundary regression coverage without changing persistence or + migrations. + ## Sprint 224 Governed GRB evolution (2026-07-16) - Enabled fail-closed object evolution between the retained regional GRB diff --git a/backend/README.md b/backend/README.md index b265bfa8..dc7eb86c 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1444,7 +1444,8 @@ docker exec geointel python /app/scripts/audit_source_freshness.py \ Use `--fail-on due` to make a planned review date fail automation, or `--fail-on never` for reporting only. The command never starts a refresh. -An operator can explicitly add the official GRB and orthophoto edition check: +An operator can explicitly add the official GRB, orthophoto and ALZ edition +check: ```bash docker exec geointel python /app/scripts/audit_source_freshness.py \ @@ -1458,17 +1459,20 @@ Use `--refresh-catalogs` to bypass the 15-minute in-memory cache and `--fail-on-catalog` only when temporary official-provider unavailability must fail an operator job. This path reads bounded WFS/WMS capabilities and their fixed ISO 19139 metadata records. It confirms GRB `GBG`, `WBN`, `WGO`, `ADP` -and orthophoto `Ortho`, `Vliegdagcontour`; it never requests feature or raster -content. The public Datavindplaats API requires an access token, so release -evidence comes from the public metadata links published by the official OGC -services rather than HTML scraping. +and orthophoto `Ortho`, `Vliegdagcontour`. It also reads the exact official ALZ +publication page and validates only allowlisted archive-link identities. It +never requests feature, raster or ALZ ZIP content. ALZ v1/v2 campaign snapshots +remain provisional; only a v3 publication is compared with a local definitive +historical edition. Runtime controls are `SOURCE_CATALOG_PROBE_ENABLED`, -`SOURCE_CATALOG_GRB_WFS_URL`, `SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS`, -`SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB` and +`SOURCE_CATALOG_GRB_WFS_URL`, `SOURCE_CATALOG_ALZ_RELEASE_URL`, +`SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS`, `SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB` and `SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS`. Metadata links remain restricted to the official HTTPS CSW path on `metadata.vlaanderen.be` even when an operator -overrides the capabilities endpoint. +overrides the capabilities endpoint. The ALZ release URL is fail-closed to the +exact HTTPS host/path and cannot be redirected to another page or download +host. ## Governed regional GRB refresh diff --git a/backend/app/core/config.py b/backend/app/core/config.py index ca22b1fd..17c40190 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -36,6 +36,10 @@ class Settings(BaseSettings): default="https://geo.api.vlaanderen.be/GRB/wfs", validation_alias="SOURCE_CATALOG_GRB_WFS_URL", ) + source_catalog_alz_release_url: str = Field( + default="https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen", + validation_alias="SOURCE_CATALOG_ALZ_RELEASE_URL", + ) source_catalog_probe_timeout_seconds: int = Field( default=10, ge=1, diff --git a/backend/app/schemas/source_catalog.py b/backend/app/schemas/source_catalog.py index 664b5baa..efe95c0b 100644 --- a/backend/app/schemas/source_catalog.py +++ b/backend/app/schemas/source_catalog.py @@ -14,7 +14,7 @@ SourceCatalogComparisonStatus = Literal["same", "different", "not_comparable", " class SourceCatalogProbeItem(BaseModel): source_name: str display_name: str - service_type: Literal["WFS", "WMS"] + service_type: Literal["WFS", "WMS", "HTML"] endpoint_url: str status: SourceCatalogProbeStatus reachable: bool diff --git a/backend/app/services/source_catalog_probe_service.py b/backend/app/services/source_catalog_probe_service.py index fae36671..6ab711e3 100644 --- a/backend/app/services/source_catalog_probe_service.py +++ b/backend/app/services/source_catalog_probe_service.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, replace from datetime import datetime, timedelta, timezone from email.utils import parsedate_to_datetime from hashlib import sha256 +from html.parser import HTMLParser import re from threading import Lock from typing import Any, Callable @@ -32,6 +33,17 @@ _XLINK = "http://www.w3.org/1999/xlink" _METADATA_HOST = "metadata.vlaanderen.be" _VERSION_DATE = re.compile(r"^(?:toestand\s+)?(\d{4}-\d{2}-\d{2})$", re.IGNORECASE) _ORTHOPHOTO_EDITION = re.compile(r"^\d{4}\.\d{2}$") +_ALZ_SOURCE_NAME = "agentschap_landbouw_zeevisserij_agricultural_parcels" +_ALZ_RELEASE_HOST = "landbouwcijfers.vlaanderen.be" +_ALZ_RELEASE_PATH = "/open-geodata-landbouwgebruikspercelen" +_ALZ_DOWNLOAD_HOST = "www.landbouwvlaanderen.be" +_ALZ_DOWNLOAD_PATH = re.compile(r"^/bestanden/gis/agpa_(20\d{2})_(\d{4}-\d{2}-\d{2})_public\.zip$") +_ALZ_SNAPSHOT = re.compile( + r"^Landbouwgebruikspercelen\s+(20\d{2})\s*-\s*(\d+)e\s+snapshot\s*" + r"\(extractie\s+(\d{2}-\d{2}-\d{4})\)(?:\s*-\s*GPKG)?$", + re.IGNORECASE, +) +_ALZ_EDITION = re.compile(r"^(20\d{2})-(?:definitive|v3)$", re.IGNORECASE) class CatalogProbeFailure(RuntimeError): @@ -88,6 +100,32 @@ class CacheEntry: probe: RemoteProbe +class _AnchorParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.anchors: list[tuple[str, str]] = [] + self._href: str | None = None + self._text: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag.lower() != "a" or self._href is not None: + return + href = dict(attrs).get("href") + if href: + self._href = href.strip() + self._text = [] + + def handle_data(self, data: str) -> None: + if self._href is not None: + self._text.append(data) + + def handle_endtag(self, tag: str) -> None: + if tag.lower() == "a" and self._href is not None: + self.anchors.append((self._href, "".join(self._text))) + self._href = None + self._text = [] + + _REMOTE_CACHE: dict[str, CacheEntry] = {} _CACHE_LOCK = Lock() @@ -124,7 +162,7 @@ def _with_capabilities_query(base_url: str, service_type: str) -> str: def _bounded_fetch(url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> FetchResult: request = Request( url, - headers={"Accept": "application/xml,text/xml,application/json", "User-Agent": "GeoIntel/0.1 source-catalog-probe"}, + headers={"Accept": "application/xml,text/xml,text/html,application/json", "User-Agent": "GeoIntel/0.1 source-catalog-probe"}, ) max_bytes = settings.source_catalog_probe_max_response_mb * 1024 * 1024 try: @@ -226,6 +264,127 @@ def _validate_capabilities_url(url: str) -> str: return url +def _validate_alz_release_url(url: str) -> str: + parsed = urlsplit(url) + if ( + parsed.scheme != "https" + or parsed.hostname != _ALZ_RELEASE_HOST + or parsed.port not in {None, 443} + or parsed.username + or parsed.password + or parsed.path.rstrip("/") != _ALZ_RELEASE_PATH + or parsed.query + or parsed.fragment + ): + raise CatalogProbeFailure( + "CATALOG_ALZ_RELEASE_URL_REJECTED", + "De ingestelde ALZ-publicatiepagina valt buiten de toegestane officiële URL.", + ) + return url + + +def _validate_alz_download_url(url: str) -> tuple[int, datetime]: + parsed = urlsplit(url) + match = _ALZ_DOWNLOAD_PATH.fullmatch(parsed.path) + if ( + parsed.scheme != "https" + or parsed.hostname != _ALZ_DOWNLOAD_HOST + or parsed.port not in {None, 443} + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + or not match + ): + raise CatalogProbeFailure( + "CATALOG_ALZ_DOWNLOAD_URL_REJECTED", + "De ALZ-publicatiepagina bevat een datasetlink buiten de toegestane officiële URL-structuur.", + ) + try: + published_at = datetime.strptime(match.group(2), "%Y-%m-%d").replace(tzinfo=timezone.utc) + except ValueError as exc: + raise CatalogProbeFailure( + "CATALOG_ALZ_DOWNLOAD_URL_REJECTED", + "De ALZ-datasetlink bevat geen geldige publicatiedatum.", + ) from exc + return int(match.group(1)), published_at + + +def _normalized_html_text(value: str) -> str: + return re.sub(r"\s+", " ", value.replace("\xad", "").replace("–", "-").replace("—", "-")).strip() + + +def _parse_alz_release_page(content: bytes) -> dict[str, Any]: + try: + html = content.decode("utf-8") + except UnicodeDecodeError as exc: + raise CatalogProbeFailure( + "CATALOG_ALZ_INVALID_HTML", + "De officiële ALZ-publicatiepagina is niet geldige UTF-8 HTML.", + ) from exc + parser = _AnchorParser() + try: + parser.feed(html) + parser.close() + except Exception as exc: + raise CatalogProbeFailure( + "CATALOG_ALZ_INVALID_HTML", + "De officiële ALZ-publicatiepagina kon niet veilig worden ontleed.", + ) from exc + + definitive: list[tuple[int, datetime]] = [] + snapshots: list[tuple[int, int, datetime]] = [] + for href, raw_text in parser.anchors: + text = _normalized_html_text(raw_text) + looks_like_alz_release = ( + text.casefold() == "downloaden" + or text.casefold().startswith("landbouwgebruikspercelen ") + or "agpa_" in href.casefold() + ) + if not looks_like_alz_release: + continue + year, file_date = _validate_alz_download_url(href) + snapshot = _ALZ_SNAPSHOT.fullmatch(text) + if snapshot: + snapshot_year = int(snapshot.group(1)) + snapshot_number = int(snapshot.group(2)) + try: + extraction_date = datetime.strptime(snapshot.group(3), "%d-%m-%Y").replace(tzinfo=timezone.utc) + except ValueError as exc: + raise CatalogProbeFailure( + "CATALOG_ALZ_SNAPSHOT_INVALID", + "De actuele ALZ-snapshot bevat geen geldige extractiedatum.", + ) from exc + if snapshot_year != year or extraction_date != file_date or snapshot_number not in {1, 2, 3}: + raise CatalogProbeFailure( + "CATALOG_ALZ_SNAPSHOT_INVALID", + "De actuele ALZ-snapshot is niet consistent met de officiële datasetlink.", + ) + snapshots.append((year, snapshot_number, extraction_date)) + elif text.casefold() == "downloaden": + definitive.append((year, file_date)) + else: + raise CatalogProbeFailure( + "CATALOG_ALZ_RELEASE_UNRECOGNIZED", + "De ALZ-publicatiepagina bevat een niet-herkende landbouwdatasetpublicatie.", + ) + + if not definitive: + raise CatalogProbeFailure( + "CATALOG_ALZ_DEFINITIVE_MISSING", + "De officiële ALZ-publicatiepagina bevat geen herkenbare definitieve landbouwperceeleditie.", + ) + latest_definitive = max(definitive, key=lambda item: (item[0], item[1])) + latest_snapshot = max(snapshots, key=lambda item: (item[0], item[1], item[2])) if snapshots else None + if latest_snapshot and latest_snapshot[1] == 3 and latest_snapshot[0] >= latest_definitive[0]: + latest_definitive = (latest_snapshot[0], latest_snapshot[2]) + return { + "definitive": latest_definitive, + "definitive_count": len(definitive), + "snapshot": latest_snapshot, + } + + def _node_text(node: ElementTree.Element | None) -> str | None: if node is None: return None @@ -282,6 +441,74 @@ def _parse_metadata(content: bytes) -> dict[str, Any]: } +def _probe_alz_remote( + contract: ProbeContract, + settings: Settings, + *, + opener: Callable[..., Any] | None, + now: datetime, +) -> RemoteProbe: + release_url = _validate_alz_release_url(contract.endpoint_url) + response = _bounded_fetch(release_url, settings, opener) + _validate_alz_release_url(response.final_url) + content_type = response.content_type.lower() + if content_type and "html" not in content_type and "text" not in content_type: + raise CatalogProbeFailure( + "CATALOG_ALZ_INVALID_CONTENT_TYPE", + "De officiële ALZ-publicatiepagina is geen HTML-respons.", + ) + release = _parse_alz_release_page(response.content) + definitive_year, definitive_date = release["definitive"] + snapshot = release["snapshot"] + matched = ["definitive_archive"] + missing: list[str] = [] + if snapshot: + matched.append("current_snapshot") + else: + missing.append("current_snapshot") + + definitive_version = f"{definitive_year}-v3" + if snapshot: + snapshot_year, snapshot_number, snapshot_date = snapshot + snapshot_version = f"{snapshot_year}-v{snapshot_number}" + if snapshot_number < 3: + message = ( + f"De officiële ALZ-publicatiepagina bevestigt definitieve editie {definitive_version}. " + f"De actuele publicatie {snapshot_version} van {snapshot_date.date().isoformat()} is voorlopig " + "en wordt niet als historische vervanging aangemerkt." + ) + else: + message = f"De officiële ALZ-publicatiepagina bevestigt definitieve editie {definitive_version}." + remote_title = f"Landbouwgebruikspercelen {definitive_version}; actuele publicatie {snapshot_version}" + else: + message = ( + f"De officiële ALZ-publicatiepagina bevestigt definitieve editie {definitive_version}, " + "maar bevat geen herkenbare actuele snapshot." + ) + remote_title = f"Landbouwgebruikspercelen {definitive_version}" + + return RemoteProbe( + status="available" if snapshot else "degraded", + reachable=True, + checked_at=now, + expected_layers=contract.expected_layers, + matched_layers=tuple(matched), + missing_layers=tuple(missing), + advertised_layer_count=release["definitive_count"] + (1 if snapshot else 0), + metadata_url=response.final_url, + metadata_identifier="alz-agricultural-use-parcels", + remote_title=remote_title, + remote_version=definitive_version, + remote_modified_at=response.last_modified_at, + remote_published_at=definitive_date, + capabilities_sha256=sha256(response.content).hexdigest(), + capabilities_etag=response.etag, + capabilities_last_modified_at=response.last_modified_at, + message=message, + error_code="CATALOG_ALZ_CURRENT_SNAPSHOT_MISSING" if not snapshot else None, + ) + + def _probe_remote( contract: ProbeContract, settings: Settings, @@ -290,6 +517,8 @@ def _probe_remote( now: datetime, ) -> RemoteProbe: try: + if contract.service_type == "HTML": + return _probe_alz_remote(contract, settings, opener=opener, now=now) capabilities = _bounded_fetch(_validate_capabilities_url(contract.endpoint_url), settings, opener) _validate_capabilities_url(capabilities.final_url) content_type = capabilities.content_type.lower() @@ -394,6 +623,18 @@ def _latest_local_version(source_name: str, datasets: list[Dataset]) -> str | No ] if explicit_current: candidates = explicit_current + elif source_name == _ALZ_SOURCE_NAME: + definitive = [item for item in candidates if _ALZ_EDITION.fullmatch((item.source_version or "").strip())] + if definitive: + latest = max( + definitive, + key=lambda item: ( + int(_ALZ_EDITION.fullmatch((item.source_version or "").strip()).group(1)), + _utc(item.observed_at) if item.observed_at else datetime.min.replace(tzinfo=timezone.utc), + str(item.id), + ), + ) + return latest.source_version if not candidates: return None latest = max( @@ -416,6 +657,9 @@ def _normalized_version(source_name: str, version: str | None) -> str | None: return match.group(1) if match else None if source_name == "digitaal_vlaanderen_orthophoto" and _ORTHOPHOTO_EDITION.fullmatch(value): return value + if source_name == _ALZ_SOURCE_NAME: + match = _ALZ_EDITION.fullmatch(value) + return f"{match.group(1)}-v3" if match else None return None @@ -467,6 +711,13 @@ class SourceCatalogProbeService: endpoint_url=_with_capabilities_query(active_settings.orthophoto_wms_url, "WMS"), expected_layers=(active_settings.orthophoto_wms_layer, "Vliegdagcontour"), ), + ProbeContract( + source_name=_ALZ_SOURCE_NAME, + display_name="Landbouwgebruikspercelen (ALZ)", + service_type="HTML", + endpoint_url=active_settings.source_catalog_alz_release_url, + expected_layers=("definitive_archive", "current_snapshot"), + ), ) items: list[SourceCatalogProbeItem] = [] for contract in contracts: @@ -538,8 +789,9 @@ class SourceCatalogProbeService: summary=summary, items=items, limitations=[ - "Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities en gekoppelde ISO 19139 metadata.", + "Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities, gekoppelde ISO 19139 metadata en de officiële ALZ-publicatiepagina.", "Er worden geen features, rasters of modelbestanden opgehaald en geen datasets aangemaakt of overschreven.", + "Voor ALZ is alleen de nieuwste definitieve v3-editie vergelijkbaar; voorlopige v1/v2-snapshots zijn uitsluitend informatief.", "Een versieverschil is controlesignaal, geen bewijs dat een lokale dataset onbruikbaar is en geen automatische importopdracht.", ], ) diff --git a/backend/tests/test_sprint222_source_catalog_probes.py b/backend/tests/test_sprint222_source_catalog_probes.py index 23451d95..f11551c2 100644 --- a/backend/tests/test_sprint222_source_catalog_probes.py +++ b/backend/tests/test_sprint222_source_catalog_probes.py @@ -21,6 +21,7 @@ ORTHO_METADATA_URL = ( "https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&service=CSW&" "id=f5304d6d-0dd4-43fd-a726-427af31e8d61&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd" ) +ALZ_RELEASE_URL = "https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen" def _wfs_capabilities(metadata_url: str = GRB_METADATA_URL) -> bytes: @@ -77,6 +78,23 @@ def _metadata(identifier: str, title: str, edition: str, modified: str, publishe """.encode() +def _alz_release_page(*, include_snapshot: bool = True, download_host: str = "www.landbouwvlaanderen.be") -> bytes: + snapshot = ( + f'' + "Landbouwgebruikspercelen 2026 – 1e snapshot (extractie 02-06-2026) - GPKG" + if include_snapshot + else "" + ) + return f""" + + {snapshot} +

Definitieve datasets

+ Downloaden + Downloaden + + """.encode() + + class _Response: def __init__(self, content: bytes, content_type: str = "text/xml", *, content_length: int | None = None) -> None: self.content = content @@ -97,8 +115,8 @@ class _Response: class _RedirectedResponse(_Response): - def __init__(self, content: bytes, final_url: str) -> None: - super().__init__(content) + def __init__(self, content: bytes, final_url: str, content_type: str = "text/xml") -> None: + super().__init__(content, content_type) self.final_url = final_url def geturl(self) -> str: @@ -144,6 +162,7 @@ def _settings(**overrides) -> Settings: values = { "SOURCE_CATALOG_PROBE_ENABLED": True, "SOURCE_CATALOG_GRB_WFS_URL": "https://geo.api.vlaanderen.be/GRB/wfs", + "SOURCE_CATALOG_ALZ_RELEASE_URL": ALZ_RELEASE_URL, "SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS": 3, "SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB": 1, "SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS": 0, @@ -157,6 +176,8 @@ def _settings(**overrides) -> Settings: def _opener(request, timeout): assert timeout == 3 url = request.full_url + if url == ALZ_RELEASE_URL: + return _Response(_alz_release_page(), "text/html; charset=utf-8") if "metadata.vlaanderen.be" in url: if "f5304d6d" in url: return _Response(_metadata("f5304d6d", "Orthofoto meest recent, 2025.04", "2025.04", "2026-04-27", "2025-12-11")) @@ -170,7 +191,13 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions SourceCatalogProbeService.clear_cache() project_id = uuid.uuid4() report = SourceCatalogProbeService.audit_project( - _Db([_dataset("grb", "2026-07-14"), _dataset("digitaal_vlaanderen_orthophoto", "most_recent_at_2026-07-14")]), + _Db( + [ + _dataset("grb", "2026-07-14"), + _dataset("digitaal_vlaanderen_orthophoto", "most_recent_at_2026-07-14"), + _dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2025-definitive"), + ] + ), project_id, settings=_settings(), opener=_opener, @@ -178,7 +205,7 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions ) by_source = {item.source_name: item for item in report.items} - assert report.summary.available_count == 2 + assert report.summary.available_count == 3 assert report.summary.different_version_count == 1 assert by_source["grb"].remote_version == "Toestand 2026-07-15" assert by_source["grb"].comparison_status == "different" @@ -187,6 +214,13 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions assert by_source["digitaal_vlaanderen_orthophoto"].remote_version == "2025.04" assert by_source["digitaal_vlaanderen_orthophoto"].comparison_status == "not_comparable" assert by_source["digitaal_vlaanderen_orthophoto"].remote_published_at.year == 2025 + alz = by_source["agentschap_landbouw_zeevisserij_agricultural_parcels"] + assert alz.service_type == "HTML" + assert alz.remote_version == "2025-v3" + assert alz.comparison_status == "same" + assert alz.matched_layers == ["definitive_archive", "current_snapshot"] + assert "2026-v1" in alz.message + assert "voorlopig" in alz.message assert all(item.capabilities_sha256 for item in report.items) @@ -275,10 +309,10 @@ def test_catalog_probe_cache_is_explicitly_bypassable() -> None: second = SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW) SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW, force=True) - assert first_call_count == 4 + assert first_call_count == 5 assert all(item.cached is False for item in first.items) assert all(item.cached is True for item in second.items) - assert len(calls) == 8 + assert len(calls) == 10 def test_catalog_probe_can_be_disabled_without_network_access() -> None: @@ -293,7 +327,7 @@ def test_catalog_probe_can_be_disabled_without_network_access() -> None: now=NOW, ) - assert report.summary.disabled_count == 2 + assert report.summary.disabled_count == 3 assert all(item.status == "disabled" for item in report.items) @@ -316,7 +350,7 @@ def test_catalog_probe_route_returns_canonical_envelope(monkeypatch) -> None: assert list(response) == ["data"] assert response["data"]["project_id"] == project_id - assert response["data"]["summary"]["provider_count"] == 2 + assert response["data"]["summary"]["provider_count"] == 3 def test_catalog_probe_remains_explicit_and_never_imports_provider_data() -> None: @@ -330,3 +364,75 @@ def test_catalog_probe_remains_explicit_and_never_imports_provider_data() -> Non assert "VectorFeatureService" not in service assert "--probe-catalogs" in operator assert "/datasets/source-catalog-probes" in operator + + +def test_alz_catalog_probe_rejects_release_page_redirect_outside_allowlist() -> None: + def redirected_opener(request, timeout): + if request.full_url == ALZ_RELEASE_URL: + return _RedirectedResponse( + _alz_release_page(), + "https://example.com/open-geodata-landbouwgebruikspercelen", + "text/html", + ) + return _opener(request, timeout) + + report = SourceCatalogProbeService.audit_project( + _Db([]), uuid.uuid4(), settings=_settings(), opener=redirected_opener, now=NOW + ) + alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels") + + assert alz.status == "unavailable" + assert alz.error_code == "CATALOG_ALZ_RELEASE_URL_REJECTED" + + +def test_alz_catalog_probe_rejects_untrusted_download_host() -> None: + def malicious_opener(request, timeout): + if request.full_url == ALZ_RELEASE_URL: + return _Response(_alz_release_page(download_host="example.com"), "text/html") + return _opener(request, timeout) + + report = SourceCatalogProbeService.audit_project( + _Db([]), uuid.uuid4(), settings=_settings(), opener=malicious_opener, now=NOW + ) + alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels") + + assert alz.status == "unavailable" + assert alz.error_code == "CATALOG_ALZ_DOWNLOAD_URL_REJECTED" + + +def test_alz_catalog_probe_degrades_without_current_snapshot_but_keeps_definitive_evidence() -> None: + def archive_only_opener(request, timeout): + if request.full_url == ALZ_RELEASE_URL: + return _Response(_alz_release_page(include_snapshot=False), "text/html") + return _opener(request, timeout) + + report = SourceCatalogProbeService.audit_project( + _Db([_dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2024-definitive")]), + uuid.uuid4(), + settings=_settings(), + opener=archive_only_opener, + now=NOW, + ) + alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels") + + assert alz.status == "degraded" + assert alz.remote_version == "2025-v3" + assert alz.comparison_status == "different" + assert alz.matched_layers == ["definitive_archive"] + assert alz.missing_layers == ["current_snapshot"] + assert alz.error_code == "CATALOG_ALZ_CURRENT_SNAPSHOT_MISSING" + + +def test_alz_catalog_probe_compares_latest_definitive_year_not_latest_import_order() -> None: + older = _dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2024-definitive") + newer = _dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2025-definitive") + older.imported_at = NOW + newer.imported_at = NOW.replace(year=2025) + + report = SourceCatalogProbeService.audit_project( + _Db([older, newer]), uuid.uuid4(), settings=_settings(), opener=_opener, now=NOW + ) + alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels") + + assert alz.local_source_version == "2025-definitive" + assert alz.comparison_status == "same" diff --git a/deploy/unraid/geointel-unraid-template.xml b/deploy/unraid/geointel-unraid-template.xml index 7f06fa13..33c24e9c 100644 --- a/deploy/unraid/geointel-unraid-template.xml +++ b/deploy/unraid/geointel-unraid-template.xml @@ -34,8 +34,9 @@ https://geo.api.vlaanderen.be/OMWRGBMRVL/wms 1.0 1024 - true + true https://geo.api.vlaanderen.be/GRB/wfs + https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen 10 2 900 diff --git a/deploy/unraid/geointel.env.example b/deploy/unraid/geointel.env.example index 50797d53..8af69ac3 100644 --- a/deploy/unraid/geointel.env.example +++ b/deploy/unraid/geointel.env.example @@ -33,10 +33,11 @@ ORTHOPHOTO_MIN_SIDE_M=128 ORTHOPHOTO_MAX_SIDE_M=1024 ORTHOPHOTO_CACHE_TTL_HOURS=24 -# Explicit read-only edition checks for GRB and the most-recent orthophoto. -# No feature or raster data is downloaded by these probes. +# Explicit read-only edition checks for GRB, orthophoto and ALZ publications. +# No feature, raster or ALZ archive is downloaded by these probes. SOURCE_CATALOG_PROBE_ENABLED=true SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs +SOURCE_CATALOG_ALZ_RELEASE_URL=https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10 SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2 SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900 diff --git a/deploy/unraid/run-dockerman-container.sh b/deploy/unraid/run-dockerman-container.sh index 2b762306..92a9f9e9 100644 --- a/deploy/unraid/run-dockerman-container.sh +++ b/deploy/unraid/run-dockerman-container.sh @@ -29,6 +29,7 @@ ORTHOPHOTO_MAX_SIDE_M="${ORTHOPHOTO_MAX_SIDE_M:-1024}" ORTHOPHOTO_CACHE_TTL_HOURS="${ORTHOPHOTO_CACHE_TTL_HOURS:-24}" SOURCE_CATALOG_PROBE_ENABLED="${SOURCE_CATALOG_PROBE_ENABLED:-true}" SOURCE_CATALOG_GRB_WFS_URL="${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}" +SOURCE_CATALOG_ALZ_RELEASE_URL="${SOURCE_CATALOG_ALZ_RELEASE_URL:-https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen}" SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS="${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}" SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB="${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}" SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}" @@ -133,6 +134,7 @@ docker run -d \ -e ORTHOPHOTO_CACHE_TTL_HOURS="$ORTHOPHOTO_CACHE_TTL_HOURS" \ -e SOURCE_CATALOG_PROBE_ENABLED="$SOURCE_CATALOG_PROBE_ENABLED" \ -e SOURCE_CATALOG_GRB_WFS_URL="$SOURCE_CATALOG_GRB_WFS_URL" \ + -e SOURCE_CATALOG_ALZ_RELEASE_URL="$SOURCE_CATALOG_ALZ_RELEASE_URL" \ -e SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS="$SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS" \ -e SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB="$SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB" \ -e SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="$SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS" \ diff --git a/docker-compose.yml b/docker-compose.yml index d2500ea5..b091a45b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,6 +32,7 @@ services: ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24} SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true} SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs} + SOURCE_CATALOG_ALZ_RELEASE_URL: ${SOURCE_CATALOG_ALZ_RELEASE_URL:-https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen} SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS: ${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10} SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB: ${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2} SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS: ${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900} diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 9252fa51..9e9ba693 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -414,11 +414,15 @@ not marked stale merely because their source date is old. Unknown sources are ### GET `/api/v1/projects/{project_id}/datasets/source-catalog-probes` -Performs an explicit, read-only release probe for the allowlisted GRB WFS and -most-recent orthophoto WMS. The endpoint reads bounded `GetCapabilities` -responses, confirms the expected layers and follows only HTTPS ISO 19139 -`GetRecordById` links on `metadata.vlaanderen.be`. Query parameter -`refresh=true` bypasses the short in-memory response cache. +Performs an explicit, read-only release probe for the allowlisted GRB WFS, +most-recent orthophoto WMS and ALZ agricultural-use parcel publication page. +For OGC services, the endpoint reads bounded `GetCapabilities` responses, +confirms the expected layers and follows only HTTPS ISO 19139 `GetRecordById` +links on `metadata.vlaanderen.be`. For ALZ, it reads only the bounded official +HTML release page and accepts only exact +`www.landbouwvlaanderen.be/bestanden/gis/agpa___public.zip` link +identities. It does not request those archives. Query parameter `refresh=true` +bypasses the short in-memory response cache. Each provider item returns service reachability, matched/missing layers, the official metadata identifier, title, edition, publication/metadata dates, @@ -427,6 +431,13 @@ local `source_version` and one comparison status: `same`, `different`, `available`, `degraded`, `unavailable` or `disabled`. A difference means only that an operator should review provenance; it is not an update instruction. +ALZ comparisons use only the latest definitive third snapshot, normalized as +`-v3`. A newer first or second snapshot is reported in the item title +and message as provisional but cannot mark a local definitive historical +edition as outdated. `service_type=HTML` uses `definitive_archive` and +`current_snapshot` as evidence markers in the existing expected/matched/missing +arrays; no parallel response shape is introduced. + The endpoint accepts no arbitrary URL, feature query, area or layer. It does not fetch vector features, raster pixels or models, create jobs/datasets, write to PostGIS or trigger an import. The normal `source-freshness` endpoint remains diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 5eb1e86a..9ebfe484 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -173,7 +173,7 @@ gebeurd. | --- | --- | --- | | GRB gebouwen/wegen/water/percelen | operationele, expliciete plan-stage-apply refresh met onveranderlijke snapshots | alleen een nieuw officieel gedateerd cataloguseditie na operatorbevestiging ophalen | | Statbel bevolking | jaarlijkse, expliciete edities in één tijdreeks | een nieuwe publicatie alleen na schema-, sectorgeometrie- en totalencontrole toevoegen | -| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; metricvergelijking zonder objectlineage | eerst een stabiele machineleesbare editiebron verifiëren, daarna dezelfde begrensde operatorflow toepassen | +| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete officiële publicatieprobe; metricvergelijking zonder objectlineage | een nieuwere definitieve v3-editie eerst handmatig beoordelen en daarna via de bestaande begrensde operatorflow toevoegen | | orthofoto | vaste lokale opname per expliciete analysezone; catalogusprobe is alleen een signaal | vluchtjaar, productvariant en dekking vergelijken voordat nieuwe pixels worden opgehaald | | landgebruik, thematische rasters, DHMV en VMM-scenario's | vaste product-/scenario-edities, geen rolling snapshot | alleen een nieuwe gedocumenteerde producteditie als afzonderlijke Dataset verwerven | | bodemkaart en historische kaarten | historische referentie-editie | niet als verouderde actuele bron labelen; alleen vervangen bij een officiële inhoudelijke heruitgave | @@ -183,6 +183,13 @@ Geen van deze regels activeert een browserfetch, scheduler of automatische vervanging. Een bron wordt pas `refreshable` wanneer versie, dekking, schema, provenance en een begrensde acquisitieroute afzonderlijk verifieerbaar zijn. +De ALZ-publicatiepagina levert drie campagnesnapshots: v1 en v2 zijn +voorlopig, v3 is de definitieve historische editie. GeoIntel vergelijkt daarom +de lokale `2025-definitive` uitsluitend met de officiële `2025-v3`, terwijl de +actuele `2026-v1` zichtbaar maar niet updategerechtigd blijft. De probe leest +alleen de allowlisted HTML-pagina en linkidentiteiten; ZIP-archieven worden pas +door de afzonderlijke operator opgehaald na menselijke editiebevestiging. + ### Mol population history `scripts/provision_mol_population_history.py` imports official Statbel diff --git a/docs/ENVIRONMENT_SPEC.md b/docs/ENVIRONMENT_SPEC.md index ea6d6087..a6b4f7da 100644 --- a/docs/ENVIRONMENT_SPEC.md +++ b/docs/ENVIRONMENT_SPEC.md @@ -29,6 +29,7 @@ GRB_WFS_URL= # Explicit read-only source-edition probes (no provider import). SOURCE_CATALOG_PROBE_ENABLED=true SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs +SOURCE_CATALOG_ALZ_RELEASE_URL=https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10 SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2 SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900 diff --git a/docs/TODO.md b/docs/TODO.md index e6e5355d..2632c6c3 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -41,7 +41,8 @@ - [x] Add a read-only source freshness/version audit with explicit publication policies, local integrity checks, operator CLI and compact Status workspace surface. - [x] Add explicit bounded GRB/orthophoto catalogue release probes using official capabilities and ISO metadata; keep every acquisition separate and operator-controlled. - [x] Add a governed regional GRB plan -> stage -> checksum-confirmed apply workflow that preserves every previous snapshot. -- [ ] Extend catalogue probes only to additional sources that publish a stable machine-readable edition contract; do not add background polling or infer releases from HTTP dates alone. +- [x] Add a fail-closed ALZ publication probe that distinguishes provisional v1/v2 snapshots from the definitive v3 historical edition and never downloads an archive. +- [ ] Extend catalogue probes only to additional sources that publish a stable official edition contract; do not add background polling or infer releases from HTTP dates alone. ## Governed source expansion backlog @@ -683,7 +684,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Show precise daily edition ranges and explain registration-date nuance in the Evolution workspace. - [x] Document refresh readiness across the complete official-source portfolio. -- [ ] Add the next governed catalog probe only after an official stable version - contract is verified; ALZ annual releases are the first candidate. +- [x] Verify and add the governed ALZ catalog probe against the official + campaign-snapshot and definitive-archive publication contract. - [ ] Keep orthophoto refresh manual until product variant, flight year and complete selected-area coverage can be compared deterministically. diff --git a/frontend/README.md b/frontend/README.md index 85aa1a79..4d7c8ae8 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -10,9 +10,12 @@ require local integrity review or are local artifacts. Only attention items are expanded by default; all source detail remains available through disclosure. The refresh button reruns the local read-only audit and never downloads or replaces source data. A separate `Officiële edities controleren` action is the -only trigger for bounded GRB/orthophoto catalog reads. It presents official and -local editions, layer-contract coverage and honest non-comparable version -markers without starting an import or background poll. +only trigger for bounded GRB/orthophoto catalog reads and the official ALZ +publication-page check. It presents official and local editions, +layer/publication evidence and honest non-comparable version markers without +starting an import or background poll. ALZ v1/v2 snapshots are labelled +provisional; only the latest definitive v3 edition is compared with local +history. After that explicit check, the same surface shows a compact GRB refresh plan for buildings, roads, water and parcels. It states whether each local regional snapshot is current, updateable or needs review and shows the current object diff --git a/frontend/src/components/status/SourceFreshnessPanel.tsx b/frontend/src/components/status/SourceFreshnessPanel.tsx index 6694b920..8c8fb35e 100644 --- a/frontend/src/components/status/SourceFreshnessPanel.tsx +++ b/frontend/src/components/status/SourceFreshnessPanel.tsx @@ -86,6 +86,7 @@ function integrityIssueCount(item: SourceFreshnessItem): number { } function CatalogRow({ item }: { item: SourceCatalogProbeItem }): JSX.Element { + const evidenceLabel = item.service_type === 'HTML' ? 'Publicaties' : 'Lagen' return (
@@ -98,7 +99,7 @@ function CatalogRow({ item }: { item: SourceCatalogProbeItem }): JSX.Element {

{item.message}

Lokaal: {item.local_source_version ?? 'niet aanwezig'} - Lagen: {item.matched_layers.length}/{item.expected_layers.length} bevestigd + {evidenceLabel}: {item.matched_layers.length}/{item.expected_layers.length} bevestigd {item.remote_modified_at ? Metadata: {formatDate(item.remote_modified_at)} : null} {item.cached ? cache gebruikt : null}
@@ -150,7 +151,7 @@ export function SourceFreshnessPanel({

Bronbeheer

Actualiteit en versiecontrole

-

Controleert lokale publicaties en kan op aanvraag de officiële GRB- en orthofoto-editie uitlezen.

+

Controleert lokale publicaties en kan op aanvraag de officiële GRB-, orthofoto- en landbouweditie uitlezen.

{catalogError ?

{catalogError}

: null} {!catalogReport && !catalogError ? ( -

Controleer GRB en orthofoto wanneer je een lokale bronversie met de officiële editie wilt vergelijken.

+

Controleer GRB, orthofoto en landbouwpercelen wanneer je lokale bronversies met officiële edities wilt vergelijken.

) : null} {catalogReport ? ( <> diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 9163943e..16eeea7c 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -715,7 +715,7 @@ export type SourceCatalogComparisonStatus = 'same' | 'different' | 'not_comparab export interface SourceCatalogProbeItem { source_name: string display_name: string - service_type: 'WFS' | 'WMS' + service_type: 'WFS' | 'WMS' | 'HTML' endpoint_url: string status: SourceCatalogProbeStatus reachable: boolean