Add governed ALZ edition probe
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-16 20:27:07 +02:00
parent 5e897206b2
commit 0a23e3484c
18 changed files with 451 additions and 38 deletions
+1
View File
@@ -14,6 +14,7 @@ ORTHOPHOTO_MAX_SIDE_M=1024
ORTHOPHOTO_CACHE_TTL_HOURS=24
SOURCE_CATALOG_PROBE_ENABLED=true
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
SOURCE_CATALOG_ALZ_RELEASE_URL=https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900
+17
View File
@@ -7,6 +7,23 @@
# Changelog
## Sprint 225 Governed ALZ edition probe (2026-07-16)
- Extended the explicit read-only source catalog audit with the official ALZ
agricultural-use parcel publication page without adding an importer,
scheduler or background provider request.
- Added strict release-page, redirect and archive-link allowlists and bounded
HTML parsing; the probe never downloads the referenced ALZ ZIP archives.
- Normalized only definitive third snapshots as comparable editions, so local
`2025-definitive` evidence matches official `2025-v3` while current
`2026-v1` remains visibly provisional.
- Reused the canonical source-catalog envelope, Status UI and operator command,
and propagated the fixed release URL through Docker and Unraid runtime
configuration.
- Added focused release parsing, version ordering, cache, failure-isolation and
trust-boundary regression coverage without changing persistence or
migrations.
## Sprint 224 Governed GRB evolution (2026-07-16)
- Enabled fail-closed object evolution between the retained regional GRB
+12 -8
View File
@@ -1444,7 +1444,8 @@ docker exec geointel python /app/scripts/audit_source_freshness.py \
Use `--fail-on due` to make a planned review date fail automation, or
`--fail-on never` for reporting only. The command never starts a refresh.
An operator can explicitly add the official GRB and orthophoto edition check:
An operator can explicitly add the official GRB, orthophoto and ALZ edition
check:
```bash
docker exec geointel python /app/scripts/audit_source_freshness.py \
@@ -1458,17 +1459,20 @@ Use `--refresh-catalogs` to bypass the 15-minute in-memory cache and
`--fail-on-catalog` only when temporary official-provider unavailability must
fail an operator job. This path reads bounded WFS/WMS capabilities and their
fixed ISO 19139 metadata records. It confirms GRB `GBG`, `WBN`, `WGO`, `ADP`
and orthophoto `Ortho`, `Vliegdagcontour`; it never requests feature or raster
content. The public Datavindplaats API requires an access token, so release
evidence comes from the public metadata links published by the official OGC
services rather than HTML scraping.
and orthophoto `Ortho`, `Vliegdagcontour`. It also reads the exact official ALZ
publication page and validates only allowlisted archive-link identities. It
never requests feature, raster or ALZ ZIP content. ALZ v1/v2 campaign snapshots
remain provisional; only a v3 publication is compared with a local definitive
historical edition.
Runtime controls are `SOURCE_CATALOG_PROBE_ENABLED`,
`SOURCE_CATALOG_GRB_WFS_URL`, `SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS`,
`SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB` and
`SOURCE_CATALOG_GRB_WFS_URL`, `SOURCE_CATALOG_ALZ_RELEASE_URL`,
`SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS`, `SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB` and
`SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS`. Metadata links remain restricted to
the official HTTPS CSW path on `metadata.vlaanderen.be` even when an operator
overrides the capabilities endpoint.
overrides the capabilities endpoint. The ALZ release URL is fail-closed to the
exact HTTPS host/path and cannot be redirected to another page or download
host.
## Governed regional GRB refresh
+4
View File
@@ -36,6 +36,10 @@ class Settings(BaseSettings):
default="https://geo.api.vlaanderen.be/GRB/wfs",
validation_alias="SOURCE_CATALOG_GRB_WFS_URL",
)
source_catalog_alz_release_url: str = Field(
default="https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen",
validation_alias="SOURCE_CATALOG_ALZ_RELEASE_URL",
)
source_catalog_probe_timeout_seconds: int = Field(
default=10,
ge=1,
+1 -1
View File
@@ -14,7 +14,7 @@ SourceCatalogComparisonStatus = Literal["same", "different", "not_comparable", "
class SourceCatalogProbeItem(BaseModel):
source_name: str
display_name: str
service_type: Literal["WFS", "WMS"]
service_type: Literal["WFS", "WMS", "HTML"]
endpoint_url: str
status: SourceCatalogProbeStatus
reachable: bool
@@ -4,6 +4,7 @@ from dataclasses import dataclass, replace
from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime
from hashlib import sha256
from html.parser import HTMLParser
import re
from threading import Lock
from typing import Any, Callable
@@ -32,6 +33,17 @@ _XLINK = "http://www.w3.org/1999/xlink"
_METADATA_HOST = "metadata.vlaanderen.be"
_VERSION_DATE = re.compile(r"^(?:toestand\s+)?(\d{4}-\d{2}-\d{2})$", re.IGNORECASE)
_ORTHOPHOTO_EDITION = re.compile(r"^\d{4}\.\d{2}$")
_ALZ_SOURCE_NAME = "agentschap_landbouw_zeevisserij_agricultural_parcels"
_ALZ_RELEASE_HOST = "landbouwcijfers.vlaanderen.be"
_ALZ_RELEASE_PATH = "/open-geodata-landbouwgebruikspercelen"
_ALZ_DOWNLOAD_HOST = "www.landbouwvlaanderen.be"
_ALZ_DOWNLOAD_PATH = re.compile(r"^/bestanden/gis/agpa_(20\d{2})_(\d{4}-\d{2}-\d{2})_public\.zip$")
_ALZ_SNAPSHOT = re.compile(
r"^Landbouwgebruikspercelen\s+(20\d{2})\s*-\s*(\d+)e\s+snapshot\s*"
r"\(extractie\s+(\d{2}-\d{2}-\d{4})\)(?:\s*-\s*GPKG)?$",
re.IGNORECASE,
)
_ALZ_EDITION = re.compile(r"^(20\d{2})-(?:definitive|v3)$", re.IGNORECASE)
class CatalogProbeFailure(RuntimeError):
@@ -88,6 +100,32 @@ class CacheEntry:
probe: RemoteProbe
class _AnchorParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.anchors: list[tuple[str, str]] = []
self._href: str | None = None
self._text: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if tag.lower() != "a" or self._href is not None:
return
href = dict(attrs).get("href")
if href:
self._href = href.strip()
self._text = []
def handle_data(self, data: str) -> None:
if self._href is not None:
self._text.append(data)
def handle_endtag(self, tag: str) -> None:
if tag.lower() == "a" and self._href is not None:
self.anchors.append((self._href, "".join(self._text)))
self._href = None
self._text = []
_REMOTE_CACHE: dict[str, CacheEntry] = {}
_CACHE_LOCK = Lock()
@@ -124,7 +162,7 @@ def _with_capabilities_query(base_url: str, service_type: str) -> str:
def _bounded_fetch(url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> FetchResult:
request = Request(
url,
headers={"Accept": "application/xml,text/xml,application/json", "User-Agent": "GeoIntel/0.1 source-catalog-probe"},
headers={"Accept": "application/xml,text/xml,text/html,application/json", "User-Agent": "GeoIntel/0.1 source-catalog-probe"},
)
max_bytes = settings.source_catalog_probe_max_response_mb * 1024 * 1024
try:
@@ -226,6 +264,127 @@ def _validate_capabilities_url(url: str) -> str:
return url
def _validate_alz_release_url(url: str) -> str:
parsed = urlsplit(url)
if (
parsed.scheme != "https"
or parsed.hostname != _ALZ_RELEASE_HOST
or parsed.port not in {None, 443}
or parsed.username
or parsed.password
or parsed.path.rstrip("/") != _ALZ_RELEASE_PATH
or parsed.query
or parsed.fragment
):
raise CatalogProbeFailure(
"CATALOG_ALZ_RELEASE_URL_REJECTED",
"De ingestelde ALZ-publicatiepagina valt buiten de toegestane officiële URL.",
)
return url
def _validate_alz_download_url(url: str) -> tuple[int, datetime]:
parsed = urlsplit(url)
match = _ALZ_DOWNLOAD_PATH.fullmatch(parsed.path)
if (
parsed.scheme != "https"
or parsed.hostname != _ALZ_DOWNLOAD_HOST
or parsed.port not in {None, 443}
or parsed.username
or parsed.password
or parsed.query
or parsed.fragment
or not match
):
raise CatalogProbeFailure(
"CATALOG_ALZ_DOWNLOAD_URL_REJECTED",
"De ALZ-publicatiepagina bevat een datasetlink buiten de toegestane officiële URL-structuur.",
)
try:
published_at = datetime.strptime(match.group(2), "%Y-%m-%d").replace(tzinfo=timezone.utc)
except ValueError as exc:
raise CatalogProbeFailure(
"CATALOG_ALZ_DOWNLOAD_URL_REJECTED",
"De ALZ-datasetlink bevat geen geldige publicatiedatum.",
) from exc
return int(match.group(1)), published_at
def _normalized_html_text(value: str) -> str:
return re.sub(r"\s+", " ", value.replace("\xad", "").replace("", "-").replace("", "-")).strip()
def _parse_alz_release_page(content: bytes) -> dict[str, Any]:
try:
html = content.decode("utf-8")
except UnicodeDecodeError as exc:
raise CatalogProbeFailure(
"CATALOG_ALZ_INVALID_HTML",
"De officiële ALZ-publicatiepagina is niet geldige UTF-8 HTML.",
) from exc
parser = _AnchorParser()
try:
parser.feed(html)
parser.close()
except Exception as exc:
raise CatalogProbeFailure(
"CATALOG_ALZ_INVALID_HTML",
"De officiële ALZ-publicatiepagina kon niet veilig worden ontleed.",
) from exc
definitive: list[tuple[int, datetime]] = []
snapshots: list[tuple[int, int, datetime]] = []
for href, raw_text in parser.anchors:
text = _normalized_html_text(raw_text)
looks_like_alz_release = (
text.casefold() == "downloaden"
or text.casefold().startswith("landbouwgebruikspercelen ")
or "agpa_" in href.casefold()
)
if not looks_like_alz_release:
continue
year, file_date = _validate_alz_download_url(href)
snapshot = _ALZ_SNAPSHOT.fullmatch(text)
if snapshot:
snapshot_year = int(snapshot.group(1))
snapshot_number = int(snapshot.group(2))
try:
extraction_date = datetime.strptime(snapshot.group(3), "%d-%m-%Y").replace(tzinfo=timezone.utc)
except ValueError as exc:
raise CatalogProbeFailure(
"CATALOG_ALZ_SNAPSHOT_INVALID",
"De actuele ALZ-snapshot bevat geen geldige extractiedatum.",
) from exc
if snapshot_year != year or extraction_date != file_date or snapshot_number not in {1, 2, 3}:
raise CatalogProbeFailure(
"CATALOG_ALZ_SNAPSHOT_INVALID",
"De actuele ALZ-snapshot is niet consistent met de officiële datasetlink.",
)
snapshots.append((year, snapshot_number, extraction_date))
elif text.casefold() == "downloaden":
definitive.append((year, file_date))
else:
raise CatalogProbeFailure(
"CATALOG_ALZ_RELEASE_UNRECOGNIZED",
"De ALZ-publicatiepagina bevat een niet-herkende landbouwdatasetpublicatie.",
)
if not definitive:
raise CatalogProbeFailure(
"CATALOG_ALZ_DEFINITIVE_MISSING",
"De officiële ALZ-publicatiepagina bevat geen herkenbare definitieve landbouwperceeleditie.",
)
latest_definitive = max(definitive, key=lambda item: (item[0], item[1]))
latest_snapshot = max(snapshots, key=lambda item: (item[0], item[1], item[2])) if snapshots else None
if latest_snapshot and latest_snapshot[1] == 3 and latest_snapshot[0] >= latest_definitive[0]:
latest_definitive = (latest_snapshot[0], latest_snapshot[2])
return {
"definitive": latest_definitive,
"definitive_count": len(definitive),
"snapshot": latest_snapshot,
}
def _node_text(node: ElementTree.Element | None) -> str | None:
if node is None:
return None
@@ -282,6 +441,74 @@ def _parse_metadata(content: bytes) -> dict[str, Any]:
}
def _probe_alz_remote(
contract: ProbeContract,
settings: Settings,
*,
opener: Callable[..., Any] | None,
now: datetime,
) -> RemoteProbe:
release_url = _validate_alz_release_url(contract.endpoint_url)
response = _bounded_fetch(release_url, settings, opener)
_validate_alz_release_url(response.final_url)
content_type = response.content_type.lower()
if content_type and "html" not in content_type and "text" not in content_type:
raise CatalogProbeFailure(
"CATALOG_ALZ_INVALID_CONTENT_TYPE",
"De officiële ALZ-publicatiepagina is geen HTML-respons.",
)
release = _parse_alz_release_page(response.content)
definitive_year, definitive_date = release["definitive"]
snapshot = release["snapshot"]
matched = ["definitive_archive"]
missing: list[str] = []
if snapshot:
matched.append("current_snapshot")
else:
missing.append("current_snapshot")
definitive_version = f"{definitive_year}-v3"
if snapshot:
snapshot_year, snapshot_number, snapshot_date = snapshot
snapshot_version = f"{snapshot_year}-v{snapshot_number}"
if snapshot_number < 3:
message = (
f"De officiële ALZ-publicatiepagina bevestigt definitieve editie {definitive_version}. "
f"De actuele publicatie {snapshot_version} van {snapshot_date.date().isoformat()} is voorlopig "
"en wordt niet als historische vervanging aangemerkt."
)
else:
message = f"De officiële ALZ-publicatiepagina bevestigt definitieve editie {definitive_version}."
remote_title = f"Landbouwgebruikspercelen {definitive_version}; actuele publicatie {snapshot_version}"
else:
message = (
f"De officiële ALZ-publicatiepagina bevestigt definitieve editie {definitive_version}, "
"maar bevat geen herkenbare actuele snapshot."
)
remote_title = f"Landbouwgebruikspercelen {definitive_version}"
return RemoteProbe(
status="available" if snapshot else "degraded",
reachable=True,
checked_at=now,
expected_layers=contract.expected_layers,
matched_layers=tuple(matched),
missing_layers=tuple(missing),
advertised_layer_count=release["definitive_count"] + (1 if snapshot else 0),
metadata_url=response.final_url,
metadata_identifier="alz-agricultural-use-parcels",
remote_title=remote_title,
remote_version=definitive_version,
remote_modified_at=response.last_modified_at,
remote_published_at=definitive_date,
capabilities_sha256=sha256(response.content).hexdigest(),
capabilities_etag=response.etag,
capabilities_last_modified_at=response.last_modified_at,
message=message,
error_code="CATALOG_ALZ_CURRENT_SNAPSHOT_MISSING" if not snapshot else None,
)
def _probe_remote(
contract: ProbeContract,
settings: Settings,
@@ -290,6 +517,8 @@ def _probe_remote(
now: datetime,
) -> RemoteProbe:
try:
if contract.service_type == "HTML":
return _probe_alz_remote(contract, settings, opener=opener, now=now)
capabilities = _bounded_fetch(_validate_capabilities_url(contract.endpoint_url), settings, opener)
_validate_capabilities_url(capabilities.final_url)
content_type = capabilities.content_type.lower()
@@ -394,6 +623,18 @@ def _latest_local_version(source_name: str, datasets: list[Dataset]) -> str | No
]
if explicit_current:
candidates = explicit_current
elif source_name == _ALZ_SOURCE_NAME:
definitive = [item for item in candidates if _ALZ_EDITION.fullmatch((item.source_version or "").strip())]
if definitive:
latest = max(
definitive,
key=lambda item: (
int(_ALZ_EDITION.fullmatch((item.source_version or "").strip()).group(1)),
_utc(item.observed_at) if item.observed_at else datetime.min.replace(tzinfo=timezone.utc),
str(item.id),
),
)
return latest.source_version
if not candidates:
return None
latest = max(
@@ -416,6 +657,9 @@ def _normalized_version(source_name: str, version: str | None) -> str | None:
return match.group(1) if match else None
if source_name == "digitaal_vlaanderen_orthophoto" and _ORTHOPHOTO_EDITION.fullmatch(value):
return value
if source_name == _ALZ_SOURCE_NAME:
match = _ALZ_EDITION.fullmatch(value)
return f"{match.group(1)}-v3" if match else None
return None
@@ -467,6 +711,13 @@ class SourceCatalogProbeService:
endpoint_url=_with_capabilities_query(active_settings.orthophoto_wms_url, "WMS"),
expected_layers=(active_settings.orthophoto_wms_layer, "Vliegdagcontour"),
),
ProbeContract(
source_name=_ALZ_SOURCE_NAME,
display_name="Landbouwgebruikspercelen (ALZ)",
service_type="HTML",
endpoint_url=active_settings.source_catalog_alz_release_url,
expected_layers=("definitive_archive", "current_snapshot"),
),
)
items: list[SourceCatalogProbeItem] = []
for contract in contracts:
@@ -538,8 +789,9 @@ class SourceCatalogProbeService:
summary=summary,
items=items,
limitations=[
"Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities en gekoppelde ISO 19139 metadata.",
"Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities, gekoppelde ISO 19139 metadata en de officiële ALZ-publicatiepagina.",
"Er worden geen features, rasters of modelbestanden opgehaald en geen datasets aangemaakt of overschreven.",
"Voor ALZ is alleen de nieuwste definitieve v3-editie vergelijkbaar; voorlopige v1/v2-snapshots zijn uitsluitend informatief.",
"Een versieverschil is controlesignaal, geen bewijs dat een lokale dataset onbruikbaar is en geen automatische importopdracht.",
],
)
@@ -21,6 +21,7 @@ ORTHO_METADATA_URL = (
"https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&service=CSW&"
"id=f5304d6d-0dd4-43fd-a726-427af31e8d61&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd"
)
ALZ_RELEASE_URL = "https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen"
def _wfs_capabilities(metadata_url: str = GRB_METADATA_URL) -> bytes:
@@ -77,6 +78,23 @@ def _metadata(identifier: str, title: str, edition: str, modified: str, publishe
""".encode()
def _alz_release_page(*, include_snapshot: bool = True, download_host: str = "www.landbouwvlaanderen.be") -> bytes:
snapshot = (
f'<a href="https://{download_host}/bestanden/gis/agpa_2026_2026-06-02_public.zip">'
"Landbouwgebruikspercelen 2026 1e snapshot (extractie 02-06-2026) - GPKG</a>"
if include_snapshot
else ""
)
return f"""
<html><body>
{snapshot}
<p>Definitieve datasets</p>
<a href="https://{download_host}/bestanden/gis/agpa_2025_2026-05-13_public.zip">Downloaden</a>
<a href="https://{download_host}/bestanden/gis/agpa_2024_2025-03-27_public.zip">Downloaden</a>
</body></html>
""".encode()
class _Response:
def __init__(self, content: bytes, content_type: str = "text/xml", *, content_length: int | None = None) -> None:
self.content = content
@@ -97,8 +115,8 @@ class _Response:
class _RedirectedResponse(_Response):
def __init__(self, content: bytes, final_url: str) -> None:
super().__init__(content)
def __init__(self, content: bytes, final_url: str, content_type: str = "text/xml") -> None:
super().__init__(content, content_type)
self.final_url = final_url
def geturl(self) -> str:
@@ -144,6 +162,7 @@ def _settings(**overrides) -> Settings:
values = {
"SOURCE_CATALOG_PROBE_ENABLED": True,
"SOURCE_CATALOG_GRB_WFS_URL": "https://geo.api.vlaanderen.be/GRB/wfs",
"SOURCE_CATALOG_ALZ_RELEASE_URL": ALZ_RELEASE_URL,
"SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS": 3,
"SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB": 1,
"SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS": 0,
@@ -157,6 +176,8 @@ def _settings(**overrides) -> Settings:
def _opener(request, timeout):
assert timeout == 3
url = request.full_url
if url == ALZ_RELEASE_URL:
return _Response(_alz_release_page(), "text/html; charset=utf-8")
if "metadata.vlaanderen.be" in url:
if "f5304d6d" in url:
return _Response(_metadata("f5304d6d", "Orthofoto meest recent, 2025.04", "2025.04", "2026-04-27", "2025-12-11"))
@@ -170,7 +191,13 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions
SourceCatalogProbeService.clear_cache()
project_id = uuid.uuid4()
report = SourceCatalogProbeService.audit_project(
_Db([_dataset("grb", "2026-07-14"), _dataset("digitaal_vlaanderen_orthophoto", "most_recent_at_2026-07-14")]),
_Db(
[
_dataset("grb", "2026-07-14"),
_dataset("digitaal_vlaanderen_orthophoto", "most_recent_at_2026-07-14"),
_dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2025-definitive"),
]
),
project_id,
settings=_settings(),
opener=_opener,
@@ -178,7 +205,7 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions
)
by_source = {item.source_name: item for item in report.items}
assert report.summary.available_count == 2
assert report.summary.available_count == 3
assert report.summary.different_version_count == 1
assert by_source["grb"].remote_version == "Toestand 2026-07-15"
assert by_source["grb"].comparison_status == "different"
@@ -187,6 +214,13 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions
assert by_source["digitaal_vlaanderen_orthophoto"].remote_version == "2025.04"
assert by_source["digitaal_vlaanderen_orthophoto"].comparison_status == "not_comparable"
assert by_source["digitaal_vlaanderen_orthophoto"].remote_published_at.year == 2025
alz = by_source["agentschap_landbouw_zeevisserij_agricultural_parcels"]
assert alz.service_type == "HTML"
assert alz.remote_version == "2025-v3"
assert alz.comparison_status == "same"
assert alz.matched_layers == ["definitive_archive", "current_snapshot"]
assert "2026-v1" in alz.message
assert "voorlopig" in alz.message
assert all(item.capabilities_sha256 for item in report.items)
@@ -275,10 +309,10 @@ def test_catalog_probe_cache_is_explicitly_bypassable() -> None:
second = SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW)
SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW, force=True)
assert first_call_count == 4
assert first_call_count == 5
assert all(item.cached is False for item in first.items)
assert all(item.cached is True for item in second.items)
assert len(calls) == 8
assert len(calls) == 10
def test_catalog_probe_can_be_disabled_without_network_access() -> None:
@@ -293,7 +327,7 @@ def test_catalog_probe_can_be_disabled_without_network_access() -> None:
now=NOW,
)
assert report.summary.disabled_count == 2
assert report.summary.disabled_count == 3
assert all(item.status == "disabled" for item in report.items)
@@ -316,7 +350,7 @@ def test_catalog_probe_route_returns_canonical_envelope(monkeypatch) -> None:
assert list(response) == ["data"]
assert response["data"]["project_id"] == project_id
assert response["data"]["summary"]["provider_count"] == 2
assert response["data"]["summary"]["provider_count"] == 3
def test_catalog_probe_remains_explicit_and_never_imports_provider_data() -> None:
@@ -330,3 +364,75 @@ def test_catalog_probe_remains_explicit_and_never_imports_provider_data() -> Non
assert "VectorFeatureService" not in service
assert "--probe-catalogs" in operator
assert "/datasets/source-catalog-probes" in operator
def test_alz_catalog_probe_rejects_release_page_redirect_outside_allowlist() -> None:
def redirected_opener(request, timeout):
if request.full_url == ALZ_RELEASE_URL:
return _RedirectedResponse(
_alz_release_page(),
"https://example.com/open-geodata-landbouwgebruikspercelen",
"text/html",
)
return _opener(request, timeout)
report = SourceCatalogProbeService.audit_project(
_Db([]), uuid.uuid4(), settings=_settings(), opener=redirected_opener, now=NOW
)
alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels")
assert alz.status == "unavailable"
assert alz.error_code == "CATALOG_ALZ_RELEASE_URL_REJECTED"
def test_alz_catalog_probe_rejects_untrusted_download_host() -> None:
def malicious_opener(request, timeout):
if request.full_url == ALZ_RELEASE_URL:
return _Response(_alz_release_page(download_host="example.com"), "text/html")
return _opener(request, timeout)
report = SourceCatalogProbeService.audit_project(
_Db([]), uuid.uuid4(), settings=_settings(), opener=malicious_opener, now=NOW
)
alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels")
assert alz.status == "unavailable"
assert alz.error_code == "CATALOG_ALZ_DOWNLOAD_URL_REJECTED"
def test_alz_catalog_probe_degrades_without_current_snapshot_but_keeps_definitive_evidence() -> None:
def archive_only_opener(request, timeout):
if request.full_url == ALZ_RELEASE_URL:
return _Response(_alz_release_page(include_snapshot=False), "text/html")
return _opener(request, timeout)
report = SourceCatalogProbeService.audit_project(
_Db([_dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2024-definitive")]),
uuid.uuid4(),
settings=_settings(),
opener=archive_only_opener,
now=NOW,
)
alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels")
assert alz.status == "degraded"
assert alz.remote_version == "2025-v3"
assert alz.comparison_status == "different"
assert alz.matched_layers == ["definitive_archive"]
assert alz.missing_layers == ["current_snapshot"]
assert alz.error_code == "CATALOG_ALZ_CURRENT_SNAPSHOT_MISSING"
def test_alz_catalog_probe_compares_latest_definitive_year_not_latest_import_order() -> None:
older = _dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2024-definitive")
newer = _dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2025-definitive")
older.imported_at = NOW
newer.imported_at = NOW.replace(year=2025)
report = SourceCatalogProbeService.audit_project(
_Db([older, newer]), uuid.uuid4(), settings=_settings(), opener=_opener, now=NOW
)
alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels")
assert alz.local_source_version == "2025-definitive"
assert alz.comparison_status == "same"
+2 -1
View File
@@ -34,8 +34,9 @@
<Config Name="Orthophoto WMS URL" Target="ORTHOPHOTO_WMS_URL" Default="https://geo.api.vlaanderen.be/OMWRGBMRVL/wms" Mode="" Description="Official Digitaal Vlaanderen most-recent winter orthophoto WMS endpoint." Type="Variable" Display="advanced" Required="true" Mask="false">https://geo.api.vlaanderen.be/OMWRGBMRVL/wms</Config>
<Config Name="Orthophoto Resolution (m)" Target="ORTHOPHOTO_RESOLUTION_M" Default="1.0" Mode="" Description="Requested analysis sampling in metres per pixel. Keep at 1.0 for the active building model profile." Type="Variable" Display="advanced" Required="true" Mask="false">1.0</Config>
<Config Name="Orthophoto Maximum Side (m)" Target="ORTHOPHOTO_MAX_SIDE_M" Default="1024" Mode="" Description="Safety limit for each selected rectangle side before external acquisition and local inference." Type="Variable" Display="advanced" Required="true" Mask="false">1024</Config>
<Config Name="Official Catalog Edition Probe" Target="SOURCE_CATALOG_PROBE_ENABLED" Default="true" Mode="" Description="Allow explicit read-only GRB and orthophoto edition checks. This never imports provider data." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
<Config Name="Official Catalog Edition Probe" Target="SOURCE_CATALOG_PROBE_ENABLED" Default="true" Mode="" Description="Allow explicit read-only GRB, orthophoto and ALZ edition checks. This never imports provider data." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
<Config Name="GRB Catalog WFS URL" Target="SOURCE_CATALOG_GRB_WFS_URL" Default="https://geo.api.vlaanderen.be/GRB/wfs" Mode="" Description="Official GRB WFS used only for capabilities and linked ISO metadata checks." Type="Variable" Display="advanced" Required="true" Mask="false">https://geo.api.vlaanderen.be/GRB/wfs</Config>
<Config Name="ALZ Release Page URL" Target="SOURCE_CATALOG_ALZ_RELEASE_URL" Default="https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen" Mode="" Description="Exact official ALZ publication page used only to identify definitive and provisional agricultural parcel editions." Type="Variable" Display="advanced" Required="true" Mask="false">https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen</Config>
<Config Name="Catalog Probe Timeout (seconds)" Target="SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS" Default="10" Mode="" Description="Per-request timeout for explicit read-only official catalog checks." Type="Variable" Display="advanced" Required="true" Mask="false">10</Config>
<Config Name="Catalog Probe Maximum Response (MiB)" Target="SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB" Default="2" Mode="" Description="Maximum capabilities or ISO metadata response size accepted by a catalog probe." Type="Variable" Display="advanced" Required="true" Mask="false">2</Config>
<Config Name="Catalog Probe Cache (seconds)" Target="SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS" Default="900" Mode="" Description="Short in-memory cache for repeated official edition checks; use zero to disable." Type="Variable" Display="advanced" Required="true" Mask="false">900</Config>
+3 -2
View File
@@ -33,10 +33,11 @@ ORTHOPHOTO_MIN_SIDE_M=128
ORTHOPHOTO_MAX_SIDE_M=1024
ORTHOPHOTO_CACHE_TTL_HOURS=24
# Explicit read-only edition checks for GRB and the most-recent orthophoto.
# No feature or raster data is downloaded by these probes.
# Explicit read-only edition checks for GRB, orthophoto and ALZ publications.
# No feature, raster or ALZ archive is downloaded by these probes.
SOURCE_CATALOG_PROBE_ENABLED=true
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
SOURCE_CATALOG_ALZ_RELEASE_URL=https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900
+2
View File
@@ -29,6 +29,7 @@ ORTHOPHOTO_MAX_SIDE_M="${ORTHOPHOTO_MAX_SIDE_M:-1024}"
ORTHOPHOTO_CACHE_TTL_HOURS="${ORTHOPHOTO_CACHE_TTL_HOURS:-24}"
SOURCE_CATALOG_PROBE_ENABLED="${SOURCE_CATALOG_PROBE_ENABLED:-true}"
SOURCE_CATALOG_GRB_WFS_URL="${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}"
SOURCE_CATALOG_ALZ_RELEASE_URL="${SOURCE_CATALOG_ALZ_RELEASE_URL:-https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen}"
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS="${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}"
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB="${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}"
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}"
@@ -133,6 +134,7 @@ docker run -d \
-e ORTHOPHOTO_CACHE_TTL_HOURS="$ORTHOPHOTO_CACHE_TTL_HOURS" \
-e SOURCE_CATALOG_PROBE_ENABLED="$SOURCE_CATALOG_PROBE_ENABLED" \
-e SOURCE_CATALOG_GRB_WFS_URL="$SOURCE_CATALOG_GRB_WFS_URL" \
-e SOURCE_CATALOG_ALZ_RELEASE_URL="$SOURCE_CATALOG_ALZ_RELEASE_URL" \
-e SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS="$SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS" \
-e SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB="$SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB" \
-e SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="$SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS" \
+1
View File
@@ -32,6 +32,7 @@ services:
ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24}
SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true}
SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}
SOURCE_CATALOG_ALZ_RELEASE_URL: ${SOURCE_CATALOG_ALZ_RELEASE_URL:-https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen}
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS: ${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB: ${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS: ${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}
+16 -5
View File
@@ -414,11 +414,15 @@ not marked stale merely because their source date is old. Unknown sources are
### GET `/api/v1/projects/{project_id}/datasets/source-catalog-probes`
Performs an explicit, read-only release probe for the allowlisted GRB WFS and
most-recent orthophoto WMS. The endpoint reads bounded `GetCapabilities`
responses, confirms the expected layers and follows only HTTPS ISO 19139
`GetRecordById` links on `metadata.vlaanderen.be`. Query parameter
`refresh=true` bypasses the short in-memory response cache.
Performs an explicit, read-only release probe for the allowlisted GRB WFS,
most-recent orthophoto WMS and ALZ agricultural-use parcel publication page.
For OGC services, the endpoint reads bounded `GetCapabilities` responses,
confirms the expected layers and follows only HTTPS ISO 19139 `GetRecordById`
links on `metadata.vlaanderen.be`. For ALZ, it reads only the bounded official
HTML release page and accepts only exact
`www.landbouwvlaanderen.be/bestanden/gis/agpa_<year>_<date>_public.zip` link
identities. It does not request those archives. Query parameter `refresh=true`
bypasses the short in-memory response cache.
Each provider item returns service reachability, matched/missing layers, the
official metadata identifier, title, edition, publication/metadata dates,
@@ -427,6 +431,13 @@ local `source_version` and one comparison status: `same`, `different`,
`available`, `degraded`, `unavailable` or `disabled`. A difference means only
that an operator should review provenance; it is not an update instruction.
ALZ comparisons use only the latest definitive third snapshot, normalized as
`<campaign>-v3`. A newer first or second snapshot is reported in the item title
and message as provisional but cannot mark a local definitive historical
edition as outdated. `service_type=HTML` uses `definitive_archive` and
`current_snapshot` as evidence markers in the existing expected/matched/missing
arrays; no parallel response shape is introduced.
The endpoint accepts no arbitrary URL, feature query, area or layer. It does
not fetch vector features, raster pixels or models, create jobs/datasets, write
to PostGIS or trigger an import. The normal `source-freshness` endpoint remains
+8 -1
View File
@@ -173,7 +173,7 @@ gebeurd.
| --- | --- | --- |
| GRB gebouwen/wegen/water/percelen | operationele, expliciete plan-stage-apply refresh met onveranderlijke snapshots | alleen een nieuw officieel gedateerd cataloguseditie na operatorbevestiging ophalen |
| Statbel bevolking | jaarlijkse, expliciete edities in één tijdreeks | een nieuwe publicatie alleen na schema-, sectorgeometrie- en totalencontrole toevoegen |
| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; metricvergelijking zonder objectlineage | eerst een stabiele machineleesbare editiebron verifiëren, daarna dezelfde begrensde operatorflow toepassen |
| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete officiële publicatieprobe; metricvergelijking zonder objectlineage | een nieuwere definitieve v3-editie eerst handmatig beoordelen en daarna via de bestaande begrensde operatorflow toevoegen |
| orthofoto | vaste lokale opname per expliciete analysezone; catalogusprobe is alleen een signaal | vluchtjaar, productvariant en dekking vergelijken voordat nieuwe pixels worden opgehaald |
| landgebruik, thematische rasters, DHMV en VMM-scenario's | vaste product-/scenario-edities, geen rolling snapshot | alleen een nieuwe gedocumenteerde producteditie als afzonderlijke Dataset verwerven |
| bodemkaart en historische kaarten | historische referentie-editie | niet als verouderde actuele bron labelen; alleen vervangen bij een officiële inhoudelijke heruitgave |
@@ -183,6 +183,13 @@ Geen van deze regels activeert een browserfetch, scheduler of automatische
vervanging. Een bron wordt pas `refreshable` wanneer versie, dekking, schema,
provenance en een begrensde acquisitieroute afzonderlijk verifieerbaar zijn.
De ALZ-publicatiepagina levert drie campagnesnapshots: v1 en v2 zijn
voorlopig, v3 is de definitieve historische editie. GeoIntel vergelijkt daarom
de lokale `2025-definitive` uitsluitend met de officiële `2025-v3`, terwijl de
actuele `2026-v1` zichtbaar maar niet updategerechtigd blijft. De probe leest
alleen de allowlisted HTML-pagina en linkidentiteiten; ZIP-archieven worden pas
door de afzonderlijke operator opgehaald na menselijke editiebevestiging.
### Mol population history
`scripts/provision_mol_population_history.py` imports official Statbel
+1
View File
@@ -29,6 +29,7 @@ GRB_WFS_URL=
# Explicit read-only source-edition probes (no provider import).
SOURCE_CATALOG_PROBE_ENABLED=true
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
SOURCE_CATALOG_ALZ_RELEASE_URL=https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900
+4 -3
View File
@@ -41,7 +41,8 @@
- [x] Add a read-only source freshness/version audit with explicit publication policies, local integrity checks, operator CLI and compact Status workspace surface.
- [x] Add explicit bounded GRB/orthophoto catalogue release probes using official capabilities and ISO metadata; keep every acquisition separate and operator-controlled.
- [x] Add a governed regional GRB plan -> stage -> checksum-confirmed apply workflow that preserves every previous snapshot.
- [ ] Extend catalogue probes only to additional sources that publish a stable machine-readable edition contract; do not add background polling or infer releases from HTTP dates alone.
- [x] Add a fail-closed ALZ publication probe that distinguishes provisional v1/v2 snapshots from the definitive v3 historical edition and never downloads an archive.
- [ ] Extend catalogue probes only to additional sources that publish a stable official edition contract; do not add background polling or infer releases from HTTP dates alone.
## Governed source expansion backlog
@@ -683,7 +684,7 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Show precise daily edition ranges and explain registration-date nuance in
the Evolution workspace.
- [x] Document refresh readiness across the complete official-source portfolio.
- [ ] Add the next governed catalog probe only after an official stable version
contract is verified; ALZ annual releases are the first candidate.
- [x] Verify and add the governed ALZ catalog probe against the official
campaign-snapshot and definitive-archive publication contract.
- [ ] Keep orthophoto refresh manual until product variant, flight year and
complete selected-area coverage can be compared deterministically.
+6 -3
View File
@@ -10,9 +10,12 @@ require local integrity review or are local artifacts. Only attention items are
expanded by default; all source detail remains available through disclosure.
The refresh button reruns the local read-only audit and never downloads or
replaces source data. A separate `Officiële edities controleren` action is the
only trigger for bounded GRB/orthophoto catalog reads. It presents official and
local editions, layer-contract coverage and honest non-comparable version
markers without starting an import or background poll.
only trigger for bounded GRB/orthophoto catalog reads and the official ALZ
publication-page check. It presents official and local editions,
layer/publication evidence and honest non-comparable version markers without
starting an import or background poll. ALZ v1/v2 snapshots are labelled
provisional; only the latest definitive v3 edition is compared with local
history.
After that explicit check, the same surface shows a compact GRB refresh plan
for buildings, roads, water and parcels. It states whether each local regional
snapshot is current, updateable or needs review and shows the current object
@@ -86,6 +86,7 @@ function integrityIssueCount(item: SourceFreshnessItem): number {
}
function CatalogRow({ item }: { item: SourceCatalogProbeItem }): JSX.Element {
const evidenceLabel = item.service_type === 'HTML' ? 'Publicaties' : 'Lagen'
return (
<div className={`source-catalog-row source-catalog-row-${item.status}`}>
<div className="source-freshness-main">
@@ -98,7 +99,7 @@ function CatalogRow({ item }: { item: SourceCatalogProbeItem }): JSX.Element {
<p>{item.message}</p>
<div className="source-freshness-meta">
<span>Lokaal: {item.local_source_version ?? 'niet aanwezig'}</span>
<span>Lagen: {item.matched_layers.length}/{item.expected_layers.length} bevestigd</span>
<span>{evidenceLabel}: {item.matched_layers.length}/{item.expected_layers.length} bevestigd</span>
{item.remote_modified_at ? <span>Metadata: {formatDate(item.remote_modified_at)}</span> : null}
{item.cached ? <span>cache gebruikt</span> : null}
</div>
@@ -150,7 +151,7 @@ export function SourceFreshnessPanel({
<div>
<p className="eyebrow">Bronbeheer</p>
<h2>Actualiteit en versiecontrole</h2>
<p>Controleert lokale publicaties en kan op aanvraag de officiële GRB- en orthofoto-editie uitlezen.</p>
<p>Controleert lokale publicaties en kan op aanvraag de officiële GRB-, orthofoto- en landbouweditie uitlezen.</p>
</div>
<div className="source-freshness-actions">
<button className="secondary-action" type="button" onClick={onRefresh} disabled={loading}>
@@ -206,7 +207,7 @@ export function SourceFreshnessPanel({
</div>
{catalogError ? <p className="inline-error">{catalogError}</p> : null}
{!catalogReport && !catalogError ? (
<p className="source-catalog-empty">Controleer GRB en orthofoto wanneer je een lokale bronversie met de officiële editie wilt vergelijken.</p>
<p className="source-catalog-empty">Controleer GRB, orthofoto en landbouwpercelen wanneer je lokale bronversies met officiële edities wilt vergelijken.</p>
) : null}
{catalogReport ? (
<>
+1 -1
View File
@@ -715,7 +715,7 @@ export type SourceCatalogComparisonStatus = 'same' | 'different' | 'not_comparab
export interface SourceCatalogProbeItem {
source_name: string
display_name: string
service_type: 'WFS' | 'WMS'
service_type: 'WFS' | 'WMS' | 'HTML'
endpoint_url: string
status: SourceCatalogProbeStatus
reachable: boolean