Add governed ALZ edition probe
This commit is contained in:
@@ -14,6 +14,7 @@ ORTHOPHOTO_MAX_SIDE_M=1024
|
||||
ORTHOPHOTO_CACHE_TTL_HOURS=24
|
||||
SOURCE_CATALOG_PROBE_ENABLED=true
|
||||
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
|
||||
SOURCE_CATALOG_ALZ_RELEASE_URL=https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen
|
||||
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10
|
||||
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2
|
||||
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900
|
||||
|
||||
@@ -7,6 +7,23 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 225 Governed ALZ edition probe (2026-07-16)
|
||||
|
||||
- Extended the explicit read-only source catalog audit with the official ALZ
|
||||
agricultural-use parcel publication page without adding an importer,
|
||||
scheduler or background provider request.
|
||||
- Added strict release-page, redirect and archive-link allowlists and bounded
|
||||
HTML parsing; the probe never downloads the referenced ALZ ZIP archives.
|
||||
- Normalized only definitive third snapshots as comparable editions, so local
|
||||
`2025-definitive` evidence matches official `2025-v3` while current
|
||||
`2026-v1` remains visibly provisional.
|
||||
- Reused the canonical source-catalog envelope, Status UI and operator command,
|
||||
and propagated the fixed release URL through Docker and Unraid runtime
|
||||
configuration.
|
||||
- Added focused release parsing, version ordering, cache, failure-isolation and
|
||||
trust-boundary regression coverage without changing persistence or
|
||||
migrations.
|
||||
|
||||
## Sprint 224 Governed GRB evolution (2026-07-16)
|
||||
|
||||
- Enabled fail-closed object evolution between the retained regional GRB
|
||||
|
||||
+12
-8
@@ -1444,7 +1444,8 @@ docker exec geointel python /app/scripts/audit_source_freshness.py \
|
||||
Use `--fail-on due` to make a planned review date fail automation, or
|
||||
`--fail-on never` for reporting only. The command never starts a refresh.
|
||||
|
||||
An operator can explicitly add the official GRB and orthophoto edition check:
|
||||
An operator can explicitly add the official GRB, orthophoto and ALZ edition
|
||||
check:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/audit_source_freshness.py \
|
||||
@@ -1458,17 +1459,20 @@ Use `--refresh-catalogs` to bypass the 15-minute in-memory cache and
|
||||
`--fail-on-catalog` only when temporary official-provider unavailability must
|
||||
fail an operator job. This path reads bounded WFS/WMS capabilities and their
|
||||
fixed ISO 19139 metadata records. It confirms GRB `GBG`, `WBN`, `WGO`, `ADP`
|
||||
and orthophoto `Ortho`, `Vliegdagcontour`; it never requests feature or raster
|
||||
content. The public Datavindplaats API requires an access token, so release
|
||||
evidence comes from the public metadata links published by the official OGC
|
||||
services rather than HTML scraping.
|
||||
and orthophoto `Ortho`, `Vliegdagcontour`. It also reads the exact official ALZ
|
||||
publication page and validates only allowlisted archive-link identities. It
|
||||
never requests feature, raster or ALZ ZIP content. ALZ v1/v2 campaign snapshots
|
||||
remain provisional; only a v3 publication is compared with a local definitive
|
||||
historical edition.
|
||||
|
||||
Runtime controls are `SOURCE_CATALOG_PROBE_ENABLED`,
|
||||
`SOURCE_CATALOG_GRB_WFS_URL`, `SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS`,
|
||||
`SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB` and
|
||||
`SOURCE_CATALOG_GRB_WFS_URL`, `SOURCE_CATALOG_ALZ_RELEASE_URL`,
|
||||
`SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS`, `SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB` and
|
||||
`SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS`. Metadata links remain restricted to
|
||||
the official HTTPS CSW path on `metadata.vlaanderen.be` even when an operator
|
||||
overrides the capabilities endpoint.
|
||||
overrides the capabilities endpoint. The ALZ release URL is fail-closed to the
|
||||
exact HTTPS host/path and cannot be redirected to another page or download
|
||||
host.
|
||||
|
||||
## Governed regional GRB refresh
|
||||
|
||||
|
||||
@@ -36,6 +36,10 @@ class Settings(BaseSettings):
|
||||
default="https://geo.api.vlaanderen.be/GRB/wfs",
|
||||
validation_alias="SOURCE_CATALOG_GRB_WFS_URL",
|
||||
)
|
||||
source_catalog_alz_release_url: str = Field(
|
||||
default="https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen",
|
||||
validation_alias="SOURCE_CATALOG_ALZ_RELEASE_URL",
|
||||
)
|
||||
source_catalog_probe_timeout_seconds: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
|
||||
@@ -14,7 +14,7 @@ SourceCatalogComparisonStatus = Literal["same", "different", "not_comparable", "
|
||||
class SourceCatalogProbeItem(BaseModel):
|
||||
source_name: str
|
||||
display_name: str
|
||||
service_type: Literal["WFS", "WMS"]
|
||||
service_type: Literal["WFS", "WMS", "HTML"]
|
||||
endpoint_url: str
|
||||
status: SourceCatalogProbeStatus
|
||||
reachable: bool
|
||||
|
||||
@@ -4,6 +4,7 @@ from dataclasses import dataclass, replace
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
from hashlib import sha256
|
||||
from html.parser import HTMLParser
|
||||
import re
|
||||
from threading import Lock
|
||||
from typing import Any, Callable
|
||||
@@ -32,6 +33,17 @@ _XLINK = "http://www.w3.org/1999/xlink"
|
||||
_METADATA_HOST = "metadata.vlaanderen.be"
|
||||
_VERSION_DATE = re.compile(r"^(?:toestand\s+)?(\d{4}-\d{2}-\d{2})$", re.IGNORECASE)
|
||||
_ORTHOPHOTO_EDITION = re.compile(r"^\d{4}\.\d{2}$")
|
||||
_ALZ_SOURCE_NAME = "agentschap_landbouw_zeevisserij_agricultural_parcels"
|
||||
_ALZ_RELEASE_HOST = "landbouwcijfers.vlaanderen.be"
|
||||
_ALZ_RELEASE_PATH = "/open-geodata-landbouwgebruikspercelen"
|
||||
_ALZ_DOWNLOAD_HOST = "www.landbouwvlaanderen.be"
|
||||
_ALZ_DOWNLOAD_PATH = re.compile(r"^/bestanden/gis/agpa_(20\d{2})_(\d{4}-\d{2}-\d{2})_public\.zip$")
|
||||
_ALZ_SNAPSHOT = re.compile(
|
||||
r"^Landbouwgebruikspercelen\s+(20\d{2})\s*-\s*(\d+)e\s+snapshot\s*"
|
||||
r"\(extractie\s+(\d{2}-\d{2}-\d{4})\)(?:\s*-\s*GPKG)?$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_ALZ_EDITION = re.compile(r"^(20\d{2})-(?:definitive|v3)$", re.IGNORECASE)
|
||||
|
||||
|
||||
class CatalogProbeFailure(RuntimeError):
|
||||
@@ -88,6 +100,32 @@ class CacheEntry:
|
||||
probe: RemoteProbe
|
||||
|
||||
|
||||
class _AnchorParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.anchors: list[tuple[str, str]] = []
|
||||
self._href: str | None = None
|
||||
self._text: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if tag.lower() != "a" or self._href is not None:
|
||||
return
|
||||
href = dict(attrs).get("href")
|
||||
if href:
|
||||
self._href = href.strip()
|
||||
self._text = []
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._href is not None:
|
||||
self._text.append(data)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag.lower() == "a" and self._href is not None:
|
||||
self.anchors.append((self._href, "".join(self._text)))
|
||||
self._href = None
|
||||
self._text = []
|
||||
|
||||
|
||||
_REMOTE_CACHE: dict[str, CacheEntry] = {}
|
||||
_CACHE_LOCK = Lock()
|
||||
|
||||
@@ -124,7 +162,7 @@ def _with_capabilities_query(base_url: str, service_type: str) -> str:
|
||||
def _bounded_fetch(url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> FetchResult:
|
||||
request = Request(
|
||||
url,
|
||||
headers={"Accept": "application/xml,text/xml,application/json", "User-Agent": "GeoIntel/0.1 source-catalog-probe"},
|
||||
headers={"Accept": "application/xml,text/xml,text/html,application/json", "User-Agent": "GeoIntel/0.1 source-catalog-probe"},
|
||||
)
|
||||
max_bytes = settings.source_catalog_probe_max_response_mb * 1024 * 1024
|
||||
try:
|
||||
@@ -226,6 +264,127 @@ def _validate_capabilities_url(url: str) -> str:
|
||||
return url
|
||||
|
||||
|
||||
def _validate_alz_release_url(url: str) -> str:
|
||||
parsed = urlsplit(url)
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or parsed.hostname != _ALZ_RELEASE_HOST
|
||||
or parsed.port not in {None, 443}
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or parsed.path.rstrip("/") != _ALZ_RELEASE_PATH
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
raise CatalogProbeFailure(
|
||||
"CATALOG_ALZ_RELEASE_URL_REJECTED",
|
||||
"De ingestelde ALZ-publicatiepagina valt buiten de toegestane officiële URL.",
|
||||
)
|
||||
return url
|
||||
|
||||
|
||||
def _validate_alz_download_url(url: str) -> tuple[int, datetime]:
|
||||
parsed = urlsplit(url)
|
||||
match = _ALZ_DOWNLOAD_PATH.fullmatch(parsed.path)
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or parsed.hostname != _ALZ_DOWNLOAD_HOST
|
||||
or parsed.port not in {None, 443}
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
or not match
|
||||
):
|
||||
raise CatalogProbeFailure(
|
||||
"CATALOG_ALZ_DOWNLOAD_URL_REJECTED",
|
||||
"De ALZ-publicatiepagina bevat een datasetlink buiten de toegestane officiële URL-structuur.",
|
||||
)
|
||||
try:
|
||||
published_at = datetime.strptime(match.group(2), "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
||||
except ValueError as exc:
|
||||
raise CatalogProbeFailure(
|
||||
"CATALOG_ALZ_DOWNLOAD_URL_REJECTED",
|
||||
"De ALZ-datasetlink bevat geen geldige publicatiedatum.",
|
||||
) from exc
|
||||
return int(match.group(1)), published_at
|
||||
|
||||
|
||||
def _normalized_html_text(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", value.replace("\xad", "").replace("–", "-").replace("—", "-")).strip()
|
||||
|
||||
|
||||
def _parse_alz_release_page(content: bytes) -> dict[str, Any]:
|
||||
try:
|
||||
html = content.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise CatalogProbeFailure(
|
||||
"CATALOG_ALZ_INVALID_HTML",
|
||||
"De officiële ALZ-publicatiepagina is niet geldige UTF-8 HTML.",
|
||||
) from exc
|
||||
parser = _AnchorParser()
|
||||
try:
|
||||
parser.feed(html)
|
||||
parser.close()
|
||||
except Exception as exc:
|
||||
raise CatalogProbeFailure(
|
||||
"CATALOG_ALZ_INVALID_HTML",
|
||||
"De officiële ALZ-publicatiepagina kon niet veilig worden ontleed.",
|
||||
) from exc
|
||||
|
||||
definitive: list[tuple[int, datetime]] = []
|
||||
snapshots: list[tuple[int, int, datetime]] = []
|
||||
for href, raw_text in parser.anchors:
|
||||
text = _normalized_html_text(raw_text)
|
||||
looks_like_alz_release = (
|
||||
text.casefold() == "downloaden"
|
||||
or text.casefold().startswith("landbouwgebruikspercelen ")
|
||||
or "agpa_" in href.casefold()
|
||||
)
|
||||
if not looks_like_alz_release:
|
||||
continue
|
||||
year, file_date = _validate_alz_download_url(href)
|
||||
snapshot = _ALZ_SNAPSHOT.fullmatch(text)
|
||||
if snapshot:
|
||||
snapshot_year = int(snapshot.group(1))
|
||||
snapshot_number = int(snapshot.group(2))
|
||||
try:
|
||||
extraction_date = datetime.strptime(snapshot.group(3), "%d-%m-%Y").replace(tzinfo=timezone.utc)
|
||||
except ValueError as exc:
|
||||
raise CatalogProbeFailure(
|
||||
"CATALOG_ALZ_SNAPSHOT_INVALID",
|
||||
"De actuele ALZ-snapshot bevat geen geldige extractiedatum.",
|
||||
) from exc
|
||||
if snapshot_year != year or extraction_date != file_date or snapshot_number not in {1, 2, 3}:
|
||||
raise CatalogProbeFailure(
|
||||
"CATALOG_ALZ_SNAPSHOT_INVALID",
|
||||
"De actuele ALZ-snapshot is niet consistent met de officiële datasetlink.",
|
||||
)
|
||||
snapshots.append((year, snapshot_number, extraction_date))
|
||||
elif text.casefold() == "downloaden":
|
||||
definitive.append((year, file_date))
|
||||
else:
|
||||
raise CatalogProbeFailure(
|
||||
"CATALOG_ALZ_RELEASE_UNRECOGNIZED",
|
||||
"De ALZ-publicatiepagina bevat een niet-herkende landbouwdatasetpublicatie.",
|
||||
)
|
||||
|
||||
if not definitive:
|
||||
raise CatalogProbeFailure(
|
||||
"CATALOG_ALZ_DEFINITIVE_MISSING",
|
||||
"De officiële ALZ-publicatiepagina bevat geen herkenbare definitieve landbouwperceeleditie.",
|
||||
)
|
||||
latest_definitive = max(definitive, key=lambda item: (item[0], item[1]))
|
||||
latest_snapshot = max(snapshots, key=lambda item: (item[0], item[1], item[2])) if snapshots else None
|
||||
if latest_snapshot and latest_snapshot[1] == 3 and latest_snapshot[0] >= latest_definitive[0]:
|
||||
latest_definitive = (latest_snapshot[0], latest_snapshot[2])
|
||||
return {
|
||||
"definitive": latest_definitive,
|
||||
"definitive_count": len(definitive),
|
||||
"snapshot": latest_snapshot,
|
||||
}
|
||||
|
||||
|
||||
def _node_text(node: ElementTree.Element | None) -> str | None:
|
||||
if node is None:
|
||||
return None
|
||||
@@ -282,6 +441,74 @@ def _parse_metadata(content: bytes) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _probe_alz_remote(
|
||||
contract: ProbeContract,
|
||||
settings: Settings,
|
||||
*,
|
||||
opener: Callable[..., Any] | None,
|
||||
now: datetime,
|
||||
) -> RemoteProbe:
|
||||
release_url = _validate_alz_release_url(contract.endpoint_url)
|
||||
response = _bounded_fetch(release_url, settings, opener)
|
||||
_validate_alz_release_url(response.final_url)
|
||||
content_type = response.content_type.lower()
|
||||
if content_type and "html" not in content_type and "text" not in content_type:
|
||||
raise CatalogProbeFailure(
|
||||
"CATALOG_ALZ_INVALID_CONTENT_TYPE",
|
||||
"De officiële ALZ-publicatiepagina is geen HTML-respons.",
|
||||
)
|
||||
release = _parse_alz_release_page(response.content)
|
||||
definitive_year, definitive_date = release["definitive"]
|
||||
snapshot = release["snapshot"]
|
||||
matched = ["definitive_archive"]
|
||||
missing: list[str] = []
|
||||
if snapshot:
|
||||
matched.append("current_snapshot")
|
||||
else:
|
||||
missing.append("current_snapshot")
|
||||
|
||||
definitive_version = f"{definitive_year}-v3"
|
||||
if snapshot:
|
||||
snapshot_year, snapshot_number, snapshot_date = snapshot
|
||||
snapshot_version = f"{snapshot_year}-v{snapshot_number}"
|
||||
if snapshot_number < 3:
|
||||
message = (
|
||||
f"De officiële ALZ-publicatiepagina bevestigt definitieve editie {definitive_version}. "
|
||||
f"De actuele publicatie {snapshot_version} van {snapshot_date.date().isoformat()} is voorlopig "
|
||||
"en wordt niet als historische vervanging aangemerkt."
|
||||
)
|
||||
else:
|
||||
message = f"De officiële ALZ-publicatiepagina bevestigt definitieve editie {definitive_version}."
|
||||
remote_title = f"Landbouwgebruikspercelen {definitive_version}; actuele publicatie {snapshot_version}"
|
||||
else:
|
||||
message = (
|
||||
f"De officiële ALZ-publicatiepagina bevestigt definitieve editie {definitive_version}, "
|
||||
"maar bevat geen herkenbare actuele snapshot."
|
||||
)
|
||||
remote_title = f"Landbouwgebruikspercelen {definitive_version}"
|
||||
|
||||
return RemoteProbe(
|
||||
status="available" if snapshot else "degraded",
|
||||
reachable=True,
|
||||
checked_at=now,
|
||||
expected_layers=contract.expected_layers,
|
||||
matched_layers=tuple(matched),
|
||||
missing_layers=tuple(missing),
|
||||
advertised_layer_count=release["definitive_count"] + (1 if snapshot else 0),
|
||||
metadata_url=response.final_url,
|
||||
metadata_identifier="alz-agricultural-use-parcels",
|
||||
remote_title=remote_title,
|
||||
remote_version=definitive_version,
|
||||
remote_modified_at=response.last_modified_at,
|
||||
remote_published_at=definitive_date,
|
||||
capabilities_sha256=sha256(response.content).hexdigest(),
|
||||
capabilities_etag=response.etag,
|
||||
capabilities_last_modified_at=response.last_modified_at,
|
||||
message=message,
|
||||
error_code="CATALOG_ALZ_CURRENT_SNAPSHOT_MISSING" if not snapshot else None,
|
||||
)
|
||||
|
||||
|
||||
def _probe_remote(
|
||||
contract: ProbeContract,
|
||||
settings: Settings,
|
||||
@@ -290,6 +517,8 @@ def _probe_remote(
|
||||
now: datetime,
|
||||
) -> RemoteProbe:
|
||||
try:
|
||||
if contract.service_type == "HTML":
|
||||
return _probe_alz_remote(contract, settings, opener=opener, now=now)
|
||||
capabilities = _bounded_fetch(_validate_capabilities_url(contract.endpoint_url), settings, opener)
|
||||
_validate_capabilities_url(capabilities.final_url)
|
||||
content_type = capabilities.content_type.lower()
|
||||
@@ -394,6 +623,18 @@ def _latest_local_version(source_name: str, datasets: list[Dataset]) -> str | No
|
||||
]
|
||||
if explicit_current:
|
||||
candidates = explicit_current
|
||||
elif source_name == _ALZ_SOURCE_NAME:
|
||||
definitive = [item for item in candidates if _ALZ_EDITION.fullmatch((item.source_version or "").strip())]
|
||||
if definitive:
|
||||
latest = max(
|
||||
definitive,
|
||||
key=lambda item: (
|
||||
int(_ALZ_EDITION.fullmatch((item.source_version or "").strip()).group(1)),
|
||||
_utc(item.observed_at) if item.observed_at else datetime.min.replace(tzinfo=timezone.utc),
|
||||
str(item.id),
|
||||
),
|
||||
)
|
||||
return latest.source_version
|
||||
if not candidates:
|
||||
return None
|
||||
latest = max(
|
||||
@@ -416,6 +657,9 @@ def _normalized_version(source_name: str, version: str | None) -> str | None:
|
||||
return match.group(1) if match else None
|
||||
if source_name == "digitaal_vlaanderen_orthophoto" and _ORTHOPHOTO_EDITION.fullmatch(value):
|
||||
return value
|
||||
if source_name == _ALZ_SOURCE_NAME:
|
||||
match = _ALZ_EDITION.fullmatch(value)
|
||||
return f"{match.group(1)}-v3" if match else None
|
||||
return None
|
||||
|
||||
|
||||
@@ -467,6 +711,13 @@ class SourceCatalogProbeService:
|
||||
endpoint_url=_with_capabilities_query(active_settings.orthophoto_wms_url, "WMS"),
|
||||
expected_layers=(active_settings.orthophoto_wms_layer, "Vliegdagcontour"),
|
||||
),
|
||||
ProbeContract(
|
||||
source_name=_ALZ_SOURCE_NAME,
|
||||
display_name="Landbouwgebruikspercelen (ALZ)",
|
||||
service_type="HTML",
|
||||
endpoint_url=active_settings.source_catalog_alz_release_url,
|
||||
expected_layers=("definitive_archive", "current_snapshot"),
|
||||
),
|
||||
)
|
||||
items: list[SourceCatalogProbeItem] = []
|
||||
for contract in contracts:
|
||||
@@ -538,8 +789,9 @@ class SourceCatalogProbeService:
|
||||
summary=summary,
|
||||
items=items,
|
||||
limitations=[
|
||||
"Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities en gekoppelde ISO 19139 metadata.",
|
||||
"Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities, gekoppelde ISO 19139 metadata en de officiële ALZ-publicatiepagina.",
|
||||
"Er worden geen features, rasters of modelbestanden opgehaald en geen datasets aangemaakt of overschreven.",
|
||||
"Voor ALZ is alleen de nieuwste definitieve v3-editie vergelijkbaar; voorlopige v1/v2-snapshots zijn uitsluitend informatief.",
|
||||
"Een versieverschil is controlesignaal, geen bewijs dat een lokale dataset onbruikbaar is en geen automatische importopdracht.",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -21,6 +21,7 @@ ORTHO_METADATA_URL = (
|
||||
"https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&service=CSW&"
|
||||
"id=f5304d6d-0dd4-43fd-a726-427af31e8d61&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd"
|
||||
)
|
||||
ALZ_RELEASE_URL = "https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen"
|
||||
|
||||
|
||||
def _wfs_capabilities(metadata_url: str = GRB_METADATA_URL) -> bytes:
|
||||
@@ -77,6 +78,23 @@ def _metadata(identifier: str, title: str, edition: str, modified: str, publishe
|
||||
""".encode()
|
||||
|
||||
|
||||
def _alz_release_page(*, include_snapshot: bool = True, download_host: str = "www.landbouwvlaanderen.be") -> bytes:
|
||||
snapshot = (
|
||||
f'<a href="https://{download_host}/bestanden/gis/agpa_2026_2026-06-02_public.zip">'
|
||||
"Landbouwgebruikspercelen 2026 – 1e snapshot (extractie 02-06-2026) - GPKG</a>"
|
||||
if include_snapshot
|
||||
else ""
|
||||
)
|
||||
return f"""
|
||||
<html><body>
|
||||
{snapshot}
|
||||
<p>Definitieve datasets</p>
|
||||
<a href="https://{download_host}/bestanden/gis/agpa_2025_2026-05-13_public.zip">Downloaden</a>
|
||||
<a href="https://{download_host}/bestanden/gis/agpa_2024_2025-03-27_public.zip">Downloaden</a>
|
||||
</body></html>
|
||||
""".encode()
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, content: bytes, content_type: str = "text/xml", *, content_length: int | None = None) -> None:
|
||||
self.content = content
|
||||
@@ -97,8 +115,8 @@ class _Response:
|
||||
|
||||
|
||||
class _RedirectedResponse(_Response):
|
||||
def __init__(self, content: bytes, final_url: str) -> None:
|
||||
super().__init__(content)
|
||||
def __init__(self, content: bytes, final_url: str, content_type: str = "text/xml") -> None:
|
||||
super().__init__(content, content_type)
|
||||
self.final_url = final_url
|
||||
|
||||
def geturl(self) -> str:
|
||||
@@ -144,6 +162,7 @@ def _settings(**overrides) -> Settings:
|
||||
values = {
|
||||
"SOURCE_CATALOG_PROBE_ENABLED": True,
|
||||
"SOURCE_CATALOG_GRB_WFS_URL": "https://geo.api.vlaanderen.be/GRB/wfs",
|
||||
"SOURCE_CATALOG_ALZ_RELEASE_URL": ALZ_RELEASE_URL,
|
||||
"SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS": 3,
|
||||
"SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB": 1,
|
||||
"SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS": 0,
|
||||
@@ -157,6 +176,8 @@ def _settings(**overrides) -> Settings:
|
||||
def _opener(request, timeout):
|
||||
assert timeout == 3
|
||||
url = request.full_url
|
||||
if url == ALZ_RELEASE_URL:
|
||||
return _Response(_alz_release_page(), "text/html; charset=utf-8")
|
||||
if "metadata.vlaanderen.be" in url:
|
||||
if "f5304d6d" in url:
|
||||
return _Response(_metadata("f5304d6d", "Orthofoto meest recent, 2025.04", "2025.04", "2026-04-27", "2025-12-11"))
|
||||
@@ -170,7 +191,13 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions
|
||||
SourceCatalogProbeService.clear_cache()
|
||||
project_id = uuid.uuid4()
|
||||
report = SourceCatalogProbeService.audit_project(
|
||||
_Db([_dataset("grb", "2026-07-14"), _dataset("digitaal_vlaanderen_orthophoto", "most_recent_at_2026-07-14")]),
|
||||
_Db(
|
||||
[
|
||||
_dataset("grb", "2026-07-14"),
|
||||
_dataset("digitaal_vlaanderen_orthophoto", "most_recent_at_2026-07-14"),
|
||||
_dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2025-definitive"),
|
||||
]
|
||||
),
|
||||
project_id,
|
||||
settings=_settings(),
|
||||
opener=_opener,
|
||||
@@ -178,7 +205,7 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions
|
||||
)
|
||||
by_source = {item.source_name: item for item in report.items}
|
||||
|
||||
assert report.summary.available_count == 2
|
||||
assert report.summary.available_count == 3
|
||||
assert report.summary.different_version_count == 1
|
||||
assert by_source["grb"].remote_version == "Toestand 2026-07-15"
|
||||
assert by_source["grb"].comparison_status == "different"
|
||||
@@ -187,6 +214,13 @@ def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions
|
||||
assert by_source["digitaal_vlaanderen_orthophoto"].remote_version == "2025.04"
|
||||
assert by_source["digitaal_vlaanderen_orthophoto"].comparison_status == "not_comparable"
|
||||
assert by_source["digitaal_vlaanderen_orthophoto"].remote_published_at.year == 2025
|
||||
alz = by_source["agentschap_landbouw_zeevisserij_agricultural_parcels"]
|
||||
assert alz.service_type == "HTML"
|
||||
assert alz.remote_version == "2025-v3"
|
||||
assert alz.comparison_status == "same"
|
||||
assert alz.matched_layers == ["definitive_archive", "current_snapshot"]
|
||||
assert "2026-v1" in alz.message
|
||||
assert "voorlopig" in alz.message
|
||||
assert all(item.capabilities_sha256 for item in report.items)
|
||||
|
||||
|
||||
@@ -275,10 +309,10 @@ def test_catalog_probe_cache_is_explicitly_bypassable() -> None:
|
||||
second = SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW)
|
||||
SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW, force=True)
|
||||
|
||||
assert first_call_count == 4
|
||||
assert first_call_count == 5
|
||||
assert all(item.cached is False for item in first.items)
|
||||
assert all(item.cached is True for item in second.items)
|
||||
assert len(calls) == 8
|
||||
assert len(calls) == 10
|
||||
|
||||
|
||||
def test_catalog_probe_can_be_disabled_without_network_access() -> None:
|
||||
@@ -293,7 +327,7 @@ def test_catalog_probe_can_be_disabled_without_network_access() -> None:
|
||||
now=NOW,
|
||||
)
|
||||
|
||||
assert report.summary.disabled_count == 2
|
||||
assert report.summary.disabled_count == 3
|
||||
assert all(item.status == "disabled" for item in report.items)
|
||||
|
||||
|
||||
@@ -316,7 +350,7 @@ def test_catalog_probe_route_returns_canonical_envelope(monkeypatch) -> None:
|
||||
|
||||
assert list(response) == ["data"]
|
||||
assert response["data"]["project_id"] == project_id
|
||||
assert response["data"]["summary"]["provider_count"] == 2
|
||||
assert response["data"]["summary"]["provider_count"] == 3
|
||||
|
||||
|
||||
def test_catalog_probe_remains_explicit_and_never_imports_provider_data() -> None:
|
||||
@@ -330,3 +364,75 @@ def test_catalog_probe_remains_explicit_and_never_imports_provider_data() -> Non
|
||||
assert "VectorFeatureService" not in service
|
||||
assert "--probe-catalogs" in operator
|
||||
assert "/datasets/source-catalog-probes" in operator
|
||||
|
||||
|
||||
def test_alz_catalog_probe_rejects_release_page_redirect_outside_allowlist() -> None:
|
||||
def redirected_opener(request, timeout):
|
||||
if request.full_url == ALZ_RELEASE_URL:
|
||||
return _RedirectedResponse(
|
||||
_alz_release_page(),
|
||||
"https://example.com/open-geodata-landbouwgebruikspercelen",
|
||||
"text/html",
|
||||
)
|
||||
return _opener(request, timeout)
|
||||
|
||||
report = SourceCatalogProbeService.audit_project(
|
||||
_Db([]), uuid.uuid4(), settings=_settings(), opener=redirected_opener, now=NOW
|
||||
)
|
||||
alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels")
|
||||
|
||||
assert alz.status == "unavailable"
|
||||
assert alz.error_code == "CATALOG_ALZ_RELEASE_URL_REJECTED"
|
||||
|
||||
|
||||
def test_alz_catalog_probe_rejects_untrusted_download_host() -> None:
|
||||
def malicious_opener(request, timeout):
|
||||
if request.full_url == ALZ_RELEASE_URL:
|
||||
return _Response(_alz_release_page(download_host="example.com"), "text/html")
|
||||
return _opener(request, timeout)
|
||||
|
||||
report = SourceCatalogProbeService.audit_project(
|
||||
_Db([]), uuid.uuid4(), settings=_settings(), opener=malicious_opener, now=NOW
|
||||
)
|
||||
alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels")
|
||||
|
||||
assert alz.status == "unavailable"
|
||||
assert alz.error_code == "CATALOG_ALZ_DOWNLOAD_URL_REJECTED"
|
||||
|
||||
|
||||
def test_alz_catalog_probe_degrades_without_current_snapshot_but_keeps_definitive_evidence() -> None:
|
||||
def archive_only_opener(request, timeout):
|
||||
if request.full_url == ALZ_RELEASE_URL:
|
||||
return _Response(_alz_release_page(include_snapshot=False), "text/html")
|
||||
return _opener(request, timeout)
|
||||
|
||||
report = SourceCatalogProbeService.audit_project(
|
||||
_Db([_dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2024-definitive")]),
|
||||
uuid.uuid4(),
|
||||
settings=_settings(),
|
||||
opener=archive_only_opener,
|
||||
now=NOW,
|
||||
)
|
||||
alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels")
|
||||
|
||||
assert alz.status == "degraded"
|
||||
assert alz.remote_version == "2025-v3"
|
||||
assert alz.comparison_status == "different"
|
||||
assert alz.matched_layers == ["definitive_archive"]
|
||||
assert alz.missing_layers == ["current_snapshot"]
|
||||
assert alz.error_code == "CATALOG_ALZ_CURRENT_SNAPSHOT_MISSING"
|
||||
|
||||
|
||||
def test_alz_catalog_probe_compares_latest_definitive_year_not_latest_import_order() -> None:
|
||||
older = _dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2024-definitive")
|
||||
newer = _dataset("agentschap_landbouw_zeevisserij_agricultural_parcels", "2025-definitive")
|
||||
older.imported_at = NOW
|
||||
newer.imported_at = NOW.replace(year=2025)
|
||||
|
||||
report = SourceCatalogProbeService.audit_project(
|
||||
_Db([older, newer]), uuid.uuid4(), settings=_settings(), opener=_opener, now=NOW
|
||||
)
|
||||
alz = next(item for item in report.items if item.source_name == "agentschap_landbouw_zeevisserij_agricultural_parcels")
|
||||
|
||||
assert alz.local_source_version == "2025-definitive"
|
||||
assert alz.comparison_status == "same"
|
||||
|
||||
@@ -34,8 +34,9 @@
|
||||
<Config Name="Orthophoto WMS URL" Target="ORTHOPHOTO_WMS_URL" Default="https://geo.api.vlaanderen.be/OMWRGBMRVL/wms" Mode="" Description="Official Digitaal Vlaanderen most-recent winter orthophoto WMS endpoint." Type="Variable" Display="advanced" Required="true" Mask="false">https://geo.api.vlaanderen.be/OMWRGBMRVL/wms</Config>
|
||||
<Config Name="Orthophoto Resolution (m)" Target="ORTHOPHOTO_RESOLUTION_M" Default="1.0" Mode="" Description="Requested analysis sampling in metres per pixel. Keep at 1.0 for the active building model profile." Type="Variable" Display="advanced" Required="true" Mask="false">1.0</Config>
|
||||
<Config Name="Orthophoto Maximum Side (m)" Target="ORTHOPHOTO_MAX_SIDE_M" Default="1024" Mode="" Description="Safety limit for each selected rectangle side before external acquisition and local inference." Type="Variable" Display="advanced" Required="true" Mask="false">1024</Config>
|
||||
<Config Name="Official Catalog Edition Probe" Target="SOURCE_CATALOG_PROBE_ENABLED" Default="true" Mode="" Description="Allow explicit read-only GRB and orthophoto edition checks. This never imports provider data." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||
<Config Name="Official Catalog Edition Probe" Target="SOURCE_CATALOG_PROBE_ENABLED" Default="true" Mode="" Description="Allow explicit read-only GRB, orthophoto and ALZ edition checks. This never imports provider data." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||
<Config Name="GRB Catalog WFS URL" Target="SOURCE_CATALOG_GRB_WFS_URL" Default="https://geo.api.vlaanderen.be/GRB/wfs" Mode="" Description="Official GRB WFS used only for capabilities and linked ISO metadata checks." Type="Variable" Display="advanced" Required="true" Mask="false">https://geo.api.vlaanderen.be/GRB/wfs</Config>
|
||||
<Config Name="ALZ Release Page URL" Target="SOURCE_CATALOG_ALZ_RELEASE_URL" Default="https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen" Mode="" Description="Exact official ALZ publication page used only to identify definitive and provisional agricultural parcel editions." Type="Variable" Display="advanced" Required="true" Mask="false">https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen</Config>
|
||||
<Config Name="Catalog Probe Timeout (seconds)" Target="SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS" Default="10" Mode="" Description="Per-request timeout for explicit read-only official catalog checks." Type="Variable" Display="advanced" Required="true" Mask="false">10</Config>
|
||||
<Config Name="Catalog Probe Maximum Response (MiB)" Target="SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB" Default="2" Mode="" Description="Maximum capabilities or ISO metadata response size accepted by a catalog probe." Type="Variable" Display="advanced" Required="true" Mask="false">2</Config>
|
||||
<Config Name="Catalog Probe Cache (seconds)" Target="SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS" Default="900" Mode="" Description="Short in-memory cache for repeated official edition checks; use zero to disable." Type="Variable" Display="advanced" Required="true" Mask="false">900</Config>
|
||||
|
||||
@@ -33,10 +33,11 @@ ORTHOPHOTO_MIN_SIDE_M=128
|
||||
ORTHOPHOTO_MAX_SIDE_M=1024
|
||||
ORTHOPHOTO_CACHE_TTL_HOURS=24
|
||||
|
||||
# Explicit read-only edition checks for GRB and the most-recent orthophoto.
|
||||
# No feature or raster data is downloaded by these probes.
|
||||
# Explicit read-only edition checks for GRB, orthophoto and ALZ publications.
|
||||
# No feature, raster or ALZ archive is downloaded by these probes.
|
||||
SOURCE_CATALOG_PROBE_ENABLED=true
|
||||
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
|
||||
SOURCE_CATALOG_ALZ_RELEASE_URL=https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen
|
||||
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10
|
||||
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2
|
||||
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900
|
||||
|
||||
@@ -29,6 +29,7 @@ ORTHOPHOTO_MAX_SIDE_M="${ORTHOPHOTO_MAX_SIDE_M:-1024}"
|
||||
ORTHOPHOTO_CACHE_TTL_HOURS="${ORTHOPHOTO_CACHE_TTL_HOURS:-24}"
|
||||
SOURCE_CATALOG_PROBE_ENABLED="${SOURCE_CATALOG_PROBE_ENABLED:-true}"
|
||||
SOURCE_CATALOG_GRB_WFS_URL="${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}"
|
||||
SOURCE_CATALOG_ALZ_RELEASE_URL="${SOURCE_CATALOG_ALZ_RELEASE_URL:-https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen}"
|
||||
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS="${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}"
|
||||
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB="${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}"
|
||||
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}"
|
||||
@@ -133,6 +134,7 @@ docker run -d \
|
||||
-e ORTHOPHOTO_CACHE_TTL_HOURS="$ORTHOPHOTO_CACHE_TTL_HOURS" \
|
||||
-e SOURCE_CATALOG_PROBE_ENABLED="$SOURCE_CATALOG_PROBE_ENABLED" \
|
||||
-e SOURCE_CATALOG_GRB_WFS_URL="$SOURCE_CATALOG_GRB_WFS_URL" \
|
||||
-e SOURCE_CATALOG_ALZ_RELEASE_URL="$SOURCE_CATALOG_ALZ_RELEASE_URL" \
|
||||
-e SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS="$SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS" \
|
||||
-e SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB="$SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB" \
|
||||
-e SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="$SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS" \
|
||||
|
||||
@@ -32,6 +32,7 @@ services:
|
||||
ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24}
|
||||
SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true}
|
||||
SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}
|
||||
SOURCE_CATALOG_ALZ_RELEASE_URL: ${SOURCE_CATALOG_ALZ_RELEASE_URL:-https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen}
|
||||
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS: ${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}
|
||||
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB: ${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}
|
||||
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS: ${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}
|
||||
|
||||
+16
-5
@@ -414,11 +414,15 @@ not marked stale merely because their source date is old. Unknown sources are
|
||||
|
||||
### GET `/api/v1/projects/{project_id}/datasets/source-catalog-probes`
|
||||
|
||||
Performs an explicit, read-only release probe for the allowlisted GRB WFS and
|
||||
most-recent orthophoto WMS. The endpoint reads bounded `GetCapabilities`
|
||||
responses, confirms the expected layers and follows only HTTPS ISO 19139
|
||||
`GetRecordById` links on `metadata.vlaanderen.be`. Query parameter
|
||||
`refresh=true` bypasses the short in-memory response cache.
|
||||
Performs an explicit, read-only release probe for the allowlisted GRB WFS,
|
||||
most-recent orthophoto WMS and ALZ agricultural-use parcel publication page.
|
||||
For OGC services, the endpoint reads bounded `GetCapabilities` responses,
|
||||
confirms the expected layers and follows only HTTPS ISO 19139 `GetRecordById`
|
||||
links on `metadata.vlaanderen.be`. For ALZ, it reads only the bounded official
|
||||
HTML release page and accepts only exact
|
||||
`www.landbouwvlaanderen.be/bestanden/gis/agpa_<year>_<date>_public.zip` link
|
||||
identities. It does not request those archives. Query parameter `refresh=true`
|
||||
bypasses the short in-memory response cache.
|
||||
|
||||
Each provider item returns service reachability, matched/missing layers, the
|
||||
official metadata identifier, title, edition, publication/metadata dates,
|
||||
@@ -427,6 +431,13 @@ local `source_version` and one comparison status: `same`, `different`,
|
||||
`available`, `degraded`, `unavailable` or `disabled`. A difference means only
|
||||
that an operator should review provenance; it is not an update instruction.
|
||||
|
||||
ALZ comparisons use only the latest definitive third snapshot, normalized as
|
||||
`<campaign>-v3`. A newer first or second snapshot is reported in the item title
|
||||
and message as provisional but cannot mark a local definitive historical
|
||||
edition as outdated. `service_type=HTML` uses `definitive_archive` and
|
||||
`current_snapshot` as evidence markers in the existing expected/matched/missing
|
||||
arrays; no parallel response shape is introduced.
|
||||
|
||||
The endpoint accepts no arbitrary URL, feature query, area or layer. It does
|
||||
not fetch vector features, raster pixels or models, create jobs/datasets, write
|
||||
to PostGIS or trigger an import. The normal `source-freshness` endpoint remains
|
||||
|
||||
@@ -173,7 +173,7 @@ gebeurd.
|
||||
| --- | --- | --- |
|
||||
| GRB gebouwen/wegen/water/percelen | operationele, expliciete plan-stage-apply refresh met onveranderlijke snapshots | alleen een nieuw officieel gedateerd cataloguseditie na operatorbevestiging ophalen |
|
||||
| Statbel bevolking | jaarlijkse, expliciete edities in één tijdreeks | een nieuwe publicatie alleen na schema-, sectorgeometrie- en totalencontrole toevoegen |
|
||||
| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; metricvergelijking zonder objectlineage | eerst een stabiele machineleesbare editiebron verifiëren, daarna dezelfde begrensde operatorflow toepassen |
|
||||
| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete officiële publicatieprobe; metricvergelijking zonder objectlineage | een nieuwere definitieve v3-editie eerst handmatig beoordelen en daarna via de bestaande begrensde operatorflow toevoegen |
|
||||
| orthofoto | vaste lokale opname per expliciete analysezone; catalogusprobe is alleen een signaal | vluchtjaar, productvariant en dekking vergelijken voordat nieuwe pixels worden opgehaald |
|
||||
| landgebruik, thematische rasters, DHMV en VMM-scenario's | vaste product-/scenario-edities, geen rolling snapshot | alleen een nieuwe gedocumenteerde producteditie als afzonderlijke Dataset verwerven |
|
||||
| bodemkaart en historische kaarten | historische referentie-editie | niet als verouderde actuele bron labelen; alleen vervangen bij een officiële inhoudelijke heruitgave |
|
||||
@@ -183,6 +183,13 @@ Geen van deze regels activeert een browserfetch, scheduler of automatische
|
||||
vervanging. Een bron wordt pas `refreshable` wanneer versie, dekking, schema,
|
||||
provenance en een begrensde acquisitieroute afzonderlijk verifieerbaar zijn.
|
||||
|
||||
De ALZ-publicatiepagina levert drie campagnesnapshots: v1 en v2 zijn
|
||||
voorlopig, v3 is de definitieve historische editie. GeoIntel vergelijkt daarom
|
||||
de lokale `2025-definitive` uitsluitend met de officiële `2025-v3`, terwijl de
|
||||
actuele `2026-v1` zichtbaar maar niet updategerechtigd blijft. De probe leest
|
||||
alleen de allowlisted HTML-pagina en linkidentiteiten; ZIP-archieven worden pas
|
||||
door de afzonderlijke operator opgehaald na menselijke editiebevestiging.
|
||||
|
||||
### Mol population history
|
||||
|
||||
`scripts/provision_mol_population_history.py` imports official Statbel
|
||||
|
||||
@@ -29,6 +29,7 @@ GRB_WFS_URL=
|
||||
# Explicit read-only source-edition probes (no provider import).
|
||||
SOURCE_CATALOG_PROBE_ENABLED=true
|
||||
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
|
||||
SOURCE_CATALOG_ALZ_RELEASE_URL=https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen
|
||||
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10
|
||||
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2
|
||||
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900
|
||||
|
||||
+4
-3
@@ -41,7 +41,8 @@
|
||||
- [x] Add a read-only source freshness/version audit with explicit publication policies, local integrity checks, operator CLI and compact Status workspace surface.
|
||||
- [x] Add explicit bounded GRB/orthophoto catalogue release probes using official capabilities and ISO metadata; keep every acquisition separate and operator-controlled.
|
||||
- [x] Add a governed regional GRB plan -> stage -> checksum-confirmed apply workflow that preserves every previous snapshot.
|
||||
- [ ] Extend catalogue probes only to additional sources that publish a stable machine-readable edition contract; do not add background polling or infer releases from HTTP dates alone.
|
||||
- [x] Add a fail-closed ALZ publication probe that distinguishes provisional v1/v2 snapshots from the definitive v3 historical edition and never downloads an archive.
|
||||
- [ ] Extend catalogue probes only to additional sources that publish a stable official edition contract; do not add background polling or infer releases from HTTP dates alone.
|
||||
|
||||
## Governed source expansion backlog
|
||||
|
||||
@@ -683,7 +684,7 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
- [x] Show precise daily edition ranges and explain registration-date nuance in
|
||||
the Evolution workspace.
|
||||
- [x] Document refresh readiness across the complete official-source portfolio.
|
||||
- [ ] Add the next governed catalog probe only after an official stable version
|
||||
contract is verified; ALZ annual releases are the first candidate.
|
||||
- [x] Verify and add the governed ALZ catalog probe against the official
|
||||
campaign-snapshot and definitive-archive publication contract.
|
||||
- [ ] Keep orthophoto refresh manual until product variant, flight year and
|
||||
complete selected-area coverage can be compared deterministically.
|
||||
|
||||
+6
-3
@@ -10,9 +10,12 @@ require local integrity review or are local artifacts. Only attention items are
|
||||
expanded by default; all source detail remains available through disclosure.
|
||||
The refresh button reruns the local read-only audit and never downloads or
|
||||
replaces source data. A separate `Officiële edities controleren` action is the
|
||||
only trigger for bounded GRB/orthophoto catalog reads. It presents official and
|
||||
local editions, layer-contract coverage and honest non-comparable version
|
||||
markers without starting an import or background poll.
|
||||
only trigger for bounded GRB/orthophoto catalog reads and the official ALZ
|
||||
publication-page check. It presents official and local editions,
|
||||
layer/publication evidence and honest non-comparable version markers without
|
||||
starting an import or background poll. ALZ v1/v2 snapshots are labelled
|
||||
provisional; only the latest definitive v3 edition is compared with local
|
||||
history.
|
||||
After that explicit check, the same surface shows a compact GRB refresh plan
|
||||
for buildings, roads, water and parcels. It states whether each local regional
|
||||
snapshot is current, updateable or needs review and shows the current object
|
||||
|
||||
@@ -86,6 +86,7 @@ function integrityIssueCount(item: SourceFreshnessItem): number {
|
||||
}
|
||||
|
||||
function CatalogRow({ item }: { item: SourceCatalogProbeItem }): JSX.Element {
|
||||
const evidenceLabel = item.service_type === 'HTML' ? 'Publicaties' : 'Lagen'
|
||||
return (
|
||||
<div className={`source-catalog-row source-catalog-row-${item.status}`}>
|
||||
<div className="source-freshness-main">
|
||||
@@ -98,7 +99,7 @@ function CatalogRow({ item }: { item: SourceCatalogProbeItem }): JSX.Element {
|
||||
<p>{item.message}</p>
|
||||
<div className="source-freshness-meta">
|
||||
<span>Lokaal: {item.local_source_version ?? 'niet aanwezig'}</span>
|
||||
<span>Lagen: {item.matched_layers.length}/{item.expected_layers.length} bevestigd</span>
|
||||
<span>{evidenceLabel}: {item.matched_layers.length}/{item.expected_layers.length} bevestigd</span>
|
||||
{item.remote_modified_at ? <span>Metadata: {formatDate(item.remote_modified_at)}</span> : null}
|
||||
{item.cached ? <span>cache gebruikt</span> : null}
|
||||
</div>
|
||||
@@ -150,7 +151,7 @@ export function SourceFreshnessPanel({
|
||||
<div>
|
||||
<p className="eyebrow">Bronbeheer</p>
|
||||
<h2>Actualiteit en versiecontrole</h2>
|
||||
<p>Controleert lokale publicaties en kan op aanvraag de officiële GRB- en orthofoto-editie uitlezen.</p>
|
||||
<p>Controleert lokale publicaties en kan op aanvraag de officiële GRB-, orthofoto- en landbouweditie uitlezen.</p>
|
||||
</div>
|
||||
<div className="source-freshness-actions">
|
||||
<button className="secondary-action" type="button" onClick={onRefresh} disabled={loading}>
|
||||
@@ -206,7 +207,7 @@ export function SourceFreshnessPanel({
|
||||
</div>
|
||||
{catalogError ? <p className="inline-error">{catalogError}</p> : null}
|
||||
{!catalogReport && !catalogError ? (
|
||||
<p className="source-catalog-empty">Controleer GRB en orthofoto wanneer je een lokale bronversie met de officiële editie wilt vergelijken.</p>
|
||||
<p className="source-catalog-empty">Controleer GRB, orthofoto en landbouwpercelen wanneer je lokale bronversies met officiële edities wilt vergelijken.</p>
|
||||
) : null}
|
||||
{catalogReport ? (
|
||||
<>
|
||||
|
||||
@@ -715,7 +715,7 @@ export type SourceCatalogComparisonStatus = 'same' | 'different' | 'not_comparab
|
||||
export interface SourceCatalogProbeItem {
|
||||
source_name: string
|
||||
display_name: string
|
||||
service_type: 'WFS' | 'WMS'
|
||||
service_type: 'WFS' | 'WMS' | 'HTML'
|
||||
endpoint_url: string
|
||||
status: SourceCatalogProbeStatus
|
||||
reachable: boolean
|
||||
|
||||
Reference in New Issue
Block a user