Add official source edition probes
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-16 17:33:40 +02:00
parent fd05c46e2a
commit 595967f892
27 changed files with 1513 additions and 18 deletions
+5
View File
@@ -12,6 +12,11 @@ ORTHOPHOTO_RESOLUTION_M=1.0
ORTHOPHOTO_MIN_SIDE_M=128
ORTHOPHOTO_MAX_SIDE_M=1024
ORTHOPHOTO_CACHE_TTL_HOURS=24
SOURCE_CATALOG_PROBE_ENABLED=true
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900
DHMV_ENABLED=true
DHMV_WCS_URL=https://geo.api.vlaanderen.be/DHMV/wcs
DHMV_RESOLUTION_M=5.0
+12
View File
@@ -7,6 +7,18 @@
# Changelog
## Sprint 222 Official source edition probes (2026-07-16)
- Added explicit, read-only GRB and most-recent orthophoto catalog probes using
official WFS/WMS capabilities and linked ISO 19139 CSW metadata records.
- Added bounded response sizes, timeouts, provider-isolated failures, a short
cache and strict metadata-host/path validation. No provider features, raster
pixels or application data are fetched or modified.
- Added canonical API, compact opt-in Status UI and operator CLI support. The
local source-freshness audit remains automatic and provider-free.
- Added deterministic XML/network-boundary tests and editable Compose/Unraid
controls without changing migrations or persistence contracts.
## Sprint 221 Governed source freshness audit (2026-07-16)
- Added a project-wide read-only source report derived from Dataset,
+26
View File
@@ -1443,3 +1443,29 @@ docker exec geointel python /app/scripts/audit_source_freshness.py \
Use `--fail-on due` to make a planned review date fail automation, or
`--fail-on never` for reporting only. The command never starts a refresh.
An operator can explicitly add the official GRB and orthophoto edition check:
```bash
docker exec geointel python /app/scripts/audit_source_freshness.py \
--project-id 82a85913-c522-45d7-84a1-02b393d89e55 \
--api-url http://127.0.0.1/api/v1 \
--probe-catalogs \
--output /app/storage/operator-evidence/source-freshness/with-catalogs.json
```
Use `--refresh-catalogs` to bypass the 15-minute in-memory cache and
`--fail-on-catalog` only when temporary official-provider unavailability must
fail an operator job. This path reads bounded WFS/WMS capabilities and their
fixed ISO 19139 metadata records. It confirms GRB `GBG`, `WBN`, `WGO`, `ADP`
and orthophoto `Ortho`, `Vliegdagcontour`; it never requests feature or raster
content. The public Datavindplaats API requires an access token, so release
evidence comes from the public metadata links published by the official OGC
services rather than HTML scraping.
Runtime controls are `SOURCE_CATALOG_PROBE_ENABLED`,
`SOURCE_CATALOG_GRB_WFS_URL`, `SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS`,
`SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB` and
`SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS`. Metadata links remain restricted to
the official HTTPS CSW path on `metadata.vlaanderen.be` even when an operator
overrides the capabilities endpoint.
+11
View File
@@ -45,6 +45,7 @@ from app.services.vector_operations_service import VectorOperationsService
from app.services.vector_feature_service import VectorFeatureService
from app.services.dataset_service import DatasetService
from app.services.source_freshness_service import SourceFreshnessService
from app.services.source_catalog_probe_service import SourceCatalogProbeService
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.terrain_analysis_service import TerrainAnalysisService
@@ -255,6 +256,16 @@ def audit_dataset_source_freshness(
return envelope(report.model_dump())
@router.get("/datasets/source-catalog-probes", response_model=dict)
def probe_dataset_source_catalogs(
project_id: UUID,
refresh: bool = Query(default=False),
db: Session = Depends(get_db),
):
report = SourceCatalogProbeService.audit_project(db, project_id, force=refresh)
return envelope(report.model_dump())
@router.get("/datasets/{dataset_id}", response_model=dict)
def get_dataset(
project_id: UUID,
+23
View File
@@ -31,6 +31,29 @@ class Settings(BaseSettings):
orthophoto_timeout_seconds: int = Field(default=120, ge=1, validation_alias="ORTHOPHOTO_TIMEOUT_SECONDS")
orthophoto_max_response_mb: int = Field(default=32, ge=1, validation_alias="ORTHOPHOTO_MAX_RESPONSE_MB")
orthophoto_cache_ttl_hours: int = Field(default=24, ge=0, validation_alias="ORTHOPHOTO_CACHE_TTL_HOURS")
source_catalog_probe_enabled: bool = Field(default=True, validation_alias="SOURCE_CATALOG_PROBE_ENABLED")
source_catalog_grb_wfs_url: str = Field(
default="https://geo.api.vlaanderen.be/GRB/wfs",
validation_alias="SOURCE_CATALOG_GRB_WFS_URL",
)
source_catalog_probe_timeout_seconds: int = Field(
default=10,
ge=1,
le=60,
validation_alias="SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS",
)
source_catalog_probe_max_response_mb: int = Field(
default=2,
ge=1,
le=10,
validation_alias="SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB",
)
source_catalog_probe_cache_ttl_seconds: int = Field(
default=900,
ge=0,
le=86_400,
validation_alias="SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS",
)
dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED")
dhmv_wcs_url: str = Field(
default="https://geo.api.vlaanderen.be/DHMV/wcs",
+8
View File
@@ -11,6 +11,11 @@ from .source_freshness import (
SourceFreshnessSummary,
SourceIntegritySummary,
)
from .source_catalog import (
SourceCatalogProbeItem,
SourceCatalogProbeReport,
SourceCatalogProbeSummary,
)
from .detection import (
DetectionListResponse,
DetectionModelCapability,
@@ -141,6 +146,9 @@ __all__ = [
"SourceFreshnessReport",
"SourceFreshnessSummary",
"SourceIntegritySummary",
"SourceCatalogProbeItem",
"SourceCatalogProbeReport",
"SourceCatalogProbeSummary",
"DetectionListResponse",
"DetectionModelCapability",
"DetectionModelsResponse",
+56
View File
@@ -0,0 +1,56 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel
SourceCatalogProbeStatus = Literal["available", "degraded", "unavailable", "disabled"]
SourceCatalogComparisonStatus = Literal["same", "different", "not_comparable", "no_local_data", "unavailable"]
class SourceCatalogProbeItem(BaseModel):
source_name: str
display_name: str
service_type: Literal["WFS", "WMS"]
endpoint_url: str
status: SourceCatalogProbeStatus
reachable: bool
checked_at: datetime
cached: bool = False
expected_layers: list[str]
matched_layers: list[str]
missing_layers: list[str]
advertised_layer_count: int
metadata_url: str | None = None
metadata_identifier: str | None = None
remote_title: str | None = None
remote_version: str | None = None
remote_modified_at: datetime | None = None
remote_published_at: datetime | None = None
local_source_version: str | None = None
comparison_status: SourceCatalogComparisonStatus
capabilities_sha256: str | None = None
capabilities_etag: str | None = None
capabilities_last_modified_at: datetime | None = None
message: str
error_code: str | None = None
class SourceCatalogProbeSummary(BaseModel):
provider_count: int
available_count: int
degraded_count: int
unavailable_count: int
disabled_count: int
different_version_count: int
class SourceCatalogProbeReport(BaseModel):
project_id: UUID
generated_at: datetime
summary: SourceCatalogProbeSummary
items: list[SourceCatalogProbeItem]
limitations: list[str]
@@ -0,0 +1,545 @@
from __future__ import annotations
from dataclasses import dataclass, replace
from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime
from hashlib import sha256
import re
from threading import Lock
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from urllib.request import Request, urlopen
from uuid import UUID
from xml.etree import ElementTree
from sqlalchemy.orm import Session
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.models import Dataset, Project
from app.schemas.source_catalog import (
SourceCatalogProbeItem,
SourceCatalogProbeReport,
SourceCatalogProbeSummary,
)
_GMD = "http://www.isotc211.org/2005/gmd"
_GCO = "http://www.isotc211.org/2005/gco"
_WFS = "http://www.opengis.net/wfs/2.0"
_XLINK = "http://www.w3.org/1999/xlink"
_METADATA_HOST = "metadata.vlaanderen.be"
_VERSION_DATE = re.compile(r"^(?:toestand\s+)?(\d{4}-\d{2}-\d{2})$", re.IGNORECASE)
_ORTHOPHOTO_EDITION = re.compile(r"^\d{4}\.\d{2}$")
class CatalogProbeFailure(RuntimeError):
def __init__(self, code: str, message: str) -> None:
super().__init__(message)
self.code = code
self.message = message
@dataclass(frozen=True)
class ProbeContract:
source_name: str
display_name: str
service_type: str
endpoint_url: str
expected_layers: tuple[str, ...]
@dataclass(frozen=True)
class FetchResult:
content: bytes
content_type: str
etag: str | None
last_modified_at: datetime | None
final_url: str
@dataclass(frozen=True)
class RemoteProbe:
status: str
reachable: bool
checked_at: datetime
expected_layers: tuple[str, ...]
matched_layers: tuple[str, ...] = ()
missing_layers: tuple[str, ...] = ()
advertised_layer_count: int = 0
metadata_url: str | None = None
metadata_identifier: str | None = None
remote_title: str | None = None
remote_version: str | None = None
remote_modified_at: datetime | None = None
remote_published_at: datetime | None = None
capabilities_sha256: str | None = None
capabilities_etag: str | None = None
capabilities_last_modified_at: datetime | None = None
message: str = ""
error_code: str | None = None
cached: bool = False
@dataclass(frozen=True)
class CacheEntry:
expires_at: datetime
probe: RemoteProbe
_REMOTE_CACHE: dict[str, CacheEntry] = {}
_CACHE_LOCK = Lock()
def _utc(value: datetime | None = None) -> datetime:
current = value or datetime.now(timezone.utc)
if current.tzinfo is None:
return current.replace(tzinfo=timezone.utc)
return current.astimezone(timezone.utc)
def _header(headers: Any, name: str) -> str | None:
value = headers.get(name) if headers is not None else None
return str(value).strip() if value is not None and str(value).strip() else None
def _http_date(value: str | None) -> datetime | None:
if not value:
return None
try:
return _utc(parsedate_to_datetime(value))
except (TypeError, ValueError, OverflowError):
return None
def _with_capabilities_query(base_url: str, service_type: str) -> str:
parsed = urlsplit(base_url)
retained = [(key, value) for key, value in parse_qsl(parsed.query) if key.lower() not in {"service", "request", "version"}]
version = "2.0.0" if service_type == "WFS" else "1.3.0"
query = urlencode([*retained, ("SERVICE", service_type), ("VERSION", version), ("REQUEST", "GetCapabilities")])
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, query, ""))
def _bounded_fetch(url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> FetchResult:
request = Request(
url,
headers={"Accept": "application/xml,text/xml,application/json", "User-Agent": "GeoIntel/0.1 source-catalog-probe"},
)
max_bytes = settings.source_catalog_probe_max_response_mb * 1024 * 1024
try:
with (opener or urlopen)(request, timeout=settings.source_catalog_probe_timeout_seconds) as response:
content_length = _header(response.headers, "Content-Length")
if content_length:
try:
if int(content_length) > max_bytes:
raise CatalogProbeFailure("CATALOG_RESPONSE_TOO_LARGE", "De officiële metadatarespons overschrijdt de ingestelde limiet.")
except ValueError as exc:
raise CatalogProbeFailure("CATALOG_INVALID_RESPONSE", "De officiële metadatarespons bevat een ongeldige Content-Length.") from exc
content = response.read(max_bytes + 1)
result = FetchResult(
content=content,
content_type=_header(response.headers, "Content-Type") or "",
etag=_header(response.headers, "ETag"),
last_modified_at=_http_date(_header(response.headers, "Last-Modified")),
final_url=str(response.geturl()) if hasattr(response, "geturl") else url,
)
except CatalogProbeFailure:
raise
except (HTTPError, URLError, TimeoutError, OSError) as exc:
raise CatalogProbeFailure("CATALOG_PROVIDER_UNAVAILABLE", "De officiële catalogus kon niet tijdig worden gelezen.") from exc
if len(result.content) > max_bytes:
raise CatalogProbeFailure("CATALOG_RESPONSE_TOO_LARGE", "De officiële metadatarespons overschrijdt de ingestelde limiet.")
return result
def _local_name(tag: str) -> str:
return tag.rsplit("}", 1)[-1]
def _direct_text(parent: ElementTree.Element, name: str) -> str | None:
for child in parent:
if _local_name(child.tag) == name and child.text and child.text.strip():
return child.text.strip()
return None
def _metadata_urls(parent: ElementTree.Element) -> list[str]:
urls: list[str] = []
for node in parent.iter():
if _local_name(node.tag) not in {"MetadataURL", "MetadataUrl"}:
continue
href = node.attrib.get(f"{{{_XLINK}}}href")
if href:
urls.append(href.strip())
for child in node.iter():
if _local_name(child.tag) in {"URL", "OnlineResource"}:
nested = child.attrib.get(f"{{{_XLINK}}}href") or (child.text or "").strip()
if nested:
urls.append(nested)
return urls
def _parse_capabilities(content: bytes, contract: ProbeContract) -> tuple[list[str], str | None]:
try:
root = ElementTree.fromstring(content)
except ElementTree.ParseError as exc:
raise CatalogProbeFailure("CATALOG_INVALID_XML", "De capabilities-respons is geen geldige XML.") from exc
layers: list[str] = []
metadata_urls: list[str] = []
node_name = "FeatureType" if contract.service_type == "WFS" else "Layer"
for node in root.iter():
if _local_name(node.tag) != node_name:
continue
raw_name = _direct_text(node, "Name")
if not raw_name:
continue
name = raw_name.rsplit(":", 1)[-1]
layers.append(name)
if name in contract.expected_layers:
metadata_urls.extend(_metadata_urls(node))
expected = set(contract.expected_layers)
if not expected.intersection(layers):
raise CatalogProbeFailure("CATALOG_EXPECTED_LAYERS_MISSING", "De officiële service bevat geen van de verwachte lagen.")
xml_metadata = next((url for url in metadata_urls if "GetRecordById" in url and "OUTPUTSCHEMA" in url.upper()), None)
return sorted(set(layers)), xml_metadata
def _validate_metadata_url(url: str) -> str:
parsed = urlsplit(url)
if parsed.scheme != "https" or parsed.hostname != _METADATA_HOST or parsed.username or parsed.password:
raise CatalogProbeFailure("CATALOG_METADATA_URL_REJECTED", "De capabilities verwijzen niet naar de toegestane officiële metadatahost.")
if not parsed.path.startswith("/srv/dut/csw"):
raise CatalogProbeFailure("CATALOG_METADATA_URL_REJECTED", "De capabilities verwijzen niet naar het toegestane CSW-pad.")
query = {key.lower(): value for key, value in parse_qsl(parsed.query)}
if query.get("request", "").lower() != "getrecordbyid" or not query.get("id"):
raise CatalogProbeFailure("CATALOG_METADATA_URL_REJECTED", "De capabilities bevatten geen begrensde GetRecordById-verwijzing.")
return url
def _validate_capabilities_url(url: str) -> str:
parsed = urlsplit(url)
if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
raise CatalogProbeFailure("CATALOG_ENDPOINT_REJECTED", "De ingestelde capabilities-URL moet een geldige HTTP(S)-URL zonder credentials zijn.")
return url
def _node_text(node: ElementTree.Element | None) -> str | None:
if node is None:
return None
for descendant in node.iter():
if descendant is not node and descendant.text and descendant.text.strip():
return descendant.text.strip()
return node.text.strip() if node.text and node.text.strip() else None
def _parse_iso_datetime(value: str | None) -> datetime | None:
if not value:
return None
normalized = value.strip().replace("Z", "+00:00")
try:
return _utc(datetime.fromisoformat(normalized))
except ValueError:
try:
return datetime.strptime(normalized, "%Y-%m-%d").replace(tzinfo=timezone.utc)
except ValueError:
return None
def _parse_metadata(content: bytes) -> dict[str, Any]:
try:
root = ElementTree.fromstring(content)
except ElementTree.ParseError as exc:
raise CatalogProbeFailure("CATALOG_INVALID_METADATA_XML", "Het officiële metadatarecord is geen geldige XML.") from exc
namespaces = {"gmd": _GMD, "gco": _GCO}
metadata = root.find(".//gmd:MD_Metadata", namespaces)
if metadata is None and _local_name(root.tag) == "MD_Metadata":
metadata = root
if metadata is None:
raise CatalogProbeFailure("CATALOG_METADATA_MISSING", "Het CSW-antwoord bevat geen ISO 19139 metadatarecord.")
citation = metadata.find(".//gmd:identificationInfo/*/gmd:citation/gmd:CI_Citation", namespaces)
title = _node_text(citation.find("gmd:title", namespaces) if citation is not None else None)
edition = _node_text(citation.find("gmd:edition", namespaces) if citation is not None else None)
identifier = _node_text(metadata.find("gmd:fileIdentifier", namespaces))
modified = _parse_iso_datetime(_node_text(metadata.find("gmd:dateStamp", namespaces)))
published = None
if citation is not None:
for date_node in citation.findall("gmd:date/gmd:CI_Date", namespaces):
date_type = date_node.find("gmd:dateType/gmd:CI_DateTypeCode", namespaces)
if date_type is not None and date_type.attrib.get("codeListValue") == "publication":
published = _parse_iso_datetime(_node_text(date_node.find("gmd:date", namespaces)))
break
if not title or not edition:
raise CatalogProbeFailure("CATALOG_VERSION_MISSING", "Het officiële metadatarecord bevat geen herkenbare titel en editie.")
return {
"identifier": identifier,
"title": title,
"version": edition,
"modified_at": modified,
"published_at": published,
}
def _probe_remote(
contract: ProbeContract,
settings: Settings,
*,
opener: Callable[..., Any] | None,
now: datetime,
) -> RemoteProbe:
try:
capabilities = _bounded_fetch(_validate_capabilities_url(contract.endpoint_url), settings, opener)
_validate_capabilities_url(capabilities.final_url)
content_type = capabilities.content_type.lower()
if content_type and "xml" not in content_type and "text" not in content_type:
raise CatalogProbeFailure("CATALOG_INVALID_CONTENT_TYPE", "De officiële capabilities-respons is geen XML.")
layers, metadata_url = _parse_capabilities(capabilities.content, contract)
matched = tuple(layer for layer in contract.expected_layers if layer in layers)
missing = tuple(layer for layer in contract.expected_layers if layer not in layers)
digest = sha256(capabilities.content).hexdigest()
if not metadata_url:
return RemoteProbe(
status="degraded",
reachable=True,
checked_at=now,
expected_layers=contract.expected_layers,
matched_layers=matched,
missing_layers=missing,
advertised_layer_count=len(layers),
capabilities_sha256=digest,
capabilities_etag=capabilities.etag,
capabilities_last_modified_at=capabilities.last_modified_at,
message="De service is bereikbaar, maar publiceert geen machineleesbare ISO-metadata voor de verwachte lagen.",
error_code="CATALOG_METADATA_LINK_MISSING",
)
metadata_url = _validate_metadata_url(metadata_url)
metadata_response = _bounded_fetch(metadata_url, settings, opener)
_validate_metadata_url(metadata_response.final_url)
metadata = _parse_metadata(metadata_response.content)
status = "degraded" if missing else "available"
message = (
f"De officiële catalogus is bereikbaar en publiceert editie {metadata['version']}."
if not missing
else f"Editie {metadata['version']} is gevonden, maar niet alle verwachte lagen worden aangeboden."
)
return RemoteProbe(
status=status,
reachable=True,
checked_at=now,
expected_layers=contract.expected_layers,
matched_layers=matched,
missing_layers=missing,
advertised_layer_count=len(layers),
metadata_url=metadata_url,
metadata_identifier=metadata["identifier"],
remote_title=metadata["title"],
remote_version=metadata["version"],
remote_modified_at=metadata["modified_at"],
remote_published_at=metadata["published_at"],
capabilities_sha256=digest,
capabilities_etag=capabilities.etag,
capabilities_last_modified_at=capabilities.last_modified_at,
message=message,
error_code="CATALOG_EXPECTED_LAYERS_INCOMPLETE" if missing else None,
)
except CatalogProbeFailure as exc:
return RemoteProbe(
status="unavailable",
reachable=False,
checked_at=now,
expected_layers=contract.expected_layers,
missing_layers=contract.expected_layers,
message=exc.message,
error_code=exc.code,
)
def _remote_with_cache(
contract: ProbeContract,
settings: Settings,
*,
force: bool,
opener: Callable[..., Any] | None,
now: datetime,
) -> RemoteProbe:
cache_key = f"{contract.source_name}|{contract.endpoint_url}|{','.join(contract.expected_layers)}"
if not force and settings.source_catalog_probe_cache_ttl_seconds > 0:
with _CACHE_LOCK:
entry = _REMOTE_CACHE.get(cache_key)
if entry and entry.expires_at > now:
return replace(entry.probe, cached=True)
probe = _probe_remote(contract, settings, opener=opener, now=now)
if settings.source_catalog_probe_cache_ttl_seconds > 0:
with _CACHE_LOCK:
_REMOTE_CACHE[cache_key] = CacheEntry(
expires_at=now + timedelta(seconds=settings.source_catalog_probe_cache_ttl_seconds),
probe=probe,
)
return probe
def _dataset_source_name(dataset: Dataset) -> str:
return (dataset.source_name or dataset.source or "").strip().lower()
def _latest_local_version(source_name: str, datasets: list[Dataset]) -> str | None:
candidates = [item for item in datasets if _dataset_source_name(item) == source_name and item.source_version]
if source_name == "digitaal_vlaanderen_orthophoto":
explicit_current = [
item
for item in candidates
if any(token in (item.source_version or "").lower() for token in ("most_recent", "latest", "current"))
]
if explicit_current:
candidates = explicit_current
if not candidates:
return None
latest = max(
candidates,
key=lambda item: (
_utc(item.imported_at) if item.imported_at else datetime.min.replace(tzinfo=timezone.utc),
_utc(item.observed_at) if item.observed_at else datetime.min.replace(tzinfo=timezone.utc),
str(item.id),
),
)
return latest.source_version
def _normalized_version(source_name: str, version: str | None) -> str | None:
if not version:
return None
value = version.strip()
if source_name == "grb":
match = _VERSION_DATE.fullmatch(value)
return match.group(1) if match else None
if source_name == "digitaal_vlaanderen_orthophoto" and _ORTHOPHOTO_EDITION.fullmatch(value):
return value
return None
def _comparison(source_name: str, local: str | None, remote: str | None, remote_status: str) -> str:
if remote_status in {"unavailable", "disabled"}:
return "unavailable"
if not local:
return "no_local_data"
normalized_local = _normalized_version(source_name, local)
normalized_remote = _normalized_version(source_name, remote)
if normalized_local is None or normalized_remote is None:
return "not_comparable"
return "same" if normalized_local == normalized_remote else "different"
class SourceCatalogProbeService:
@staticmethod
def clear_cache() -> None:
with _CACHE_LOCK:
_REMOTE_CACHE.clear()
@staticmethod
def audit_project(
db: Session,
project_id: UUID,
*,
force: bool = False,
opener: Callable[..., Any] | None = None,
now: datetime | None = None,
settings: Settings | None = None,
) -> SourceCatalogProbeReport:
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
active_settings = settings or get_settings()
generated_at = _utc(now)
datasets = db.query(Dataset).filter(Dataset.project_id == project_id).all()
contracts = (
ProbeContract(
source_name="grb",
display_name="Basiskaart Vlaanderen (GRB)",
service_type="WFS",
endpoint_url=_with_capabilities_query(active_settings.source_catalog_grb_wfs_url, "WFS"),
expected_layers=("GBG", "WBN", "WGO", "ADP"),
),
ProbeContract(
source_name="digitaal_vlaanderen_orthophoto",
display_name="Orthofoto Vlaanderen",
service_type="WMS",
endpoint_url=_with_capabilities_query(active_settings.orthophoto_wms_url, "WMS"),
expected_layers=(active_settings.orthophoto_wms_layer, "Vliegdagcontour"),
),
)
items: list[SourceCatalogProbeItem] = []
for contract in contracts:
local_version = _latest_local_version(contract.source_name, datasets)
if active_settings.source_catalog_probe_enabled:
remote = _remote_with_cache(
contract,
active_settings,
force=force,
opener=opener,
now=generated_at,
)
else:
remote = RemoteProbe(
status="disabled",
reachable=False,
checked_at=generated_at,
expected_layers=contract.expected_layers,
missing_layers=contract.expected_layers,
message="Officiële catalogusprobes zijn uitgeschakeld in de runtimeconfiguratie.",
error_code="CATALOG_PROBE_DISABLED",
)
comparison = _comparison(contract.source_name, local_version, remote.remote_version, remote.status)
message = remote.message
if comparison == "different":
message += " De officiële editie verschilt van de lokaal vastgelegde bronversie; controleer dit handmatig vóór een begrensde verversing."
elif comparison == "not_comparable" and local_version:
message += " De lokale waarde is een opname- of importmarkering en kan niet eerlijk als officiële cataloguseditie worden vergeleken."
items.append(
SourceCatalogProbeItem(
source_name=contract.source_name,
display_name=contract.display_name,
service_type=contract.service_type,
endpoint_url=contract.endpoint_url,
status=remote.status,
reachable=remote.reachable,
checked_at=remote.checked_at,
cached=remote.cached,
expected_layers=list(remote.expected_layers),
matched_layers=list(remote.matched_layers),
missing_layers=list(remote.missing_layers),
advertised_layer_count=remote.advertised_layer_count,
metadata_url=remote.metadata_url,
metadata_identifier=remote.metadata_identifier,
remote_title=remote.remote_title,
remote_version=remote.remote_version,
remote_modified_at=remote.remote_modified_at,
remote_published_at=remote.remote_published_at,
local_source_version=local_version,
comparison_status=comparison,
capabilities_sha256=remote.capabilities_sha256,
capabilities_etag=remote.capabilities_etag,
capabilities_last_modified_at=remote.capabilities_last_modified_at,
message=message,
error_code=remote.error_code,
)
)
summary = SourceCatalogProbeSummary(
provider_count=len(items),
available_count=sum(item.status == "available" for item in items),
degraded_count=sum(item.status == "degraded" for item in items),
unavailable_count=sum(item.status == "unavailable" for item in items),
disabled_count=sum(item.status == "disabled" for item in items),
different_version_count=sum(item.comparison_status == "different" for item in items),
)
return SourceCatalogProbeReport(
project_id=project_id,
generated_at=generated_at,
summary=summary,
items=items,
limitations=[
"Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities en gekoppelde ISO 19139 metadata.",
"Er worden geen features, rasters of modelbestanden opgehaald en geen datasets aangemaakt of overschreven.",
"Een versieverschil is controlesignaal, geen bewijs dat een lokale dataset onbruikbaar is en geen automatische importopdracht.",
],
)
@@ -0,0 +1,332 @@
from __future__ import annotations
from datetime import datetime, timezone
from email.message import Message
from pathlib import Path
from types import SimpleNamespace
from urllib.error import URLError
import uuid
from app.core.config import Settings
from app.models import Dataset
from app.services.source_catalog_probe_service import SourceCatalogProbeService
NOW = datetime(2026, 7, 16, 15, 0, tzinfo=timezone.utc)
GRB_METADATA_URL = (
"https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&service=CSW&"
"id=7C823055-7BBF-4D62-B55E-F85C30D53162&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd"
)
ORTHO_METADATA_URL = (
"https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&service=CSW&"
"id=f5304d6d-0dd4-43fd-a726-427af31e8d61&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd"
)
def _wfs_capabilities(metadata_url: str = GRB_METADATA_URL) -> bytes:
layers = "".join(
f"""
<wfs:FeatureType>
<wfs:Name>GRB:{name}</wfs:Name>
<wfs:MetadataURL xlink:href="{metadata_url.replace('&', '&amp;')}" />
</wfs:FeatureType>
"""
for name in ("GBG", "WBN", "WGO", "ADP", "WTZ")
)
return f"""
<wfs:WFS_Capabilities xmlns:wfs="http://www.opengis.net/wfs/2.0"
xmlns:xlink="http://www.w3.org/1999/xlink">
<wfs:FeatureTypeList>{layers}</wfs:FeatureTypeList>
</wfs:WFS_Capabilities>
""".encode()
def _wms_capabilities(metadata_url: str = ORTHO_METADATA_URL) -> bytes:
return f"""
<WMS_Capabilities xmlns="http://www.opengis.net/wms"
xmlns:xlink="http://www.w3.org/1999/xlink">
<Capability><Layer><Title>Orthofoto</Title>
<Layer><Name>Ortho</Name><MetadataURL><Format>text/xml</Format>
<OnlineResource xlink:href="{metadata_url.replace('&', '&amp;')}" />
</MetadataURL></Layer>
<Layer><Name>Vliegdagcontour</Name><MetadataURL><Format>text/xml</Format>
<OnlineResource xlink:href="{metadata_url.replace('&', '&amp;')}" />
</MetadataURL></Layer>
</Layer></Capability>
</WMS_Capabilities>
""".encode()
def _metadata(identifier: str, title: str, edition: str, modified: str, published: str) -> bytes:
return f"""
<csw:GetRecordByIdResponse xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
xmlns:gmd="http://www.isotc211.org/2005/gmd"
xmlns:gco="http://www.isotc211.org/2005/gco">
<gmd:MD_Metadata>
<gmd:fileIdentifier><gco:CharacterString>{identifier}</gco:CharacterString></gmd:fileIdentifier>
<gmd:dateStamp><gco:Date>{modified}</gco:Date></gmd:dateStamp>
<gmd:identificationInfo><gmd:MD_DataIdentification><gmd:citation><gmd:CI_Citation>
<gmd:title><gco:CharacterString>{title}</gco:CharacterString></gmd:title>
<gmd:edition><gco:CharacterString>{edition}</gco:CharacterString></gmd:edition>
<gmd:date><gmd:CI_Date><gmd:date><gco:Date>{published}</gco:Date></gmd:date>
<gmd:dateType><gmd:CI_DateTypeCode codeListValue="publication">publication</gmd:CI_DateTypeCode></gmd:dateType>
</gmd:CI_Date></gmd:date>
</gmd:CI_Citation></gmd:citation></gmd:MD_DataIdentification></gmd:identificationInfo>
</gmd:MD_Metadata>
</csw:GetRecordByIdResponse>
""".encode()
class _Response:
def __init__(self, content: bytes, content_type: str = "text/xml", *, content_length: int | None = None) -> None:
self.content = content
self.headers = Message()
self.headers["Content-Type"] = content_type
self.headers["Content-Length"] = str(content_length if content_length is not None else len(content))
self.headers["ETag"] = '"catalog-test"'
self.headers["Last-Modified"] = "Wed, 15 Jul 2026 10:00:00 GMT"
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self, size: int = -1) -> bytes:
return self.content if size < 0 else self.content[:size]
class _RedirectedResponse(_Response):
def __init__(self, content: bytes, final_url: str) -> None:
super().__init__(content)
self.final_url = final_url
def geturl(self) -> str:
return self.final_url
class _Query:
def __init__(self, datasets: list[Dataset]) -> None:
self.datasets = datasets
def filter(self, *_args):
return self
def all(self) -> list[Dataset]:
return self.datasets
class _Db:
def __init__(self, datasets: list[Dataset]) -> None:
self.datasets = datasets
def get(self, _model, _identifier):
return SimpleNamespace(id=_identifier)
def query(self, _model):
return _Query(self.datasets)
def _dataset(source_name: str, version: str) -> Dataset:
return Dataset(
id=uuid.uuid4(),
project_id=uuid.uuid4(),
name=f"{source_name} source",
dataset_type="vector" if source_name == "grb" else "raster",
source=source_name,
source_name=source_name,
source_version=version,
imported_at=NOW,
status="ready",
)
def _settings(**overrides) -> Settings:
values = {
"SOURCE_CATALOG_PROBE_ENABLED": True,
"SOURCE_CATALOG_GRB_WFS_URL": "https://geo.api.vlaanderen.be/GRB/wfs",
"SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS": 3,
"SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB": 1,
"SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS": 0,
"ORTHOPHOTO_WMS_URL": "https://geo.api.vlaanderen.be/OMWRGBMRVL/wms",
"ORTHOPHOTO_WMS_LAYER": "Ortho",
}
values.update(overrides)
return Settings(**values)
def _opener(request, timeout):
assert timeout == 3
url = request.full_url
if "metadata.vlaanderen.be" in url:
if "f5304d6d" in url:
return _Response(_metadata("f5304d6d", "Orthofoto meest recent, 2025.04", "2025.04", "2026-04-27", "2025-12-11"))
return _Response(_metadata("7C823055", "GRBgis", "Toestand 2026-07-15", "2026-07-15", "2026-07-15"))
if "/GRB/" in url:
return _Response(_wfs_capabilities())
return _Response(_wms_capabilities())
def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions() -> None:
SourceCatalogProbeService.clear_cache()
project_id = uuid.uuid4()
report = SourceCatalogProbeService.audit_project(
_Db([_dataset("grb", "2026-07-14"), _dataset("digitaal_vlaanderen_orthophoto", "most_recent_at_2026-07-14")]),
project_id,
settings=_settings(),
opener=_opener,
now=NOW,
)
by_source = {item.source_name: item for item in report.items}
assert report.summary.available_count == 2
assert report.summary.different_version_count == 1
assert by_source["grb"].remote_version == "Toestand 2026-07-15"
assert by_source["grb"].comparison_status == "different"
assert by_source["grb"].matched_layers == ["GBG", "WBN", "WGO", "ADP"]
assert by_source["grb"].advertised_layer_count == 5
assert by_source["digitaal_vlaanderen_orthophoto"].remote_version == "2025.04"
assert by_source["digitaal_vlaanderen_orthophoto"].comparison_status == "not_comparable"
assert by_source["digitaal_vlaanderen_orthophoto"].remote_published_at.year == 2025
assert all(item.capabilities_sha256 for item in report.items)
def test_catalog_probe_isolates_provider_failure() -> None:
def partial_opener(request, timeout):
if "/GRB/" in request.full_url:
raise URLError("offline")
return _opener(request, timeout)
report = SourceCatalogProbeService.audit_project(
_Db([]), uuid.uuid4(), settings=_settings(), opener=partial_opener, now=NOW
)
by_source = {item.source_name: item for item in report.items}
assert by_source["grb"].status == "unavailable"
assert by_source["grb"].error_code == "CATALOG_PROVIDER_UNAVAILABLE"
assert by_source["digitaal_vlaanderen_orthophoto"].status == "available"
assert report.summary.unavailable_count == 1
def test_catalog_probe_rejects_metadata_redirect_outside_allowlist() -> None:
def malicious_opener(request, timeout):
if "/GRB/" in request.full_url:
return _Response(
_wfs_capabilities(
"https://example.com/csw?request=GetRecordById&id=evil&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd"
)
)
return _opener(request, timeout)
report = SourceCatalogProbeService.audit_project(
_Db([]), uuid.uuid4(), settings=_settings(), opener=malicious_opener, now=NOW
)
grb = next(item for item in report.items if item.source_name == "grb")
assert grb.status == "unavailable"
assert grb.error_code == "CATALOG_METADATA_URL_REJECTED"
def test_catalog_probe_enforces_response_limit_without_reading_external_data() -> None:
def oversized_opener(request, timeout):
if "/GRB/" in request.full_url:
return _Response(b"<xml />", content_length=2 * 1024 * 1024)
return _opener(request, timeout)
report = SourceCatalogProbeService.audit_project(
_Db([]), uuid.uuid4(), settings=_settings(), opener=oversized_opener, now=NOW
)
grb = next(item for item in report.items if item.source_name == "grb")
assert grb.status == "unavailable"
assert grb.error_code == "CATALOG_RESPONSE_TOO_LARGE"
def test_catalog_probe_revalidates_metadata_host_after_redirect() -> None:
def redirected_opener(request, timeout):
if "metadata.vlaanderen.be" in request.full_url and "7C823055" in request.full_url:
return _RedirectedResponse(
_metadata("7C823055", "GRBgis", "Toestand 2026-07-15", "2026-07-15", "2026-07-15"),
"https://example.com/redirected-metadata",
)
return _opener(request, timeout)
report = SourceCatalogProbeService.audit_project(
_Db([]), uuid.uuid4(), settings=_settings(), opener=redirected_opener, now=NOW
)
grb = next(item for item in report.items if item.source_name == "grb")
assert grb.status == "unavailable"
assert grb.error_code == "CATALOG_METADATA_URL_REJECTED"
def test_catalog_probe_cache_is_explicitly_bypassable() -> None:
calls: list[str] = []
def counting_opener(request, timeout):
calls.append(request.full_url)
return _opener(request, timeout)
SourceCatalogProbeService.clear_cache()
settings = _settings(SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900)
db = _Db([])
project_id = uuid.uuid4()
first = SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW)
first_call_count = len(calls)
second = SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW)
SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW, force=True)
assert first_call_count == 4
assert all(item.cached is False for item in first.items)
assert all(item.cached is True for item in second.items)
assert len(calls) == 8
def test_catalog_probe_can_be_disabled_without_network_access() -> None:
def forbidden_opener(*_args, **_kwargs):
raise AssertionError("network must not be called")
report = SourceCatalogProbeService.audit_project(
_Db([]),
uuid.uuid4(),
settings=_settings(SOURCE_CATALOG_PROBE_ENABLED=False),
opener=forbidden_opener,
now=NOW,
)
assert report.summary.disabled_count == 2
assert all(item.status == "disabled" for item in report.items)
def test_catalog_probe_route_returns_canonical_envelope(monkeypatch) -> None:
from app.api.routes import datasets as dataset_routes
project_id = uuid.uuid4()
expected = SourceCatalogProbeService.audit_project(
_Db([]), project_id, settings=_settings(SOURCE_CATALOG_PROBE_ENABLED=False), now=NOW
)
monkeypatch.setattr(
dataset_routes.SourceCatalogProbeService,
"audit_project",
lambda db, selected_project_id, force=False: expected,
)
response = dataset_routes.probe_dataset_source_catalogs(
project_id=project_id, refresh=False, db=SimpleNamespace()
)
assert list(response) == ["data"]
assert response["data"]["project_id"] == project_id
assert response["data"]["summary"]["provider_count"] == 2
def test_catalog_probe_remains_explicit_and_never_imports_provider_data() -> None:
root = Path(__file__).resolve().parents[2]
hook = (root / "frontend" / "src" / "hooks" / "useSourceFreshness.ts").read_text(encoding="utf-8")
service = (root / "backend" / "app" / "services" / "source_catalog_probe_service.py").read_text(encoding="utf-8")
operator = (root / "scripts" / "audit_source_freshness.py").read_text(encoding="utf-8")
assert "void probeCatalogs(" not in hook
assert "DatasetService" not in service
assert "VectorFeatureService" not in service
assert "--probe-catalogs" in operator
assert "/datasets/source-catalog-probes" in operator
@@ -34,6 +34,11 @@
<Config Name="Orthophoto WMS URL" Target="ORTHOPHOTO_WMS_URL" Default="https://geo.api.vlaanderen.be/OMWRGBMRVL/wms" Mode="" Description="Official Digitaal Vlaanderen most-recent winter orthophoto WMS endpoint." Type="Variable" Display="advanced" Required="true" Mask="false">https://geo.api.vlaanderen.be/OMWRGBMRVL/wms</Config>
<Config Name="Orthophoto Resolution (m)" Target="ORTHOPHOTO_RESOLUTION_M" Default="1.0" Mode="" Description="Requested analysis sampling in metres per pixel. Keep at 1.0 for the active building model profile." Type="Variable" Display="advanced" Required="true" Mask="false">1.0</Config>
<Config Name="Orthophoto Maximum Side (m)" Target="ORTHOPHOTO_MAX_SIDE_M" Default="1024" Mode="" Description="Safety limit for each selected rectangle side before external acquisition and local inference." Type="Variable" Display="advanced" Required="true" Mask="false">1024</Config>
<Config Name="Official Catalog Edition Probe" Target="SOURCE_CATALOG_PROBE_ENABLED" Default="true" Mode="" Description="Allow explicit read-only GRB and orthophoto edition checks. This never imports provider data." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
<Config Name="GRB Catalog WFS URL" Target="SOURCE_CATALOG_GRB_WFS_URL" Default="https://geo.api.vlaanderen.be/GRB/wfs" Mode="" Description="Official GRB WFS used only for capabilities and linked ISO metadata checks." Type="Variable" Display="advanced" Required="true" Mask="false">https://geo.api.vlaanderen.be/GRB/wfs</Config>
<Config Name="Catalog Probe Timeout (seconds)" Target="SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS" Default="10" Mode="" Description="Per-request timeout for explicit read-only official catalog checks." Type="Variable" Display="advanced" Required="true" Mask="false">10</Config>
<Config Name="Catalog Probe Maximum Response (MiB)" Target="SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB" Default="2" Mode="" Description="Maximum capabilities or ISO metadata response size accepted by a catalog probe." Type="Variable" Display="advanced" Required="true" Mask="false">2</Config>
<Config Name="Catalog Probe Cache (seconds)" Target="SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS" Default="900" Mode="" Description="Short in-memory cache for repeated official edition checks; use zero to disable." Type="Variable" Display="advanced" Required="true" Mask="false">900</Config>
<Config Name="Official DHMV Acquisition" Target="DHMV_ENABLED" Default="true" Mode="" Description="Allow bounded official DHMV II terrain and surface raster acquisition." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
<Config Name="DHMV WCS URL" Target="DHMV_WCS_URL" Default="https://geo.api.vlaanderen.be/DHMV/wcs" Mode="" Description="Official Digitaal Vlaanderen DHMV WCS endpoint." Type="Variable" Display="advanced" Required="true" Mask="false">https://geo.api.vlaanderen.be/DHMV/wcs</Config>
<Config Name="DHMV Analysis Resolution (m)" Target="DHMV_RESOLUTION_M" Default="5.0" Mode="" Description="Stored analysis grid resolution. Native source resolution remains recorded as 1 metre." Type="Variable" Display="advanced" Required="true" Mask="false">5.0</Config>
+8
View File
@@ -32,6 +32,14 @@ ORTHOPHOTO_RESOLUTION_M=1.0
ORTHOPHOTO_MIN_SIDE_M=128
ORTHOPHOTO_MAX_SIDE_M=1024
ORTHOPHOTO_CACHE_TTL_HOURS=24
# Explicit read-only edition checks for GRB and the most-recent orthophoto.
# No feature or raster data is downloaded by these probes.
SOURCE_CATALOG_PROBE_ENABLED=true
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900
DHMV_ENABLED=true
DHMV_WCS_URL=https://geo.api.vlaanderen.be/DHMV/wcs
DHMV_RESOLUTION_M=5.0
+10
View File
@@ -27,6 +27,11 @@ ORTHOPHOTO_RESOLUTION_M="${ORTHOPHOTO_RESOLUTION_M:-1.0}"
ORTHOPHOTO_MIN_SIDE_M="${ORTHOPHOTO_MIN_SIDE_M:-128}"
ORTHOPHOTO_MAX_SIDE_M="${ORTHOPHOTO_MAX_SIDE_M:-1024}"
ORTHOPHOTO_CACHE_TTL_HOURS="${ORTHOPHOTO_CACHE_TTL_HOURS:-24}"
SOURCE_CATALOG_PROBE_ENABLED="${SOURCE_CATALOG_PROBE_ENABLED:-true}"
SOURCE_CATALOG_GRB_WFS_URL="${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}"
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS="${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}"
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB="${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}"
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}"
DHMV_ENABLED="${DHMV_ENABLED:-true}"
DHMV_WCS_URL="${DHMV_WCS_URL:-https://geo.api.vlaanderen.be/DHMV/wcs}"
DHMV_RESOLUTION_M="${DHMV_RESOLUTION_M:-5.0}"
@@ -126,6 +131,11 @@ docker run -d \
-e ORTHOPHOTO_MIN_SIDE_M="$ORTHOPHOTO_MIN_SIDE_M" \
-e ORTHOPHOTO_MAX_SIDE_M="$ORTHOPHOTO_MAX_SIDE_M" \
-e ORTHOPHOTO_CACHE_TTL_HOURS="$ORTHOPHOTO_CACHE_TTL_HOURS" \
-e SOURCE_CATALOG_PROBE_ENABLED="$SOURCE_CATALOG_PROBE_ENABLED" \
-e SOURCE_CATALOG_GRB_WFS_URL="$SOURCE_CATALOG_GRB_WFS_URL" \
-e SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS="$SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS" \
-e SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB="$SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB" \
-e SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="$SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS" \
-e DHMV_ENABLED="$DHMV_ENABLED" \
-e DHMV_WCS_URL="$DHMV_WCS_URL" \
-e DHMV_RESOLUTION_M="$DHMV_RESOLUTION_M" \
+5
View File
@@ -25,6 +25,11 @@ services:
ORTHOPHOTO_MIN_SIDE_M: ${ORTHOPHOTO_MIN_SIDE_M:-128}
ORTHOPHOTO_MAX_SIDE_M: ${ORTHOPHOTO_MAX_SIDE_M:-1024}
ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24}
SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true}
SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS: ${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB: ${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS: ${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}
DHMV_ENABLED: ${DHMV_ENABLED:-true}
DHMV_WCS_URL: ${DHMV_WCS_URL:-https://geo.api.vlaanderen.be/DHMV/wcs}
DHMV_RESOLUTION_M: ${DHMV_RESOLUTION_M:-5.0}
+5
View File
@@ -30,6 +30,11 @@ services:
ORTHOPHOTO_MIN_SIDE_M: ${ORTHOPHOTO_MIN_SIDE_M:-128}
ORTHOPHOTO_MAX_SIDE_M: ${ORTHOPHOTO_MAX_SIDE_M:-1024}
ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24}
SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true}
SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS: ${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB: ${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS: ${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}
DHMV_ENABLED: ${DHMV_ENABLED:-true}
DHMV_WCS_URL: ${DHMV_WCS_URL:-https://geo.api.vlaanderen.be/DHMV/wcs}
DHMV_RESOLUTION_M: ${DHMV_RESOLUTION_M:-5.0}
+20
View File
@@ -412,6 +412,26 @@ Dataset or silently refreshes a publication. Fixed editions and scenarios are
not marked stale merely because their source date is old. Unknown sources are
`review_required` until an explicit publication policy is defined.
### GET `/api/v1/projects/{project_id}/datasets/source-catalog-probes`
Performs an explicit, read-only release probe for the allowlisted GRB WFS and
most-recent orthophoto WMS. The endpoint reads bounded `GetCapabilities`
responses, confirms the expected layers and follows only HTTPS ISO 19139
`GetRecordById` links on `metadata.vlaanderen.be`. Query parameter
`refresh=true` bypasses the short in-memory response cache.
Each provider item returns service reachability, matched/missing layers, the
official metadata identifier, title, edition, publication/metadata dates,
local `source_version` and one comparison status: `same`, `different`,
`not_comparable`, `no_local_data` or `unavailable`. Provider status is
`available`, `degraded`, `unavailable` or `disabled`. A difference means only
that an operator should review provenance; it is not an update instruction.
The endpoint accepts no arbitrary URL, feature query, area or layer. It does
not fetch vector features, raster pixels or models, create jobs/datasets, write
to PostGIS or trigger an import. The normal `source-freshness` endpoint remains
local-only and never invokes this probe implicitly.
### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}`
Return metadata.
+37
View File
@@ -9302,3 +9302,40 @@ Next:
- Add machine-readable catalogue release probes only for providers with stable
official version endpoints, keeping every acquisition an explicit bounded
operator action.
## Sprint 222 - Official source edition probes (2026-07-16)
Implemented:
- Verified the public machine-readable contracts published by Digitaal
Vlaanderen. GRB WFS and OMWRGBMRVL WMS capabilities expose stable ISO 19139
CSW records containing real source editions and metadata dates.
- Added a canonical project endpoint that reads only allowlisted capabilities
and linked `metadata.vlaanderen.be` `GetRecordById` responses. It confirms
GRB `GBG/WBN/WGO/ADP` and orthophoto `Ortho/Vliegdagcontour` availability,
then reports the official edition beside local `source_version` evidence.
- Added strict HTTP(S)/metadata allowlists, timeout and byte limits,
post-redirect validation, SHA-256 capabilities evidence, provider-isolated
degraded/unavailable states and a short in-memory cache.
- Added an explicit Status action and operator CLI flags. Neither path runs on
page load, starts a job, downloads source data or changes a Dataset.
- Added Compose, all-in-one Unraid and DockerMan controls for the bounded probe.
Validation:
- Deterministic tests cover official XML parsing, edition comparison,
provider failure isolation, metadata URL rejection, size limits, caching,
disabled mode and canonical envelopes.
- Frontend typecheck/build and targeted backend tests pass before the full
release and Tower validation recorded in the delivery result.
Boundaries:
- The general Datavindplaats API requires authentication and is not scraped or
silently bypassed. GeoIntel uses only public metadata records advertised by
the official OGC services.
- A version difference remains a manual provenance-review signal. No automatic
import, refresh scheduler, GRB feature fetch or orthophoto pixel request was
added.
Next:
- Extend the same probe pattern only where another official source publishes a
stable machine-readable edition. Do not infer release versions from service
protocol versions, ETags or HTTP modification dates.
+7
View File
@@ -25,6 +25,13 @@ YOLO_MAX_DETECTIONS=1000
YOLO_DUPLICATE_IOU_THRESHOLD=0.5
ENABLE_GRB_WFS=false
GRB_WFS_URL=
# Explicit read-only source-edition probes (no provider import).
SOURCE_CATALOG_PROBE_ENABLED=true
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900
OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter
```
+2 -1
View File
@@ -39,7 +39,8 @@
- [x] Execute and audit the complete 336-product VMM and 56-product DHMV regional runtime matrices.
- [x] Present municipal DHMV/VMM partitions as logical regional layers and analyse cross-boundary rectangles without a municipality prerequisite.
- [x] Add a read-only source freshness/version audit with explicit publication policies, local integrity checks, operator CLI and compact Status workspace surface.
- [ ] Add external catalogue release probes only after each official provider exposes a stable machine-readable version contract; keep every refresh explicit and bounded.
- [x] Add explicit bounded GRB/orthophoto catalogue release probes using official capabilities and ISO metadata; keep every acquisition separate and operator-controlled.
- [ ] Extend catalogue probes only to additional sources that publish a stable machine-readable edition contract; do not add background polling or infer releases from HTTP dates alone.
## Governed source expansion backlog
+4 -1
View File
@@ -9,7 +9,10 @@ surface. It separates sources that are current, due for a catalogue review,
require local integrity review or are local artifacts. Only attention items are
expanded by default; all source detail remains available through disclosure.
The refresh button reruns the local read-only audit and never downloads or
replaces source data.
replaces source data. A separate `Officiële edities controleren` action is the
only trigger for bounded GRB/orthophoto catalog reads. It presents official and
local editions, layer-contract coverage and honest non-comparable version
markers without starting an import or background poll.
The user-facing shell is task based: `Kaart`, `Bronnen`, `Kwaliteit`, `Beeldanalyse`, `Downloads`, `Status` and `Beheer`. Internal benchmark projects, raw dataset metadata, provider capabilities, model registry details and QA evidence remain accessible through labelled advanced disclosures instead of competing with the normal workflow.
+4
View File
@@ -860,6 +860,10 @@ function App(): JSX.Element {
loading={sourceFreshness.loading}
error={sourceFreshness.error}
onRefresh={() => { void sourceFreshness.refresh() }}
catalogReport={sourceFreshness.catalogReport}
catalogLoading={sourceFreshness.catalogLoading}
catalogError={sourceFreshness.catalogError}
onProbeCatalogs={(force) => { void sourceFreshness.probeCatalogs(force) }}
/>
<details className="status-details-disclosure">
<summary>
@@ -1,10 +1,22 @@
import type { SourceFreshnessItem, SourceFreshnessReport, SourceFreshnessStatus } from '../../types'
import type {
SourceCatalogComparisonStatus,
SourceCatalogProbeItem,
SourceCatalogProbeReport,
SourceCatalogProbeStatus,
SourceFreshnessItem,
SourceFreshnessReport,
SourceFreshnessStatus,
} from '../../types'
interface SourceFreshnessPanelProps {
report: SourceFreshnessReport | null
loading: boolean
error: string | null
onRefresh: () => void
catalogReport: SourceCatalogProbeReport | null
catalogLoading: boolean
catalogError: string | null
onProbeCatalogs: (force: boolean) => void
}
function statusLabel(status: SourceFreshnessStatus): string {
@@ -14,6 +26,21 @@ function statusLabel(status: SourceFreshnessStatus): string {
return 'lokaal'
}
function catalogStatusLabel(status: SourceCatalogProbeStatus): string {
if (status === 'available') return 'bereikbaar'
if (status === 'degraded') return 'beperkt'
if (status === 'unavailable') return 'niet bereikbaar'
return 'uitgeschakeld'
}
function comparisonLabel(status: SourceCatalogComparisonStatus): string {
if (status === 'same') return 'zelfde editie'
if (status === 'different') return 'verschil controleren'
if (status === 'no_local_data') return 'nog niet lokaal ingeladen'
if (status === 'not_comparable') return 'andere versienotatie'
return 'niet vergelijkbaar'
}
function formatDate(value?: string | null): string {
if (!value) return 'geen datum'
return new Intl.DateTimeFormat('nl-BE', { dateStyle: 'medium' }).format(new Date(value))
@@ -23,6 +50,27 @@ function integrityIssueCount(item: SourceFreshnessItem): number {
return Object.values(item.integrity).reduce((total, value) => total + value, 0)
}
function CatalogRow({ item }: { item: SourceCatalogProbeItem }): JSX.Element {
return (
<div className={`source-catalog-row source-catalog-row-${item.status}`}>
<div className="source-freshness-main">
<div>
<strong>{item.display_name}</strong>
<span>{catalogStatusLabel(item.status)} / {comparisonLabel(item.comparison_status)}</span>
</div>
<span className="source-freshness-status">{item.remote_version ?? 'geen editie'}</span>
</div>
<p>{item.message}</p>
<div className="source-freshness-meta">
<span>Lokaal: {item.local_source_version ?? 'niet aanwezig'}</span>
<span>Lagen: {item.matched_layers.length}/{item.expected_layers.length} bevestigd</span>
{item.remote_modified_at ? <span>Metadata: {formatDate(item.remote_modified_at)}</span> : null}
{item.cached ? <span>cache gebruikt</span> : null}
</div>
</div>
)
}
function SourceRow({ item }: { item: SourceFreshnessItem }): JSX.Element {
const issueCount = integrityIssueCount(item)
return (
@@ -30,7 +78,7 @@ function SourceRow({ item }: { item: SourceFreshnessItem }): JSX.Element {
<div className="source-freshness-main">
<div>
<strong>{item.display_name}</strong>
<span>{item.dataset_count} datasets · {item.version_count} versies</span>
<span>{item.dataset_count} datasets / {item.version_count} versies</span>
</div>
<span className="source-freshness-status">{statusLabel(item.status)}</span>
</div>
@@ -45,7 +93,16 @@ function SourceRow({ item }: { item: SourceFreshnessItem }): JSX.Element {
)
}
export function SourceFreshnessPanel({ report, loading, error, onRefresh }: SourceFreshnessPanelProps): JSX.Element {
export function SourceFreshnessPanel({
report,
loading,
error,
onRefresh,
catalogReport,
catalogLoading,
catalogError,
onProbeCatalogs,
}: SourceFreshnessPanelProps): JSX.Element {
const attentionItems = report?.items.filter((item) => item.status === 'due' || item.status === 'review_required') ?? []
const summary = report?.summary
@@ -55,11 +112,21 @@ export function SourceFreshnessPanel({ report, loading, error, onRefresh }: Sour
<div>
<p className="eyebrow">Bronbeheer</p>
<h2>Actualiteit en versiecontrole</h2>
<p>Controleert lokale publicaties, versies en bestanden zonder externe bronnen automatisch te wijzigen.</p>
<p>Controleert lokale publicaties en kan op aanvraag de officiële GRB- en orthofoto-editie uitlezen.</p>
</div>
<div className="source-freshness-actions">
<button className="secondary-action" type="button" onClick={onRefresh} disabled={loading}>
{loading ? 'Controleren...' : 'Lokale status vernieuwen'}
</button>
<button
className="primary-action"
type="button"
onClick={() => onProbeCatalogs(Boolean(catalogReport))}
disabled={catalogLoading}
>
{catalogLoading ? 'Officiële bronnen lezen...' : catalogReport ? 'Opnieuw bij bron controleren' : 'Officiële edities controleren'}
</button>
</div>
<button className="secondary-action" type="button" onClick={onRefresh} disabled={loading}>
{loading ? 'Controleren...' : 'Opnieuw controleren'}
</button>
</div>
{error ? <p className="inline-error">{error}</p> : null}
@@ -89,6 +156,31 @@ export function SourceFreshnessPanel({ report, loading, error, onRefresh }: Sour
</div>
<p className="source-freshness-limitation">{report.limitations.join(' ')}</p>
</details>
<div className="source-catalog-section" aria-live="polite">
<div className="source-catalog-heading">
<div>
<strong>Officiële catalogusedities</strong>
<span>Alleen na jouw expliciete controle; er wordt niets ingeladen of vervangen.</span>
</div>
{catalogReport ? (
<span>{catalogReport.summary.available_count}/{catalogReport.summary.provider_count} bereikbaar</span>
) : null}
</div>
{catalogError ? <p className="inline-error">{catalogError}</p> : null}
{!catalogReport && !catalogError ? (
<p className="source-catalog-empty">Controleer GRB en orthofoto wanneer je een lokale bronversie met de officiële editie wilt vergelijken.</p>
) : null}
{catalogReport ? (
<>
<div className="source-catalog-list">
{catalogReport.items.map((item) => <CatalogRow item={item} key={item.source_name} />)}
</div>
<p className="source-freshness-limitation">
Gecontroleerd {formatDate(catalogReport.generated_at)}. Een verschil vraagt menselijke beoordeling en start nooit automatisch een import.
</p>
</>
) : null}
</div>
</>
) : null}
</section>
+48 -2
View File
@@ -1,13 +1,17 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { datasetsApi } from '../services/api'
import type { SourceFreshnessReport } from '../types'
import type { SourceCatalogProbeReport, SourceFreshnessReport } from '../types'
export function useSourceFreshness(selectedProjectId: string | null) {
const [report, setReport] = useState<SourceFreshnessReport | null>(null)
const [loading, setLoading] = useState(false)
const [error, setError] = useState<string | null>(null)
const requestSequence = useRef(0)
const catalogRequestSequence = useRef(0)
const [catalogReport, setCatalogReport] = useState<SourceCatalogProbeReport | null>(null)
const [catalogLoading, setCatalogLoading] = useState(false)
const [catalogError, setCatalogError] = useState<string | null>(null)
const refresh = useCallback(async () => {
const requestId = ++requestSequence.current
@@ -35,6 +39,32 @@ export function useSourceFreshness(selectedProjectId: string | null) {
}
}, [selectedProjectId])
const probeCatalogs = useCallback(async (force = false) => {
const requestId = ++catalogRequestSequence.current
if (!selectedProjectId) {
setCatalogReport(null)
setCatalogError(null)
setCatalogLoading(false)
return
}
setCatalogLoading(true)
setCatalogError(null)
try {
const nextReport = await datasetsApi.sourceCatalogProbes(selectedProjectId, force)
if (catalogRequestSequence.current === requestId) {
setCatalogReport(nextReport)
}
} catch (caught) {
if (catalogRequestSequence.current === requestId) {
setCatalogError(caught instanceof Error ? caught.message : 'De officiële broncatalogi konden niet worden gecontroleerd.')
}
} finally {
if (catalogRequestSequence.current === requestId) {
setCatalogLoading(false)
}
}
}, [selectedProjectId])
useEffect(() => {
void refresh()
return () => {
@@ -42,5 +72,21 @@ export function useSourceFreshness(selectedProjectId: string | null) {
}
}, [refresh])
return { report, loading, error, refresh }
useEffect(() => {
catalogRequestSequence.current += 1
setCatalogReport(null)
setCatalogError(null)
setCatalogLoading(false)
}, [selectedProjectId])
return {
report,
loading,
error,
refresh,
catalogReport,
catalogLoading,
catalogError,
probeCatalogs,
}
}
+5
View File
@@ -28,6 +28,7 @@ import type {
ThematicRasterProductRead,
ThematicRasterSelectionResponse,
SourceFreshnessReport,
SourceCatalogProbeReport,
} from '../../types'
const DATASET_PAGE_SIZE = 200
@@ -61,6 +62,10 @@ export const datasetsApi = {
list: listProjectDatasets,
sourceFreshness: (projectId: string): Promise<SourceFreshnessReport> =>
apiGet<SourceFreshnessReport>(`/api/v1/projects/${projectId}/datasets/source-freshness`),
sourceCatalogProbes: (projectId: string, refresh = false): Promise<SourceCatalogProbeReport> =>
apiGet<SourceCatalogProbeReport>(
`/api/v1/projects/${projectId}/datasets/source-catalog-probes?refresh=${refresh ? 'true' : 'false'}`,
),
upload: (
projectId: string,
payload: {
+102 -2
View File
@@ -836,13 +836,20 @@ details.ai-lab-model-surface > summary strong {
padding: 1rem 1.2rem;
}
.source-freshness-header .secondary-action {
.source-freshness-header .secondary-action,
.source-freshness-header .primary-action {
flex: 0 0 auto;
width: auto;
min-width: 10rem;
margin-top: 0;
}
.source-freshness-actions {
display: flex;
flex: 0 0 auto;
gap: 0.55rem;
}
.source-freshness-header h2 {
margin: 0;
font-size: 1.05rem;
@@ -959,6 +966,81 @@ details.ai-lab-model-surface > summary strong {
border-top: 1px solid var(--line);
}
.source-catalog-section {
border-top: 1px solid var(--line);
background: #fbfcfd;
}
.source-catalog-heading {
display: flex;
gap: 1rem;
align-items: center;
justify-content: space-between;
padding: 0.8rem 1rem;
}
.source-catalog-heading > div {
display: grid;
gap: 0.12rem;
}
.source-catalog-heading strong {
color: #23313b;
font-size: 0.84rem;
}
.source-catalog-heading span,
.source-catalog-empty {
color: var(--muted);
font-size: 0.75rem;
}
.source-catalog-heading > span {
flex: 0 0 auto;
font-weight: 700;
}
.source-catalog-empty {
margin: 0;
padding: 0 1rem 0.85rem;
}
.source-catalog-list {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
border-top: 1px solid var(--line);
}
.source-catalog-row {
min-width: 0;
padding: 0.8rem 1rem;
border-right: 1px solid var(--line);
box-shadow: inset 0 3px 0 #8a98a3;
}
.source-catalog-row:last-child {
border-right: 0;
}
.source-catalog-row-available {
box-shadow: inset 0 3px 0 #2d9272;
}
.source-catalog-row-degraded {
box-shadow: inset 0 3px 0 #ca7a18;
}
.source-catalog-row-unavailable,
.source-catalog-row-disabled {
box-shadow: inset 0 3px 0 #b84e4e;
}
.source-catalog-row p {
margin: 0.45rem 0;
color: var(--muted);
font-size: 0.75rem;
}
.source-freshness-details > summary {
display: flex;
cursor: pointer;
@@ -982,10 +1064,15 @@ details.ai-lab-model-surface > summary strong {
flex-direction: column;
}
.source-freshness-header .secondary-action {
.source-freshness-header .secondary-action,
.source-freshness-header .primary-action {
width: 100%;
}
.source-freshness-actions {
flex-direction: column;
}
.source-freshness-summary {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
@@ -997,6 +1084,19 @@ details.ai-lab-model-surface > summary strong {
.source-freshness-summary span:nth-child(-n + 2) {
border-bottom: 1px solid var(--line);
}
.source-catalog-list {
grid-template-columns: 1fr;
}
.source-catalog-row {
border-right: 0;
border-bottom: 1px solid var(--line);
}
.source-catalog-row:last-child {
border-bottom: 0;
}
}
.workflow-guidance-panel {
+46
View File
@@ -709,6 +709,52 @@ export interface SourceFreshnessReport {
limitations: string[]
}
export type SourceCatalogProbeStatus = 'available' | 'degraded' | 'unavailable' | 'disabled'
export type SourceCatalogComparisonStatus = 'same' | 'different' | 'not_comparable' | 'no_local_data' | 'unavailable'
export interface SourceCatalogProbeItem {
source_name: string
display_name: string
service_type: 'WFS' | 'WMS'
endpoint_url: string
status: SourceCatalogProbeStatus
reachable: boolean
checked_at: string
cached: boolean
expected_layers: string[]
matched_layers: string[]
missing_layers: string[]
advertised_layer_count: number
metadata_url?: string | null
metadata_identifier?: string | null
remote_title?: string | null
remote_version?: string | null
remote_modified_at?: string | null
remote_published_at?: string | null
local_source_version?: string | null
comparison_status: SourceCatalogComparisonStatus
capabilities_sha256?: string | null
capabilities_etag?: string | null
capabilities_last_modified_at?: string | null
message: string
error_code?: string | null
}
export interface SourceCatalogProbeReport {
project_id: string
generated_at: string
summary: {
provider_count: number
available_count: number
degraded_count: number
unavailable_count: number
disabled_count: number
different_version_count: number
}
items: SourceCatalogProbeItem[]
limitations: string[]
}
export interface GeojsonEnvelopeResponse {
data: object
}
+8
View File
@@ -1708,6 +1708,14 @@ uses one canonical `GET`, changes no application data and performs no source
download. It can therefore be scheduled explicitly through Unraid cron without
turning GeoIntel into a real-time monitoring system.
Add `--probe-catalogs` to explicitly read the allowlisted official GRB and
orthophoto capabilities plus linked ISO metadata. `--refresh-catalogs` bypasses
the short server cache and implies the probe. `--fail-on-catalog` makes a
degraded/unavailable official metadata service fail the command. With a probe,
JSON output contains `source_freshness` and `catalog_probes`; without it, the
original local report shape is unchanged. No flag downloads provider features
or imagery and no flag writes a Dataset.
## Tower deployment
Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime:
+80 -5
View File
@@ -26,6 +26,21 @@ def parse_args() -> argparse.Namespace:
default="integrity",
help="Non-zero exit policy for cron or release automation",
)
parser.add_argument(
"--probe-catalogs",
action="store_true",
help="Explicitly query the allowlisted official GRB and orthophoto metadata catalogs",
)
parser.add_argument(
"--refresh-catalogs",
action="store_true",
help="Bypass the short server-side catalog cache (implies --probe-catalogs)",
)
parser.add_argument(
"--fail-on-catalog",
action="store_true",
help="Exit non-zero when an explicitly requested official catalog is degraded or unavailable",
)
return parser.parse_args()
@@ -45,7 +60,24 @@ def fetch_report(api_url: str, project_id: str, timeout: float) -> dict:
return payload["data"]
def print_text(report: dict) -> None:
def fetch_catalog_report(api_url: str, project_id: str, timeout: float, refresh: bool) -> dict:
query = "true" if refresh else "false"
endpoint = f"{api_url.rstrip('/')}/projects/{project_id}/datasets/source-catalog-probes?refresh={query}"
request = Request(endpoint, headers={"Accept": "application/json", "User-Agent": "GeoIntel-source-audit/1.0"})
try:
with urlopen(request, timeout=timeout) as response:
payload = json.load(response)
except HTTPError as exc:
body = exc.read().decode("utf-8", errors="replace")
raise RuntimeError(f"GeoIntel catalog probe returned HTTP {exc.code}: {body}") from exc
except URLError as exc:
raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc
if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
raise RuntimeError("Catalog probe response is not a canonical GeoIntel data envelope")
return payload["data"]
def print_text(report: dict, *, catalog_queried: bool = False) -> None:
summary = report.get("summary", {})
print(
"GeoIntel source audit: "
@@ -59,7 +91,30 @@ def print_text(report: dict) -> None:
status = item.get("status", "unknown")
if status not in {"current", "local"} or item.get("integrity", {}).get("missing_version_count", 0):
print(f"- {status:15} {item.get('display_name', item.get('source_name'))}: {item.get('reason', '')}")
print("No external catalog was queried and no dataset was modified.")
if catalog_queried:
print("The official catalog was queried read-only; no feature, raster or dataset was modified.")
else:
print("No external catalog was queried and no dataset was modified.")
def print_catalog_text(report: dict) -> None:
summary = report.get("summary", {})
print(
"Official catalog probe: "
f"{summary.get('available_count', 0)}/{summary.get('provider_count', 0)} available, "
f"{summary.get('degraded_count', 0)} degraded, "
f"{summary.get('unavailable_count', 0)} unavailable, "
f"{summary.get('different_version_count', 0)} version differences"
)
for item in report.get("items", []):
remote = item.get("remote_version") or "no official edition"
local = item.get("local_source_version") or "not loaded locally"
print(
f"- {item.get('display_name', item.get('source_name'))}: "
f"{item.get('status')} / official={remote} / local={local} / "
f"comparison={item.get('comparison_status')}"
)
print(f" {item.get('message', '')}")
def should_fail(report: dict, fail_on: str) -> bool:
@@ -76,22 +131,42 @@ def should_fail(report: dict, fail_on: str) -> bool:
return integrity > 0 or due > 0 or review > 0
def catalog_should_fail(report: dict) -> bool:
summary = report.get("summary", {})
return int(summary.get("degraded_count", 0)) > 0 or int(summary.get("unavailable_count", 0)) > 0
def main() -> int:
args = parse_args()
try:
report = fetch_report(args.api_url, args.project_id, args.timeout)
catalog_report = (
fetch_catalog_report(args.api_url, args.project_id, args.timeout, args.refresh_catalogs)
if args.probe_catalogs or args.refresh_catalogs
else None
)
except RuntimeError as exc:
print(str(exc), file=sys.stderr)
return 2
serialized = json.dumps(report, indent=2, sort_keys=True)
output_payload = (
{"source_freshness": report, "catalog_probes": catalog_report}
if catalog_report is not None
else report
)
serialized = json.dumps(output_payload, indent=2, sort_keys=True)
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(serialized + "\n", encoding="utf-8")
if args.json:
print(serialized)
else:
print_text(report)
return 1 if should_fail(report, args.fail_on) else 0
print_text(report, catalog_queried=catalog_report is not None)
if catalog_report is not None:
print_catalog_text(catalog_report)
failed = should_fail(report, args.fail_on)
if args.fail_on_catalog and catalog_report is not None:
failed = failed or catalog_should_fail(catalog_report)
return 1 if failed else 0
if __name__ == "__main__":