From 595967f8926d278b3032ee8a7ddd0520d3dc5985 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 16 Jul 2026 17:33:40 +0200 Subject: [PATCH] Add official source edition probes --- .env.example | 5 + CHANGELOG.md | 12 + backend/README.md | 26 + backend/app/api/routes/datasets.py | 11 + backend/app/core/config.py | 23 + backend/app/schemas/__init__.py | 8 + backend/app/schemas/source_catalog.py | 56 ++ .../services/source_catalog_probe_service.py | 545 ++++++++++++++++++ .../test_sprint222_source_catalog_probes.py | 332 +++++++++++ deploy/unraid/geointel-unraid-template.xml | 5 + deploy/unraid/geointel.env.example | 8 + deploy/unraid/run-dockerman-container.sh | 10 + docker-compose.unraid.yml | 5 + docker-compose.yml | 5 + docs/API_CONTRACTS.md | 20 + docs/CODEX_EXECUTION_LOG.md | 37 ++ docs/ENVIRONMENT_SPEC.md | 7 + docs/TODO.md | 3 +- frontend/README.md | 5 +- frontend/src/App.tsx | 4 + .../status/SourceFreshnessPanel.tsx | 106 +++- frontend/src/hooks/useSourceFreshness.ts | 50 +- frontend/src/services/api/datasets.ts | 5 + frontend/src/styles/premium.css | 104 +++- frontend/src/types.ts | 46 ++ scripts/README.md | 8 + scripts/audit_source_freshness.py | 85 ++- 27 files changed, 1513 insertions(+), 18 deletions(-) create mode 100644 backend/app/schemas/source_catalog.py create mode 100644 backend/app/services/source_catalog_probe_service.py create mode 100644 backend/tests/test_sprint222_source_catalog_probes.py diff --git a/.env.example b/.env.example index 2b1feca7..3875ece0 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,11 @@ ORTHOPHOTO_RESOLUTION_M=1.0 ORTHOPHOTO_MIN_SIDE_M=128 ORTHOPHOTO_MAX_SIDE_M=1024 ORTHOPHOTO_CACHE_TTL_HOURS=24 +SOURCE_CATALOG_PROBE_ENABLED=true +SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs +SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10 +SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2 +SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900 DHMV_ENABLED=true DHMV_WCS_URL=https://geo.api.vlaanderen.be/DHMV/wcs DHMV_RESOLUTION_M=5.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 19d8330b..e623c765 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ # Changelog +## Sprint 222 Official source edition probes (2026-07-16) + +- Added explicit, read-only GRB and most-recent orthophoto catalog probes using + official WFS/WMS capabilities and linked ISO 19139 CSW metadata records. +- Added bounded response sizes, timeouts, provider-isolated failures, a short + cache and strict metadata-host/path validation. No provider features, raster + pixels or application data are fetched or modified. +- Added canonical API, compact opt-in Status UI and operator CLI support. The + local source-freshness audit remains automatic and provider-free. +- Added deterministic XML/network-boundary tests and editable Compose/Unraid + controls without changing migrations or persistence contracts. + ## Sprint 221 Governed source freshness audit (2026-07-16) - Added a project-wide read-only source report derived from Dataset, diff --git a/backend/README.md b/backend/README.md index 8ce811bc..af97d7d3 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1443,3 +1443,29 @@ docker exec geointel python /app/scripts/audit_source_freshness.py \ Use `--fail-on due` to make a planned review date fail automation, or `--fail-on never` for reporting only. The command never starts a refresh. + +An operator can explicitly add the official GRB and orthophoto edition check: + +```bash +docker exec geointel python /app/scripts/audit_source_freshness.py \ + --project-id 82a85913-c522-45d7-84a1-02b393d89e55 \ + --api-url http://127.0.0.1/api/v1 \ + --probe-catalogs \ + --output /app/storage/operator-evidence/source-freshness/with-catalogs.json +``` + +Use `--refresh-catalogs` to bypass the 15-minute in-memory cache and +`--fail-on-catalog` only when temporary official-provider unavailability must +fail an operator job. This path reads bounded WFS/WMS capabilities and their +fixed ISO 19139 metadata records. It confirms GRB `GBG`, `WBN`, `WGO`, `ADP` +and orthophoto `Ortho`, `Vliegdagcontour`; it never requests feature or raster +content. The public Datavindplaats API requires an access token, so release +evidence comes from the public metadata links published by the official OGC +services rather than HTML scraping. + +Runtime controls are `SOURCE_CATALOG_PROBE_ENABLED`, +`SOURCE_CATALOG_GRB_WFS_URL`, `SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS`, +`SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB` and +`SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS`. Metadata links remain restricted to +the official HTTPS CSW path on `metadata.vlaanderen.be` even when an operator +overrides the capabilities endpoint. diff --git a/backend/app/api/routes/datasets.py b/backend/app/api/routes/datasets.py index a6c31b40..212c4cff 100644 --- a/backend/app/api/routes/datasets.py +++ b/backend/app/api/routes/datasets.py @@ -45,6 +45,7 @@ from app.services.vector_operations_service import VectorOperationsService from app.services.vector_feature_service import VectorFeatureService from app.services.dataset_service import DatasetService from app.services.source_freshness_service import SourceFreshnessService +from app.services.source_catalog_probe_service import SourceCatalogProbeService from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService from app.services.dhmv_acquisition_service import DhmvAcquisitionService from app.services.terrain_analysis_service import TerrainAnalysisService @@ -255,6 +256,16 @@ def audit_dataset_source_freshness( return envelope(report.model_dump()) +@router.get("/datasets/source-catalog-probes", response_model=dict) +def probe_dataset_source_catalogs( + project_id: UUID, + refresh: bool = Query(default=False), + db: Session = Depends(get_db), +): + report = SourceCatalogProbeService.audit_project(db, project_id, force=refresh) + return envelope(report.model_dump()) + + @router.get("/datasets/{dataset_id}", response_model=dict) def get_dataset( project_id: UUID, diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 83af58c3..ca22b1fd 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -31,6 +31,29 @@ class Settings(BaseSettings): orthophoto_timeout_seconds: int = Field(default=120, ge=1, validation_alias="ORTHOPHOTO_TIMEOUT_SECONDS") orthophoto_max_response_mb: int = Field(default=32, ge=1, validation_alias="ORTHOPHOTO_MAX_RESPONSE_MB") orthophoto_cache_ttl_hours: int = Field(default=24, ge=0, validation_alias="ORTHOPHOTO_CACHE_TTL_HOURS") + source_catalog_probe_enabled: bool = Field(default=True, validation_alias="SOURCE_CATALOG_PROBE_ENABLED") + source_catalog_grb_wfs_url: str = Field( + default="https://geo.api.vlaanderen.be/GRB/wfs", + validation_alias="SOURCE_CATALOG_GRB_WFS_URL", + ) + source_catalog_probe_timeout_seconds: int = Field( + default=10, + ge=1, + le=60, + validation_alias="SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS", + ) + source_catalog_probe_max_response_mb: int = Field( + default=2, + ge=1, + le=10, + validation_alias="SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB", + ) + source_catalog_probe_cache_ttl_seconds: int = Field( + default=900, + ge=0, + le=86_400, + validation_alias="SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS", + ) dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED") dhmv_wcs_url: str = Field( default="https://geo.api.vlaanderen.be/DHMV/wcs", diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index 610db024..9bc80ab4 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -11,6 +11,11 @@ from .source_freshness import ( SourceFreshnessSummary, SourceIntegritySummary, ) +from .source_catalog import ( + SourceCatalogProbeItem, + SourceCatalogProbeReport, + SourceCatalogProbeSummary, +) from .detection import ( DetectionListResponse, DetectionModelCapability, @@ -141,6 +146,9 @@ __all__ = [ "SourceFreshnessReport", "SourceFreshnessSummary", "SourceIntegritySummary", + "SourceCatalogProbeItem", + "SourceCatalogProbeReport", + "SourceCatalogProbeSummary", "DetectionListResponse", "DetectionModelCapability", "DetectionModelsResponse", diff --git a/backend/app/schemas/source_catalog.py b/backend/app/schemas/source_catalog.py new file mode 100644 index 00000000..664b5baa --- /dev/null +++ b/backend/app/schemas/source_catalog.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel + + +SourceCatalogProbeStatus = Literal["available", "degraded", "unavailable", "disabled"] +SourceCatalogComparisonStatus = Literal["same", "different", "not_comparable", "no_local_data", "unavailable"] + + +class SourceCatalogProbeItem(BaseModel): + source_name: str + display_name: str + service_type: Literal["WFS", "WMS"] + endpoint_url: str + status: SourceCatalogProbeStatus + reachable: bool + checked_at: datetime + cached: bool = False + expected_layers: list[str] + matched_layers: list[str] + missing_layers: list[str] + advertised_layer_count: int + metadata_url: str | None = None + metadata_identifier: str | None = None + remote_title: str | None = None + remote_version: str | None = None + remote_modified_at: datetime | None = None + remote_published_at: datetime | None = None + local_source_version: str | None = None + comparison_status: SourceCatalogComparisonStatus + capabilities_sha256: str | None = None + capabilities_etag: str | None = None + capabilities_last_modified_at: datetime | None = None + message: str + error_code: str | None = None + + +class SourceCatalogProbeSummary(BaseModel): + provider_count: int + available_count: int + degraded_count: int + unavailable_count: int + disabled_count: int + different_version_count: int + + +class SourceCatalogProbeReport(BaseModel): + project_id: UUID + generated_at: datetime + summary: SourceCatalogProbeSummary + items: list[SourceCatalogProbeItem] + limitations: list[str] diff --git a/backend/app/services/source_catalog_probe_service.py b/backend/app/services/source_catalog_probe_service.py new file mode 100644 index 00000000..fae36671 --- /dev/null +++ b/backend/app/services/source_catalog_probe_service.py @@ -0,0 +1,545 @@ +from __future__ import annotations + +from dataclasses import dataclass, replace +from datetime import datetime, timedelta, timezone +from email.utils import parsedate_to_datetime +from hashlib import sha256 +import re +from threading import Lock +from typing import Any, Callable +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from urllib.request import Request, urlopen +from uuid import UUID +from xml.etree import ElementTree + +from sqlalchemy.orm import Session + +from app.core.config import Settings, get_settings +from app.core.errors import AppError +from app.models import Dataset, Project +from app.schemas.source_catalog import ( + SourceCatalogProbeItem, + SourceCatalogProbeReport, + SourceCatalogProbeSummary, +) + + +_GMD = "http://www.isotc211.org/2005/gmd" +_GCO = "http://www.isotc211.org/2005/gco" +_WFS = "http://www.opengis.net/wfs/2.0" +_XLINK = "http://www.w3.org/1999/xlink" +_METADATA_HOST = "metadata.vlaanderen.be" +_VERSION_DATE = re.compile(r"^(?:toestand\s+)?(\d{4}-\d{2}-\d{2})$", re.IGNORECASE) +_ORTHOPHOTO_EDITION = re.compile(r"^\d{4}\.\d{2}$") + + +class CatalogProbeFailure(RuntimeError): + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +@dataclass(frozen=True) +class ProbeContract: + source_name: str + display_name: str + service_type: str + endpoint_url: str + expected_layers: tuple[str, ...] + + +@dataclass(frozen=True) +class FetchResult: + content: bytes + content_type: str + etag: str | None + last_modified_at: datetime | None + final_url: str + + +@dataclass(frozen=True) +class RemoteProbe: + status: str + reachable: bool + checked_at: datetime + expected_layers: tuple[str, ...] + matched_layers: tuple[str, ...] = () + missing_layers: tuple[str, ...] = () + advertised_layer_count: int = 0 + metadata_url: str | None = None + metadata_identifier: str | None = None + remote_title: str | None = None + remote_version: str | None = None + remote_modified_at: datetime | None = None + remote_published_at: datetime | None = None + capabilities_sha256: str | None = None + capabilities_etag: str | None = None + capabilities_last_modified_at: datetime | None = None + message: str = "" + error_code: str | None = None + cached: bool = False + + +@dataclass(frozen=True) +class CacheEntry: + expires_at: datetime + probe: RemoteProbe + + +_REMOTE_CACHE: dict[str, CacheEntry] = {} +_CACHE_LOCK = Lock() + + +def _utc(value: datetime | None = None) -> datetime: + current = value or datetime.now(timezone.utc) + if current.tzinfo is None: + return current.replace(tzinfo=timezone.utc) + return current.astimezone(timezone.utc) + + +def _header(headers: Any, name: str) -> str | None: + value = headers.get(name) if headers is not None else None + return str(value).strip() if value is not None and str(value).strip() else None + + +def _http_date(value: str | None) -> datetime | None: + if not value: + return None + try: + return _utc(parsedate_to_datetime(value)) + except (TypeError, ValueError, OverflowError): + return None + + +def _with_capabilities_query(base_url: str, service_type: str) -> str: + parsed = urlsplit(base_url) + retained = [(key, value) for key, value in parse_qsl(parsed.query) if key.lower() not in {"service", "request", "version"}] + version = "2.0.0" if service_type == "WFS" else "1.3.0" + query = urlencode([*retained, ("SERVICE", service_type), ("VERSION", version), ("REQUEST", "GetCapabilities")]) + return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, query, "")) + + +def _bounded_fetch(url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> FetchResult: + request = Request( + url, + headers={"Accept": "application/xml,text/xml,application/json", "User-Agent": "GeoIntel/0.1 source-catalog-probe"}, + ) + max_bytes = settings.source_catalog_probe_max_response_mb * 1024 * 1024 + try: + with (opener or urlopen)(request, timeout=settings.source_catalog_probe_timeout_seconds) as response: + content_length = _header(response.headers, "Content-Length") + if content_length: + try: + if int(content_length) > max_bytes: + raise CatalogProbeFailure("CATALOG_RESPONSE_TOO_LARGE", "De officiële metadatarespons overschrijdt de ingestelde limiet.") + except ValueError as exc: + raise CatalogProbeFailure("CATALOG_INVALID_RESPONSE", "De officiële metadatarespons bevat een ongeldige Content-Length.") from exc + content = response.read(max_bytes + 1) + result = FetchResult( + content=content, + content_type=_header(response.headers, "Content-Type") or "", + etag=_header(response.headers, "ETag"), + last_modified_at=_http_date(_header(response.headers, "Last-Modified")), + final_url=str(response.geturl()) if hasattr(response, "geturl") else url, + ) + except CatalogProbeFailure: + raise + except (HTTPError, URLError, TimeoutError, OSError) as exc: + raise CatalogProbeFailure("CATALOG_PROVIDER_UNAVAILABLE", "De officiële catalogus kon niet tijdig worden gelezen.") from exc + if len(result.content) > max_bytes: + raise CatalogProbeFailure("CATALOG_RESPONSE_TOO_LARGE", "De officiële metadatarespons overschrijdt de ingestelde limiet.") + return result + + +def _local_name(tag: str) -> str: + return tag.rsplit("}", 1)[-1] + + +def _direct_text(parent: ElementTree.Element, name: str) -> str | None: + for child in parent: + if _local_name(child.tag) == name and child.text and child.text.strip(): + return child.text.strip() + return None + + +def _metadata_urls(parent: ElementTree.Element) -> list[str]: + urls: list[str] = [] + for node in parent.iter(): + if _local_name(node.tag) not in {"MetadataURL", "MetadataUrl"}: + continue + href = node.attrib.get(f"{{{_XLINK}}}href") + if href: + urls.append(href.strip()) + for child in node.iter(): + if _local_name(child.tag) in {"URL", "OnlineResource"}: + nested = child.attrib.get(f"{{{_XLINK}}}href") or (child.text or "").strip() + if nested: + urls.append(nested) + return urls + + +def _parse_capabilities(content: bytes, contract: ProbeContract) -> tuple[list[str], str | None]: + try: + root = ElementTree.fromstring(content) + except ElementTree.ParseError as exc: + raise CatalogProbeFailure("CATALOG_INVALID_XML", "De capabilities-respons is geen geldige XML.") from exc + + layers: list[str] = [] + metadata_urls: list[str] = [] + node_name = "FeatureType" if contract.service_type == "WFS" else "Layer" + for node in root.iter(): + if _local_name(node.tag) != node_name: + continue + raw_name = _direct_text(node, "Name") + if not raw_name: + continue + name = raw_name.rsplit(":", 1)[-1] + layers.append(name) + if name in contract.expected_layers: + metadata_urls.extend(_metadata_urls(node)) + + expected = set(contract.expected_layers) + if not expected.intersection(layers): + raise CatalogProbeFailure("CATALOG_EXPECTED_LAYERS_MISSING", "De officiële service bevat geen van de verwachte lagen.") + xml_metadata = next((url for url in metadata_urls if "GetRecordById" in url and "OUTPUTSCHEMA" in url.upper()), None) + return sorted(set(layers)), xml_metadata + + +def _validate_metadata_url(url: str) -> str: + parsed = urlsplit(url) + if parsed.scheme != "https" or parsed.hostname != _METADATA_HOST or parsed.username or parsed.password: + raise CatalogProbeFailure("CATALOG_METADATA_URL_REJECTED", "De capabilities verwijzen niet naar de toegestane officiële metadatahost.") + if not parsed.path.startswith("/srv/dut/csw"): + raise CatalogProbeFailure("CATALOG_METADATA_URL_REJECTED", "De capabilities verwijzen niet naar het toegestane CSW-pad.") + query = {key.lower(): value for key, value in parse_qsl(parsed.query)} + if query.get("request", "").lower() != "getrecordbyid" or not query.get("id"): + raise CatalogProbeFailure("CATALOG_METADATA_URL_REJECTED", "De capabilities bevatten geen begrensde GetRecordById-verwijzing.") + return url + + +def _validate_capabilities_url(url: str) -> str: + parsed = urlsplit(url) + if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password: + raise CatalogProbeFailure("CATALOG_ENDPOINT_REJECTED", "De ingestelde capabilities-URL moet een geldige HTTP(S)-URL zonder credentials zijn.") + return url + + +def _node_text(node: ElementTree.Element | None) -> str | None: + if node is None: + return None + for descendant in node.iter(): + if descendant is not node and descendant.text and descendant.text.strip(): + return descendant.text.strip() + return node.text.strip() if node.text and node.text.strip() else None + + +def _parse_iso_datetime(value: str | None) -> datetime | None: + if not value: + return None + normalized = value.strip().replace("Z", "+00:00") + try: + return _utc(datetime.fromisoformat(normalized)) + except ValueError: + try: + return datetime.strptime(normalized, "%Y-%m-%d").replace(tzinfo=timezone.utc) + except ValueError: + return None + + +def _parse_metadata(content: bytes) -> dict[str, Any]: + try: + root = ElementTree.fromstring(content) + except ElementTree.ParseError as exc: + raise CatalogProbeFailure("CATALOG_INVALID_METADATA_XML", "Het officiële metadatarecord is geen geldige XML.") from exc + namespaces = {"gmd": _GMD, "gco": _GCO} + metadata = root.find(".//gmd:MD_Metadata", namespaces) + if metadata is None and _local_name(root.tag) == "MD_Metadata": + metadata = root + if metadata is None: + raise CatalogProbeFailure("CATALOG_METADATA_MISSING", "Het CSW-antwoord bevat geen ISO 19139 metadatarecord.") + citation = metadata.find(".//gmd:identificationInfo/*/gmd:citation/gmd:CI_Citation", namespaces) + title = _node_text(citation.find("gmd:title", namespaces) if citation is not None else None) + edition = _node_text(citation.find("gmd:edition", namespaces) if citation is not None else None) + identifier = _node_text(metadata.find("gmd:fileIdentifier", namespaces)) + modified = _parse_iso_datetime(_node_text(metadata.find("gmd:dateStamp", namespaces))) + published = None + if citation is not None: + for date_node in citation.findall("gmd:date/gmd:CI_Date", namespaces): + date_type = date_node.find("gmd:dateType/gmd:CI_DateTypeCode", namespaces) + if date_type is not None and date_type.attrib.get("codeListValue") == "publication": + published = _parse_iso_datetime(_node_text(date_node.find("gmd:date", namespaces))) + break + if not title or not edition: + raise CatalogProbeFailure("CATALOG_VERSION_MISSING", "Het officiële metadatarecord bevat geen herkenbare titel en editie.") + return { + "identifier": identifier, + "title": title, + "version": edition, + "modified_at": modified, + "published_at": published, + } + + +def _probe_remote( + contract: ProbeContract, + settings: Settings, + *, + opener: Callable[..., Any] | None, + now: datetime, +) -> RemoteProbe: + try: + capabilities = _bounded_fetch(_validate_capabilities_url(contract.endpoint_url), settings, opener) + _validate_capabilities_url(capabilities.final_url) + content_type = capabilities.content_type.lower() + if content_type and "xml" not in content_type and "text" not in content_type: + raise CatalogProbeFailure("CATALOG_INVALID_CONTENT_TYPE", "De officiële capabilities-respons is geen XML.") + layers, metadata_url = _parse_capabilities(capabilities.content, contract) + matched = tuple(layer for layer in contract.expected_layers if layer in layers) + missing = tuple(layer for layer in contract.expected_layers if layer not in layers) + digest = sha256(capabilities.content).hexdigest() + if not metadata_url: + return RemoteProbe( + status="degraded", + reachable=True, + checked_at=now, + expected_layers=contract.expected_layers, + matched_layers=matched, + missing_layers=missing, + advertised_layer_count=len(layers), + capabilities_sha256=digest, + capabilities_etag=capabilities.etag, + capabilities_last_modified_at=capabilities.last_modified_at, + message="De service is bereikbaar, maar publiceert geen machineleesbare ISO-metadata voor de verwachte lagen.", + error_code="CATALOG_METADATA_LINK_MISSING", + ) + metadata_url = _validate_metadata_url(metadata_url) + metadata_response = _bounded_fetch(metadata_url, settings, opener) + _validate_metadata_url(metadata_response.final_url) + metadata = _parse_metadata(metadata_response.content) + status = "degraded" if missing else "available" + message = ( + f"De officiële catalogus is bereikbaar en publiceert editie {metadata['version']}." + if not missing + else f"Editie {metadata['version']} is gevonden, maar niet alle verwachte lagen worden aangeboden." + ) + return RemoteProbe( + status=status, + reachable=True, + checked_at=now, + expected_layers=contract.expected_layers, + matched_layers=matched, + missing_layers=missing, + advertised_layer_count=len(layers), + metadata_url=metadata_url, + metadata_identifier=metadata["identifier"], + remote_title=metadata["title"], + remote_version=metadata["version"], + remote_modified_at=metadata["modified_at"], + remote_published_at=metadata["published_at"], + capabilities_sha256=digest, + capabilities_etag=capabilities.etag, + capabilities_last_modified_at=capabilities.last_modified_at, + message=message, + error_code="CATALOG_EXPECTED_LAYERS_INCOMPLETE" if missing else None, + ) + except CatalogProbeFailure as exc: + return RemoteProbe( + status="unavailable", + reachable=False, + checked_at=now, + expected_layers=contract.expected_layers, + missing_layers=contract.expected_layers, + message=exc.message, + error_code=exc.code, + ) + + +def _remote_with_cache( + contract: ProbeContract, + settings: Settings, + *, + force: bool, + opener: Callable[..., Any] | None, + now: datetime, +) -> RemoteProbe: + cache_key = f"{contract.source_name}|{contract.endpoint_url}|{','.join(contract.expected_layers)}" + if not force and settings.source_catalog_probe_cache_ttl_seconds > 0: + with _CACHE_LOCK: + entry = _REMOTE_CACHE.get(cache_key) + if entry and entry.expires_at > now: + return replace(entry.probe, cached=True) + probe = _probe_remote(contract, settings, opener=opener, now=now) + if settings.source_catalog_probe_cache_ttl_seconds > 0: + with _CACHE_LOCK: + _REMOTE_CACHE[cache_key] = CacheEntry( + expires_at=now + timedelta(seconds=settings.source_catalog_probe_cache_ttl_seconds), + probe=probe, + ) + return probe + + +def _dataset_source_name(dataset: Dataset) -> str: + return (dataset.source_name or dataset.source or "").strip().lower() + + +def _latest_local_version(source_name: str, datasets: list[Dataset]) -> str | None: + candidates = [item for item in datasets if _dataset_source_name(item) == source_name and item.source_version] + if source_name == "digitaal_vlaanderen_orthophoto": + explicit_current = [ + item + for item in candidates + if any(token in (item.source_version or "").lower() for token in ("most_recent", "latest", "current")) + ] + if explicit_current: + candidates = explicit_current + if not candidates: + return None + latest = max( + candidates, + key=lambda item: ( + _utc(item.imported_at) if item.imported_at else datetime.min.replace(tzinfo=timezone.utc), + _utc(item.observed_at) if item.observed_at else datetime.min.replace(tzinfo=timezone.utc), + str(item.id), + ), + ) + return latest.source_version + + +def _normalized_version(source_name: str, version: str | None) -> str | None: + if not version: + return None + value = version.strip() + if source_name == "grb": + match = _VERSION_DATE.fullmatch(value) + return match.group(1) if match else None + if source_name == "digitaal_vlaanderen_orthophoto" and _ORTHOPHOTO_EDITION.fullmatch(value): + return value + return None + + +def _comparison(source_name: str, local: str | None, remote: str | None, remote_status: str) -> str: + if remote_status in {"unavailable", "disabled"}: + return "unavailable" + if not local: + return "no_local_data" + normalized_local = _normalized_version(source_name, local) + normalized_remote = _normalized_version(source_name, remote) + if normalized_local is None or normalized_remote is None: + return "not_comparable" + return "same" if normalized_local == normalized_remote else "different" + + +class SourceCatalogProbeService: + @staticmethod + def clear_cache() -> None: + with _CACHE_LOCK: + _REMOTE_CACHE.clear() + + @staticmethod + def audit_project( + db: Session, + project_id: UUID, + *, + force: bool = False, + opener: Callable[..., Any] | None = None, + now: datetime | None = None, + settings: Settings | None = None, + ) -> SourceCatalogProbeReport: + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + active_settings = settings or get_settings() + generated_at = _utc(now) + datasets = db.query(Dataset).filter(Dataset.project_id == project_id).all() + contracts = ( + ProbeContract( + source_name="grb", + display_name="Basiskaart Vlaanderen (GRB)", + service_type="WFS", + endpoint_url=_with_capabilities_query(active_settings.source_catalog_grb_wfs_url, "WFS"), + expected_layers=("GBG", "WBN", "WGO", "ADP"), + ), + ProbeContract( + source_name="digitaal_vlaanderen_orthophoto", + display_name="Orthofoto Vlaanderen", + service_type="WMS", + endpoint_url=_with_capabilities_query(active_settings.orthophoto_wms_url, "WMS"), + expected_layers=(active_settings.orthophoto_wms_layer, "Vliegdagcontour"), + ), + ) + items: list[SourceCatalogProbeItem] = [] + for contract in contracts: + local_version = _latest_local_version(contract.source_name, datasets) + if active_settings.source_catalog_probe_enabled: + remote = _remote_with_cache( + contract, + active_settings, + force=force, + opener=opener, + now=generated_at, + ) + else: + remote = RemoteProbe( + status="disabled", + reachable=False, + checked_at=generated_at, + expected_layers=contract.expected_layers, + missing_layers=contract.expected_layers, + message="Officiële catalogusprobes zijn uitgeschakeld in de runtimeconfiguratie.", + error_code="CATALOG_PROBE_DISABLED", + ) + comparison = _comparison(contract.source_name, local_version, remote.remote_version, remote.status) + message = remote.message + if comparison == "different": + message += " De officiële editie verschilt van de lokaal vastgelegde bronversie; controleer dit handmatig vóór een begrensde verversing." + elif comparison == "not_comparable" and local_version: + message += " De lokale waarde is een opname- of importmarkering en kan niet eerlijk als officiële cataloguseditie worden vergeleken." + items.append( + SourceCatalogProbeItem( + source_name=contract.source_name, + display_name=contract.display_name, + service_type=contract.service_type, + endpoint_url=contract.endpoint_url, + status=remote.status, + reachable=remote.reachable, + checked_at=remote.checked_at, + cached=remote.cached, + expected_layers=list(remote.expected_layers), + matched_layers=list(remote.matched_layers), + missing_layers=list(remote.missing_layers), + advertised_layer_count=remote.advertised_layer_count, + metadata_url=remote.metadata_url, + metadata_identifier=remote.metadata_identifier, + remote_title=remote.remote_title, + remote_version=remote.remote_version, + remote_modified_at=remote.remote_modified_at, + remote_published_at=remote.remote_published_at, + local_source_version=local_version, + comparison_status=comparison, + capabilities_sha256=remote.capabilities_sha256, + capabilities_etag=remote.capabilities_etag, + capabilities_last_modified_at=remote.capabilities_last_modified_at, + message=message, + error_code=remote.error_code, + ) + ) + summary = SourceCatalogProbeSummary( + provider_count=len(items), + available_count=sum(item.status == "available" for item in items), + degraded_count=sum(item.status == "degraded" for item in items), + unavailable_count=sum(item.status == "unavailable" for item in items), + disabled_count=sum(item.status == "disabled" for item in items), + different_version_count=sum(item.comparison_status == "different" for item in items), + ) + return SourceCatalogProbeReport( + project_id=project_id, + generated_at=generated_at, + summary=summary, + items=items, + limitations=[ + "Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities en gekoppelde ISO 19139 metadata.", + "Er worden geen features, rasters of modelbestanden opgehaald en geen datasets aangemaakt of overschreven.", + "Een versieverschil is controlesignaal, geen bewijs dat een lokale dataset onbruikbaar is en geen automatische importopdracht.", + ], + ) diff --git a/backend/tests/test_sprint222_source_catalog_probes.py b/backend/tests/test_sprint222_source_catalog_probes.py new file mode 100644 index 00000000..23451d95 --- /dev/null +++ b/backend/tests/test_sprint222_source_catalog_probes.py @@ -0,0 +1,332 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from email.message import Message +from pathlib import Path +from types import SimpleNamespace +from urllib.error import URLError +import uuid + +from app.core.config import Settings +from app.models import Dataset +from app.services.source_catalog_probe_service import SourceCatalogProbeService + + +NOW = datetime(2026, 7, 16, 15, 0, tzinfo=timezone.utc) +GRB_METADATA_URL = ( + "https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&service=CSW&" + "id=7C823055-7BBF-4D62-B55E-F85C30D53162&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd" +) +ORTHO_METADATA_URL = ( + "https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&service=CSW&" + "id=f5304d6d-0dd4-43fd-a726-427af31e8d61&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd" +) + + +def _wfs_capabilities(metadata_url: str = GRB_METADATA_URL) -> bytes: + layers = "".join( + f""" + + GRB:{name} + + + """ + for name in ("GBG", "WBN", "WGO", "ADP", "WTZ") + ) + return f""" + + {layers} + + """.encode() + + +def _wms_capabilities(metadata_url: str = ORTHO_METADATA_URL) -> bytes: + return f""" + + Orthofoto + Orthotext/xml + + + Vliegdagcontourtext/xml + + + + + """.encode() + + +def _metadata(identifier: str, title: str, edition: str, modified: str, published: str) -> bytes: + return f""" + + + {identifier} + {modified} + + {title} + {edition} + {published} + publication + + + + + """.encode() + + +class _Response: + def __init__(self, content: bytes, content_type: str = "text/xml", *, content_length: int | None = None) -> None: + self.content = content + self.headers = Message() + self.headers["Content-Type"] = content_type + self.headers["Content-Length"] = str(content_length if content_length is not None else len(content)) + self.headers["ETag"] = '"catalog-test"' + self.headers["Last-Modified"] = "Wed, 15 Jul 2026 10:00:00 GMT" + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, size: int = -1) -> bytes: + return self.content if size < 0 else self.content[:size] + + +class _RedirectedResponse(_Response): + def __init__(self, content: bytes, final_url: str) -> None: + super().__init__(content) + self.final_url = final_url + + def geturl(self) -> str: + return self.final_url + +class _Query: + def __init__(self, datasets: list[Dataset]) -> None: + self.datasets = datasets + + def filter(self, *_args): + return self + + def all(self) -> list[Dataset]: + return self.datasets + + +class _Db: + def __init__(self, datasets: list[Dataset]) -> None: + self.datasets = datasets + + def get(self, _model, _identifier): + return SimpleNamespace(id=_identifier) + + def query(self, _model): + return _Query(self.datasets) + + +def _dataset(source_name: str, version: str) -> Dataset: + return Dataset( + id=uuid.uuid4(), + project_id=uuid.uuid4(), + name=f"{source_name} source", + dataset_type="vector" if source_name == "grb" else "raster", + source=source_name, + source_name=source_name, + source_version=version, + imported_at=NOW, + status="ready", + ) + + +def _settings(**overrides) -> Settings: + values = { + "SOURCE_CATALOG_PROBE_ENABLED": True, + "SOURCE_CATALOG_GRB_WFS_URL": "https://geo.api.vlaanderen.be/GRB/wfs", + "SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS": 3, + "SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB": 1, + "SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS": 0, + "ORTHOPHOTO_WMS_URL": "https://geo.api.vlaanderen.be/OMWRGBMRVL/wms", + "ORTHOPHOTO_WMS_LAYER": "Ortho", + } + values.update(overrides) + return Settings(**values) + + +def _opener(request, timeout): + assert timeout == 3 + url = request.full_url + if "metadata.vlaanderen.be" in url: + if "f5304d6d" in url: + return _Response(_metadata("f5304d6d", "Orthofoto meest recent, 2025.04", "2025.04", "2026-04-27", "2025-12-11")) + return _Response(_metadata("7C823055", "GRBgis", "Toestand 2026-07-15", "2026-07-15", "2026-07-15")) + if "/GRB/" in url: + return _Response(_wfs_capabilities()) + return _Response(_wms_capabilities()) + + +def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions() -> None: + SourceCatalogProbeService.clear_cache() + project_id = uuid.uuid4() + report = SourceCatalogProbeService.audit_project( + _Db([_dataset("grb", "2026-07-14"), _dataset("digitaal_vlaanderen_orthophoto", "most_recent_at_2026-07-14")]), + project_id, + settings=_settings(), + opener=_opener, + now=NOW, + ) + by_source = {item.source_name: item for item in report.items} + + assert report.summary.available_count == 2 + assert report.summary.different_version_count == 1 + assert by_source["grb"].remote_version == "Toestand 2026-07-15" + assert by_source["grb"].comparison_status == "different" + assert by_source["grb"].matched_layers == ["GBG", "WBN", "WGO", "ADP"] + assert by_source["grb"].advertised_layer_count == 5 + assert by_source["digitaal_vlaanderen_orthophoto"].remote_version == "2025.04" + assert by_source["digitaal_vlaanderen_orthophoto"].comparison_status == "not_comparable" + assert by_source["digitaal_vlaanderen_orthophoto"].remote_published_at.year == 2025 + assert all(item.capabilities_sha256 for item in report.items) + + +def test_catalog_probe_isolates_provider_failure() -> None: + def partial_opener(request, timeout): + if "/GRB/" in request.full_url: + raise URLError("offline") + return _opener(request, timeout) + + report = SourceCatalogProbeService.audit_project( + _Db([]), uuid.uuid4(), settings=_settings(), opener=partial_opener, now=NOW + ) + by_source = {item.source_name: item for item in report.items} + + assert by_source["grb"].status == "unavailable" + assert by_source["grb"].error_code == "CATALOG_PROVIDER_UNAVAILABLE" + assert by_source["digitaal_vlaanderen_orthophoto"].status == "available" + assert report.summary.unavailable_count == 1 + + +def test_catalog_probe_rejects_metadata_redirect_outside_allowlist() -> None: + def malicious_opener(request, timeout): + if "/GRB/" in request.full_url: + return _Response( + _wfs_capabilities( + "https://example.com/csw?request=GetRecordById&id=evil&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd" + ) + ) + return _opener(request, timeout) + + report = SourceCatalogProbeService.audit_project( + _Db([]), uuid.uuid4(), settings=_settings(), opener=malicious_opener, now=NOW + ) + grb = next(item for item in report.items if item.source_name == "grb") + + assert grb.status == "unavailable" + assert grb.error_code == "CATALOG_METADATA_URL_REJECTED" + + +def test_catalog_probe_enforces_response_limit_without_reading_external_data() -> None: + def oversized_opener(request, timeout): + if "/GRB/" in request.full_url: + return _Response(b"", content_length=2 * 1024 * 1024) + return _opener(request, timeout) + + report = SourceCatalogProbeService.audit_project( + _Db([]), uuid.uuid4(), settings=_settings(), opener=oversized_opener, now=NOW + ) + grb = next(item for item in report.items if item.source_name == "grb") + + assert grb.status == "unavailable" + assert grb.error_code == "CATALOG_RESPONSE_TOO_LARGE" + + +def test_catalog_probe_revalidates_metadata_host_after_redirect() -> None: + def redirected_opener(request, timeout): + if "metadata.vlaanderen.be" in request.full_url and "7C823055" in request.full_url: + return _RedirectedResponse( + _metadata("7C823055", "GRBgis", "Toestand 2026-07-15", "2026-07-15", "2026-07-15"), + "https://example.com/redirected-metadata", + ) + return _opener(request, timeout) + + report = SourceCatalogProbeService.audit_project( + _Db([]), uuid.uuid4(), settings=_settings(), opener=redirected_opener, now=NOW + ) + grb = next(item for item in report.items if item.source_name == "grb") + + assert grb.status == "unavailable" + assert grb.error_code == "CATALOG_METADATA_URL_REJECTED" + + +def test_catalog_probe_cache_is_explicitly_bypassable() -> None: + calls: list[str] = [] + + def counting_opener(request, timeout): + calls.append(request.full_url) + return _opener(request, timeout) + + SourceCatalogProbeService.clear_cache() + settings = _settings(SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900) + db = _Db([]) + project_id = uuid.uuid4() + first = SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW) + first_call_count = len(calls) + second = SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW) + SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW, force=True) + + assert first_call_count == 4 + assert all(item.cached is False for item in first.items) + assert all(item.cached is True for item in second.items) + assert len(calls) == 8 + + +def test_catalog_probe_can_be_disabled_without_network_access() -> None: + def forbidden_opener(*_args, **_kwargs): + raise AssertionError("network must not be called") + + report = SourceCatalogProbeService.audit_project( + _Db([]), + uuid.uuid4(), + settings=_settings(SOURCE_CATALOG_PROBE_ENABLED=False), + opener=forbidden_opener, + now=NOW, + ) + + assert report.summary.disabled_count == 2 + assert all(item.status == "disabled" for item in report.items) + + +def test_catalog_probe_route_returns_canonical_envelope(monkeypatch) -> None: + from app.api.routes import datasets as dataset_routes + + project_id = uuid.uuid4() + expected = SourceCatalogProbeService.audit_project( + _Db([]), project_id, settings=_settings(SOURCE_CATALOG_PROBE_ENABLED=False), now=NOW + ) + monkeypatch.setattr( + dataset_routes.SourceCatalogProbeService, + "audit_project", + lambda db, selected_project_id, force=False: expected, + ) + + response = dataset_routes.probe_dataset_source_catalogs( + project_id=project_id, refresh=False, db=SimpleNamespace() + ) + + assert list(response) == ["data"] + assert response["data"]["project_id"] == project_id + assert response["data"]["summary"]["provider_count"] == 2 + + +def test_catalog_probe_remains_explicit_and_never_imports_provider_data() -> None: + root = Path(__file__).resolve().parents[2] + hook = (root / "frontend" / "src" / "hooks" / "useSourceFreshness.ts").read_text(encoding="utf-8") + service = (root / "backend" / "app" / "services" / "source_catalog_probe_service.py").read_text(encoding="utf-8") + operator = (root / "scripts" / "audit_source_freshness.py").read_text(encoding="utf-8") + + assert "void probeCatalogs(" not in hook + assert "DatasetService" not in service + assert "VectorFeatureService" not in service + assert "--probe-catalogs" in operator + assert "/datasets/source-catalog-probes" in operator diff --git a/deploy/unraid/geointel-unraid-template.xml b/deploy/unraid/geointel-unraid-template.xml index fd577449..7f06fa13 100644 --- a/deploy/unraid/geointel-unraid-template.xml +++ b/deploy/unraid/geointel-unraid-template.xml @@ -34,6 +34,11 @@ https://geo.api.vlaanderen.be/OMWRGBMRVL/wms 1.0 1024 + true + https://geo.api.vlaanderen.be/GRB/wfs + 10 + 2 + 900 true https://geo.api.vlaanderen.be/DHMV/wcs 5.0 diff --git a/deploy/unraid/geointel.env.example b/deploy/unraid/geointel.env.example index 1e01892d..50797d53 100644 --- a/deploy/unraid/geointel.env.example +++ b/deploy/unraid/geointel.env.example @@ -32,6 +32,14 @@ ORTHOPHOTO_RESOLUTION_M=1.0 ORTHOPHOTO_MIN_SIDE_M=128 ORTHOPHOTO_MAX_SIDE_M=1024 ORTHOPHOTO_CACHE_TTL_HOURS=24 + +# Explicit read-only edition checks for GRB and the most-recent orthophoto. +# No feature or raster data is downloaded by these probes. +SOURCE_CATALOG_PROBE_ENABLED=true +SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs +SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10 +SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2 +SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900 DHMV_ENABLED=true DHMV_WCS_URL=https://geo.api.vlaanderen.be/DHMV/wcs DHMV_RESOLUTION_M=5.0 diff --git a/deploy/unraid/run-dockerman-container.sh b/deploy/unraid/run-dockerman-container.sh index fd241acd..2b762306 100644 --- a/deploy/unraid/run-dockerman-container.sh +++ b/deploy/unraid/run-dockerman-container.sh @@ -27,6 +27,11 @@ ORTHOPHOTO_RESOLUTION_M="${ORTHOPHOTO_RESOLUTION_M:-1.0}" ORTHOPHOTO_MIN_SIDE_M="${ORTHOPHOTO_MIN_SIDE_M:-128}" ORTHOPHOTO_MAX_SIDE_M="${ORTHOPHOTO_MAX_SIDE_M:-1024}" ORTHOPHOTO_CACHE_TTL_HOURS="${ORTHOPHOTO_CACHE_TTL_HOURS:-24}" +SOURCE_CATALOG_PROBE_ENABLED="${SOURCE_CATALOG_PROBE_ENABLED:-true}" +SOURCE_CATALOG_GRB_WFS_URL="${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}" +SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS="${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}" +SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB="${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}" +SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}" DHMV_ENABLED="${DHMV_ENABLED:-true}" DHMV_WCS_URL="${DHMV_WCS_URL:-https://geo.api.vlaanderen.be/DHMV/wcs}" DHMV_RESOLUTION_M="${DHMV_RESOLUTION_M:-5.0}" @@ -126,6 +131,11 @@ docker run -d \ -e ORTHOPHOTO_MIN_SIDE_M="$ORTHOPHOTO_MIN_SIDE_M" \ -e ORTHOPHOTO_MAX_SIDE_M="$ORTHOPHOTO_MAX_SIDE_M" \ -e ORTHOPHOTO_CACHE_TTL_HOURS="$ORTHOPHOTO_CACHE_TTL_HOURS" \ + -e SOURCE_CATALOG_PROBE_ENABLED="$SOURCE_CATALOG_PROBE_ENABLED" \ + -e SOURCE_CATALOG_GRB_WFS_URL="$SOURCE_CATALOG_GRB_WFS_URL" \ + -e SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS="$SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS" \ + -e SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB="$SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB" \ + -e SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="$SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS" \ -e DHMV_ENABLED="$DHMV_ENABLED" \ -e DHMV_WCS_URL="$DHMV_WCS_URL" \ -e DHMV_RESOLUTION_M="$DHMV_RESOLUTION_M" \ diff --git a/docker-compose.unraid.yml b/docker-compose.unraid.yml index e307df27..b68dc52a 100644 --- a/docker-compose.unraid.yml +++ b/docker-compose.unraid.yml @@ -25,6 +25,11 @@ services: ORTHOPHOTO_MIN_SIDE_M: ${ORTHOPHOTO_MIN_SIDE_M:-128} ORTHOPHOTO_MAX_SIDE_M: ${ORTHOPHOTO_MAX_SIDE_M:-1024} ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24} + SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true} + SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs} + SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS: ${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10} + SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB: ${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2} + SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS: ${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900} DHMV_ENABLED: ${DHMV_ENABLED:-true} DHMV_WCS_URL: ${DHMV_WCS_URL:-https://geo.api.vlaanderen.be/DHMV/wcs} DHMV_RESOLUTION_M: ${DHMV_RESOLUTION_M:-5.0} diff --git a/docker-compose.yml b/docker-compose.yml index 1c253397..d2500ea5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,11 @@ services: ORTHOPHOTO_MIN_SIDE_M: ${ORTHOPHOTO_MIN_SIDE_M:-128} ORTHOPHOTO_MAX_SIDE_M: ${ORTHOPHOTO_MAX_SIDE_M:-1024} ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24} + SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true} + SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs} + SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS: ${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10} + SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB: ${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2} + SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS: ${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900} DHMV_ENABLED: ${DHMV_ENABLED:-true} DHMV_WCS_URL: ${DHMV_WCS_URL:-https://geo.api.vlaanderen.be/DHMV/wcs} DHMV_RESOLUTION_M: ${DHMV_RESOLUTION_M:-5.0} diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 921dae81..744c08dc 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -412,6 +412,26 @@ Dataset or silently refreshes a publication. Fixed editions and scenarios are not marked stale merely because their source date is old. Unknown sources are `review_required` until an explicit publication policy is defined. +### GET `/api/v1/projects/{project_id}/datasets/source-catalog-probes` + +Performs an explicit, read-only release probe for the allowlisted GRB WFS and +most-recent orthophoto WMS. The endpoint reads bounded `GetCapabilities` +responses, confirms the expected layers and follows only HTTPS ISO 19139 +`GetRecordById` links on `metadata.vlaanderen.be`. Query parameter +`refresh=true` bypasses the short in-memory response cache. + +Each provider item returns service reachability, matched/missing layers, the +official metadata identifier, title, edition, publication/metadata dates, +local `source_version` and one comparison status: `same`, `different`, +`not_comparable`, `no_local_data` or `unavailable`. Provider status is +`available`, `degraded`, `unavailable` or `disabled`. A difference means only +that an operator should review provenance; it is not an update instruction. + +The endpoint accepts no arbitrary URL, feature query, area or layer. It does +not fetch vector features, raster pixels or models, create jobs/datasets, write +to PostGIS or trigger an import. The normal `source-freshness` endpoint remains +local-only and never invokes this probe implicitly. + ### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}` Return metadata. diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 915e7338..3696182c 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -9302,3 +9302,40 @@ Next: - Add machine-readable catalogue release probes only for providers with stable official version endpoints, keeping every acquisition an explicit bounded operator action. + +## Sprint 222 - Official source edition probes (2026-07-16) + +Implemented: +- Verified the public machine-readable contracts published by Digitaal + Vlaanderen. GRB WFS and OMWRGBMRVL WMS capabilities expose stable ISO 19139 + CSW records containing real source editions and metadata dates. +- Added a canonical project endpoint that reads only allowlisted capabilities + and linked `metadata.vlaanderen.be` `GetRecordById` responses. It confirms + GRB `GBG/WBN/WGO/ADP` and orthophoto `Ortho/Vliegdagcontour` availability, + then reports the official edition beside local `source_version` evidence. +- Added strict HTTP(S)/metadata allowlists, timeout and byte limits, + post-redirect validation, SHA-256 capabilities evidence, provider-isolated + degraded/unavailable states and a short in-memory cache. +- Added an explicit Status action and operator CLI flags. Neither path runs on + page load, starts a job, downloads source data or changes a Dataset. +- Added Compose, all-in-one Unraid and DockerMan controls for the bounded probe. + +Validation: +- Deterministic tests cover official XML parsing, edition comparison, + provider failure isolation, metadata URL rejection, size limits, caching, + disabled mode and canonical envelopes. +- Frontend typecheck/build and targeted backend tests pass before the full + release and Tower validation recorded in the delivery result. + +Boundaries: +- The general Datavindplaats API requires authentication and is not scraped or + silently bypassed. GeoIntel uses only public metadata records advertised by + the official OGC services. +- A version difference remains a manual provenance-review signal. No automatic + import, refresh scheduler, GRB feature fetch or orthophoto pixel request was + added. + +Next: +- Extend the same probe pattern only where another official source publishes a + stable machine-readable edition. Do not infer release versions from service + protocol versions, ETags or HTTP modification dates. diff --git a/docs/ENVIRONMENT_SPEC.md b/docs/ENVIRONMENT_SPEC.md index bda34307..ea6d6087 100644 --- a/docs/ENVIRONMENT_SPEC.md +++ b/docs/ENVIRONMENT_SPEC.md @@ -25,6 +25,13 @@ YOLO_MAX_DETECTIONS=1000 YOLO_DUPLICATE_IOU_THRESHOLD=0.5 ENABLE_GRB_WFS=false GRB_WFS_URL= + +# Explicit read-only source-edition probes (no provider import). +SOURCE_CATALOG_PROBE_ENABLED=true +SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs +SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10 +SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2 +SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900 OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter ``` diff --git a/docs/TODO.md b/docs/TODO.md index 20479f04..08d26d53 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -39,7 +39,8 @@ - [x] Execute and audit the complete 336-product VMM and 56-product DHMV regional runtime matrices. - [x] Present municipal DHMV/VMM partitions as logical regional layers and analyse cross-boundary rectangles without a municipality prerequisite. - [x] Add a read-only source freshness/version audit with explicit publication policies, local integrity checks, operator CLI and compact Status workspace surface. -- [ ] Add external catalogue release probes only after each official provider exposes a stable machine-readable version contract; keep every refresh explicit and bounded. +- [x] Add explicit bounded GRB/orthophoto catalogue release probes using official capabilities and ISO metadata; keep every acquisition separate and operator-controlled. +- [ ] Extend catalogue probes only to additional sources that publish a stable machine-readable edition contract; do not add background polling or infer releases from HTTP dates alone. ## Governed source expansion backlog diff --git a/frontend/README.md b/frontend/README.md index b0c53a34..0eb91901 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -9,7 +9,10 @@ surface. It separates sources that are current, due for a catalogue review, require local integrity review or are local artifacts. Only attention items are expanded by default; all source detail remains available through disclosure. The refresh button reruns the local read-only audit and never downloads or -replaces source data. +replaces source data. A separate `Officiële edities controleren` action is the +only trigger for bounded GRB/orthophoto catalog reads. It presents official and +local editions, layer-contract coverage and honest non-comparable version +markers without starting an import or background poll. The user-facing shell is task based: `Kaart`, `Bronnen`, `Kwaliteit`, `Beeldanalyse`, `Downloads`, `Status` and `Beheer`. Internal benchmark projects, raw dataset metadata, provider capabilities, model registry details and QA evidence remain accessible through labelled advanced disclosures instead of competing with the normal workflow. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 72d4b885..9d01c1a9 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -860,6 +860,10 @@ function App(): JSX.Element { loading={sourceFreshness.loading} error={sourceFreshness.error} onRefresh={() => { void sourceFreshness.refresh() }} + catalogReport={sourceFreshness.catalogReport} + catalogLoading={sourceFreshness.catalogLoading} + catalogError={sourceFreshness.catalogError} + onProbeCatalogs={(force) => { void sourceFreshness.probeCatalogs(force) }} />
diff --git a/frontend/src/components/status/SourceFreshnessPanel.tsx b/frontend/src/components/status/SourceFreshnessPanel.tsx index 8fec2012..2e089ded 100644 --- a/frontend/src/components/status/SourceFreshnessPanel.tsx +++ b/frontend/src/components/status/SourceFreshnessPanel.tsx @@ -1,10 +1,22 @@ -import type { SourceFreshnessItem, SourceFreshnessReport, SourceFreshnessStatus } from '../../types' +import type { + SourceCatalogComparisonStatus, + SourceCatalogProbeItem, + SourceCatalogProbeReport, + SourceCatalogProbeStatus, + SourceFreshnessItem, + SourceFreshnessReport, + SourceFreshnessStatus, +} from '../../types' interface SourceFreshnessPanelProps { report: SourceFreshnessReport | null loading: boolean error: string | null onRefresh: () => void + catalogReport: SourceCatalogProbeReport | null + catalogLoading: boolean + catalogError: string | null + onProbeCatalogs: (force: boolean) => void } function statusLabel(status: SourceFreshnessStatus): string { @@ -14,6 +26,21 @@ function statusLabel(status: SourceFreshnessStatus): string { return 'lokaal' } +function catalogStatusLabel(status: SourceCatalogProbeStatus): string { + if (status === 'available') return 'bereikbaar' + if (status === 'degraded') return 'beperkt' + if (status === 'unavailable') return 'niet bereikbaar' + return 'uitgeschakeld' +} + +function comparisonLabel(status: SourceCatalogComparisonStatus): string { + if (status === 'same') return 'zelfde editie' + if (status === 'different') return 'verschil controleren' + if (status === 'no_local_data') return 'nog niet lokaal ingeladen' + if (status === 'not_comparable') return 'andere versienotatie' + return 'niet vergelijkbaar' +} + function formatDate(value?: string | null): string { if (!value) return 'geen datum' return new Intl.DateTimeFormat('nl-BE', { dateStyle: 'medium' }).format(new Date(value)) @@ -23,6 +50,27 @@ function integrityIssueCount(item: SourceFreshnessItem): number { return Object.values(item.integrity).reduce((total, value) => total + value, 0) } +function CatalogRow({ item }: { item: SourceCatalogProbeItem }): JSX.Element { + return ( +
+
+
+ {item.display_name} + {catalogStatusLabel(item.status)} / {comparisonLabel(item.comparison_status)} +
+ {item.remote_version ?? 'geen editie'} +
+

{item.message}

+
+ Lokaal: {item.local_source_version ?? 'niet aanwezig'} + Lagen: {item.matched_layers.length}/{item.expected_layers.length} bevestigd + {item.remote_modified_at ? Metadata: {formatDate(item.remote_modified_at)} : null} + {item.cached ? cache gebruikt : null} +
+
+ ) +} + function SourceRow({ item }: { item: SourceFreshnessItem }): JSX.Element { const issueCount = integrityIssueCount(item) return ( @@ -30,7 +78,7 @@ function SourceRow({ item }: { item: SourceFreshnessItem }): JSX.Element {
{item.display_name} - {item.dataset_count} datasets · {item.version_count} versies + {item.dataset_count} datasets / {item.version_count} versies
{statusLabel(item.status)}
@@ -45,7 +93,16 @@ function SourceRow({ item }: { item: SourceFreshnessItem }): JSX.Element { ) } -export function SourceFreshnessPanel({ report, loading, error, onRefresh }: SourceFreshnessPanelProps): JSX.Element { +export function SourceFreshnessPanel({ + report, + loading, + error, + onRefresh, + catalogReport, + catalogLoading, + catalogError, + onProbeCatalogs, +}: SourceFreshnessPanelProps): JSX.Element { const attentionItems = report?.items.filter((item) => item.status === 'due' || item.status === 'review_required') ?? [] const summary = report?.summary @@ -55,11 +112,21 @@ export function SourceFreshnessPanel({ report, loading, error, onRefresh }: Sour

Bronbeheer

Actualiteit en versiecontrole

-

Controleert lokale publicaties, versies en bestanden zonder externe bronnen automatisch te wijzigen.

+

Controleert lokale publicaties en kan op aanvraag de officiële GRB- en orthofoto-editie uitlezen.

+
+
+ +
- {error ?

{error}

: null} @@ -89,6 +156,31 @@ export function SourceFreshnessPanel({ report, loading, error, onRefresh }: Sour

{report.limitations.join(' ')}

+
+
+
+ Officiële catalogusedities + Alleen na jouw expliciete controle; er wordt niets ingeladen of vervangen. +
+ {catalogReport ? ( + {catalogReport.summary.available_count}/{catalogReport.summary.provider_count} bereikbaar + ) : null} +
+ {catalogError ?

{catalogError}

: null} + {!catalogReport && !catalogError ? ( +

Controleer GRB en orthofoto wanneer je een lokale bronversie met de officiële editie wilt vergelijken.

+ ) : null} + {catalogReport ? ( + <> +
+ {catalogReport.items.map((item) => )} +
+

+ Gecontroleerd {formatDate(catalogReport.generated_at)}. Een verschil vraagt menselijke beoordeling en start nooit automatisch een import. +

+ + ) : null} +
) : null} diff --git a/frontend/src/hooks/useSourceFreshness.ts b/frontend/src/hooks/useSourceFreshness.ts index 4c1a2c9c..ec3549a1 100644 --- a/frontend/src/hooks/useSourceFreshness.ts +++ b/frontend/src/hooks/useSourceFreshness.ts @@ -1,13 +1,17 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { datasetsApi } from '../services/api' -import type { SourceFreshnessReport } from '../types' +import type { SourceCatalogProbeReport, SourceFreshnessReport } from '../types' export function useSourceFreshness(selectedProjectId: string | null) { const [report, setReport] = useState(null) const [loading, setLoading] = useState(false) const [error, setError] = useState(null) const requestSequence = useRef(0) + const catalogRequestSequence = useRef(0) + const [catalogReport, setCatalogReport] = useState(null) + const [catalogLoading, setCatalogLoading] = useState(false) + const [catalogError, setCatalogError] = useState(null) const refresh = useCallback(async () => { const requestId = ++requestSequence.current @@ -35,6 +39,32 @@ export function useSourceFreshness(selectedProjectId: string | null) { } }, [selectedProjectId]) + const probeCatalogs = useCallback(async (force = false) => { + const requestId = ++catalogRequestSequence.current + if (!selectedProjectId) { + setCatalogReport(null) + setCatalogError(null) + setCatalogLoading(false) + return + } + setCatalogLoading(true) + setCatalogError(null) + try { + const nextReport = await datasetsApi.sourceCatalogProbes(selectedProjectId, force) + if (catalogRequestSequence.current === requestId) { + setCatalogReport(nextReport) + } + } catch (caught) { + if (catalogRequestSequence.current === requestId) { + setCatalogError(caught instanceof Error ? caught.message : 'De officiële broncatalogi konden niet worden gecontroleerd.') + } + } finally { + if (catalogRequestSequence.current === requestId) { + setCatalogLoading(false) + } + } + }, [selectedProjectId]) + useEffect(() => { void refresh() return () => { @@ -42,5 +72,21 @@ export function useSourceFreshness(selectedProjectId: string | null) { } }, [refresh]) - return { report, loading, error, refresh } + useEffect(() => { + catalogRequestSequence.current += 1 + setCatalogReport(null) + setCatalogError(null) + setCatalogLoading(false) + }, [selectedProjectId]) + + return { + report, + loading, + error, + refresh, + catalogReport, + catalogLoading, + catalogError, + probeCatalogs, + } } diff --git a/frontend/src/services/api/datasets.ts b/frontend/src/services/api/datasets.ts index 3bdb78b5..6c17eab1 100644 --- a/frontend/src/services/api/datasets.ts +++ b/frontend/src/services/api/datasets.ts @@ -28,6 +28,7 @@ import type { ThematicRasterProductRead, ThematicRasterSelectionResponse, SourceFreshnessReport, + SourceCatalogProbeReport, } from '../../types' const DATASET_PAGE_SIZE = 200 @@ -61,6 +62,10 @@ export const datasetsApi = { list: listProjectDatasets, sourceFreshness: (projectId: string): Promise => apiGet(`/api/v1/projects/${projectId}/datasets/source-freshness`), + sourceCatalogProbes: (projectId: string, refresh = false): Promise => + apiGet( + `/api/v1/projects/${projectId}/datasets/source-catalog-probes?refresh=${refresh ? 'true' : 'false'}`, + ), upload: ( projectId: string, payload: { diff --git a/frontend/src/styles/premium.css b/frontend/src/styles/premium.css index 46d13a20..74d8080e 100644 --- a/frontend/src/styles/premium.css +++ b/frontend/src/styles/premium.css @@ -836,13 +836,20 @@ details.ai-lab-model-surface > summary strong { padding: 1rem 1.2rem; } -.source-freshness-header .secondary-action { +.source-freshness-header .secondary-action, +.source-freshness-header .primary-action { flex: 0 0 auto; width: auto; min-width: 10rem; margin-top: 0; } +.source-freshness-actions { + display: flex; + flex: 0 0 auto; + gap: 0.55rem; +} + .source-freshness-header h2 { margin: 0; font-size: 1.05rem; @@ -959,6 +966,81 @@ details.ai-lab-model-surface > summary strong { border-top: 1px solid var(--line); } +.source-catalog-section { + border-top: 1px solid var(--line); + background: #fbfcfd; +} + +.source-catalog-heading { + display: flex; + gap: 1rem; + align-items: center; + justify-content: space-between; + padding: 0.8rem 1rem; +} + +.source-catalog-heading > div { + display: grid; + gap: 0.12rem; +} + +.source-catalog-heading strong { + color: #23313b; + font-size: 0.84rem; +} + +.source-catalog-heading span, +.source-catalog-empty { + color: var(--muted); + font-size: 0.75rem; +} + +.source-catalog-heading > span { + flex: 0 0 auto; + font-weight: 700; +} + +.source-catalog-empty { + margin: 0; + padding: 0 1rem 0.85rem; +} + +.source-catalog-list { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + border-top: 1px solid var(--line); +} + +.source-catalog-row { + min-width: 0; + padding: 0.8rem 1rem; + border-right: 1px solid var(--line); + box-shadow: inset 0 3px 0 #8a98a3; +} + +.source-catalog-row:last-child { + border-right: 0; +} + +.source-catalog-row-available { + box-shadow: inset 0 3px 0 #2d9272; +} + +.source-catalog-row-degraded { + box-shadow: inset 0 3px 0 #ca7a18; +} + +.source-catalog-row-unavailable, +.source-catalog-row-disabled { + box-shadow: inset 0 3px 0 #b84e4e; +} + +.source-catalog-row p { + margin: 0.45rem 0; + color: var(--muted); + font-size: 0.75rem; +} + .source-freshness-details > summary { display: flex; cursor: pointer; @@ -982,10 +1064,15 @@ details.ai-lab-model-surface > summary strong { flex-direction: column; } - .source-freshness-header .secondary-action { + .source-freshness-header .secondary-action, + .source-freshness-header .primary-action { width: 100%; } + .source-freshness-actions { + flex-direction: column; + } + .source-freshness-summary { grid-template-columns: repeat(2, minmax(0, 1fr)); } @@ -997,6 +1084,19 @@ details.ai-lab-model-surface > summary strong { .source-freshness-summary span:nth-child(-n + 2) { border-bottom: 1px solid var(--line); } + + .source-catalog-list { + grid-template-columns: 1fr; + } + + .source-catalog-row { + border-right: 0; + border-bottom: 1px solid var(--line); + } + + .source-catalog-row:last-child { + border-bottom: 0; + } } .workflow-guidance-panel { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 8bc93812..e8c4915e 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -709,6 +709,52 @@ export interface SourceFreshnessReport { limitations: string[] } +export type SourceCatalogProbeStatus = 'available' | 'degraded' | 'unavailable' | 'disabled' +export type SourceCatalogComparisonStatus = 'same' | 'different' | 'not_comparable' | 'no_local_data' | 'unavailable' + +export interface SourceCatalogProbeItem { + source_name: string + display_name: string + service_type: 'WFS' | 'WMS' + endpoint_url: string + status: SourceCatalogProbeStatus + reachable: boolean + checked_at: string + cached: boolean + expected_layers: string[] + matched_layers: string[] + missing_layers: string[] + advertised_layer_count: number + metadata_url?: string | null + metadata_identifier?: string | null + remote_title?: string | null + remote_version?: string | null + remote_modified_at?: string | null + remote_published_at?: string | null + local_source_version?: string | null + comparison_status: SourceCatalogComparisonStatus + capabilities_sha256?: string | null + capabilities_etag?: string | null + capabilities_last_modified_at?: string | null + message: string + error_code?: string | null +} + +export interface SourceCatalogProbeReport { + project_id: string + generated_at: string + summary: { + provider_count: number + available_count: number + degraded_count: number + unavailable_count: number + disabled_count: number + different_version_count: number + } + items: SourceCatalogProbeItem[] + limitations: string[] +} + export interface GeojsonEnvelopeResponse { data: object } diff --git a/scripts/README.md b/scripts/README.md index e5ce9c4e..b8a30e13 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1708,6 +1708,14 @@ uses one canonical `GET`, changes no application data and performs no source download. It can therefore be scheduled explicitly through Unraid cron without turning GeoIntel into a real-time monitoring system. +Add `--probe-catalogs` to explicitly read the allowlisted official GRB and +orthophoto capabilities plus linked ISO metadata. `--refresh-catalogs` bypasses +the short server cache and implies the probe. `--fail-on-catalog` makes a +degraded/unavailable official metadata service fail the command. With a probe, +JSON output contains `source_freshness` and `catalog_probes`; without it, the +original local report shape is unchanged. No flag downloads provider features +or imagery and no flag writes a Dataset. + ## Tower deployment Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime: diff --git a/scripts/audit_source_freshness.py b/scripts/audit_source_freshness.py index c98bdd8e..40c2769a 100644 --- a/scripts/audit_source_freshness.py +++ b/scripts/audit_source_freshness.py @@ -26,6 +26,21 @@ def parse_args() -> argparse.Namespace: default="integrity", help="Non-zero exit policy for cron or release automation", ) + parser.add_argument( + "--probe-catalogs", + action="store_true", + help="Explicitly query the allowlisted official GRB and orthophoto metadata catalogs", + ) + parser.add_argument( + "--refresh-catalogs", + action="store_true", + help="Bypass the short server-side catalog cache (implies --probe-catalogs)", + ) + parser.add_argument( + "--fail-on-catalog", + action="store_true", + help="Exit non-zero when an explicitly requested official catalog is degraded or unavailable", + ) return parser.parse_args() @@ -45,7 +60,24 @@ def fetch_report(api_url: str, project_id: str, timeout: float) -> dict: return payload["data"] -def print_text(report: dict) -> None: +def fetch_catalog_report(api_url: str, project_id: str, timeout: float, refresh: bool) -> dict: + query = "true" if refresh else "false" + endpoint = f"{api_url.rstrip('/')}/projects/{project_id}/datasets/source-catalog-probes?refresh={query}" + request = Request(endpoint, headers={"Accept": "application/json", "User-Agent": "GeoIntel-source-audit/1.0"}) + try: + with urlopen(request, timeout=timeout) as response: + payload = json.load(response) + except HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"GeoIntel catalog probe returned HTTP {exc.code}: {body}") from exc + except URLError as exc: + raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc + if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict): + raise RuntimeError("Catalog probe response is not a canonical GeoIntel data envelope") + return payload["data"] + + +def print_text(report: dict, *, catalog_queried: bool = False) -> None: summary = report.get("summary", {}) print( "GeoIntel source audit: " @@ -59,7 +91,30 @@ def print_text(report: dict) -> None: status = item.get("status", "unknown") if status not in {"current", "local"} or item.get("integrity", {}).get("missing_version_count", 0): print(f"- {status:15} {item.get('display_name', item.get('source_name'))}: {item.get('reason', '')}") - print("No external catalog was queried and no dataset was modified.") + if catalog_queried: + print("The official catalog was queried read-only; no feature, raster or dataset was modified.") + else: + print("No external catalog was queried and no dataset was modified.") + + +def print_catalog_text(report: dict) -> None: + summary = report.get("summary", {}) + print( + "Official catalog probe: " + f"{summary.get('available_count', 0)}/{summary.get('provider_count', 0)} available, " + f"{summary.get('degraded_count', 0)} degraded, " + f"{summary.get('unavailable_count', 0)} unavailable, " + f"{summary.get('different_version_count', 0)} version differences" + ) + for item in report.get("items", []): + remote = item.get("remote_version") or "no official edition" + local = item.get("local_source_version") or "not loaded locally" + print( + f"- {item.get('display_name', item.get('source_name'))}: " + f"{item.get('status')} / official={remote} / local={local} / " + f"comparison={item.get('comparison_status')}" + ) + print(f" {item.get('message', '')}") def should_fail(report: dict, fail_on: str) -> bool: @@ -76,22 +131,42 @@ def should_fail(report: dict, fail_on: str) -> bool: return integrity > 0 or due > 0 or review > 0 +def catalog_should_fail(report: dict) -> bool: + summary = report.get("summary", {}) + return int(summary.get("degraded_count", 0)) > 0 or int(summary.get("unavailable_count", 0)) > 0 + + def main() -> int: args = parse_args() try: report = fetch_report(args.api_url, args.project_id, args.timeout) + catalog_report = ( + fetch_catalog_report(args.api_url, args.project_id, args.timeout, args.refresh_catalogs) + if args.probe_catalogs or args.refresh_catalogs + else None + ) except RuntimeError as exc: print(str(exc), file=sys.stderr) return 2 - serialized = json.dumps(report, indent=2, sort_keys=True) + output_payload = ( + {"source_freshness": report, "catalog_probes": catalog_report} + if catalog_report is not None + else report + ) + serialized = json.dumps(output_payload, indent=2, sort_keys=True) if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(serialized + "\n", encoding="utf-8") if args.json: print(serialized) else: - print_text(report) - return 1 if should_fail(report, args.fail_on) else 0 + print_text(report, catalog_queried=catalog_report is not None) + if catalog_report is not None: + print_catalog_text(catalog_report) + failed = should_fail(report, args.fail_on) + if args.fail_on_catalog and catalog_report is not None: + failed = failed or catalog_should_fail(catalog_report) + return 1 if failed else 0 if __name__ == "__main__":