Add official source edition probes
This commit is contained in:
@@ -12,6 +12,11 @@ ORTHOPHOTO_RESOLUTION_M=1.0
|
||||
ORTHOPHOTO_MIN_SIDE_M=128
|
||||
ORTHOPHOTO_MAX_SIDE_M=1024
|
||||
ORTHOPHOTO_CACHE_TTL_HOURS=24
|
||||
SOURCE_CATALOG_PROBE_ENABLED=true
|
||||
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
|
||||
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10
|
||||
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2
|
||||
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900
|
||||
DHMV_ENABLED=true
|
||||
DHMV_WCS_URL=https://geo.api.vlaanderen.be/DHMV/wcs
|
||||
DHMV_RESOLUTION_M=5.0
|
||||
|
||||
@@ -7,6 +7,18 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 222 Official source edition probes (2026-07-16)
|
||||
|
||||
- Added explicit, read-only GRB and most-recent orthophoto catalog probes using
|
||||
official WFS/WMS capabilities and linked ISO 19139 CSW metadata records.
|
||||
- Added bounded response sizes, timeouts, provider-isolated failures, a short
|
||||
cache and strict metadata-host/path validation. No provider features, raster
|
||||
pixels or application data are fetched or modified.
|
||||
- Added canonical API, compact opt-in Status UI and operator CLI support. The
|
||||
local source-freshness audit remains automatic and provider-free.
|
||||
- Added deterministic XML/network-boundary tests and editable Compose/Unraid
|
||||
controls without changing migrations or persistence contracts.
|
||||
|
||||
## Sprint 221 Governed source freshness audit (2026-07-16)
|
||||
|
||||
- Added a project-wide read-only source report derived from Dataset,
|
||||
|
||||
@@ -1443,3 +1443,29 @@ docker exec geointel python /app/scripts/audit_source_freshness.py \
|
||||
|
||||
Use `--fail-on due` to make a planned review date fail automation, or
|
||||
`--fail-on never` for reporting only. The command never starts a refresh.
|
||||
|
||||
An operator can explicitly add the official GRB and orthophoto edition check:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/audit_source_freshness.py \
|
||||
--project-id 82a85913-c522-45d7-84a1-02b393d89e55 \
|
||||
--api-url http://127.0.0.1/api/v1 \
|
||||
--probe-catalogs \
|
||||
--output /app/storage/operator-evidence/source-freshness/with-catalogs.json
|
||||
```
|
||||
|
||||
Use `--refresh-catalogs` to bypass the 15-minute in-memory cache and
|
||||
`--fail-on-catalog` only when temporary official-provider unavailability must
|
||||
fail an operator job. This path reads bounded WFS/WMS capabilities and their
|
||||
fixed ISO 19139 metadata records. It confirms GRB `GBG`, `WBN`, `WGO`, `ADP`
|
||||
and orthophoto `Ortho`, `Vliegdagcontour`; it never requests feature or raster
|
||||
content. The public Datavindplaats API requires an access token, so release
|
||||
evidence comes from the public metadata links published by the official OGC
|
||||
services rather than HTML scraping.
|
||||
|
||||
Runtime controls are `SOURCE_CATALOG_PROBE_ENABLED`,
|
||||
`SOURCE_CATALOG_GRB_WFS_URL`, `SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS`,
|
||||
`SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB` and
|
||||
`SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS`. Metadata links remain restricted to
|
||||
the official HTTPS CSW path on `metadata.vlaanderen.be` even when an operator
|
||||
overrides the capabilities endpoint.
|
||||
|
||||
@@ -45,6 +45,7 @@ from app.services.vector_operations_service import VectorOperationsService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.source_freshness_service import SourceFreshnessService
|
||||
from app.services.source_catalog_probe_service import SourceCatalogProbeService
|
||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
||||
from app.services.terrain_analysis_service import TerrainAnalysisService
|
||||
@@ -255,6 +256,16 @@ def audit_dataset_source_freshness(
|
||||
return envelope(report.model_dump())
|
||||
|
||||
|
||||
@router.get("/datasets/source-catalog-probes", response_model=dict)
|
||||
def probe_dataset_source_catalogs(
|
||||
project_id: UUID,
|
||||
refresh: bool = Query(default=False),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
report = SourceCatalogProbeService.audit_project(db, project_id, force=refresh)
|
||||
return envelope(report.model_dump())
|
||||
|
||||
|
||||
@router.get("/datasets/{dataset_id}", response_model=dict)
|
||||
def get_dataset(
|
||||
project_id: UUID,
|
||||
|
||||
@@ -31,6 +31,29 @@ class Settings(BaseSettings):
|
||||
orthophoto_timeout_seconds: int = Field(default=120, ge=1, validation_alias="ORTHOPHOTO_TIMEOUT_SECONDS")
|
||||
orthophoto_max_response_mb: int = Field(default=32, ge=1, validation_alias="ORTHOPHOTO_MAX_RESPONSE_MB")
|
||||
orthophoto_cache_ttl_hours: int = Field(default=24, ge=0, validation_alias="ORTHOPHOTO_CACHE_TTL_HOURS")
|
||||
source_catalog_probe_enabled: bool = Field(default=True, validation_alias="SOURCE_CATALOG_PROBE_ENABLED")
|
||||
source_catalog_grb_wfs_url: str = Field(
|
||||
default="https://geo.api.vlaanderen.be/GRB/wfs",
|
||||
validation_alias="SOURCE_CATALOG_GRB_WFS_URL",
|
||||
)
|
||||
source_catalog_probe_timeout_seconds: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=60,
|
||||
validation_alias="SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS",
|
||||
)
|
||||
source_catalog_probe_max_response_mb: int = Field(
|
||||
default=2,
|
||||
ge=1,
|
||||
le=10,
|
||||
validation_alias="SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB",
|
||||
)
|
||||
source_catalog_probe_cache_ttl_seconds: int = Field(
|
||||
default=900,
|
||||
ge=0,
|
||||
le=86_400,
|
||||
validation_alias="SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS",
|
||||
)
|
||||
dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED")
|
||||
dhmv_wcs_url: str = Field(
|
||||
default="https://geo.api.vlaanderen.be/DHMV/wcs",
|
||||
|
||||
@@ -11,6 +11,11 @@ from .source_freshness import (
|
||||
SourceFreshnessSummary,
|
||||
SourceIntegritySummary,
|
||||
)
|
||||
from .source_catalog import (
|
||||
SourceCatalogProbeItem,
|
||||
SourceCatalogProbeReport,
|
||||
SourceCatalogProbeSummary,
|
||||
)
|
||||
from .detection import (
|
||||
DetectionListResponse,
|
||||
DetectionModelCapability,
|
||||
@@ -141,6 +146,9 @@ __all__ = [
|
||||
"SourceFreshnessReport",
|
||||
"SourceFreshnessSummary",
|
||||
"SourceIntegritySummary",
|
||||
"SourceCatalogProbeItem",
|
||||
"SourceCatalogProbeReport",
|
||||
"SourceCatalogProbeSummary",
|
||||
"DetectionListResponse",
|
||||
"DetectionModelCapability",
|
||||
"DetectionModelsResponse",
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
SourceCatalogProbeStatus = Literal["available", "degraded", "unavailable", "disabled"]
|
||||
SourceCatalogComparisonStatus = Literal["same", "different", "not_comparable", "no_local_data", "unavailable"]
|
||||
|
||||
|
||||
class SourceCatalogProbeItem(BaseModel):
|
||||
source_name: str
|
||||
display_name: str
|
||||
service_type: Literal["WFS", "WMS"]
|
||||
endpoint_url: str
|
||||
status: SourceCatalogProbeStatus
|
||||
reachable: bool
|
||||
checked_at: datetime
|
||||
cached: bool = False
|
||||
expected_layers: list[str]
|
||||
matched_layers: list[str]
|
||||
missing_layers: list[str]
|
||||
advertised_layer_count: int
|
||||
metadata_url: str | None = None
|
||||
metadata_identifier: str | None = None
|
||||
remote_title: str | None = None
|
||||
remote_version: str | None = None
|
||||
remote_modified_at: datetime | None = None
|
||||
remote_published_at: datetime | None = None
|
||||
local_source_version: str | None = None
|
||||
comparison_status: SourceCatalogComparisonStatus
|
||||
capabilities_sha256: str | None = None
|
||||
capabilities_etag: str | None = None
|
||||
capabilities_last_modified_at: datetime | None = None
|
||||
message: str
|
||||
error_code: str | None = None
|
||||
|
||||
|
||||
class SourceCatalogProbeSummary(BaseModel):
|
||||
provider_count: int
|
||||
available_count: int
|
||||
degraded_count: int
|
||||
unavailable_count: int
|
||||
disabled_count: int
|
||||
different_version_count: int
|
||||
|
||||
|
||||
class SourceCatalogProbeReport(BaseModel):
|
||||
project_id: UUID
|
||||
generated_at: datetime
|
||||
summary: SourceCatalogProbeSummary
|
||||
items: list[SourceCatalogProbeItem]
|
||||
limitations: list[str]
|
||||
@@ -0,0 +1,545 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
from hashlib import sha256
|
||||
import re
|
||||
from threading import Lock
|
||||
from typing import Any, Callable
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
from urllib.request import Request, urlopen
|
||||
from uuid import UUID
|
||||
from xml.etree import ElementTree
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, Project
|
||||
from app.schemas.source_catalog import (
|
||||
SourceCatalogProbeItem,
|
||||
SourceCatalogProbeReport,
|
||||
SourceCatalogProbeSummary,
|
||||
)
|
||||
|
||||
|
||||
_GMD = "http://www.isotc211.org/2005/gmd"
|
||||
_GCO = "http://www.isotc211.org/2005/gco"
|
||||
_WFS = "http://www.opengis.net/wfs/2.0"
|
||||
_XLINK = "http://www.w3.org/1999/xlink"
|
||||
_METADATA_HOST = "metadata.vlaanderen.be"
|
||||
_VERSION_DATE = re.compile(r"^(?:toestand\s+)?(\d{4}-\d{2}-\d{2})$", re.IGNORECASE)
|
||||
_ORTHOPHOTO_EDITION = re.compile(r"^\d{4}\.\d{2}$")
|
||||
|
||||
|
||||
class CatalogProbeFailure(RuntimeError):
|
||||
def __init__(self, code: str, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProbeContract:
|
||||
source_name: str
|
||||
display_name: str
|
||||
service_type: str
|
||||
endpoint_url: str
|
||||
expected_layers: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FetchResult:
|
||||
content: bytes
|
||||
content_type: str
|
||||
etag: str | None
|
||||
last_modified_at: datetime | None
|
||||
final_url: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RemoteProbe:
|
||||
status: str
|
||||
reachable: bool
|
||||
checked_at: datetime
|
||||
expected_layers: tuple[str, ...]
|
||||
matched_layers: tuple[str, ...] = ()
|
||||
missing_layers: tuple[str, ...] = ()
|
||||
advertised_layer_count: int = 0
|
||||
metadata_url: str | None = None
|
||||
metadata_identifier: str | None = None
|
||||
remote_title: str | None = None
|
||||
remote_version: str | None = None
|
||||
remote_modified_at: datetime | None = None
|
||||
remote_published_at: datetime | None = None
|
||||
capabilities_sha256: str | None = None
|
||||
capabilities_etag: str | None = None
|
||||
capabilities_last_modified_at: datetime | None = None
|
||||
message: str = ""
|
||||
error_code: str | None = None
|
||||
cached: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CacheEntry:
|
||||
expires_at: datetime
|
||||
probe: RemoteProbe
|
||||
|
||||
|
||||
_REMOTE_CACHE: dict[str, CacheEntry] = {}
|
||||
_CACHE_LOCK = Lock()
|
||||
|
||||
|
||||
def _utc(value: datetime | None = None) -> datetime:
|
||||
current = value or datetime.now(timezone.utc)
|
||||
if current.tzinfo is None:
|
||||
return current.replace(tzinfo=timezone.utc)
|
||||
return current.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _header(headers: Any, name: str) -> str | None:
|
||||
value = headers.get(name) if headers is not None else None
|
||||
return str(value).strip() if value is not None and str(value).strip() else None
|
||||
|
||||
|
||||
def _http_date(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return _utc(parsedate_to_datetime(value))
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
|
||||
|
||||
def _with_capabilities_query(base_url: str, service_type: str) -> str:
|
||||
parsed = urlsplit(base_url)
|
||||
retained = [(key, value) for key, value in parse_qsl(parsed.query) if key.lower() not in {"service", "request", "version"}]
|
||||
version = "2.0.0" if service_type == "WFS" else "1.3.0"
|
||||
query = urlencode([*retained, ("SERVICE", service_type), ("VERSION", version), ("REQUEST", "GetCapabilities")])
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, query, ""))
|
||||
|
||||
|
||||
def _bounded_fetch(url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> FetchResult:
|
||||
request = Request(
|
||||
url,
|
||||
headers={"Accept": "application/xml,text/xml,application/json", "User-Agent": "GeoIntel/0.1 source-catalog-probe"},
|
||||
)
|
||||
max_bytes = settings.source_catalog_probe_max_response_mb * 1024 * 1024
|
||||
try:
|
||||
with (opener or urlopen)(request, timeout=settings.source_catalog_probe_timeout_seconds) as response:
|
||||
content_length = _header(response.headers, "Content-Length")
|
||||
if content_length:
|
||||
try:
|
||||
if int(content_length) > max_bytes:
|
||||
raise CatalogProbeFailure("CATALOG_RESPONSE_TOO_LARGE", "De officiële metadatarespons overschrijdt de ingestelde limiet.")
|
||||
except ValueError as exc:
|
||||
raise CatalogProbeFailure("CATALOG_INVALID_RESPONSE", "De officiële metadatarespons bevat een ongeldige Content-Length.") from exc
|
||||
content = response.read(max_bytes + 1)
|
||||
result = FetchResult(
|
||||
content=content,
|
||||
content_type=_header(response.headers, "Content-Type") or "",
|
||||
etag=_header(response.headers, "ETag"),
|
||||
last_modified_at=_http_date(_header(response.headers, "Last-Modified")),
|
||||
final_url=str(response.geturl()) if hasattr(response, "geturl") else url,
|
||||
)
|
||||
except CatalogProbeFailure:
|
||||
raise
|
||||
except (HTTPError, URLError, TimeoutError, OSError) as exc:
|
||||
raise CatalogProbeFailure("CATALOG_PROVIDER_UNAVAILABLE", "De officiële catalogus kon niet tijdig worden gelezen.") from exc
|
||||
if len(result.content) > max_bytes:
|
||||
raise CatalogProbeFailure("CATALOG_RESPONSE_TOO_LARGE", "De officiële metadatarespons overschrijdt de ingestelde limiet.")
|
||||
return result
|
||||
|
||||
|
||||
def _local_name(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def _direct_text(parent: ElementTree.Element, name: str) -> str | None:
|
||||
for child in parent:
|
||||
if _local_name(child.tag) == name and child.text and child.text.strip():
|
||||
return child.text.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _metadata_urls(parent: ElementTree.Element) -> list[str]:
|
||||
urls: list[str] = []
|
||||
for node in parent.iter():
|
||||
if _local_name(node.tag) not in {"MetadataURL", "MetadataUrl"}:
|
||||
continue
|
||||
href = node.attrib.get(f"{{{_XLINK}}}href")
|
||||
if href:
|
||||
urls.append(href.strip())
|
||||
for child in node.iter():
|
||||
if _local_name(child.tag) in {"URL", "OnlineResource"}:
|
||||
nested = child.attrib.get(f"{{{_XLINK}}}href") or (child.text or "").strip()
|
||||
if nested:
|
||||
urls.append(nested)
|
||||
return urls
|
||||
|
||||
|
||||
def _parse_capabilities(content: bytes, contract: ProbeContract) -> tuple[list[str], str | None]:
|
||||
try:
|
||||
root = ElementTree.fromstring(content)
|
||||
except ElementTree.ParseError as exc:
|
||||
raise CatalogProbeFailure("CATALOG_INVALID_XML", "De capabilities-respons is geen geldige XML.") from exc
|
||||
|
||||
layers: list[str] = []
|
||||
metadata_urls: list[str] = []
|
||||
node_name = "FeatureType" if contract.service_type == "WFS" else "Layer"
|
||||
for node in root.iter():
|
||||
if _local_name(node.tag) != node_name:
|
||||
continue
|
||||
raw_name = _direct_text(node, "Name")
|
||||
if not raw_name:
|
||||
continue
|
||||
name = raw_name.rsplit(":", 1)[-1]
|
||||
layers.append(name)
|
||||
if name in contract.expected_layers:
|
||||
metadata_urls.extend(_metadata_urls(node))
|
||||
|
||||
expected = set(contract.expected_layers)
|
||||
if not expected.intersection(layers):
|
||||
raise CatalogProbeFailure("CATALOG_EXPECTED_LAYERS_MISSING", "De officiële service bevat geen van de verwachte lagen.")
|
||||
xml_metadata = next((url for url in metadata_urls if "GetRecordById" in url and "OUTPUTSCHEMA" in url.upper()), None)
|
||||
return sorted(set(layers)), xml_metadata
|
||||
|
||||
|
||||
def _validate_metadata_url(url: str) -> str:
|
||||
parsed = urlsplit(url)
|
||||
if parsed.scheme != "https" or parsed.hostname != _METADATA_HOST or parsed.username or parsed.password:
|
||||
raise CatalogProbeFailure("CATALOG_METADATA_URL_REJECTED", "De capabilities verwijzen niet naar de toegestane officiële metadatahost.")
|
||||
if not parsed.path.startswith("/srv/dut/csw"):
|
||||
raise CatalogProbeFailure("CATALOG_METADATA_URL_REJECTED", "De capabilities verwijzen niet naar het toegestane CSW-pad.")
|
||||
query = {key.lower(): value for key, value in parse_qsl(parsed.query)}
|
||||
if query.get("request", "").lower() != "getrecordbyid" or not query.get("id"):
|
||||
raise CatalogProbeFailure("CATALOG_METADATA_URL_REJECTED", "De capabilities bevatten geen begrensde GetRecordById-verwijzing.")
|
||||
return url
|
||||
|
||||
|
||||
def _validate_capabilities_url(url: str) -> str:
|
||||
parsed = urlsplit(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
|
||||
raise CatalogProbeFailure("CATALOG_ENDPOINT_REJECTED", "De ingestelde capabilities-URL moet een geldige HTTP(S)-URL zonder credentials zijn.")
|
||||
return url
|
||||
|
||||
|
||||
def _node_text(node: ElementTree.Element | None) -> str | None:
|
||||
if node is None:
|
||||
return None
|
||||
for descendant in node.iter():
|
||||
if descendant is not node and descendant.text and descendant.text.strip():
|
||||
return descendant.text.strip()
|
||||
return node.text.strip() if node.text and node.text.strip() else None
|
||||
|
||||
|
||||
def _parse_iso_datetime(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
normalized = value.strip().replace("Z", "+00:00")
|
||||
try:
|
||||
return _utc(datetime.fromisoformat(normalized))
|
||||
except ValueError:
|
||||
try:
|
||||
return datetime.strptime(normalized, "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_metadata(content: bytes) -> dict[str, Any]:
|
||||
try:
|
||||
root = ElementTree.fromstring(content)
|
||||
except ElementTree.ParseError as exc:
|
||||
raise CatalogProbeFailure("CATALOG_INVALID_METADATA_XML", "Het officiële metadatarecord is geen geldige XML.") from exc
|
||||
namespaces = {"gmd": _GMD, "gco": _GCO}
|
||||
metadata = root.find(".//gmd:MD_Metadata", namespaces)
|
||||
if metadata is None and _local_name(root.tag) == "MD_Metadata":
|
||||
metadata = root
|
||||
if metadata is None:
|
||||
raise CatalogProbeFailure("CATALOG_METADATA_MISSING", "Het CSW-antwoord bevat geen ISO 19139 metadatarecord.")
|
||||
citation = metadata.find(".//gmd:identificationInfo/*/gmd:citation/gmd:CI_Citation", namespaces)
|
||||
title = _node_text(citation.find("gmd:title", namespaces) if citation is not None else None)
|
||||
edition = _node_text(citation.find("gmd:edition", namespaces) if citation is not None else None)
|
||||
identifier = _node_text(metadata.find("gmd:fileIdentifier", namespaces))
|
||||
modified = _parse_iso_datetime(_node_text(metadata.find("gmd:dateStamp", namespaces)))
|
||||
published = None
|
||||
if citation is not None:
|
||||
for date_node in citation.findall("gmd:date/gmd:CI_Date", namespaces):
|
||||
date_type = date_node.find("gmd:dateType/gmd:CI_DateTypeCode", namespaces)
|
||||
if date_type is not None and date_type.attrib.get("codeListValue") == "publication":
|
||||
published = _parse_iso_datetime(_node_text(date_node.find("gmd:date", namespaces)))
|
||||
break
|
||||
if not title or not edition:
|
||||
raise CatalogProbeFailure("CATALOG_VERSION_MISSING", "Het officiële metadatarecord bevat geen herkenbare titel en editie.")
|
||||
return {
|
||||
"identifier": identifier,
|
||||
"title": title,
|
||||
"version": edition,
|
||||
"modified_at": modified,
|
||||
"published_at": published,
|
||||
}
|
||||
|
||||
|
||||
def _probe_remote(
|
||||
contract: ProbeContract,
|
||||
settings: Settings,
|
||||
*,
|
||||
opener: Callable[..., Any] | None,
|
||||
now: datetime,
|
||||
) -> RemoteProbe:
|
||||
try:
|
||||
capabilities = _bounded_fetch(_validate_capabilities_url(contract.endpoint_url), settings, opener)
|
||||
_validate_capabilities_url(capabilities.final_url)
|
||||
content_type = capabilities.content_type.lower()
|
||||
if content_type and "xml" not in content_type and "text" not in content_type:
|
||||
raise CatalogProbeFailure("CATALOG_INVALID_CONTENT_TYPE", "De officiële capabilities-respons is geen XML.")
|
||||
layers, metadata_url = _parse_capabilities(capabilities.content, contract)
|
||||
matched = tuple(layer for layer in contract.expected_layers if layer in layers)
|
||||
missing = tuple(layer for layer in contract.expected_layers if layer not in layers)
|
||||
digest = sha256(capabilities.content).hexdigest()
|
||||
if not metadata_url:
|
||||
return RemoteProbe(
|
||||
status="degraded",
|
||||
reachable=True,
|
||||
checked_at=now,
|
||||
expected_layers=contract.expected_layers,
|
||||
matched_layers=matched,
|
||||
missing_layers=missing,
|
||||
advertised_layer_count=len(layers),
|
||||
capabilities_sha256=digest,
|
||||
capabilities_etag=capabilities.etag,
|
||||
capabilities_last_modified_at=capabilities.last_modified_at,
|
||||
message="De service is bereikbaar, maar publiceert geen machineleesbare ISO-metadata voor de verwachte lagen.",
|
||||
error_code="CATALOG_METADATA_LINK_MISSING",
|
||||
)
|
||||
metadata_url = _validate_metadata_url(metadata_url)
|
||||
metadata_response = _bounded_fetch(metadata_url, settings, opener)
|
||||
_validate_metadata_url(metadata_response.final_url)
|
||||
metadata = _parse_metadata(metadata_response.content)
|
||||
status = "degraded" if missing else "available"
|
||||
message = (
|
||||
f"De officiële catalogus is bereikbaar en publiceert editie {metadata['version']}."
|
||||
if not missing
|
||||
else f"Editie {metadata['version']} is gevonden, maar niet alle verwachte lagen worden aangeboden."
|
||||
)
|
||||
return RemoteProbe(
|
||||
status=status,
|
||||
reachable=True,
|
||||
checked_at=now,
|
||||
expected_layers=contract.expected_layers,
|
||||
matched_layers=matched,
|
||||
missing_layers=missing,
|
||||
advertised_layer_count=len(layers),
|
||||
metadata_url=metadata_url,
|
||||
metadata_identifier=metadata["identifier"],
|
||||
remote_title=metadata["title"],
|
||||
remote_version=metadata["version"],
|
||||
remote_modified_at=metadata["modified_at"],
|
||||
remote_published_at=metadata["published_at"],
|
||||
capabilities_sha256=digest,
|
||||
capabilities_etag=capabilities.etag,
|
||||
capabilities_last_modified_at=capabilities.last_modified_at,
|
||||
message=message,
|
||||
error_code="CATALOG_EXPECTED_LAYERS_INCOMPLETE" if missing else None,
|
||||
)
|
||||
except CatalogProbeFailure as exc:
|
||||
return RemoteProbe(
|
||||
status="unavailable",
|
||||
reachable=False,
|
||||
checked_at=now,
|
||||
expected_layers=contract.expected_layers,
|
||||
missing_layers=contract.expected_layers,
|
||||
message=exc.message,
|
||||
error_code=exc.code,
|
||||
)
|
||||
|
||||
|
||||
def _remote_with_cache(
|
||||
contract: ProbeContract,
|
||||
settings: Settings,
|
||||
*,
|
||||
force: bool,
|
||||
opener: Callable[..., Any] | None,
|
||||
now: datetime,
|
||||
) -> RemoteProbe:
|
||||
cache_key = f"{contract.source_name}|{contract.endpoint_url}|{','.join(contract.expected_layers)}"
|
||||
if not force and settings.source_catalog_probe_cache_ttl_seconds > 0:
|
||||
with _CACHE_LOCK:
|
||||
entry = _REMOTE_CACHE.get(cache_key)
|
||||
if entry and entry.expires_at > now:
|
||||
return replace(entry.probe, cached=True)
|
||||
probe = _probe_remote(contract, settings, opener=opener, now=now)
|
||||
if settings.source_catalog_probe_cache_ttl_seconds > 0:
|
||||
with _CACHE_LOCK:
|
||||
_REMOTE_CACHE[cache_key] = CacheEntry(
|
||||
expires_at=now + timedelta(seconds=settings.source_catalog_probe_cache_ttl_seconds),
|
||||
probe=probe,
|
||||
)
|
||||
return probe
|
||||
|
||||
|
||||
def _dataset_source_name(dataset: Dataset) -> str:
|
||||
return (dataset.source_name or dataset.source or "").strip().lower()
|
||||
|
||||
|
||||
def _latest_local_version(source_name: str, datasets: list[Dataset]) -> str | None:
|
||||
candidates = [item for item in datasets if _dataset_source_name(item) == source_name and item.source_version]
|
||||
if source_name == "digitaal_vlaanderen_orthophoto":
|
||||
explicit_current = [
|
||||
item
|
||||
for item in candidates
|
||||
if any(token in (item.source_version or "").lower() for token in ("most_recent", "latest", "current"))
|
||||
]
|
||||
if explicit_current:
|
||||
candidates = explicit_current
|
||||
if not candidates:
|
||||
return None
|
||||
latest = max(
|
||||
candidates,
|
||||
key=lambda item: (
|
||||
_utc(item.imported_at) if item.imported_at else datetime.min.replace(tzinfo=timezone.utc),
|
||||
_utc(item.observed_at) if item.observed_at else datetime.min.replace(tzinfo=timezone.utc),
|
||||
str(item.id),
|
||||
),
|
||||
)
|
||||
return latest.source_version
|
||||
|
||||
|
||||
def _normalized_version(source_name: str, version: str | None) -> str | None:
|
||||
if not version:
|
||||
return None
|
||||
value = version.strip()
|
||||
if source_name == "grb":
|
||||
match = _VERSION_DATE.fullmatch(value)
|
||||
return match.group(1) if match else None
|
||||
if source_name == "digitaal_vlaanderen_orthophoto" and _ORTHOPHOTO_EDITION.fullmatch(value):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _comparison(source_name: str, local: str | None, remote: str | None, remote_status: str) -> str:
|
||||
if remote_status in {"unavailable", "disabled"}:
|
||||
return "unavailable"
|
||||
if not local:
|
||||
return "no_local_data"
|
||||
normalized_local = _normalized_version(source_name, local)
|
||||
normalized_remote = _normalized_version(source_name, remote)
|
||||
if normalized_local is None or normalized_remote is None:
|
||||
return "not_comparable"
|
||||
return "same" if normalized_local == normalized_remote else "different"
|
||||
|
||||
|
||||
class SourceCatalogProbeService:
|
||||
@staticmethod
|
||||
def clear_cache() -> None:
|
||||
with _CACHE_LOCK:
|
||||
_REMOTE_CACHE.clear()
|
||||
|
||||
@staticmethod
|
||||
def audit_project(
|
||||
db: Session,
|
||||
project_id: UUID,
|
||||
*,
|
||||
force: bool = False,
|
||||
opener: Callable[..., Any] | None = None,
|
||||
now: datetime | None = None,
|
||||
settings: Settings | None = None,
|
||||
) -> SourceCatalogProbeReport:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
active_settings = settings or get_settings()
|
||||
generated_at = _utc(now)
|
||||
datasets = db.query(Dataset).filter(Dataset.project_id == project_id).all()
|
||||
contracts = (
|
||||
ProbeContract(
|
||||
source_name="grb",
|
||||
display_name="Basiskaart Vlaanderen (GRB)",
|
||||
service_type="WFS",
|
||||
endpoint_url=_with_capabilities_query(active_settings.source_catalog_grb_wfs_url, "WFS"),
|
||||
expected_layers=("GBG", "WBN", "WGO", "ADP"),
|
||||
),
|
||||
ProbeContract(
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
display_name="Orthofoto Vlaanderen",
|
||||
service_type="WMS",
|
||||
endpoint_url=_with_capabilities_query(active_settings.orthophoto_wms_url, "WMS"),
|
||||
expected_layers=(active_settings.orthophoto_wms_layer, "Vliegdagcontour"),
|
||||
),
|
||||
)
|
||||
items: list[SourceCatalogProbeItem] = []
|
||||
for contract in contracts:
|
||||
local_version = _latest_local_version(contract.source_name, datasets)
|
||||
if active_settings.source_catalog_probe_enabled:
|
||||
remote = _remote_with_cache(
|
||||
contract,
|
||||
active_settings,
|
||||
force=force,
|
||||
opener=opener,
|
||||
now=generated_at,
|
||||
)
|
||||
else:
|
||||
remote = RemoteProbe(
|
||||
status="disabled",
|
||||
reachable=False,
|
||||
checked_at=generated_at,
|
||||
expected_layers=contract.expected_layers,
|
||||
missing_layers=contract.expected_layers,
|
||||
message="Officiële catalogusprobes zijn uitgeschakeld in de runtimeconfiguratie.",
|
||||
error_code="CATALOG_PROBE_DISABLED",
|
||||
)
|
||||
comparison = _comparison(contract.source_name, local_version, remote.remote_version, remote.status)
|
||||
message = remote.message
|
||||
if comparison == "different":
|
||||
message += " De officiële editie verschilt van de lokaal vastgelegde bronversie; controleer dit handmatig vóór een begrensde verversing."
|
||||
elif comparison == "not_comparable" and local_version:
|
||||
message += " De lokale waarde is een opname- of importmarkering en kan niet eerlijk als officiële cataloguseditie worden vergeleken."
|
||||
items.append(
|
||||
SourceCatalogProbeItem(
|
||||
source_name=contract.source_name,
|
||||
display_name=contract.display_name,
|
||||
service_type=contract.service_type,
|
||||
endpoint_url=contract.endpoint_url,
|
||||
status=remote.status,
|
||||
reachable=remote.reachable,
|
||||
checked_at=remote.checked_at,
|
||||
cached=remote.cached,
|
||||
expected_layers=list(remote.expected_layers),
|
||||
matched_layers=list(remote.matched_layers),
|
||||
missing_layers=list(remote.missing_layers),
|
||||
advertised_layer_count=remote.advertised_layer_count,
|
||||
metadata_url=remote.metadata_url,
|
||||
metadata_identifier=remote.metadata_identifier,
|
||||
remote_title=remote.remote_title,
|
||||
remote_version=remote.remote_version,
|
||||
remote_modified_at=remote.remote_modified_at,
|
||||
remote_published_at=remote.remote_published_at,
|
||||
local_source_version=local_version,
|
||||
comparison_status=comparison,
|
||||
capabilities_sha256=remote.capabilities_sha256,
|
||||
capabilities_etag=remote.capabilities_etag,
|
||||
capabilities_last_modified_at=remote.capabilities_last_modified_at,
|
||||
message=message,
|
||||
error_code=remote.error_code,
|
||||
)
|
||||
)
|
||||
summary = SourceCatalogProbeSummary(
|
||||
provider_count=len(items),
|
||||
available_count=sum(item.status == "available" for item in items),
|
||||
degraded_count=sum(item.status == "degraded" for item in items),
|
||||
unavailable_count=sum(item.status == "unavailable" for item in items),
|
||||
disabled_count=sum(item.status == "disabled" for item in items),
|
||||
different_version_count=sum(item.comparison_status == "different" for item in items),
|
||||
)
|
||||
return SourceCatalogProbeReport(
|
||||
project_id=project_id,
|
||||
generated_at=generated_at,
|
||||
summary=summary,
|
||||
items=items,
|
||||
limitations=[
|
||||
"Deze expliciete controle leest alleen allowlisted WFS/WMS-capabilities en gekoppelde ISO 19139 metadata.",
|
||||
"Er worden geen features, rasters of modelbestanden opgehaald en geen datasets aangemaakt of overschreven.",
|
||||
"Een versieverschil is controlesignaal, geen bewijs dat een lokale dataset onbruikbaar is en geen automatische importopdracht.",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,332 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from email.message import Message
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.error import URLError
|
||||
import uuid
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.models import Dataset
|
||||
from app.services.source_catalog_probe_service import SourceCatalogProbeService
|
||||
|
||||
|
||||
NOW = datetime(2026, 7, 16, 15, 0, tzinfo=timezone.utc)
|
||||
GRB_METADATA_URL = (
|
||||
"https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&service=CSW&"
|
||||
"id=7C823055-7BBF-4D62-B55E-F85C30D53162&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd"
|
||||
)
|
||||
ORTHO_METADATA_URL = (
|
||||
"https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&service=CSW&"
|
||||
"id=f5304d6d-0dd4-43fd-a726-427af31e8d61&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd"
|
||||
)
|
||||
|
||||
|
||||
def _wfs_capabilities(metadata_url: str = GRB_METADATA_URL) -> bytes:
|
||||
layers = "".join(
|
||||
f"""
|
||||
<wfs:FeatureType>
|
||||
<wfs:Name>GRB:{name}</wfs:Name>
|
||||
<wfs:MetadataURL xlink:href="{metadata_url.replace('&', '&')}" />
|
||||
</wfs:FeatureType>
|
||||
"""
|
||||
for name in ("GBG", "WBN", "WGO", "ADP", "WTZ")
|
||||
)
|
||||
return f"""
|
||||
<wfs:WFS_Capabilities xmlns:wfs="http://www.opengis.net/wfs/2.0"
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<wfs:FeatureTypeList>{layers}</wfs:FeatureTypeList>
|
||||
</wfs:WFS_Capabilities>
|
||||
""".encode()
|
||||
|
||||
|
||||
def _wms_capabilities(metadata_url: str = ORTHO_METADATA_URL) -> bytes:
|
||||
return f"""
|
||||
<WMS_Capabilities xmlns="http://www.opengis.net/wms"
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<Capability><Layer><Title>Orthofoto</Title>
|
||||
<Layer><Name>Ortho</Name><MetadataURL><Format>text/xml</Format>
|
||||
<OnlineResource xlink:href="{metadata_url.replace('&', '&')}" />
|
||||
</MetadataURL></Layer>
|
||||
<Layer><Name>Vliegdagcontour</Name><MetadataURL><Format>text/xml</Format>
|
||||
<OnlineResource xlink:href="{metadata_url.replace('&', '&')}" />
|
||||
</MetadataURL></Layer>
|
||||
</Layer></Capability>
|
||||
</WMS_Capabilities>
|
||||
""".encode()
|
||||
|
||||
|
||||
def _metadata(identifier: str, title: str, edition: str, modified: str, published: str) -> bytes:
|
||||
return f"""
|
||||
<csw:GetRecordByIdResponse xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
|
||||
xmlns:gmd="http://www.isotc211.org/2005/gmd"
|
||||
xmlns:gco="http://www.isotc211.org/2005/gco">
|
||||
<gmd:MD_Metadata>
|
||||
<gmd:fileIdentifier><gco:CharacterString>{identifier}</gco:CharacterString></gmd:fileIdentifier>
|
||||
<gmd:dateStamp><gco:Date>{modified}</gco:Date></gmd:dateStamp>
|
||||
<gmd:identificationInfo><gmd:MD_DataIdentification><gmd:citation><gmd:CI_Citation>
|
||||
<gmd:title><gco:CharacterString>{title}</gco:CharacterString></gmd:title>
|
||||
<gmd:edition><gco:CharacterString>{edition}</gco:CharacterString></gmd:edition>
|
||||
<gmd:date><gmd:CI_Date><gmd:date><gco:Date>{published}</gco:Date></gmd:date>
|
||||
<gmd:dateType><gmd:CI_DateTypeCode codeListValue="publication">publication</gmd:CI_DateTypeCode></gmd:dateType>
|
||||
</gmd:CI_Date></gmd:date>
|
||||
</gmd:CI_Citation></gmd:citation></gmd:MD_DataIdentification></gmd:identificationInfo>
|
||||
</gmd:MD_Metadata>
|
||||
</csw:GetRecordByIdResponse>
|
||||
""".encode()
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, content: bytes, content_type: str = "text/xml", *, content_length: int | None = None) -> None:
|
||||
self.content = content
|
||||
self.headers = Message()
|
||||
self.headers["Content-Type"] = content_type
|
||||
self.headers["Content-Length"] = str(content_length if content_length is not None else len(content))
|
||||
self.headers["ETag"] = '"catalog-test"'
|
||||
self.headers["Last-Modified"] = "Wed, 15 Jul 2026 10:00:00 GMT"
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
return self.content if size < 0 else self.content[:size]
|
||||
|
||||
|
||||
class _RedirectedResponse(_Response):
|
||||
def __init__(self, content: bytes, final_url: str) -> None:
|
||||
super().__init__(content)
|
||||
self.final_url = final_url
|
||||
|
||||
def geturl(self) -> str:
|
||||
return self.final_url
|
||||
|
||||
class _Query:
|
||||
def __init__(self, datasets: list[Dataset]) -> None:
|
||||
self.datasets = datasets
|
||||
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
|
||||
def all(self) -> list[Dataset]:
|
||||
return self.datasets
|
||||
|
||||
|
||||
class _Db:
|
||||
def __init__(self, datasets: list[Dataset]) -> None:
|
||||
self.datasets = datasets
|
||||
|
||||
def get(self, _model, _identifier):
|
||||
return SimpleNamespace(id=_identifier)
|
||||
|
||||
def query(self, _model):
|
||||
return _Query(self.datasets)
|
||||
|
||||
|
||||
def _dataset(source_name: str, version: str) -> Dataset:
|
||||
return Dataset(
|
||||
id=uuid.uuid4(),
|
||||
project_id=uuid.uuid4(),
|
||||
name=f"{source_name} source",
|
||||
dataset_type="vector" if source_name == "grb" else "raster",
|
||||
source=source_name,
|
||||
source_name=source_name,
|
||||
source_version=version,
|
||||
imported_at=NOW,
|
||||
status="ready",
|
||||
)
|
||||
|
||||
|
||||
def _settings(**overrides) -> Settings:
|
||||
values = {
|
||||
"SOURCE_CATALOG_PROBE_ENABLED": True,
|
||||
"SOURCE_CATALOG_GRB_WFS_URL": "https://geo.api.vlaanderen.be/GRB/wfs",
|
||||
"SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS": 3,
|
||||
"SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB": 1,
|
||||
"SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS": 0,
|
||||
"ORTHOPHOTO_WMS_URL": "https://geo.api.vlaanderen.be/OMWRGBMRVL/wms",
|
||||
"ORTHOPHOTO_WMS_LAYER": "Ortho",
|
||||
}
|
||||
values.update(overrides)
|
||||
return Settings(**values)
|
||||
|
||||
|
||||
def _opener(request, timeout):
|
||||
assert timeout == 3
|
||||
url = request.full_url
|
||||
if "metadata.vlaanderen.be" in url:
|
||||
if "f5304d6d" in url:
|
||||
return _Response(_metadata("f5304d6d", "Orthofoto meest recent, 2025.04", "2025.04", "2026-04-27", "2025-12-11"))
|
||||
return _Response(_metadata("7C823055", "GRBgis", "Toestand 2026-07-15", "2026-07-15", "2026-07-15"))
|
||||
if "/GRB/" in url:
|
||||
return _Response(_wfs_capabilities())
|
||||
return _Response(_wms_capabilities())
|
||||
|
||||
|
||||
def test_catalog_probe_reads_real_editions_and_compares_only_compatible_versions() -> None:
|
||||
SourceCatalogProbeService.clear_cache()
|
||||
project_id = uuid.uuid4()
|
||||
report = SourceCatalogProbeService.audit_project(
|
||||
_Db([_dataset("grb", "2026-07-14"), _dataset("digitaal_vlaanderen_orthophoto", "most_recent_at_2026-07-14")]),
|
||||
project_id,
|
||||
settings=_settings(),
|
||||
opener=_opener,
|
||||
now=NOW,
|
||||
)
|
||||
by_source = {item.source_name: item for item in report.items}
|
||||
|
||||
assert report.summary.available_count == 2
|
||||
assert report.summary.different_version_count == 1
|
||||
assert by_source["grb"].remote_version == "Toestand 2026-07-15"
|
||||
assert by_source["grb"].comparison_status == "different"
|
||||
assert by_source["grb"].matched_layers == ["GBG", "WBN", "WGO", "ADP"]
|
||||
assert by_source["grb"].advertised_layer_count == 5
|
||||
assert by_source["digitaal_vlaanderen_orthophoto"].remote_version == "2025.04"
|
||||
assert by_source["digitaal_vlaanderen_orthophoto"].comparison_status == "not_comparable"
|
||||
assert by_source["digitaal_vlaanderen_orthophoto"].remote_published_at.year == 2025
|
||||
assert all(item.capabilities_sha256 for item in report.items)
|
||||
|
||||
|
||||
def test_catalog_probe_isolates_provider_failure() -> None:
|
||||
def partial_opener(request, timeout):
|
||||
if "/GRB/" in request.full_url:
|
||||
raise URLError("offline")
|
||||
return _opener(request, timeout)
|
||||
|
||||
report = SourceCatalogProbeService.audit_project(
|
||||
_Db([]), uuid.uuid4(), settings=_settings(), opener=partial_opener, now=NOW
|
||||
)
|
||||
by_source = {item.source_name: item for item in report.items}
|
||||
|
||||
assert by_source["grb"].status == "unavailable"
|
||||
assert by_source["grb"].error_code == "CATALOG_PROVIDER_UNAVAILABLE"
|
||||
assert by_source["digitaal_vlaanderen_orthophoto"].status == "available"
|
||||
assert report.summary.unavailable_count == 1
|
||||
|
||||
|
||||
def test_catalog_probe_rejects_metadata_redirect_outside_allowlist() -> None:
|
||||
def malicious_opener(request, timeout):
|
||||
if "/GRB/" in request.full_url:
|
||||
return _Response(
|
||||
_wfs_capabilities(
|
||||
"https://example.com/csw?request=GetRecordById&id=evil&OUTPUTSCHEMA=http://www.isotc211.org/2005/gmd"
|
||||
)
|
||||
)
|
||||
return _opener(request, timeout)
|
||||
|
||||
report = SourceCatalogProbeService.audit_project(
|
||||
_Db([]), uuid.uuid4(), settings=_settings(), opener=malicious_opener, now=NOW
|
||||
)
|
||||
grb = next(item for item in report.items if item.source_name == "grb")
|
||||
|
||||
assert grb.status == "unavailable"
|
||||
assert grb.error_code == "CATALOG_METADATA_URL_REJECTED"
|
||||
|
||||
|
||||
def test_catalog_probe_enforces_response_limit_without_reading_external_data() -> None:
|
||||
def oversized_opener(request, timeout):
|
||||
if "/GRB/" in request.full_url:
|
||||
return _Response(b"<xml />", content_length=2 * 1024 * 1024)
|
||||
return _opener(request, timeout)
|
||||
|
||||
report = SourceCatalogProbeService.audit_project(
|
||||
_Db([]), uuid.uuid4(), settings=_settings(), opener=oversized_opener, now=NOW
|
||||
)
|
||||
grb = next(item for item in report.items if item.source_name == "grb")
|
||||
|
||||
assert grb.status == "unavailable"
|
||||
assert grb.error_code == "CATALOG_RESPONSE_TOO_LARGE"
|
||||
|
||||
|
||||
def test_catalog_probe_revalidates_metadata_host_after_redirect() -> None:
|
||||
def redirected_opener(request, timeout):
|
||||
if "metadata.vlaanderen.be" in request.full_url and "7C823055" in request.full_url:
|
||||
return _RedirectedResponse(
|
||||
_metadata("7C823055", "GRBgis", "Toestand 2026-07-15", "2026-07-15", "2026-07-15"),
|
||||
"https://example.com/redirected-metadata",
|
||||
)
|
||||
return _opener(request, timeout)
|
||||
|
||||
report = SourceCatalogProbeService.audit_project(
|
||||
_Db([]), uuid.uuid4(), settings=_settings(), opener=redirected_opener, now=NOW
|
||||
)
|
||||
grb = next(item for item in report.items if item.source_name == "grb")
|
||||
|
||||
assert grb.status == "unavailable"
|
||||
assert grb.error_code == "CATALOG_METADATA_URL_REJECTED"
|
||||
|
||||
|
||||
def test_catalog_probe_cache_is_explicitly_bypassable() -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
def counting_opener(request, timeout):
|
||||
calls.append(request.full_url)
|
||||
return _opener(request, timeout)
|
||||
|
||||
SourceCatalogProbeService.clear_cache()
|
||||
settings = _settings(SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900)
|
||||
db = _Db([])
|
||||
project_id = uuid.uuid4()
|
||||
first = SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW)
|
||||
first_call_count = len(calls)
|
||||
second = SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW)
|
||||
SourceCatalogProbeService.audit_project(db, project_id, settings=settings, opener=counting_opener, now=NOW, force=True)
|
||||
|
||||
assert first_call_count == 4
|
||||
assert all(item.cached is False for item in first.items)
|
||||
assert all(item.cached is True for item in second.items)
|
||||
assert len(calls) == 8
|
||||
|
||||
|
||||
def test_catalog_probe_can_be_disabled_without_network_access() -> None:
|
||||
def forbidden_opener(*_args, **_kwargs):
|
||||
raise AssertionError("network must not be called")
|
||||
|
||||
report = SourceCatalogProbeService.audit_project(
|
||||
_Db([]),
|
||||
uuid.uuid4(),
|
||||
settings=_settings(SOURCE_CATALOG_PROBE_ENABLED=False),
|
||||
opener=forbidden_opener,
|
||||
now=NOW,
|
||||
)
|
||||
|
||||
assert report.summary.disabled_count == 2
|
||||
assert all(item.status == "disabled" for item in report.items)
|
||||
|
||||
|
||||
def test_catalog_probe_route_returns_canonical_envelope(monkeypatch) -> None:
|
||||
from app.api.routes import datasets as dataset_routes
|
||||
|
||||
project_id = uuid.uuid4()
|
||||
expected = SourceCatalogProbeService.audit_project(
|
||||
_Db([]), project_id, settings=_settings(SOURCE_CATALOG_PROBE_ENABLED=False), now=NOW
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dataset_routes.SourceCatalogProbeService,
|
||||
"audit_project",
|
||||
lambda db, selected_project_id, force=False: expected,
|
||||
)
|
||||
|
||||
response = dataset_routes.probe_dataset_source_catalogs(
|
||||
project_id=project_id, refresh=False, db=SimpleNamespace()
|
||||
)
|
||||
|
||||
assert list(response) == ["data"]
|
||||
assert response["data"]["project_id"] == project_id
|
||||
assert response["data"]["summary"]["provider_count"] == 2
|
||||
|
||||
|
||||
def test_catalog_probe_remains_explicit_and_never_imports_provider_data() -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
hook = (root / "frontend" / "src" / "hooks" / "useSourceFreshness.ts").read_text(encoding="utf-8")
|
||||
service = (root / "backend" / "app" / "services" / "source_catalog_probe_service.py").read_text(encoding="utf-8")
|
||||
operator = (root / "scripts" / "audit_source_freshness.py").read_text(encoding="utf-8")
|
||||
|
||||
assert "void probeCatalogs(" not in hook
|
||||
assert "DatasetService" not in service
|
||||
assert "VectorFeatureService" not in service
|
||||
assert "--probe-catalogs" in operator
|
||||
assert "/datasets/source-catalog-probes" in operator
|
||||
@@ -34,6 +34,11 @@
|
||||
<Config Name="Orthophoto WMS URL" Target="ORTHOPHOTO_WMS_URL" Default="https://geo.api.vlaanderen.be/OMWRGBMRVL/wms" Mode="" Description="Official Digitaal Vlaanderen most-recent winter orthophoto WMS endpoint." Type="Variable" Display="advanced" Required="true" Mask="false">https://geo.api.vlaanderen.be/OMWRGBMRVL/wms</Config>
|
||||
<Config Name="Orthophoto Resolution (m)" Target="ORTHOPHOTO_RESOLUTION_M" Default="1.0" Mode="" Description="Requested analysis sampling in metres per pixel. Keep at 1.0 for the active building model profile." Type="Variable" Display="advanced" Required="true" Mask="false">1.0</Config>
|
||||
<Config Name="Orthophoto Maximum Side (m)" Target="ORTHOPHOTO_MAX_SIDE_M" Default="1024" Mode="" Description="Safety limit for each selected rectangle side before external acquisition and local inference." Type="Variable" Display="advanced" Required="true" Mask="false">1024</Config>
|
||||
<Config Name="Official Catalog Edition Probe" Target="SOURCE_CATALOG_PROBE_ENABLED" Default="true" Mode="" Description="Allow explicit read-only GRB and orthophoto edition checks. This never imports provider data." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||
<Config Name="GRB Catalog WFS URL" Target="SOURCE_CATALOG_GRB_WFS_URL" Default="https://geo.api.vlaanderen.be/GRB/wfs" Mode="" Description="Official GRB WFS used only for capabilities and linked ISO metadata checks." Type="Variable" Display="advanced" Required="true" Mask="false">https://geo.api.vlaanderen.be/GRB/wfs</Config>
|
||||
<Config Name="Catalog Probe Timeout (seconds)" Target="SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS" Default="10" Mode="" Description="Per-request timeout for explicit read-only official catalog checks." Type="Variable" Display="advanced" Required="true" Mask="false">10</Config>
|
||||
<Config Name="Catalog Probe Maximum Response (MiB)" Target="SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB" Default="2" Mode="" Description="Maximum capabilities or ISO metadata response size accepted by a catalog probe." Type="Variable" Display="advanced" Required="true" Mask="false">2</Config>
|
||||
<Config Name="Catalog Probe Cache (seconds)" Target="SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS" Default="900" Mode="" Description="Short in-memory cache for repeated official edition checks; use zero to disable." Type="Variable" Display="advanced" Required="true" Mask="false">900</Config>
|
||||
<Config Name="Official DHMV Acquisition" Target="DHMV_ENABLED" Default="true" Mode="" Description="Allow bounded official DHMV II terrain and surface raster acquisition." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||
<Config Name="DHMV WCS URL" Target="DHMV_WCS_URL" Default="https://geo.api.vlaanderen.be/DHMV/wcs" Mode="" Description="Official Digitaal Vlaanderen DHMV WCS endpoint." Type="Variable" Display="advanced" Required="true" Mask="false">https://geo.api.vlaanderen.be/DHMV/wcs</Config>
|
||||
<Config Name="DHMV Analysis Resolution (m)" Target="DHMV_RESOLUTION_M" Default="5.0" Mode="" Description="Stored analysis grid resolution. Native source resolution remains recorded as 1 metre." Type="Variable" Display="advanced" Required="true" Mask="false">5.0</Config>
|
||||
|
||||
@@ -32,6 +32,14 @@ ORTHOPHOTO_RESOLUTION_M=1.0
|
||||
ORTHOPHOTO_MIN_SIDE_M=128
|
||||
ORTHOPHOTO_MAX_SIDE_M=1024
|
||||
ORTHOPHOTO_CACHE_TTL_HOURS=24
|
||||
|
||||
# Explicit read-only edition checks for GRB and the most-recent orthophoto.
|
||||
# No feature or raster data is downloaded by these probes.
|
||||
SOURCE_CATALOG_PROBE_ENABLED=true
|
||||
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
|
||||
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10
|
||||
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2
|
||||
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900
|
||||
DHMV_ENABLED=true
|
||||
DHMV_WCS_URL=https://geo.api.vlaanderen.be/DHMV/wcs
|
||||
DHMV_RESOLUTION_M=5.0
|
||||
|
||||
@@ -27,6 +27,11 @@ ORTHOPHOTO_RESOLUTION_M="${ORTHOPHOTO_RESOLUTION_M:-1.0}"
|
||||
ORTHOPHOTO_MIN_SIDE_M="${ORTHOPHOTO_MIN_SIDE_M:-128}"
|
||||
ORTHOPHOTO_MAX_SIDE_M="${ORTHOPHOTO_MAX_SIDE_M:-1024}"
|
||||
ORTHOPHOTO_CACHE_TTL_HOURS="${ORTHOPHOTO_CACHE_TTL_HOURS:-24}"
|
||||
SOURCE_CATALOG_PROBE_ENABLED="${SOURCE_CATALOG_PROBE_ENABLED:-true}"
|
||||
SOURCE_CATALOG_GRB_WFS_URL="${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}"
|
||||
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS="${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}"
|
||||
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB="${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}"
|
||||
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}"
|
||||
DHMV_ENABLED="${DHMV_ENABLED:-true}"
|
||||
DHMV_WCS_URL="${DHMV_WCS_URL:-https://geo.api.vlaanderen.be/DHMV/wcs}"
|
||||
DHMV_RESOLUTION_M="${DHMV_RESOLUTION_M:-5.0}"
|
||||
@@ -126,6 +131,11 @@ docker run -d \
|
||||
-e ORTHOPHOTO_MIN_SIDE_M="$ORTHOPHOTO_MIN_SIDE_M" \
|
||||
-e ORTHOPHOTO_MAX_SIDE_M="$ORTHOPHOTO_MAX_SIDE_M" \
|
||||
-e ORTHOPHOTO_CACHE_TTL_HOURS="$ORTHOPHOTO_CACHE_TTL_HOURS" \
|
||||
-e SOURCE_CATALOG_PROBE_ENABLED="$SOURCE_CATALOG_PROBE_ENABLED" \
|
||||
-e SOURCE_CATALOG_GRB_WFS_URL="$SOURCE_CATALOG_GRB_WFS_URL" \
|
||||
-e SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS="$SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS" \
|
||||
-e SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB="$SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB" \
|
||||
-e SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="$SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS" \
|
||||
-e DHMV_ENABLED="$DHMV_ENABLED" \
|
||||
-e DHMV_WCS_URL="$DHMV_WCS_URL" \
|
||||
-e DHMV_RESOLUTION_M="$DHMV_RESOLUTION_M" \
|
||||
|
||||
@@ -25,6 +25,11 @@ services:
|
||||
ORTHOPHOTO_MIN_SIDE_M: ${ORTHOPHOTO_MIN_SIDE_M:-128}
|
||||
ORTHOPHOTO_MAX_SIDE_M: ${ORTHOPHOTO_MAX_SIDE_M:-1024}
|
||||
ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24}
|
||||
SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true}
|
||||
SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}
|
||||
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS: ${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}
|
||||
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB: ${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}
|
||||
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS: ${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}
|
||||
DHMV_ENABLED: ${DHMV_ENABLED:-true}
|
||||
DHMV_WCS_URL: ${DHMV_WCS_URL:-https://geo.api.vlaanderen.be/DHMV/wcs}
|
||||
DHMV_RESOLUTION_M: ${DHMV_RESOLUTION_M:-5.0}
|
||||
|
||||
@@ -30,6 +30,11 @@ services:
|
||||
ORTHOPHOTO_MIN_SIDE_M: ${ORTHOPHOTO_MIN_SIDE_M:-128}
|
||||
ORTHOPHOTO_MAX_SIDE_M: ${ORTHOPHOTO_MAX_SIDE_M:-1024}
|
||||
ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24}
|
||||
SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true}
|
||||
SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}
|
||||
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS: ${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}
|
||||
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB: ${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}
|
||||
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS: ${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}
|
||||
DHMV_ENABLED: ${DHMV_ENABLED:-true}
|
||||
DHMV_WCS_URL: ${DHMV_WCS_URL:-https://geo.api.vlaanderen.be/DHMV/wcs}
|
||||
DHMV_RESOLUTION_M: ${DHMV_RESOLUTION_M:-5.0}
|
||||
|
||||
@@ -412,6 +412,26 @@ Dataset or silently refreshes a publication. Fixed editions and scenarios are
|
||||
not marked stale merely because their source date is old. Unknown sources are
|
||||
`review_required` until an explicit publication policy is defined.
|
||||
|
||||
### GET `/api/v1/projects/{project_id}/datasets/source-catalog-probes`
|
||||
|
||||
Performs an explicit, read-only release probe for the allowlisted GRB WFS and
|
||||
most-recent orthophoto WMS. The endpoint reads bounded `GetCapabilities`
|
||||
responses, confirms the expected layers and follows only HTTPS ISO 19139
|
||||
`GetRecordById` links on `metadata.vlaanderen.be`. Query parameter
|
||||
`refresh=true` bypasses the short in-memory response cache.
|
||||
|
||||
Each provider item returns service reachability, matched/missing layers, the
|
||||
official metadata identifier, title, edition, publication/metadata dates,
|
||||
local `source_version` and one comparison status: `same`, `different`,
|
||||
`not_comparable`, `no_local_data` or `unavailable`. Provider status is
|
||||
`available`, `degraded`, `unavailable` or `disabled`. A difference means only
|
||||
that an operator should review provenance; it is not an update instruction.
|
||||
|
||||
The endpoint accepts no arbitrary URL, feature query, area or layer. It does
|
||||
not fetch vector features, raster pixels or models, create jobs/datasets, write
|
||||
to PostGIS or trigger an import. The normal `source-freshness` endpoint remains
|
||||
local-only and never invokes this probe implicitly.
|
||||
|
||||
### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}`
|
||||
|
||||
Return metadata.
|
||||
|
||||
@@ -9302,3 +9302,40 @@ Next:
|
||||
- Add machine-readable catalogue release probes only for providers with stable
|
||||
official version endpoints, keeping every acquisition an explicit bounded
|
||||
operator action.
|
||||
|
||||
## Sprint 222 - Official source edition probes (2026-07-16)
|
||||
|
||||
Implemented:
|
||||
- Verified the public machine-readable contracts published by Digitaal
|
||||
Vlaanderen. GRB WFS and OMWRGBMRVL WMS capabilities expose stable ISO 19139
|
||||
CSW records containing real source editions and metadata dates.
|
||||
- Added a canonical project endpoint that reads only allowlisted capabilities
|
||||
and linked `metadata.vlaanderen.be` `GetRecordById` responses. It confirms
|
||||
GRB `GBG/WBN/WGO/ADP` and orthophoto `Ortho/Vliegdagcontour` availability,
|
||||
then reports the official edition beside local `source_version` evidence.
|
||||
- Added strict HTTP(S)/metadata allowlists, timeout and byte limits,
|
||||
post-redirect validation, SHA-256 capabilities evidence, provider-isolated
|
||||
degraded/unavailable states and a short in-memory cache.
|
||||
- Added an explicit Status action and operator CLI flags. Neither path runs on
|
||||
page load, starts a job, downloads source data or changes a Dataset.
|
||||
- Added Compose, all-in-one Unraid and DockerMan controls for the bounded probe.
|
||||
|
||||
Validation:
|
||||
- Deterministic tests cover official XML parsing, edition comparison,
|
||||
provider failure isolation, metadata URL rejection, size limits, caching,
|
||||
disabled mode and canonical envelopes.
|
||||
- Frontend typecheck/build and targeted backend tests pass before the full
|
||||
release and Tower validation recorded in the delivery result.
|
||||
|
||||
Boundaries:
|
||||
- The general Datavindplaats API requires authentication and is not scraped or
|
||||
silently bypassed. GeoIntel uses only public metadata records advertised by
|
||||
the official OGC services.
|
||||
- A version difference remains a manual provenance-review signal. No automatic
|
||||
import, refresh scheduler, GRB feature fetch or orthophoto pixel request was
|
||||
added.
|
||||
|
||||
Next:
|
||||
- Extend the same probe pattern only where another official source publishes a
|
||||
stable machine-readable edition. Do not infer release versions from service
|
||||
protocol versions, ETags or HTTP modification dates.
|
||||
|
||||
@@ -25,6 +25,13 @@ YOLO_MAX_DETECTIONS=1000
|
||||
YOLO_DUPLICATE_IOU_THRESHOLD=0.5
|
||||
ENABLE_GRB_WFS=false
|
||||
GRB_WFS_URL=
|
||||
|
||||
# Explicit read-only source-edition probes (no provider import).
|
||||
SOURCE_CATALOG_PROBE_ENABLED=true
|
||||
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
|
||||
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10
|
||||
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2
|
||||
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900
|
||||
OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter
|
||||
```
|
||||
|
||||
|
||||
+2
-1
@@ -39,7 +39,8 @@
|
||||
- [x] Execute and audit the complete 336-product VMM and 56-product DHMV regional runtime matrices.
|
||||
- [x] Present municipal DHMV/VMM partitions as logical regional layers and analyse cross-boundary rectangles without a municipality prerequisite.
|
||||
- [x] Add a read-only source freshness/version audit with explicit publication policies, local integrity checks, operator CLI and compact Status workspace surface.
|
||||
- [ ] Add external catalogue release probes only after each official provider exposes a stable machine-readable version contract; keep every refresh explicit and bounded.
|
||||
- [x] Add explicit bounded GRB/orthophoto catalogue release probes using official capabilities and ISO metadata; keep every acquisition separate and operator-controlled.
|
||||
- [ ] Extend catalogue probes only to additional sources that publish a stable machine-readable edition contract; do not add background polling or infer releases from HTTP dates alone.
|
||||
|
||||
## Governed source expansion backlog
|
||||
|
||||
|
||||
+4
-1
@@ -9,7 +9,10 @@ surface. It separates sources that are current, due for a catalogue review,
|
||||
require local integrity review or are local artifacts. Only attention items are
|
||||
expanded by default; all source detail remains available through disclosure.
|
||||
The refresh button reruns the local read-only audit and never downloads or
|
||||
replaces source data.
|
||||
replaces source data. A separate `Officiële edities controleren` action is the
|
||||
only trigger for bounded GRB/orthophoto catalog reads. It presents official and
|
||||
local editions, layer-contract coverage and honest non-comparable version
|
||||
markers without starting an import or background poll.
|
||||
|
||||
The user-facing shell is task based: `Kaart`, `Bronnen`, `Kwaliteit`, `Beeldanalyse`, `Downloads`, `Status` and `Beheer`. Internal benchmark projects, raw dataset metadata, provider capabilities, model registry details and QA evidence remain accessible through labelled advanced disclosures instead of competing with the normal workflow.
|
||||
|
||||
|
||||
@@ -860,6 +860,10 @@ function App(): JSX.Element {
|
||||
loading={sourceFreshness.loading}
|
||||
error={sourceFreshness.error}
|
||||
onRefresh={() => { void sourceFreshness.refresh() }}
|
||||
catalogReport={sourceFreshness.catalogReport}
|
||||
catalogLoading={sourceFreshness.catalogLoading}
|
||||
catalogError={sourceFreshness.catalogError}
|
||||
onProbeCatalogs={(force) => { void sourceFreshness.probeCatalogs(force) }}
|
||||
/>
|
||||
<details className="status-details-disclosure">
|
||||
<summary>
|
||||
|
||||
@@ -1,10 +1,22 @@
|
||||
import type { SourceFreshnessItem, SourceFreshnessReport, SourceFreshnessStatus } from '../../types'
|
||||
import type {
|
||||
SourceCatalogComparisonStatus,
|
||||
SourceCatalogProbeItem,
|
||||
SourceCatalogProbeReport,
|
||||
SourceCatalogProbeStatus,
|
||||
SourceFreshnessItem,
|
||||
SourceFreshnessReport,
|
||||
SourceFreshnessStatus,
|
||||
} from '../../types'
|
||||
|
||||
interface SourceFreshnessPanelProps {
|
||||
report: SourceFreshnessReport | null
|
||||
loading: boolean
|
||||
error: string | null
|
||||
onRefresh: () => void
|
||||
catalogReport: SourceCatalogProbeReport | null
|
||||
catalogLoading: boolean
|
||||
catalogError: string | null
|
||||
onProbeCatalogs: (force: boolean) => void
|
||||
}
|
||||
|
||||
function statusLabel(status: SourceFreshnessStatus): string {
|
||||
@@ -14,6 +26,21 @@ function statusLabel(status: SourceFreshnessStatus): string {
|
||||
return 'lokaal'
|
||||
}
|
||||
|
||||
function catalogStatusLabel(status: SourceCatalogProbeStatus): string {
|
||||
if (status === 'available') return 'bereikbaar'
|
||||
if (status === 'degraded') return 'beperkt'
|
||||
if (status === 'unavailable') return 'niet bereikbaar'
|
||||
return 'uitgeschakeld'
|
||||
}
|
||||
|
||||
function comparisonLabel(status: SourceCatalogComparisonStatus): string {
|
||||
if (status === 'same') return 'zelfde editie'
|
||||
if (status === 'different') return 'verschil controleren'
|
||||
if (status === 'no_local_data') return 'nog niet lokaal ingeladen'
|
||||
if (status === 'not_comparable') return 'andere versienotatie'
|
||||
return 'niet vergelijkbaar'
|
||||
}
|
||||
|
||||
function formatDate(value?: string | null): string {
|
||||
if (!value) return 'geen datum'
|
||||
return new Intl.DateTimeFormat('nl-BE', { dateStyle: 'medium' }).format(new Date(value))
|
||||
@@ -23,6 +50,27 @@ function integrityIssueCount(item: SourceFreshnessItem): number {
|
||||
return Object.values(item.integrity).reduce((total, value) => total + value, 0)
|
||||
}
|
||||
|
||||
function CatalogRow({ item }: { item: SourceCatalogProbeItem }): JSX.Element {
|
||||
return (
|
||||
<div className={`source-catalog-row source-catalog-row-${item.status}`}>
|
||||
<div className="source-freshness-main">
|
||||
<div>
|
||||
<strong>{item.display_name}</strong>
|
||||
<span>{catalogStatusLabel(item.status)} / {comparisonLabel(item.comparison_status)}</span>
|
||||
</div>
|
||||
<span className="source-freshness-status">{item.remote_version ?? 'geen editie'}</span>
|
||||
</div>
|
||||
<p>{item.message}</p>
|
||||
<div className="source-freshness-meta">
|
||||
<span>Lokaal: {item.local_source_version ?? 'niet aanwezig'}</span>
|
||||
<span>Lagen: {item.matched_layers.length}/{item.expected_layers.length} bevestigd</span>
|
||||
{item.remote_modified_at ? <span>Metadata: {formatDate(item.remote_modified_at)}</span> : null}
|
||||
{item.cached ? <span>cache gebruikt</span> : null}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function SourceRow({ item }: { item: SourceFreshnessItem }): JSX.Element {
|
||||
const issueCount = integrityIssueCount(item)
|
||||
return (
|
||||
@@ -30,7 +78,7 @@ function SourceRow({ item }: { item: SourceFreshnessItem }): JSX.Element {
|
||||
<div className="source-freshness-main">
|
||||
<div>
|
||||
<strong>{item.display_name}</strong>
|
||||
<span>{item.dataset_count} datasets · {item.version_count} versies</span>
|
||||
<span>{item.dataset_count} datasets / {item.version_count} versies</span>
|
||||
</div>
|
||||
<span className="source-freshness-status">{statusLabel(item.status)}</span>
|
||||
</div>
|
||||
@@ -45,7 +93,16 @@ function SourceRow({ item }: { item: SourceFreshnessItem }): JSX.Element {
|
||||
)
|
||||
}
|
||||
|
||||
export function SourceFreshnessPanel({ report, loading, error, onRefresh }: SourceFreshnessPanelProps): JSX.Element {
|
||||
export function SourceFreshnessPanel({
|
||||
report,
|
||||
loading,
|
||||
error,
|
||||
onRefresh,
|
||||
catalogReport,
|
||||
catalogLoading,
|
||||
catalogError,
|
||||
onProbeCatalogs,
|
||||
}: SourceFreshnessPanelProps): JSX.Element {
|
||||
const attentionItems = report?.items.filter((item) => item.status === 'due' || item.status === 'review_required') ?? []
|
||||
const summary = report?.summary
|
||||
|
||||
@@ -55,11 +112,21 @@ export function SourceFreshnessPanel({ report, loading, error, onRefresh }: Sour
|
||||
<div>
|
||||
<p className="eyebrow">Bronbeheer</p>
|
||||
<h2>Actualiteit en versiecontrole</h2>
|
||||
<p>Controleert lokale publicaties, versies en bestanden zonder externe bronnen automatisch te wijzigen.</p>
|
||||
<p>Controleert lokale publicaties en kan op aanvraag de officiële GRB- en orthofoto-editie uitlezen.</p>
|
||||
</div>
|
||||
<div className="source-freshness-actions">
|
||||
<button className="secondary-action" type="button" onClick={onRefresh} disabled={loading}>
|
||||
{loading ? 'Controleren...' : 'Lokale status vernieuwen'}
|
||||
</button>
|
||||
<button
|
||||
className="primary-action"
|
||||
type="button"
|
||||
onClick={() => onProbeCatalogs(Boolean(catalogReport))}
|
||||
disabled={catalogLoading}
|
||||
>
|
||||
{catalogLoading ? 'Officiële bronnen lezen...' : catalogReport ? 'Opnieuw bij bron controleren' : 'Officiële edities controleren'}
|
||||
</button>
|
||||
</div>
|
||||
<button className="secondary-action" type="button" onClick={onRefresh} disabled={loading}>
|
||||
{loading ? 'Controleren...' : 'Opnieuw controleren'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{error ? <p className="inline-error">{error}</p> : null}
|
||||
@@ -89,6 +156,31 @@ export function SourceFreshnessPanel({ report, loading, error, onRefresh }: Sour
|
||||
</div>
|
||||
<p className="source-freshness-limitation">{report.limitations.join(' ')}</p>
|
||||
</details>
|
||||
<div className="source-catalog-section" aria-live="polite">
|
||||
<div className="source-catalog-heading">
|
||||
<div>
|
||||
<strong>Officiële catalogusedities</strong>
|
||||
<span>Alleen na jouw expliciete controle; er wordt niets ingeladen of vervangen.</span>
|
||||
</div>
|
||||
{catalogReport ? (
|
||||
<span>{catalogReport.summary.available_count}/{catalogReport.summary.provider_count} bereikbaar</span>
|
||||
) : null}
|
||||
</div>
|
||||
{catalogError ? <p className="inline-error">{catalogError}</p> : null}
|
||||
{!catalogReport && !catalogError ? (
|
||||
<p className="source-catalog-empty">Controleer GRB en orthofoto wanneer je een lokale bronversie met de officiële editie wilt vergelijken.</p>
|
||||
) : null}
|
||||
{catalogReport ? (
|
||||
<>
|
||||
<div className="source-catalog-list">
|
||||
{catalogReport.items.map((item) => <CatalogRow item={item} key={item.source_name} />)}
|
||||
</div>
|
||||
<p className="source-freshness-limitation">
|
||||
Gecontroleerd {formatDate(catalogReport.generated_at)}. Een verschil vraagt menselijke beoordeling en start nooit automatisch een import.
|
||||
</p>
|
||||
</>
|
||||
) : null}
|
||||
</div>
|
||||
</>
|
||||
) : null}
|
||||
</section>
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
|
||||
import { datasetsApi } from '../services/api'
|
||||
import type { SourceFreshnessReport } from '../types'
|
||||
import type { SourceCatalogProbeReport, SourceFreshnessReport } from '../types'
|
||||
|
||||
export function useSourceFreshness(selectedProjectId: string | null) {
|
||||
const [report, setReport] = useState<SourceFreshnessReport | null>(null)
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const requestSequence = useRef(0)
|
||||
const catalogRequestSequence = useRef(0)
|
||||
const [catalogReport, setCatalogReport] = useState<SourceCatalogProbeReport | null>(null)
|
||||
const [catalogLoading, setCatalogLoading] = useState(false)
|
||||
const [catalogError, setCatalogError] = useState<string | null>(null)
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
const requestId = ++requestSequence.current
|
||||
@@ -35,6 +39,32 @@ export function useSourceFreshness(selectedProjectId: string | null) {
|
||||
}
|
||||
}, [selectedProjectId])
|
||||
|
||||
const probeCatalogs = useCallback(async (force = false) => {
|
||||
const requestId = ++catalogRequestSequence.current
|
||||
if (!selectedProjectId) {
|
||||
setCatalogReport(null)
|
||||
setCatalogError(null)
|
||||
setCatalogLoading(false)
|
||||
return
|
||||
}
|
||||
setCatalogLoading(true)
|
||||
setCatalogError(null)
|
||||
try {
|
||||
const nextReport = await datasetsApi.sourceCatalogProbes(selectedProjectId, force)
|
||||
if (catalogRequestSequence.current === requestId) {
|
||||
setCatalogReport(nextReport)
|
||||
}
|
||||
} catch (caught) {
|
||||
if (catalogRequestSequence.current === requestId) {
|
||||
setCatalogError(caught instanceof Error ? caught.message : 'De officiële broncatalogi konden niet worden gecontroleerd.')
|
||||
}
|
||||
} finally {
|
||||
if (catalogRequestSequence.current === requestId) {
|
||||
setCatalogLoading(false)
|
||||
}
|
||||
}
|
||||
}, [selectedProjectId])
|
||||
|
||||
useEffect(() => {
|
||||
void refresh()
|
||||
return () => {
|
||||
@@ -42,5 +72,21 @@ export function useSourceFreshness(selectedProjectId: string | null) {
|
||||
}
|
||||
}, [refresh])
|
||||
|
||||
return { report, loading, error, refresh }
|
||||
useEffect(() => {
|
||||
catalogRequestSequence.current += 1
|
||||
setCatalogReport(null)
|
||||
setCatalogError(null)
|
||||
setCatalogLoading(false)
|
||||
}, [selectedProjectId])
|
||||
|
||||
return {
|
||||
report,
|
||||
loading,
|
||||
error,
|
||||
refresh,
|
||||
catalogReport,
|
||||
catalogLoading,
|
||||
catalogError,
|
||||
probeCatalogs,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ import type {
|
||||
ThematicRasterProductRead,
|
||||
ThematicRasterSelectionResponse,
|
||||
SourceFreshnessReport,
|
||||
SourceCatalogProbeReport,
|
||||
} from '../../types'
|
||||
|
||||
const DATASET_PAGE_SIZE = 200
|
||||
@@ -61,6 +62,10 @@ export const datasetsApi = {
|
||||
list: listProjectDatasets,
|
||||
sourceFreshness: (projectId: string): Promise<SourceFreshnessReport> =>
|
||||
apiGet<SourceFreshnessReport>(`/api/v1/projects/${projectId}/datasets/source-freshness`),
|
||||
sourceCatalogProbes: (projectId: string, refresh = false): Promise<SourceCatalogProbeReport> =>
|
||||
apiGet<SourceCatalogProbeReport>(
|
||||
`/api/v1/projects/${projectId}/datasets/source-catalog-probes?refresh=${refresh ? 'true' : 'false'}`,
|
||||
),
|
||||
upload: (
|
||||
projectId: string,
|
||||
payload: {
|
||||
|
||||
@@ -836,13 +836,20 @@ details.ai-lab-model-surface > summary strong {
|
||||
padding: 1rem 1.2rem;
|
||||
}
|
||||
|
||||
.source-freshness-header .secondary-action {
|
||||
.source-freshness-header .secondary-action,
|
||||
.source-freshness-header .primary-action {
|
||||
flex: 0 0 auto;
|
||||
width: auto;
|
||||
min-width: 10rem;
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.source-freshness-actions {
|
||||
display: flex;
|
||||
flex: 0 0 auto;
|
||||
gap: 0.55rem;
|
||||
}
|
||||
|
||||
.source-freshness-header h2 {
|
||||
margin: 0;
|
||||
font-size: 1.05rem;
|
||||
@@ -959,6 +966,81 @@ details.ai-lab-model-surface > summary strong {
|
||||
border-top: 1px solid var(--line);
|
||||
}
|
||||
|
||||
.source-catalog-section {
|
||||
border-top: 1px solid var(--line);
|
||||
background: #fbfcfd;
|
||||
}
|
||||
|
||||
.source-catalog-heading {
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 0.8rem 1rem;
|
||||
}
|
||||
|
||||
.source-catalog-heading > div {
|
||||
display: grid;
|
||||
gap: 0.12rem;
|
||||
}
|
||||
|
||||
.source-catalog-heading strong {
|
||||
color: #23313b;
|
||||
font-size: 0.84rem;
|
||||
}
|
||||
|
||||
.source-catalog-heading span,
|
||||
.source-catalog-empty {
|
||||
color: var(--muted);
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.source-catalog-heading > span {
|
||||
flex: 0 0 auto;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.source-catalog-empty {
|
||||
margin: 0;
|
||||
padding: 0 1rem 0.85rem;
|
||||
}
|
||||
|
||||
.source-catalog-list {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
border-top: 1px solid var(--line);
|
||||
}
|
||||
|
||||
.source-catalog-row {
|
||||
min-width: 0;
|
||||
padding: 0.8rem 1rem;
|
||||
border-right: 1px solid var(--line);
|
||||
box-shadow: inset 0 3px 0 #8a98a3;
|
||||
}
|
||||
|
||||
.source-catalog-row:last-child {
|
||||
border-right: 0;
|
||||
}
|
||||
|
||||
.source-catalog-row-available {
|
||||
box-shadow: inset 0 3px 0 #2d9272;
|
||||
}
|
||||
|
||||
.source-catalog-row-degraded {
|
||||
box-shadow: inset 0 3px 0 #ca7a18;
|
||||
}
|
||||
|
||||
.source-catalog-row-unavailable,
|
||||
.source-catalog-row-disabled {
|
||||
box-shadow: inset 0 3px 0 #b84e4e;
|
||||
}
|
||||
|
||||
.source-catalog-row p {
|
||||
margin: 0.45rem 0;
|
||||
color: var(--muted);
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.source-freshness-details > summary {
|
||||
display: flex;
|
||||
cursor: pointer;
|
||||
@@ -982,10 +1064,15 @@ details.ai-lab-model-surface > summary strong {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.source-freshness-header .secondary-action {
|
||||
.source-freshness-header .secondary-action,
|
||||
.source-freshness-header .primary-action {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.source-freshness-actions {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.source-freshness-summary {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
@@ -997,6 +1084,19 @@ details.ai-lab-model-surface > summary strong {
|
||||
.source-freshness-summary span:nth-child(-n + 2) {
|
||||
border-bottom: 1px solid var(--line);
|
||||
}
|
||||
|
||||
.source-catalog-list {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.source-catalog-row {
|
||||
border-right: 0;
|
||||
border-bottom: 1px solid var(--line);
|
||||
}
|
||||
|
||||
.source-catalog-row:last-child {
|
||||
border-bottom: 0;
|
||||
}
|
||||
}
|
||||
|
||||
.workflow-guidance-panel {
|
||||
|
||||
@@ -709,6 +709,52 @@ export interface SourceFreshnessReport {
|
||||
limitations: string[]
|
||||
}
|
||||
|
||||
export type SourceCatalogProbeStatus = 'available' | 'degraded' | 'unavailable' | 'disabled'
|
||||
export type SourceCatalogComparisonStatus = 'same' | 'different' | 'not_comparable' | 'no_local_data' | 'unavailable'
|
||||
|
||||
export interface SourceCatalogProbeItem {
|
||||
source_name: string
|
||||
display_name: string
|
||||
service_type: 'WFS' | 'WMS'
|
||||
endpoint_url: string
|
||||
status: SourceCatalogProbeStatus
|
||||
reachable: boolean
|
||||
checked_at: string
|
||||
cached: boolean
|
||||
expected_layers: string[]
|
||||
matched_layers: string[]
|
||||
missing_layers: string[]
|
||||
advertised_layer_count: number
|
||||
metadata_url?: string | null
|
||||
metadata_identifier?: string | null
|
||||
remote_title?: string | null
|
||||
remote_version?: string | null
|
||||
remote_modified_at?: string | null
|
||||
remote_published_at?: string | null
|
||||
local_source_version?: string | null
|
||||
comparison_status: SourceCatalogComparisonStatus
|
||||
capabilities_sha256?: string | null
|
||||
capabilities_etag?: string | null
|
||||
capabilities_last_modified_at?: string | null
|
||||
message: string
|
||||
error_code?: string | null
|
||||
}
|
||||
|
||||
export interface SourceCatalogProbeReport {
|
||||
project_id: string
|
||||
generated_at: string
|
||||
summary: {
|
||||
provider_count: number
|
||||
available_count: number
|
||||
degraded_count: number
|
||||
unavailable_count: number
|
||||
disabled_count: number
|
||||
different_version_count: number
|
||||
}
|
||||
items: SourceCatalogProbeItem[]
|
||||
limitations: string[]
|
||||
}
|
||||
|
||||
export interface GeojsonEnvelopeResponse {
|
||||
data: object
|
||||
}
|
||||
|
||||
@@ -1708,6 +1708,14 @@ uses one canonical `GET`, changes no application data and performs no source
|
||||
download. It can therefore be scheduled explicitly through Unraid cron without
|
||||
turning GeoIntel into a real-time monitoring system.
|
||||
|
||||
Add `--probe-catalogs` to explicitly read the allowlisted official GRB and
|
||||
orthophoto capabilities plus linked ISO metadata. `--refresh-catalogs` bypasses
|
||||
the short server cache and implies the probe. `--fail-on-catalog` makes a
|
||||
degraded/unavailable official metadata service fail the command. With a probe,
|
||||
JSON output contains `source_freshness` and `catalog_probes`; without it, the
|
||||
original local report shape is unchanged. No flag downloads provider features
|
||||
or imagery and no flag writes a Dataset.
|
||||
|
||||
## Tower deployment
|
||||
|
||||
Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime:
|
||||
|
||||
@@ -26,6 +26,21 @@ def parse_args() -> argparse.Namespace:
|
||||
default="integrity",
|
||||
help="Non-zero exit policy for cron or release automation",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--probe-catalogs",
|
||||
action="store_true",
|
||||
help="Explicitly query the allowlisted official GRB and orthophoto metadata catalogs",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--refresh-catalogs",
|
||||
action="store_true",
|
||||
help="Bypass the short server-side catalog cache (implies --probe-catalogs)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fail-on-catalog",
|
||||
action="store_true",
|
||||
help="Exit non-zero when an explicitly requested official catalog is degraded or unavailable",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -45,7 +60,24 @@ def fetch_report(api_url: str, project_id: str, timeout: float) -> dict:
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def print_text(report: dict) -> None:
|
||||
def fetch_catalog_report(api_url: str, project_id: str, timeout: float, refresh: bool) -> dict:
|
||||
query = "true" if refresh else "false"
|
||||
endpoint = f"{api_url.rstrip('/')}/projects/{project_id}/datasets/source-catalog-probes?refresh={query}"
|
||||
request = Request(endpoint, headers={"Accept": "application/json", "User-Agent": "GeoIntel-source-audit/1.0"})
|
||||
try:
|
||||
with urlopen(request, timeout=timeout) as response:
|
||||
payload = json.load(response)
|
||||
except HTTPError as exc:
|
||||
body = exc.read().decode("utf-8", errors="replace")
|
||||
raise RuntimeError(f"GeoIntel catalog probe returned HTTP {exc.code}: {body}") from exc
|
||||
except URLError as exc:
|
||||
raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
|
||||
raise RuntimeError("Catalog probe response is not a canonical GeoIntel data envelope")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def print_text(report: dict, *, catalog_queried: bool = False) -> None:
|
||||
summary = report.get("summary", {})
|
||||
print(
|
||||
"GeoIntel source audit: "
|
||||
@@ -59,7 +91,30 @@ def print_text(report: dict) -> None:
|
||||
status = item.get("status", "unknown")
|
||||
if status not in {"current", "local"} or item.get("integrity", {}).get("missing_version_count", 0):
|
||||
print(f"- {status:15} {item.get('display_name', item.get('source_name'))}: {item.get('reason', '')}")
|
||||
print("No external catalog was queried and no dataset was modified.")
|
||||
if catalog_queried:
|
||||
print("The official catalog was queried read-only; no feature, raster or dataset was modified.")
|
||||
else:
|
||||
print("No external catalog was queried and no dataset was modified.")
|
||||
|
||||
|
||||
def print_catalog_text(report: dict) -> None:
|
||||
summary = report.get("summary", {})
|
||||
print(
|
||||
"Official catalog probe: "
|
||||
f"{summary.get('available_count', 0)}/{summary.get('provider_count', 0)} available, "
|
||||
f"{summary.get('degraded_count', 0)} degraded, "
|
||||
f"{summary.get('unavailable_count', 0)} unavailable, "
|
||||
f"{summary.get('different_version_count', 0)} version differences"
|
||||
)
|
||||
for item in report.get("items", []):
|
||||
remote = item.get("remote_version") or "no official edition"
|
||||
local = item.get("local_source_version") or "not loaded locally"
|
||||
print(
|
||||
f"- {item.get('display_name', item.get('source_name'))}: "
|
||||
f"{item.get('status')} / official={remote} / local={local} / "
|
||||
f"comparison={item.get('comparison_status')}"
|
||||
)
|
||||
print(f" {item.get('message', '')}")
|
||||
|
||||
|
||||
def should_fail(report: dict, fail_on: str) -> bool:
|
||||
@@ -76,22 +131,42 @@ def should_fail(report: dict, fail_on: str) -> bool:
|
||||
return integrity > 0 or due > 0 or review > 0
|
||||
|
||||
|
||||
def catalog_should_fail(report: dict) -> bool:
|
||||
summary = report.get("summary", {})
|
||||
return int(summary.get("degraded_count", 0)) > 0 or int(summary.get("unavailable_count", 0)) > 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
report = fetch_report(args.api_url, args.project_id, args.timeout)
|
||||
catalog_report = (
|
||||
fetch_catalog_report(args.api_url, args.project_id, args.timeout, args.refresh_catalogs)
|
||||
if args.probe_catalogs or args.refresh_catalogs
|
||||
else None
|
||||
)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 2
|
||||
serialized = json.dumps(report, indent=2, sort_keys=True)
|
||||
output_payload = (
|
||||
{"source_freshness": report, "catalog_probes": catalog_report}
|
||||
if catalog_report is not None
|
||||
else report
|
||||
)
|
||||
serialized = json.dumps(output_payload, indent=2, sort_keys=True)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(serialized + "\n", encoding="utf-8")
|
||||
if args.json:
|
||||
print(serialized)
|
||||
else:
|
||||
print_text(report)
|
||||
return 1 if should_fail(report, args.fail_on) else 0
|
||||
print_text(report, catalog_queried=catalog_report is not None)
|
||||
if catalog_report is not None:
|
||||
print_catalog_text(catalog_report)
|
||||
failed = should_fail(report, args.fail_on)
|
||||
if args.fail_on_catalog and catalog_report is not None:
|
||||
failed = failed or catalog_should_fail(catalog_report)
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user