Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
288 lines
11 KiB
Python
288 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from email.message import Message
|
|
from types import SimpleNamespace
|
|
from urllib.error import URLError
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
from app.core.config import Settings
|
|
from app.models import Dataset
|
|
from app.services.source_catalog_probe_service import SourceCatalogProbeService
|
|
from app.services.statbel_catalog_probe import StatbelCatalogError, parse_statbel_population_catalog
|
|
|
|
|
|
NOW = datetime(2026, 7, 16, 18, 0, tzinfo=timezone.utc)
|
|
STATBEL_DCAT_URL = "https://doc.statbel.be/publications/DCAT/DCAT_opendata_datasets.ttl"
|
|
ALZ_RELEASE_URL = "https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen"
|
|
|
|
|
|
def _dataset_block(
|
|
year: int,
|
|
*,
|
|
node_id: int,
|
|
host: str = "statbel.fgov.be",
|
|
include_new_zip: bool = True,
|
|
include_standard_zip: bool = False,
|
|
) -> str:
|
|
distributions = [
|
|
f"https://{host}/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_{year}_NEW.xlsx#distribution{node_id}",
|
|
f"https://{host}/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_{year}_OLD.zip#distribution{node_id}",
|
|
]
|
|
if include_new_zip:
|
|
distributions.append(
|
|
f"https://{host}/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_{year}_NEW.zip#distribution{node_id}"
|
|
)
|
|
if include_standard_zip:
|
|
distributions.append(
|
|
f"https://{host}/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_{year}.zip#distribution{node_id}"
|
|
)
|
|
distribution_values = ",\n ".join(f"<{value}>" for value in distributions)
|
|
return f"""
|
|
<https://statbel.fgov.be/node/{node_id}#id> a dcat:Dataset ;
|
|
dct:title "Bevolking per statistische sector"@nl ;
|
|
dct:alternative "Bevolking per statistische sector [Periode: {year}]"@nl ;
|
|
dct:identifier "NodeID{node_id}" ;
|
|
dct:license <https://creativecommons.org/licenses/by/4.0/> ;
|
|
dct:temporal [ dcat:startDate "{year}-01-01"^^xsd:date ] ;
|
|
dcat:landingPage <https://statbel.fgov.be/nl/open-data/bevolking-statistische-sector-{node_id}> ;
|
|
dcat:distribution {distribution_values} .
|
|
"""
|
|
|
|
|
|
def _catalog(*blocks: str) -> bytes:
|
|
return f"""
|
|
@prefix dcat: <http://www.w3.org/ns/dcat#> .
|
|
@prefix dct: <http://purl.org/dc/terms/> .
|
|
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
|
<http://data.gov.be/catalog/statbelopen> a dcat:Catalog ;
|
|
dct:modified "2026-07-07"^^xsd:date .
|
|
{''.join(blocks)}
|
|
""".encode()
|
|
|
|
|
|
class _Response:
|
|
def __init__(
|
|
self,
|
|
content: bytes,
|
|
*,
|
|
content_type: str = "application/octet-stream",
|
|
final_url: str | None = None,
|
|
content_length: int | None = None,
|
|
) -> None:
|
|
self.content = content
|
|
self.final_url = final_url
|
|
self.headers = Message()
|
|
self.headers["Content-Type"] = content_type
|
|
self.headers["Content-Length"] = str(content_length if content_length is not None else len(content))
|
|
self.headers["ETag"] = '"statbel-test"'
|
|
self.headers["Last-Modified"] = "Mon, 13 Jul 2026 08:22:54 GMT"
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
def read(self, size: int = -1) -> bytes:
|
|
return self.content if size < 0 else self.content[:size]
|
|
|
|
def geturl(self) -> str:
|
|
return self.final_url or STATBEL_DCAT_URL
|
|
|
|
|
|
class _Query:
|
|
def __init__(self, datasets: list[Dataset]) -> None:
|
|
self.datasets = datasets
|
|
|
|
def filter(self, *_args):
|
|
return self
|
|
|
|
def all(self) -> list[Dataset]:
|
|
return self.datasets
|
|
|
|
|
|
class _Db:
|
|
def __init__(self, datasets: list[Dataset]) -> None:
|
|
self.datasets = datasets
|
|
|
|
def get(self, _model, identifier):
|
|
return SimpleNamespace(id=identifier)
|
|
|
|
def query(self, _model):
|
|
return _Query(self.datasets)
|
|
|
|
|
|
def _settings(**overrides) -> Settings:
|
|
values = {
|
|
"SOURCE_CATALOG_PROBE_ENABLED": True,
|
|
"SOURCE_CATALOG_GRB_WFS_URL": "https://geo.api.vlaanderen.be/GRB/wfs",
|
|
"SOURCE_CATALOG_ALZ_RELEASE_URL": ALZ_RELEASE_URL,
|
|
"SOURCE_CATALOG_STATBEL_DCAT_URL": STATBEL_DCAT_URL,
|
|
"SOURCE_CATALOG_STATBEL_MAX_RESPONSE_MB": 5,
|
|
"SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS": 3,
|
|
"SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB": 1,
|
|
"SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS": 0,
|
|
"ORTHOPHOTO_WMS_URL": "https://geo.api.vlaanderen.be/OMWRGBMRVL/wms",
|
|
"ORTHOPHOTO_WMS_LAYER": "Ortho",
|
|
}
|
|
values.update(overrides)
|
|
return Settings(**values)
|
|
|
|
|
|
def _statbel_item(content: bytes, *, final_url: str | None = None, content_length: int | None = None, datasets=None):
|
|
def opener(request, timeout):
|
|
if request.full_url == STATBEL_DCAT_URL:
|
|
return _Response(content, final_url=final_url, content_length=content_length)
|
|
raise URLError("not needed for selected assertion")
|
|
|
|
report = SourceCatalogProbeService.audit_project(
|
|
_Db(datasets or []), uuid.uuid4(), settings=_settings(), opener=opener, now=NOW
|
|
)
|
|
return next(item for item in report.items if item.source_name == "statbel")
|
|
|
|
|
|
def test_statbel_parser_selects_latest_population_release_and_redegeo_variant() -> None:
|
|
release = parse_statbel_population_catalog(
|
|
_catalog(_dataset_block(2024, node_id=5510), _dataset_block(2025, node_id=6475))
|
|
)
|
|
|
|
assert release.version == "2025"
|
|
assert release.identifier == "NodeID6475"
|
|
assert release.current_distribution_variant == "new"
|
|
assert release.legacy_distribution_available is True
|
|
assert release.distribution_count == 3
|
|
assert release.catalog_modified_at == datetime(2026, 7, 7, tzinfo=timezone.utc)
|
|
|
|
|
|
def test_statbel_probe_compares_latest_local_year_not_latest_import_order() -> None:
|
|
older = Dataset(
|
|
id=uuid.uuid4(), project_id=uuid.uuid4(), name="Statbel 2024", dataset_type="vector",
|
|
source="statbel", source_name="statbel", source_version="2024", imported_at=NOW, status="ready",
|
|
)
|
|
newer = Dataset(
|
|
id=uuid.uuid4(), project_id=uuid.uuid4(), name="Statbel 2025", dataset_type="vector",
|
|
source="statbel", source_name="statbel", source_version="2025", imported_at=NOW.replace(year=2025), status="ready",
|
|
)
|
|
item = _statbel_item(_catalog(_dataset_block(2025, node_id=6475)), datasets=[older, newer])
|
|
|
|
assert item.status == "available"
|
|
assert item.local_source_version == "2025"
|
|
assert item.remote_version == "2025"
|
|
assert item.comparison_status == "same"
|
|
assert item.metadata_identifier == "NodeID6475"
|
|
assert item.matched_layers == ["population_txt_current", "landing_page", "cc_by_4_0"]
|
|
|
|
|
|
def test_statbel_parser_rejects_untrusted_distribution_host() -> None:
|
|
with pytest.raises(StatbelCatalogError) as exc_info:
|
|
parse_statbel_population_catalog(_catalog(_dataset_block(2025, node_id=6475, host="example.com")))
|
|
|
|
assert exc_info.value.code == "CATALOG_STATBEL_DISTRIBUTION_REJECTED"
|
|
|
|
|
|
def test_statbel_parser_requires_new_2025_txt_distribution() -> None:
|
|
with pytest.raises(StatbelCatalogError) as exc_info:
|
|
parse_statbel_population_catalog(
|
|
_catalog(_dataset_block(2025, node_id=6475, include_new_zip=False))
|
|
)
|
|
|
|
assert exc_info.value.code == "CATALOG_STATBEL_CURRENT_DISTRIBUTION_MISSING"
|
|
|
|
|
|
def test_statbel_parser_rejects_conflicting_period_evidence() -> None:
|
|
content = _catalog(_dataset_block(2025, node_id=6475)).replace(
|
|
b'dcat:startDate "2025-01-01"',
|
|
b'dcat:startDate "2024-01-01"',
|
|
)
|
|
|
|
with pytest.raises(StatbelCatalogError) as exc_info:
|
|
parse_statbel_population_catalog(content)
|
|
|
|
assert exc_info.value.code == "CATALOG_STATBEL_PERIOD_AMBIGUOUS"
|
|
|
|
|
|
def test_statbel_parser_rejects_duplicate_latest_release() -> None:
|
|
with pytest.raises(StatbelCatalogError) as exc_info:
|
|
parse_statbel_population_catalog(
|
|
_catalog(_dataset_block(2025, node_id=6475), _dataset_block(2025, node_id=7000))
|
|
)
|
|
|
|
assert exc_info.value.code == "CATALOG_STATBEL_POPULATION_AMBIGUOUS"
|
|
|
|
|
|
def test_statbel_parser_requires_cc_by_4_license() -> None:
|
|
content = _catalog(_dataset_block(2025, node_id=6475)).replace(
|
|
b"https://creativecommons.org/licenses/by/4.0/",
|
|
b"https://example.com/unknown-license",
|
|
)
|
|
|
|
with pytest.raises(StatbelCatalogError) as exc_info:
|
|
parse_statbel_population_catalog(content)
|
|
|
|
assert exc_info.value.code == "CATALOG_STATBEL_LICENSE_MISSING"
|
|
|
|
|
|
def test_statbel_probe_revalidates_catalog_url_after_redirect() -> None:
|
|
item = _statbel_item(
|
|
_catalog(_dataset_block(2025, node_id=6475)),
|
|
final_url="https://example.com/DCAT_opendata_datasets.ttl",
|
|
)
|
|
|
|
assert item.status == "unavailable"
|
|
assert item.error_code == "CATALOG_STATBEL_URL_REJECTED"
|
|
|
|
|
|
def test_statbel_probe_rejects_noncanonical_configured_catalog_before_network() -> None:
|
|
calls: list[str] = []
|
|
|
|
def opener(request, timeout):
|
|
calls.append(request.full_url)
|
|
raise URLError("network must not be reached for the rejected Statbel URL")
|
|
|
|
report = SourceCatalogProbeService.audit_project(
|
|
_Db([]),
|
|
uuid.uuid4(),
|
|
settings=_settings(SOURCE_CATALOG_STATBEL_DCAT_URL="https://example.com/catalog.ttl"),
|
|
opener=opener,
|
|
now=NOW,
|
|
)
|
|
item = next(value for value in report.items if value.source_name == "statbel")
|
|
|
|
assert item.status == "unavailable"
|
|
assert item.error_code == "CATALOG_STATBEL_URL_REJECTED"
|
|
assert "https://example.com/catalog.ttl" not in calls
|
|
|
|
|
|
def test_statbel_probe_uses_separate_bounded_catalog_limit() -> None:
|
|
item = _statbel_item(
|
|
_catalog(_dataset_block(2025, node_id=6475)),
|
|
content_length=6 * 1024 * 1024,
|
|
)
|
|
|
|
assert item.status == "unavailable"
|
|
assert item.error_code == "CATALOG_RESPONSE_TOO_LARGE"
|
|
|
|
|
|
def test_statbel_probe_does_not_fetch_catalog_distributions() -> None:
|
|
calls: list[str] = []
|
|
|
|
def opener(request, timeout):
|
|
calls.append(request.full_url)
|
|
if request.full_url == STATBEL_DCAT_URL:
|
|
return _Response(_catalog(_dataset_block(2025, node_id=6475)))
|
|
if "OPENDATA_SECTOREN" in request.full_url:
|
|
raise AssertionError("Statbel distribution fetch attempted")
|
|
raise URLError("unrelated provider offline")
|
|
|
|
report = SourceCatalogProbeService.audit_project(
|
|
_Db([]), uuid.uuid4(), settings=_settings(), opener=opener, now=NOW
|
|
)
|
|
item = next(value for value in report.items if value.source_name == "statbel")
|
|
|
|
assert item.status == "available"
|
|
assert calls.count(STATBEL_DCAT_URL) == 1
|
|
assert all("OPENDATA_SECTOREN" not in value for value in calls)
|