Initial public release

2026-08-31 21:56:53 +02:00
commit faeb58ef6d
1386 changed files with 263203 additions and 0 deletions
@@ -0,0 +1,287 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from email.message import Message
+from types import SimpleNamespace
+from urllib.error import URLError
+import uuid
+
+import pytest
+
+from app.core.config import Settings
+from app.models import Dataset
+from app.services.source_catalog_probe_service import SourceCatalogProbeService
+from app.services.statbel_catalog_probe import StatbelCatalogError, parse_statbel_population_catalog
+
+
+NOW = datetime(2026, 7, 16, 18, 0, tzinfo=timezone.utc)
+STATBEL_DCAT_URL = "https://doc.statbel.be/publications/DCAT/DCAT_opendata_datasets.ttl"
+ALZ_RELEASE_URL = "https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen"
+
+
+def _dataset_block(
+    year: int,
+    *,
+    node_id: int,
+    host: str = "statbel.fgov.be",
+    include_new_zip: bool = True,
+    include_standard_zip: bool = False,
+) -> str:
+    distributions = [
+        f"https://{host}/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_{year}_NEW.xlsx#distribution{node_id}",
+        f"https://{host}/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_{year}_OLD.zip#distribution{node_id}",
+    ]
+    if include_new_zip:
+        distributions.append(
+            f"https://{host}/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_{year}_NEW.zip#distribution{node_id}"
+        )
+    if include_standard_zip:
+        distributions.append(
+            f"https://{host}/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_{year}.zip#distribution{node_id}"
+        )
+    distribution_values = ",\n        ".join(f"<{value}>" for value in distributions)
+    return f"""
+    <https://statbel.fgov.be/node/{node_id}#id> a dcat:Dataset ;
+      dct:title "Bevolking per statistische sector"@nl ;
+      dct:alternative "Bevolking per statistische sector [Periode: {year}]"@nl ;
+      dct:identifier "NodeID{node_id}" ;
+      dct:license <https://creativecommons.org/licenses/by/4.0/> ;
+      dct:temporal [ dcat:startDate "{year}-01-01"^^xsd:date ] ;
+      dcat:landingPage <https://statbel.fgov.be/nl/open-data/bevolking-statistische-sector-{node_id}> ;
+      dcat:distribution {distribution_values} .
+    """
+
+
+def _catalog(*blocks: str) -> bytes:
+    return f"""
+    @prefix dcat: <http://www.w3.org/ns/dcat#> .
+    @prefix dct: <http://purl.org/dc/terms/> .
+    @prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+    <http://data.gov.be/catalog/statbelopen> a dcat:Catalog ;
+      dct:modified "2026-07-07"^^xsd:date .
+    {''.join(blocks)}
+    """.encode()
+
+
+class _Response:
+    def __init__(
+        self,
+        content: bytes,
+        *,
+        content_type: str = "application/octet-stream",
+        final_url: str | None = None,
+        content_length: int | None = None,
+    ) -> None:
+        self.content = content
+        self.final_url = final_url
+        self.headers = Message()
+        self.headers["Content-Type"] = content_type
+        self.headers["Content-Length"] = str(content_length if content_length is not None else len(content))
+        self.headers["ETag"] = '"statbel-test"'
+        self.headers["Last-Modified"] = "Mon, 13 Jul 2026 08:22:54 GMT"
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args):
+        return False
+
+    def read(self, size: int = -1) -> bytes:
+        return self.content if size < 0 else self.content[:size]
+
+    def geturl(self) -> str:
+        return self.final_url or STATBEL_DCAT_URL
+
+
+class _Query:
+    def __init__(self, datasets: list[Dataset]) -> None:
+        self.datasets = datasets
+
+    def filter(self, *_args):
+        return self
+
+    def all(self) -> list[Dataset]:
+        return self.datasets
+
+
+class _Db:
+    def __init__(self, datasets: list[Dataset]) -> None:
+        self.datasets = datasets
+
+    def get(self, _model, identifier):
+        return SimpleNamespace(id=identifier)
+
+    def query(self, _model):
+        return _Query(self.datasets)
+
+
+def _settings(**overrides) -> Settings:
+    values = {
+        "SOURCE_CATALOG_PROBE_ENABLED": True,
+        "SOURCE_CATALOG_GRB_WFS_URL": "https://geo.api.vlaanderen.be/GRB/wfs",
+        "SOURCE_CATALOG_ALZ_RELEASE_URL": ALZ_RELEASE_URL,
+        "SOURCE_CATALOG_STATBEL_DCAT_URL": STATBEL_DCAT_URL,
+        "SOURCE_CATALOG_STATBEL_MAX_RESPONSE_MB": 5,
+        "SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS": 3,
+        "SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB": 1,
+        "SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS": 0,
+        "ORTHOPHOTO_WMS_URL": "https://geo.api.vlaanderen.be/OMWRGBMRVL/wms",
+        "ORTHOPHOTO_WMS_LAYER": "Ortho",
+    }
+    values.update(overrides)
+    return Settings(**values)
+
+
+def _statbel_item(content: bytes, *, final_url: str | None = None, content_length: int | None = None, datasets=None):
+    def opener(request, timeout):
+        if request.full_url == STATBEL_DCAT_URL:
+            return _Response(content, final_url=final_url, content_length=content_length)
+        raise URLError("not needed for selected assertion")
+
+    report = SourceCatalogProbeService.audit_project(
+        _Db(datasets or []), uuid.uuid4(), settings=_settings(), opener=opener, now=NOW
+    )
+    return next(item for item in report.items if item.source_name == "statbel")
+
+
+def test_statbel_parser_selects_latest_population_release_and_redegeo_variant() -> None:
+    release = parse_statbel_population_catalog(
+        _catalog(_dataset_block(2024, node_id=5510), _dataset_block(2025, node_id=6475))
+    )
+
+    assert release.version == "2025"
+    assert release.identifier == "NodeID6475"
+    assert release.current_distribution_variant == "new"
+    assert release.legacy_distribution_available is True
+    assert release.distribution_count == 3
+    assert release.catalog_modified_at == datetime(2026, 7, 7, tzinfo=timezone.utc)
+
+
+def test_statbel_probe_compares_latest_local_year_not_latest_import_order() -> None:
+    older = Dataset(
+        id=uuid.uuid4(), project_id=uuid.uuid4(), name="Statbel 2024", dataset_type="vector",
+        source="statbel", source_name="statbel", source_version="2024", imported_at=NOW, status="ready",
+    )
+    newer = Dataset(
+        id=uuid.uuid4(), project_id=uuid.uuid4(), name="Statbel 2025", dataset_type="vector",
+        source="statbel", source_name="statbel", source_version="2025", imported_at=NOW.replace(year=2025), status="ready",
+    )
+    item = _statbel_item(_catalog(_dataset_block(2025, node_id=6475)), datasets=[older, newer])
+
+    assert item.status == "available"
+    assert item.local_source_version == "2025"
+    assert item.remote_version == "2025"
+    assert item.comparison_status == "same"
+    assert item.metadata_identifier == "NodeID6475"
+    assert item.matched_layers == ["population_txt_current", "landing_page", "cc_by_4_0"]
+
+
+def test_statbel_parser_rejects_untrusted_distribution_host() -> None:
+    with pytest.raises(StatbelCatalogError) as exc_info:
+        parse_statbel_population_catalog(_catalog(_dataset_block(2025, node_id=6475, host="example.com")))
+
+    assert exc_info.value.code == "CATALOG_STATBEL_DISTRIBUTION_REJECTED"
+
+
+def test_statbel_parser_requires_new_2025_txt_distribution() -> None:
+    with pytest.raises(StatbelCatalogError) as exc_info:
+        parse_statbel_population_catalog(
+            _catalog(_dataset_block(2025, node_id=6475, include_new_zip=False))
+        )
+
+    assert exc_info.value.code == "CATALOG_STATBEL_CURRENT_DISTRIBUTION_MISSING"
+
+
+def test_statbel_parser_rejects_conflicting_period_evidence() -> None:
+    content = _catalog(_dataset_block(2025, node_id=6475)).replace(
+        b'dcat:startDate "2025-01-01"',
+        b'dcat:startDate "2024-01-01"',
+    )
+
+    with pytest.raises(StatbelCatalogError) as exc_info:
+        parse_statbel_population_catalog(content)
+
+    assert exc_info.value.code == "CATALOG_STATBEL_PERIOD_AMBIGUOUS"
+
+
+def test_statbel_parser_rejects_duplicate_latest_release() -> None:
+    with pytest.raises(StatbelCatalogError) as exc_info:
+        parse_statbel_population_catalog(
+            _catalog(_dataset_block(2025, node_id=6475), _dataset_block(2025, node_id=7000))
+        )
+
+    assert exc_info.value.code == "CATALOG_STATBEL_POPULATION_AMBIGUOUS"
+
+
+def test_statbel_parser_requires_cc_by_4_license() -> None:
+    content = _catalog(_dataset_block(2025, node_id=6475)).replace(
+        b"https://creativecommons.org/licenses/by/4.0/",
+        b"https://example.com/unknown-license",
+    )
+
+    with pytest.raises(StatbelCatalogError) as exc_info:
+        parse_statbel_population_catalog(content)
+
+    assert exc_info.value.code == "CATALOG_STATBEL_LICENSE_MISSING"
+
+
+def test_statbel_probe_revalidates_catalog_url_after_redirect() -> None:
+    item = _statbel_item(
+        _catalog(_dataset_block(2025, node_id=6475)),
+        final_url="https://example.com/DCAT_opendata_datasets.ttl",
+    )
+
+    assert item.status == "unavailable"
+    assert item.error_code == "CATALOG_STATBEL_URL_REJECTED"
+
+
+def test_statbel_probe_rejects_noncanonical_configured_catalog_before_network() -> None:
+    calls: list[str] = []
+
+    def opener(request, timeout):
+        calls.append(request.full_url)
+        raise URLError("network must not be reached for the rejected Statbel URL")
+
+    report = SourceCatalogProbeService.audit_project(
+        _Db([]),
+        uuid.uuid4(),
+        settings=_settings(SOURCE_CATALOG_STATBEL_DCAT_URL="https://example.com/catalog.ttl"),
+        opener=opener,
+        now=NOW,
+    )
+    item = next(value for value in report.items if value.source_name == "statbel")
+
+    assert item.status == "unavailable"
+    assert item.error_code == "CATALOG_STATBEL_URL_REJECTED"
+    assert "https://example.com/catalog.ttl" not in calls
+
+
+def test_statbel_probe_uses_separate_bounded_catalog_limit() -> None:
+    item = _statbel_item(
+        _catalog(_dataset_block(2025, node_id=6475)),
+        content_length=6 * 1024 * 1024,
+    )
+
+    assert item.status == "unavailable"
+    assert item.error_code == "CATALOG_RESPONSE_TOO_LARGE"
+
+
+def test_statbel_probe_does_not_fetch_catalog_distributions() -> None:
+    calls: list[str] = []
+
+    def opener(request, timeout):
+        calls.append(request.full_url)
+        if request.full_url == STATBEL_DCAT_URL:
+            return _Response(_catalog(_dataset_block(2025, node_id=6475)))
+        if "OPENDATA_SECTOREN" in request.full_url:
+            raise AssertionError("Statbel distribution fetch attempted")
+        raise URLError("unrelated provider offline")
+
+    report = SourceCatalogProbeService.audit_project(
+        _Db([]), uuid.uuid4(), settings=_settings(), opener=opener, now=NOW
+    )
+    item = next(value for value in report.items if value.source_name == "statbel")
+
+    assert item.status == "available"
+    assert calls.count(STATBEL_DCAT_URL) == 1
+    assert all("OPENDATA_SECTOREN" not in value for value in calls)