From d062edbbe1d05df254d2a5d3a3fb4e2feb810c10 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 15:31:54 +0200 Subject: [PATCH] fix: use WFS POST for historical land use --- .../test_sprint187_temporal_map_foundation.py | 1 + docs/CODEX_EXECUTION_LOG.md | 1 + docs/DATA_SOURCES.md | 4 + scripts/provision_mol_historical_landuse.py | 92 ++++++++++++------- 4 files changed, 67 insertions(+), 31 deletions(-) diff --git a/backend/tests/test_sprint187_temporal_map_foundation.py b/backend/tests/test_sprint187_temporal_map_foundation.py index 76ed41f5..85cc8bfe 100644 --- a/backend/tests/test_sprint187_temporal_map_foundation.py +++ b/backend/tests/test_sprint187_temporal_map_foundation.py @@ -245,6 +245,7 @@ def test_temporal_frontend_and_official_operator_contracts_exist() -> None: assert "/temporal/compare" in temporal_api assert "Statbel" in population and "area_weighted_sum" in population assert "HistLandgebruik" in landuse and "intersection_area" in landuse + assert " requests.Session: status=5, backoff_factor=1.0, status_forcelist=(429, 500, 502, 503, 504), - allowed_methods=frozenset({"GET"}), + allowed_methods=frozenset({"GET", "POST"}), raise_on_status=True, ) session = requests.Session() @@ -127,6 +127,54 @@ def wfs_filter_xml(definition: ThemeDefinition, bounds: tuple[float, float, floa ) +def wfs_request_xml( + *, + collection: str, + filter_xml: str, + page_size: int, + start_index: int, +) -> str: + return ( + "" + "" + f"{filter_xml}" + "" + ) + + +def fetch_wfs_page( + session: requests.Session, + *, + collection: str, + filter_xml: str, + page_size: int, + start_index: int, + timeout: int, +) -> list[dict[str, Any]]: + response = session.post( + WFS_URL, + data=wfs_request_xml( + collection=collection, + filter_xml=filter_xml, + page_size=page_size, + start_index=start_index, + ).encode("utf-8"), + headers={"Content-Type": "application/xml; charset=UTF-8", "Accept": "application/json"}, + timeout=timeout, + ) + response.raise_for_status() + payload = response.json() + features = payload.get("features") if isinstance(payload, dict) else None + if not isinstance(features, list): + raise RuntimeError("Historical land-use WFS returned an invalid FeatureCollection") + return features + + def fetch_year( session: requests.Session, year: int, @@ -145,23 +193,14 @@ def fetch_year( start_index = 0 seen: set[str] = set() while True: - response = session.get( - WFS_URL, - params={ - "service": "WFS", - "version": "2.0.0", - "request": "GetFeature", - "typeNames": collection, - "outputFormat": "application/json", - "srsName": "EPSG:4326", - "FILTER": filter_xml, - "count": page_size, - "startIndex": start_index, - }, + page = fetch_wfs_page( + session, + collection=collection, + filter_xml=filter_xml, + page_size=page_size, + start_index=start_index, timeout=timeout, ) - response.raise_for_status() - page = response.json().get("features") or [] for raw_feature in page: feature_id = str(raw_feature.get("id") or "") if not feature_id or feature_id in seen: @@ -234,23 +273,14 @@ def write_theme_snapshot( ) output.write(',"features":[') while True: - response = session.get( - WFS_URL, - params={ - "service": "WFS", - "version": "2.0.0", - "request": "GetFeature", - "typeNames": collection, - "outputFormat": "application/json", - "srsName": "EPSG:4326", - "FILTER": filter_xml, - "count": page_size, - "startIndex": start_index, - }, + page = fetch_wfs_page( + session, + collection=collection, + filter_xml=filter_xml, + page_size=page_size, + start_index=start_index, timeout=timeout, ) - response.raise_for_status() - page = response.json().get("features") or [] for raw_feature in page: feature_id = str(raw_feature.get("id") or "") if not feature_id or feature_id in seen: