fix: use WFS POST for historical land use
This commit is contained in:
@@ -245,6 +245,7 @@ def test_temporal_frontend_and_official_operator_contracts_exist() -> None:
|
|||||||
assert "/temporal/compare" in temporal_api
|
assert "/temporal/compare" in temporal_api
|
||||||
assert "Statbel" in population and "area_weighted_sum" in population
|
assert "Statbel" in population and "area_weighted_sum" in population
|
||||||
assert "HistLandgebruik" in landuse and "intersection_area" in landuse
|
assert "HistLandgebruik" in landuse and "intersection_area" in landuse
|
||||||
|
assert "<wfs:GetFeature" in landuse and "session.post(" in landuse
|
||||||
assert "provision_mol_population_history.py" in dockerfile
|
assert "provision_mol_population_history.py" in dockerfile
|
||||||
assert "provision_mol_historical_landuse.py" in dockerfile
|
assert "provision_mol_historical_landuse.py" in dockerfile
|
||||||
assert "fake" not in population.lower()
|
assert "fake" not in population.lower()
|
||||||
|
|||||||
@@ -7801,3 +7801,4 @@ Live deployment correction:
|
|||||||
- Both Tower deploy entry points now wait for the `geointel` container healthcheck, which includes completed startup migrations and backend readiness, before launching the independent migration smoke.
|
- Both Tower deploy entry points now wait for the `geointel` container healthcheck, which includes completed startup migrations and backend readiness, before launching the independent migration smoke.
|
||||||
- The first Statbel upload exposed 3D sector coordinates (`Z=0`) against the canonical 2D PostGIS vector column. The source is valid; GeoIntel now preserves the original artifact, records the Z-feature count and explicitly drops Z only for the 2D query index.
|
- The first Statbel upload exposed 3D sector coordinates (`Z=0`) against the canonical 2D PostGIS vector column. The source is valid; GeoIntel now preserves the original artifact, records the Z-feature count and explicitly drops Z only for the 2D query index.
|
||||||
- Vector upload persistence is now atomic across Dataset, DatasetVersion and VectorFeature rows, with storage cleanup on rollback. This prevents the failed-indexing orphan state observed during the live import.
|
- Vector upload persistence is now atomic across Dataset, DatasetVersion and VectorFeature rows, with storage cleanup on rollback. This prevents the failed-indexing orphan state observed during the live import.
|
||||||
|
- Tower could read the historical WFS capabilities but its gateway rejected XML `FILTER` content embedded in a GET query. The operator now sends the same read-only WFS 2.0 GetFeature request as XML POST; a live two-feature forest page returned successfully before redeployment.
|
||||||
|
|||||||
@@ -55,6 +55,10 @@ These historical map editions support exploratory area evolution, not
|
|||||||
cadastral object lineage. Their feature identities are declared unstable and
|
cadastral object lineage. Their feature identities are declared unstable and
|
||||||
GeoIntel does not fabricate added/removed object counts.
|
GeoIntel does not fabricate added/removed object counts.
|
||||||
|
|
||||||
|
The operator uses standards-compliant WFS 2.0 XML POST requests. This keeps
|
||||||
|
the spatial/class filters server-side without exposing a long XML filter in a
|
||||||
|
GET query, which the public gateway rejects.
|
||||||
|
|
||||||
## OSM
|
## OSM
|
||||||
|
|
||||||
- Naam: OpenStreetMap
|
- Naam: OpenStreetMap
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ def build_session() -> requests.Session:
|
|||||||
status=5,
|
status=5,
|
||||||
backoff_factor=1.0,
|
backoff_factor=1.0,
|
||||||
status_forcelist=(429, 500, 502, 503, 504),
|
status_forcelist=(429, 500, 502, 503, 504),
|
||||||
allowed_methods=frozenset({"GET"}),
|
allowed_methods=frozenset({"GET", "POST"}),
|
||||||
raise_on_status=True,
|
raise_on_status=True,
|
||||||
)
|
)
|
||||||
session = requests.Session()
|
session = requests.Session()
|
||||||
@@ -127,6 +127,54 @@ def wfs_filter_xml(definition: ThemeDefinition, bounds: tuple[float, float, floa
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def wfs_request_xml(
|
||||||
|
*,
|
||||||
|
collection: str,
|
||||||
|
filter_xml: str,
|
||||||
|
page_size: int,
|
||||||
|
start_index: int,
|
||||||
|
) -> str:
|
||||||
|
return (
|
||||||
|
"<?xml version='1.0' encoding='UTF-8'?>"
|
||||||
|
"<wfs:GetFeature service='WFS' version='2.0.0' outputFormat='application/json' "
|
||||||
|
f"count='{page_size}' startIndex='{start_index}' "
|
||||||
|
"xmlns:wfs='http://www.opengis.net/wfs/2.0' "
|
||||||
|
"xmlns:fes='http://www.opengis.net/fes/2.0' "
|
||||||
|
"xmlns:gml='http://www.opengis.net/gml/3.2' "
|
||||||
|
"xmlns:HistLandgebruik='https://geo.api.vlaanderen.be/HistLandgebruik'>"
|
||||||
|
f"<wfs:Query typeNames='{collection}' srsName='EPSG:4326'>{filter_xml}</wfs:Query>"
|
||||||
|
"</wfs:GetFeature>"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_wfs_page(
|
||||||
|
session: requests.Session,
|
||||||
|
*,
|
||||||
|
collection: str,
|
||||||
|
filter_xml: str,
|
||||||
|
page_size: int,
|
||||||
|
start_index: int,
|
||||||
|
timeout: int,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
response = session.post(
|
||||||
|
WFS_URL,
|
||||||
|
data=wfs_request_xml(
|
||||||
|
collection=collection,
|
||||||
|
filter_xml=filter_xml,
|
||||||
|
page_size=page_size,
|
||||||
|
start_index=start_index,
|
||||||
|
).encode("utf-8"),
|
||||||
|
headers={"Content-Type": "application/xml; charset=UTF-8", "Accept": "application/json"},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
payload = response.json()
|
||||||
|
features = payload.get("features") if isinstance(payload, dict) else None
|
||||||
|
if not isinstance(features, list):
|
||||||
|
raise RuntimeError("Historical land-use WFS returned an invalid FeatureCollection")
|
||||||
|
return features
|
||||||
|
|
||||||
|
|
||||||
def fetch_year(
|
def fetch_year(
|
||||||
session: requests.Session,
|
session: requests.Session,
|
||||||
year: int,
|
year: int,
|
||||||
@@ -145,23 +193,14 @@ def fetch_year(
|
|||||||
start_index = 0
|
start_index = 0
|
||||||
seen: set[str] = set()
|
seen: set[str] = set()
|
||||||
while True:
|
while True:
|
||||||
response = session.get(
|
page = fetch_wfs_page(
|
||||||
WFS_URL,
|
session,
|
||||||
params={
|
collection=collection,
|
||||||
"service": "WFS",
|
filter_xml=filter_xml,
|
||||||
"version": "2.0.0",
|
page_size=page_size,
|
||||||
"request": "GetFeature",
|
start_index=start_index,
|
||||||
"typeNames": collection,
|
|
||||||
"outputFormat": "application/json",
|
|
||||||
"srsName": "EPSG:4326",
|
|
||||||
"FILTER": filter_xml,
|
|
||||||
"count": page_size,
|
|
||||||
"startIndex": start_index,
|
|
||||||
},
|
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
response.raise_for_status()
|
|
||||||
page = response.json().get("features") or []
|
|
||||||
for raw_feature in page:
|
for raw_feature in page:
|
||||||
feature_id = str(raw_feature.get("id") or "")
|
feature_id = str(raw_feature.get("id") or "")
|
||||||
if not feature_id or feature_id in seen:
|
if not feature_id or feature_id in seen:
|
||||||
@@ -234,23 +273,14 @@ def write_theme_snapshot(
|
|||||||
)
|
)
|
||||||
output.write(',"features":[')
|
output.write(',"features":[')
|
||||||
while True:
|
while True:
|
||||||
response = session.get(
|
page = fetch_wfs_page(
|
||||||
WFS_URL,
|
session,
|
||||||
params={
|
collection=collection,
|
||||||
"service": "WFS",
|
filter_xml=filter_xml,
|
||||||
"version": "2.0.0",
|
page_size=page_size,
|
||||||
"request": "GetFeature",
|
start_index=start_index,
|
||||||
"typeNames": collection,
|
|
||||||
"outputFormat": "application/json",
|
|
||||||
"srsName": "EPSG:4326",
|
|
||||||
"FILTER": filter_xml,
|
|
||||||
"count": page_size,
|
|
||||||
"startIndex": start_index,
|
|
||||||
},
|
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
response.raise_for_status()
|
|
||||||
page = response.json().get("features") or []
|
|
||||||
for raw_feature in page:
|
for raw_feature in page:
|
||||||
feature_id = str(raw_feature.get("id") or "")
|
feature_id = str(raw_feature.get("id") or "")
|
||||||
if not feature_id or feature_id in seen:
|
if not feature_id or feature_id in seen:
|
||||||
|
|||||||
Reference in New Issue
Block a user