feat: provision regional Kempen context layers
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-14 20:07:03 +02:00
parent 84f695d069
commit 0ed214da02
10 changed files with 1142 additions and 0 deletions
+10
View File
@@ -7,6 +7,16 @@
# Changelog
## Sprint 191 Regional Kempen GRB context foundation (2026-07-14)
- Added one explicit regional operator for current GRB roads, water and parcels, with independent resumable municipality partitions and one normal PostGIS dataset per theme.
- Preserved official source geometry dimensions across `Wegsegment`, `WTZ`, `WLAS`, `WGR` and `ADP`; collection-qualified source IDs prevent collisions inside the combined water layer.
- Assigned cross-boundary polygons by maximum overlap area and lines by maximum overlap length, with deterministic NIS-code tie breaking and clipping only to the complete approved region.
- Reused DatasetService, VectorFeatureService, immutable dataset versions, exact selection aggregation and existing Area/bbox APIs; no API contract, migration or direct vector-feature write was added.
- Added truncation guards, atomic manifests, checksum reuse, duplicate rejection, Docker packaging, readiness compilation and focused geometry/persistence tests.
- Documented source semantics honestly: road objects are not traffic data, heterogeneous water objects are not volume metrics and ADP is not a legal cadastral survey.
- Kept all source access operator-only; the browser, startup path and public `not_configured` GRB provider perform no external fetch.
## Sprint 190 Regional Kempen GRB buildings (2026-07-14)
- Added an explicit regional GRB building operator that fetches the approved Kempen scope in 28 resumable municipality partitions and follows every OGC API pagination link.
+16
View File
@@ -972,6 +972,22 @@ exact manifest feature count or it rolls back and removes the managed copy.
No public API contract or provider readiness claim is changed by this
operator-only path.
Provision the regional current road, water and parcel context through the
same persistence boundary:
```bash
docker exec geointel python /app/scripts/provision_regional_grb_context.py \
--scope kempen-transport-region --layers roads water parcels
```
The operator keeps one resumable municipality partition set per theme and
creates one regional reference Dataset per theme. Polygon ownership uses
maximum overlap area; line ownership uses maximum overlap length. It preserves
source geometry dimensions and collection-qualified source IDs, copies the
combined artifact through StorageService and indexes bounded batches through
DatasetService/VectorFeatureService. It does not add API routes, direct SQL or
interactive provider downloads.
## Temporal Mol data and evolution
Dataset uploads accept `temporal_series_key`, `observed_at`, `valid_from`,
@@ -0,0 +1,215 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import sys
import pytest
from shapely.geometry import LineString, Polygon, shape
from shapely.ops import unary_union
ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
def load_operator():
scripts_path = str(SCRIPTS)
if scripts_path not in sys.path:
sys.path.insert(0, scripts_path)
spec = importlib.util.spec_from_file_location(
"provision_regional_grb_context_test",
SCRIPTS / "provision_regional_grb_context.py",
)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def source_feature(feature_id: str, geometry) -> dict:
return {
"type": "Feature",
"id": feature_id,
"geometry": geometry.__geo_interface__,
"properties": {"UIDN": feature_id},
}
def test_layer_registry_matches_verified_official_grb_collections() -> None:
operator = load_operator()
assert [item.key for item in operator.LAYERS] == ["roads", "water", "parcels"]
assert [(item.name, item.geometry_dimension) for item in operator.LAYER_BY_KEY["roads"].collections] == [
("Wegsegment", 1)
]
assert [(item.name, item.geometry_dimension) for item in operator.LAYER_BY_KEY["water"].collections] == [
("WTZ", 2),
("WLAS", 1),
("WGR", 1),
]
assert [(item.name, item.geometry_dimension) for item in operator.LAYER_BY_KEY["parcels"].collections] == [
("ADP", 2)
]
assert [item.key for item in operator.selected_definitions("parcels,roads")] == ["roads", "parcels"]
with pytest.raises(ValueError, match="Unsupported layers"):
operator.selected_definitions("buildings")
def test_line_owner_uses_intersection_length_and_deterministic_tie_break() -> None:
operator = load_operator()
scopes = __import__("geographic_scopes")
alpha = scopes.ScopeMember("Alpha", "10001")
beta = scopes.ScopeMember("Beta", "10002")
alpha_boundary = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)])
beta_boundary = Polygon([(1, 0), (2, 0), (2, 1), (1, 1)])
members = [(alpha, alpha_boundary), (beta, beta_boundary)]
assert operator.assign_owner_nis(
LineString([(0.7, 0.5), (1.1, 0.5)]),
members,
expected_dimension=1,
) == "10001"
assert operator.assign_owner_nis(
LineString([(0.8, 0.5), (1.2, 0.5)]),
members,
expected_dimension=1,
) == "10001"
def test_mixed_water_partition_preserves_dimensions_and_source_identity() -> None:
operator = load_operator()
scopes = __import__("geographic_scopes")
member = scopes.ScopeMember("Alpha", "10001")
scope = scopes.GeographicScope(
key="test-region",
display_name="Test region",
project_name="Test project",
project_region="Test",
area_name="Test boundary",
authority_name="Test",
authority_url="https://example.test/scope",
scope_type="policy_region",
limitation_message="Test only.",
members=(member,),
)
boundary = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)])
definition = operator.LAYER_BY_KEY["water"]
pages = [
(
definition.collections[0],
{"type": "FeatureCollection", "features": [source_feature("WTZ.1", Polygon([(0.1, 0.1), (0.3, 0.1), (0.3, 0.3), (0.1, 0.3)]))]},
"https://example.test/wtz",
),
(
definition.collections[1],
{"type": "FeatureCollection", "features": [source_feature("WLAS.1", LineString([(-0.2, 0.5), (0.5, 0.5)]))]},
"https://example.test/wlas",
),
(
definition.collections[2],
{"type": "FeatureCollection", "features": [source_feature("WGR.1", LineString([(0.4, 0.7), (0.8, 0.7)]))]},
"https://example.test/wgr",
),
]
features, summary = operator.build_partition_features(
pages,
definition=definition,
member=member,
members=[(member, boundary)],
regional_boundary=boundary,
scope=scope,
max_features=10,
)
assert [item["id"] for item in features] == ["WTZ:WTZ.1", "WLAS:WLAS.1", "WGR:WGR.1"]
assert [shape(item["geometry"]).geom_type for item in features] == ["Polygon", "LineString", "LineString"]
assert shape(features[1]["geometry"]).bounds == (0.0, 0.5, 0.5, 0.5)
assert features[1]["properties"]["clipped_to_regional_scope"] is True
assert summary["features_by_collection"] == {"WTZ": 1, "WLAS": 1, "WGR": 1}
assert summary["reference_truncated"] is False
def test_polygon_partition_assignment_does_not_duplicate_cross_boundary_parcel() -> None:
operator = load_operator()
scopes = __import__("geographic_scopes")
alpha = scopes.ScopeMember("Alpha", "10001")
beta = scopes.ScopeMember("Beta", "10002")
scope = scopes.GeographicScope(
key="test-region",
display_name="Test region",
project_name="Test project",
project_region="Test",
area_name="Test boundary",
authority_name="Test",
authority_url="https://example.test/scope",
scope_type="policy_region",
limitation_message="Test only.",
members=(alpha, beta),
)
alpha_boundary = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)])
beta_boundary = Polygon([(1, 0), (2, 0), (2, 1), (1, 1)])
members = [(alpha, alpha_boundary), (beta, beta_boundary)]
region = unary_union([alpha_boundary, beta_boundary])
definition = operator.LAYER_BY_KEY["parcels"]
crossing = source_feature("ADP.1", Polygon([(0.7, 0.2), (1.1, 0.2), (1.1, 0.6), (0.7, 0.6)]))
page = (definition.collections[0], {"type": "FeatureCollection", "features": [crossing]}, "https://example.test/adp")
alpha_features, _ = operator.build_partition_features(
[page], definition=definition, member=alpha, members=members, regional_boundary=region, scope=scope, max_features=10
)
with pytest.raises(RuntimeError, match="No GRB parcels"):
operator.build_partition_features(
[page], definition=definition, member=beta, members=members, regional_boundary=region, scope=scope, max_features=10
)
assert [item["id"] for item in alpha_features] == ["ADP:ADP.1"]
def test_combined_context_artifact_rejects_duplicate_source_identity(tmp_path: Path) -> None:
operator = load_operator()
scope = __import__("geographic_scopes").KEMPEN_TRANSPORT_REGION_SCOPE
definition = operator.LAYER_BY_KEY["roads"]
feature = source_feature("Wegsegment:Wegsegment.1", LineString([(4.9, 51.1), (4.91, 51.11)]))
first = tmp_path / "first.geojson"
second = tmp_path / "second.geojson"
first.write_text(json.dumps({"type": "FeatureCollection", "features": [feature]}), encoding="utf-8")
second.write_text(json.dumps({"type": "FeatureCollection", "features": [source_feature("Wegsegment:Wegsegment.2", LineString([(5.0, 51.2), (5.01, 51.21)]))]}), encoding="utf-8")
combined = tmp_path / "combined.geojson"
summary = operator.write_combined_artifact(
combined,
definition=definition,
scope=scope,
observed_date=operator.date(2026, 7, 14),
partition_paths=[first, second],
expected_feature_count=2,
)
assert summary["feature_count"] == 2
second.write_text(first.read_text(encoding="utf-8"), encoding="utf-8")
with pytest.raises(RuntimeError, match="Duplicate regional source feature"):
operator.write_combined_artifact(
combined,
definition=definition,
scope=scope,
observed_date=operator.date(2026, 7, 14),
partition_paths=[first, second],
expected_feature_count=2,
)
def test_context_operator_is_packaged_and_uses_existing_service_boundary() -> None:
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8")
operator = (SCRIPTS / "provision_regional_grb_context.py").read_text(encoding="utf-8")
assert "provision_regional_grb_context.py" in dockerfile
assert "py_compile scripts/provision_regional_grb_context.py" in readiness
assert "DatasetService.import_partitioned_vector_artifact" in operator
assert "insert into vector_features" not in operator.lower()
assert "db.add(VectorFeature" not in operator
+1
View File
@@ -80,6 +80,7 @@ COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_off
COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py
COPY scripts/provision_geographic_scope.py /app/scripts/provision_geographic_scope.py
COPY scripts/provision_regional_grb_buildings.py /app/scripts/provision_regional_grb_buildings.py
COPY scripts/provision_regional_grb_context.py /app/scripts/provision_regional_grb_context.py
COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py
COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
+16
View File
@@ -1,3 +1,19 @@
## Sprint 191 Regional Kempen GRB context foundation (2026-07-14)
Changed:
- Added `scripts/provision_regional_grb_context.py` for bounded, independently resumable `roads`, `water` and `parcels` source snapshots across the approved 28-municipality scope.
- Reused geographic-scope artifacts and the existing DatasetService/VectorFeatureService partition import boundary; no direct SQL, migration, API contract or interactive provider fetch was introduced.
- Preserved line and polygon source dimensions and collection-qualified identities, with length-based or area-based deterministic ownership for boundary-crossing features.
- Added Docker packaging, readiness compilation, focused source-geometry/service tests and operator/source documentation.
Tested before deployment:
- `python -m py_compile scripts/provision_regional_grb_context.py`.
- `python -m pytest backend/tests/test_sprint191_regional_grb_context.py backend/tests/test_sprint190_regional_grb_buildings.py backend/tests/test_sprint106_map_bbox_extract.py -q` (`19 passed`).
- `bash scripts/run_readiness_check.sh` (`552 passed`; one Alembic head; frontend typecheck/build and script syntax gates passed).
Next:
- Pass the complete release gate, deploy the operator and provision all three live regional snapshots before claiming runtime readiness.
## Sprint 169 Filtered YOLO candidate gate and operator hardening (2026-07-12)
Changed:
+22
View File
@@ -124,6 +124,28 @@ reference dataset uses `source_name=grb`, `reference_layer_name=buildings`,
selection aggregation. Existing viewport and bbox selection APIs remain the
only interactive delivery path.
### Regional GRB context partition policy
Roads, water and parcels reuse the building operator's 28 resumable source
partitions and one-dataset-per-theme persistence model. Polygon collections
use maximum intersection area for cross-boundary ownership; line collections
use maximum intersection length. Exact ties use the lower NIS code. Source
identities are collection-qualified because the water theme combines `WTZ`,
`WLAS` and `WGR`.
The user-facing datasets use `source_name=grb`, `dataset_role=reference`,
`coverage_scope=kempen-transport-region` and respectively
`reference_layer_name=roads`, `water` or `parcels`. They are current source
snapshots with immutable observation dates, not temporal series inferred from
successive operator runs. Feature counts retain source-object semantics and
must not be relabelled as road length, water area or legal parcel counts.
Only `scripts/provision_regional_grb_context.py` may fetch these bounded
artifacts. Application startup, map requests and the public provider registry
remain fetch-free. All combined artifacts enter PostGIS through the existing
DatasetService/VectorFeatureService boundary and existing bbox/Area selection
contracts.
## User-uploaded raster strategy
V1 must support controlled local datasets because public raster access and model compatibility can be difficult.
+21
View File
@@ -67,6 +67,27 @@ referentiedataset via DatasetService en VectorFeatureService. De publieke GRB
provider blijft `not_configured`: dit is een bewuste operatorrun en geen live
download vanuit een browseractie of applicatiestart.
De overige actuele GRB-contextlagen volgen dezelfde begrensde aanpak via
`scripts/provision_regional_grb_context.py`:
- `roads`: collectie `Wegsegment` (lijngeometrie). Dit zijn wegsegmenten uit
de basiskaart, geen verkeersmetingen en geen routeringsnetwerk.
- `water`: collecties `WTZ` (vlakken), `WLAS` en `WGR` (lijnen). De gemengde
objecttelling is geen maat voor wateroppervlakte, debiet of volume.
- `parcels`: collectie `ADP` (vlakken). Dit is de grafische ligging van het
vermoedelijke kadastrale perceel en geen juridisch landmeetkundig bewijs.
Lijnobjecten worden bij gemeentegrenzen toegewezen op basis van de grootste
intersectielengte, vlakobjecten op basis van de grootste intersectieoppervlakte
en exacte ties op de laagste NIS-code. De brongeometrie wordt alleen tegen de
volledige regiogrens gesneden. Bron-ID's worden met hun collectienaam
gekwalificeerd zodat de drie watercollecties nooit stilzwijgend botsen.
Collectiecatalogus en contract:
- https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections?f=text%2Fhtml
- https://geo.api.vlaanderen.be/GRB/ogc/features/v1/openapi?f=text%2Fhtml
### Mol population history
`scripts/provision_mol_population_history.py` imports official Statbel
+32
View File
@@ -1325,6 +1325,38 @@ buildings from 879 GRB source pages. Its 28 retained partitions produce one
`e236fa03-7fce-4b37-bc7d-8bddd4d50346`; an immediate repeat run reused the
same checksums and Dataset instead of fetching or inserting duplicates.
### Regional Kempen GRB roads, water and parcels
Provision the remaining current GRB context snapshots after the scope
foundation is present:
```bash
docker exec -it geointel python3 /app/scripts/provision_regional_grb_context.py \
--scope kempen-transport-region \
--layers roads water parcels
```
Each theme has an independent manifest and can be resumed or rerun alone:
```bash
docker exec -it geointel python3 /app/scripts/provision_regional_grb_context.py \
--scope kempen-transport-region --layers roads
```
Use `--fetch-only` to prepare checksummed artifacts without database writes.
Artifacts are retained below
`/app/storage/operator-data/regional-themes/kempen-transport-region/<theme>/<date>`.
The operator follows all OGC API pagination links, fails rather than hiding a
safety-limit truncation and persists one normal PostGIS dataset per theme via
DatasetService. Repeat runs reuse valid partitions, combined artifacts and
datasets; `--force` refreshes the selected observation date.
The semantic limits are deliberate: `Wegsegment` is road-map context rather
than traffic or routing data; `WTZ`/`WLAS`/`WGR` are heterogeneous water
objects rather than a volume metric; `ADP` is the graphical presumed cadastral
parcel location rather than a legal survey. The public GRB provider remains
`not_configured`, and no fetch runs from the browser or during startup.
## Tower deployment
Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime:
+808
View File
@@ -0,0 +1,808 @@
"""Provision regional GRB roads, water and parcel datasets.
Every source layer is fetched in resumable municipality partitions for an
approved geographic scope. Source identities are assigned to exactly one
partition, while retained geometries are clipped only to the complete region.
The resulting artifacts are indexed through DatasetService and
VectorFeatureService; this operator never writes directly to vector_features.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import sys
from dataclasses import dataclass
from datetime import date
from pathlib import Path
from typing import Any, Iterable
from urllib.parse import urlparse
from uuid import UUID
import requests
from shapely.geometry import LineString, MultiLineString, MultiPolygon, Polygon, mapping, shape
from shapely.ops import unary_union
from shapely.validation import make_valid
from geographic_scopes import GEOGRAPHIC_SCOPES, GeographicScope, ScopeMember
from provision_geographic_scope import fetch_scope_members
from provision_regional_grb_buildings import (
DEFAULT_API_URL,
DEFAULT_MAX_FEATURES_PER_MEMBER,
DEFAULT_MAX_TOTAL_FEATURES,
DEFAULT_OUTPUT_ROOT,
DEFAULT_PAGE_LIMIT,
DEFAULT_SCOPE_KEY,
GEOJSON_CRS,
GRB_ATTRIBUTION,
bounds_overlap,
build_member_geometries,
build_source_session,
ensure_backend_path,
list_paginated_items,
next_page_url,
observed_at,
response_data,
reusable_manifest,
safe_slug,
sha256_file,
utc_now,
write_json_atomic,
)
GRB_COLLECTION_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/{collection}/items"
@dataclass(frozen=True)
class CollectionDefinition:
name: str
geometry_dimension: int
@dataclass(frozen=True)
class LayerDefinition:
key: str
collections: tuple[CollectionDefinition, ...]
reference_layer_name: str
layer_type: str
geometry_types: tuple[str, ...]
metric_label: str
limitation_message: str
LAYERS = (
LayerDefinition(
key="roads",
collections=(CollectionDefinition("Wegsegment", 1),),
reference_layer_name="roads",
layer_type="road",
geometry_types=("LineString", "MultiLineString"),
metric_label="Wegen",
limitation_message="GRB Wegsegment represents road-network line segments, not traffic volume or routing suitability.",
),
LayerDefinition(
key="water",
collections=(
CollectionDefinition("WTZ", 2),
CollectionDefinition("WLAS", 1),
CollectionDefinition("WGR", 1),
),
reference_layer_name="water",
layer_type="water",
geometry_types=("LineString", "MultiLineString", "Polygon", "MultiPolygon"),
metric_label="Waterobjecten",
limitation_message="GRB water combines surface-water polygons and water-related line collections; counts are object counts, not water volume.",
),
LayerDefinition(
key="parcels",
collections=(CollectionDefinition("ADP", 2),),
reference_layer_name="parcels",
layer_type="parcel",
geometry_types=("Polygon", "MultiPolygon"),
metric_label="Percelen",
limitation_message="GRB ADP is a graphical representation of the presumed cadastral parcel location and is not a legal boundary survey.",
),
)
LAYER_BY_KEY = {definition.key: definition for definition in LAYERS}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Provision municipality-partitioned regional GRB context layers.")
parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE_KEY)
parser.add_argument("--layers", default=",".join(LAYER_BY_KEY), help="Comma-separated subset: roads,water,parcels")
parser.add_argument("--observed-date", type=date.fromisoformat, default=date.today())
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
parser.add_argument(
"--output-root",
type=Path,
default=Path(os.environ.get("GEOINTEL_REGIONAL_THEME_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
)
parser.add_argument("--page-limit", type=int, default=DEFAULT_PAGE_LIMIT)
parser.add_argument("--max-features-per-member", type=int, default=DEFAULT_MAX_FEATURES_PER_MEMBER)
parser.add_argument("--max-total-features", type=int, default=DEFAULT_MAX_TOTAL_FEATURES)
parser.add_argument("--request-timeout", type=int, default=180)
parser.add_argument("--api-timeout", type=int, default=180)
parser.add_argument("--batch-size", type=int, default=1000)
parser.add_argument("--force", action="store_true", help="Refetch every selected municipality partition.")
parser.add_argument("--fetch-only", action="store_true", help="Build and validate artifacts without persistence.")
return parser.parse_args()
def selected_definitions(raw_layers: str) -> list[LayerDefinition]:
requested = {item.strip().lower() for item in raw_layers.split(",") if item.strip()}
unknown = requested - set(LAYER_BY_KEY)
if unknown or not requested:
raise ValueError(f"Unsupported layers: {sorted(unknown)}")
return [definition for definition in LAYERS if definition.key in requested]
def source_session() -> requests.Session:
session = build_source_session()
session.headers.update({"User-Agent": "GeoIntel-Regional-GRB-Context-Operator/1.0"})
return session
def geometry_dimension(geometry) -> int:
if geometry is None or geometry.is_empty:
return -1
if "Polygon" in geometry.geom_type:
return 2
if "LineString" in geometry.geom_type or geometry.geom_type == "LinearRing":
return 1
if "Point" in geometry.geom_type:
return 0
if geometry.geom_type == "GeometryCollection":
return max((geometry_dimension(part) for part in geometry.geoms), default=-1)
return -1
def extract_dimension(geometry, expected_dimension: int):
if geometry is None or geometry.is_empty:
return None
if not geometry.is_valid:
geometry = make_valid(geometry)
parts: list[Any] = []
def collect(candidate) -> None:
if candidate is None or candidate.is_empty:
return
if expected_dimension == 2:
if isinstance(candidate, Polygon):
parts.append(candidate)
return
if isinstance(candidate, MultiPolygon):
parts.extend(part for part in candidate.geoms if not part.is_empty)
return
if expected_dimension == 1:
if isinstance(candidate, LineString):
parts.append(candidate)
return
if isinstance(candidate, MultiLineString):
parts.extend(part for part in candidate.geoms if not part.is_empty)
return
if hasattr(candidate, "geoms"):
for part in candidate.geoms:
collect(part)
collect(geometry)
if not parts:
return None
normalized = unary_union(parts)
if normalized.is_empty:
return None
if not normalized.is_valid:
normalized = make_valid(normalized)
if normalized.is_empty or not normalized.is_valid or geometry_dimension(normalized) != expected_dimension:
return None
return normalized
def normalized_geometry(payload: dict[str, Any] | None, expected_dimension: int):
if not payload:
return None
return extract_dimension(shape(payload), expected_dimension)
def geometry_measure(geometry, expected_dimension: int) -> float:
candidate = extract_dimension(geometry, expected_dimension)
if candidate is None:
return 0.0
if expected_dimension == 2:
return float(candidate.area)
if expected_dimension == 1:
return float(candidate.length)
return 1.0
def assign_owner_nis(
source_geometry,
members: list[tuple[ScopeMember, Any]],
*,
expected_dimension: int,
) -> str | None:
candidates: list[tuple[float, str]] = []
source_bounds = source_geometry.bounds
for member, boundary in members:
if not bounds_overlap(source_bounds, boundary.bounds) or not source_geometry.intersects(boundary):
continue
score = geometry_measure(source_geometry.intersection(boundary), expected_dimension)
if score > 0:
candidates.append((score, member.nis_code))
if not candidates:
return None
candidates.sort(key=lambda item: (-item[0], item[1]))
return candidates[0][1]
def iter_collection_pages(
session: requests.Session,
collection: CollectionDefinition,
bounds: tuple[float, float, float, float],
*,
page_limit: int,
timeout: int,
) -> Iterable[tuple[CollectionDefinition, dict[str, Any], str]]:
params = {
"f": "application/geo+json",
"limit": str(page_limit),
"bbox": ",".join(f"{value:.8f}" for value in bounds),
}
url: str | None = GRB_COLLECTION_URL.format(collection=collection.name)
seen_urls: set[str] = set()
first_request = True
while url:
if url in seen_urls:
raise RuntimeError(f"GRB pagination loop detected for {collection.name}: {url}")
seen_urls.add(url)
response = session.get(url, params=params if first_request else None, timeout=timeout)
first_request = False
response.raise_for_status()
payload = response.json()
if payload.get("type") != "FeatureCollection":
raise RuntimeError(f"GRB {collection.name} returned a non-FeatureCollection response")
yield collection, payload, response.url
url = next_page_url(payload)
def iter_layer_pages(
session: requests.Session,
definition: LayerDefinition,
bounds: tuple[float, float, float, float],
*,
page_limit: int,
timeout: int,
) -> Iterable[tuple[CollectionDefinition, dict[str, Any], str]]:
for collection in definition.collections:
yield from iter_collection_pages(
session,
collection,
bounds,
page_limit=page_limit,
timeout=timeout,
)
def build_partition_features(
pages: Iterable[tuple[CollectionDefinition, dict[str, Any], str]],
*,
definition: LayerDefinition,
member: ScopeMember,
members: list[tuple[ScopeMember, Any]],
regional_boundary,
scope: GeographicScope,
max_features: int,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
features: list[dict[str, Any]] = []
source_urls: list[str] = []
seen_ids: set[str] = set()
pages_by_collection: dict[str, int] = {collection.name: 0 for collection in definition.collections}
features_by_collection: dict[str, int] = {collection.name: 0 for collection in definition.collections}
geometry_types: dict[str, int] = {}
bbox_feature_count = 0
assigned_elsewhere_count = 0
outside_scope_count = 0
clipped_to_scope_count = 0
member_boundary = next(boundary for candidate, boundary in members if candidate.nis_code == member.nis_code)
for collection, payload, source_url in pages:
source_urls.append(source_url)
pages_by_collection[collection.name] += 1
for source_feature in payload.get("features") or []:
bbox_feature_count += 1
raw_id = str(source_feature.get("id") or "")
if not raw_id:
raw_id = hashlib.sha256(
json.dumps(source_feature.get("geometry"), sort_keys=True).encode("utf-8")
).hexdigest()
feature_id = f"{collection.name}:{raw_id}"
if feature_id in seen_ids:
continue
seen_ids.add(feature_id)
source_geometry = normalized_geometry(source_feature.get("geometry"), collection.geometry_dimension)
if source_geometry is None or not source_geometry.intersects(regional_boundary):
outside_scope_count += 1
continue
owner_nis = (
member.nis_code
if member_boundary.covers(source_geometry)
else assign_owner_nis(
source_geometry,
members,
expected_dimension=collection.geometry_dimension,
)
)
if owner_nis != member.nis_code:
assigned_elsewhere_count += 1
continue
clipped = not regional_boundary.covers(source_geometry)
retained_geometry = source_geometry
if clipped:
retained_geometry = extract_dimension(
source_geometry.intersection(regional_boundary),
collection.geometry_dimension,
)
clipped_to_scope_count += 1
if retained_geometry is None:
outside_scope_count += 1
continue
if len(features) >= max_features:
raise RuntimeError(
f"{member.name} {definition.key} exceeds --max-features-per-member={max_features}; "
"refusing truncated output"
)
properties = dict(source_feature.get("properties") or {})
properties.update(
{
"source_name": "grb",
"source_collection": collection.name,
"source_feature_id": feature_id,
"reference_layer_name": definition.reference_layer_name,
"layer_type": definition.layer_type,
"theme": definition.key,
"authority_level": "authoritative",
"coverage_scope": scope.key,
"scope_type": scope.scope_type,
"partition_scope": "municipality",
"partition_municipality": member.name,
"partition_nis_code": member.nis_code,
"partition_assignment": "maximum_same_dimension_intersection",
"clipped_to_regional_scope": clipped,
"attribution": GRB_ATTRIBUTION,
}
)
features.append(
{
"type": "Feature",
"id": feature_id,
"geometry": mapping(retained_geometry),
"properties": properties,
}
)
features_by_collection[collection.name] += 1
geometry_types[retained_geometry.geom_type] = geometry_types.get(retained_geometry.geom_type, 0) + 1
if not features:
raise RuntimeError(f"No GRB {definition.key} features were assigned to {member.name}")
return features, {
"municipality": member.name,
"nis_code": member.nis_code,
"pages_fetched": len(source_urls),
"pages_by_collection": pages_by_collection,
"source_urls": source_urls,
"bbox_feature_count": bbox_feature_count,
"feature_count": len(features),
"features_by_collection": features_by_collection,
"geometry_types": geometry_types,
"assigned_elsewhere_count": assigned_elsewhere_count,
"outside_scope_count": outside_scope_count,
"clipped_to_scope_count": clipped_to_scope_count,
"reference_truncated": False,
}
def partition_filename(definition: LayerDefinition, member: ScopeMember, observed_date: date) -> str:
return f"{member.nis_code}_{safe_slug(member.name)}_grb_{definition.key}_{observed_date.isoformat()}.geojson"
def combined_filename(definition: LayerDefinition, scope: GeographicScope, observed_date: date) -> str:
return f"grb_{definition.key}_{scope.key.replace('-', '_')}_{observed_date.isoformat()}.geojson"
def write_partition(
path: Path,
*,
definition: LayerDefinition,
scope: GeographicScope,
member: ScopeMember,
features: list[dict[str, Any]],
generated_at: str,
) -> None:
write_json_atomic(
path,
{
"type": "FeatureCollection",
"name": f"GRB {definition.key} - {member.name} partition of {scope.display_name}",
"crs": GEOJSON_CRS,
"features": features,
"source": f"Digitaal Vlaanderen GRB OGC API collections {', '.join(item.name for item in definition.collections)}",
"source_urls": [GRB_COLLECTION_URL.format(collection=item.name) for item in definition.collections],
"attribution": GRB_ATTRIBUTION,
"coverage_scope": scope.key,
"partition_municipality": member.name,
"partition_nis_code": member.nis_code,
"generated_at": generated_at,
},
)
def write_combined_artifact(
path: Path,
*,
definition: LayerDefinition,
scope: GeographicScope,
observed_date: date,
partition_paths: list[Path],
expected_feature_count: int,
) -> dict[str, Any]:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(f"{path.suffix}.partial")
header = {
"type": "FeatureCollection",
"name": f"GRB {definition.key} - {scope.display_name}",
"crs": GEOJSON_CRS,
"source": f"Digitaal Vlaanderen GRB OGC API collections {', '.join(item.name for item in definition.collections)}",
"source_urls": [GRB_COLLECTION_URL.format(collection=item.name) for item in definition.collections],
"attribution": GRB_ATTRIBUTION,
"coverage_scope": scope.key,
"scope_type": scope.scope_type,
"partition_count": len(partition_paths),
"partition_strategy": "municipality_bbox_maximum_same_dimension_intersection",
"observed_at": observed_date.isoformat(),
}
encoded_header = json.dumps(header, ensure_ascii=False, separators=(",", ":"))
seen_ids: set[str] = set()
written = 0
first = True
with temporary.open("w", encoding="utf-8", newline="") as output:
output.write(encoded_header[:-1])
output.write(',"features":[')
for partition_path in partition_paths:
payload = json.loads(partition_path.read_text(encoding="utf-8"))
if payload.get("type") != "FeatureCollection" or not isinstance(payload.get("features"), list):
raise RuntimeError(f"Invalid partition artifact: {partition_path}")
for feature in payload["features"]:
feature_id = str(feature.get("id") or (feature.get("properties") or {}).get("source_feature_id") or "")
if not feature_id:
raise RuntimeError(f"Partition feature without source identity in {partition_path.name}")
if feature_id in seen_ids:
raise RuntimeError(f"Duplicate regional source feature {feature_id} in {partition_path.name}")
seen_ids.add(feature_id)
if not first:
output.write(",")
output.write(json.dumps(feature, ensure_ascii=False, separators=(",", ":")))
first = False
written += 1
output.write("]}")
if written != expected_feature_count:
temporary.unlink(missing_ok=True)
raise RuntimeError(f"Expected {expected_feature_count} combined features, wrote {written}")
temporary.replace(path)
return {"feature_count": written, "size_bytes": path.stat().st_size, "sha256": sha256_file(path)}
def prepare_layer_artifacts(
args: argparse.Namespace,
scope: GeographicScope,
definition: LayerDefinition,
) -> tuple[Path, list[Path], Path, dict[str, Any]]:
observation_dir = args.output_root / scope.key / definition.key / args.observed_date.isoformat()
partition_dir = observation_dir / "partitions"
artifact_path = observation_dir / combined_filename(definition, scope, args.observed_date)
manifest_path = observation_dir / f"regional_{definition.key}_manifest.json"
partition_dir.mkdir(parents=True, exist_ok=True)
if not args.force:
existing = reusable_manifest(manifest_path, artifact_path, partition_dir, len(scope.members))
if existing:
paths = [partition_dir / summary["filename"] for summary in existing["partitions"]]
return artifact_path, paths, manifest_path, existing
existing_manifest: dict[str, Any] = {}
if manifest_path.is_file() and not args.force:
existing_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
existing_by_nis = {
str(item.get("nis_code")): item
for item in existing_manifest.get("partitions") or []
if isinstance(item, dict)
}
generated_at = utc_now()
with source_session() as session:
source_features, vrbg_source_url = fetch_scope_members(session, scope, args.request_timeout)
members, regional_boundary = build_member_geometries(scope, source_features)
summaries: list[dict[str, Any]] = []
for member, boundary in members:
path = partition_dir / partition_filename(definition, member, args.observed_date)
reusable = existing_by_nis.get(member.nis_code)
if (
reusable
and path.is_file()
and reusable.get("filename") == path.name
and reusable.get("sha256") == sha256_file(path)
):
summaries.append(reusable)
continue
features, summary = build_partition_features(
iter_layer_pages(
session,
definition,
boundary.bounds,
page_limit=args.page_limit,
timeout=args.request_timeout,
),
definition=definition,
member=member,
members=members,
regional_boundary=regional_boundary,
scope=scope,
max_features=args.max_features_per_member,
)
write_partition(
path,
definition=definition,
scope=scope,
member=member,
features=features,
generated_at=generated_at,
)
summary.update({"filename": path.name, "size_bytes": path.stat().st_size, "sha256": sha256_file(path)})
summaries.append(summary)
write_json_atomic(
manifest_path,
{
"schema_version": 1,
"status": "in_progress",
"scope": scope.key,
"theme": definition.key,
"observed_at": args.observed_date.isoformat(),
"generated_at": generated_at,
"vrbg_source_url": vrbg_source_url,
"grb_collections": [item.name for item in definition.collections],
"partitions": summaries,
},
pretty=True,
)
total_features = sum(int(summary["feature_count"]) for summary in summaries)
if total_features > args.max_total_features:
raise RuntimeError(
f"Regional {definition.key} count {total_features} exceeds --max-total-features={args.max_total_features}"
)
partition_paths = [partition_dir / summary["filename"] for summary in summaries]
artifact = write_combined_artifact(
artifact_path,
definition=definition,
scope=scope,
observed_date=args.observed_date,
partition_paths=partition_paths,
expected_feature_count=total_features,
)
aggregate_geometry_types: dict[str, int] = {}
for summary in summaries:
for geometry_type, count in (summary.get("geometry_types") or {}).items():
aggregate_geometry_types[geometry_type] = aggregate_geometry_types.get(geometry_type, 0) + int(count)
manifest = {
"schema_version": 1,
"status": "complete",
"scope": scope.key,
"scope_type": scope.scope_type,
"scope_authority_url": scope.authority_url,
"scope_limitation": scope.limitation_message,
"theme": definition.key,
"reference_layer_name": definition.reference_layer_name,
"observed_at": args.observed_date.isoformat(),
"generated_at": generated_at,
"member_count": len(scope.members),
"feature_count": total_features,
"geometry_types": aggregate_geometry_types,
"reference_truncated": False,
"partition_strategy": "municipality_bbox_maximum_same_dimension_intersection",
"partition_assignment_rule": "largest same-dimension intersection measure; NIS code resolves exact ties",
"vrbg_source_url": vrbg_source_url,
"grb_collections": [item.name for item in definition.collections],
"grb_source_urls": [GRB_COLLECTION_URL.format(collection=item.name) for item in definition.collections],
"artifact_filename": artifact_path.name,
"artifact_size_bytes": artifact["size_bytes"],
"artifact_sha256": artifact["sha256"],
"bounds_json": {
"min_x": float(regional_boundary.bounds[0]),
"min_y": float(regional_boundary.bounds[1]),
"max_x": float(regional_boundary.bounds[2]),
"max_y": float(regional_boundary.bounds[3]),
},
"limitation_message": definition.limitation_message,
"partitions": summaries,
"attribution": GRB_ATTRIBUTION,
}
write_json_atomic(manifest_path, manifest, pretty=True)
return artifact_path, partition_paths, manifest_path, manifest
def provision_dataset(
args: argparse.Namespace,
scope: GeographicScope,
definition: LayerDefinition,
artifact_path: Path,
partition_paths: list[Path],
manifest_path: Path,
manifest: dict[str, Any],
) -> dict[str, Any]:
parsed_base = urlparse(args.base_url)
if parsed_base.hostname not in {"127.0.0.1", "localhost", "::1"}:
raise RuntimeError("Partitioned service import must run inside the GeoIntel container against its local backend")
with requests.Session() as session:
projects = list_paginated_items(session, f"{args.base_url.rstrip('/')}/api/v1/projects", args.api_timeout)
project = next((item for item in projects if item.get("name") == scope.project_name), None)
if not project:
raise RuntimeError("Regional scope project is missing; run provision_geographic_scope.py first")
project_id = str(project["id"])
areas = list_paginated_items(
session,
f"{args.base_url.rstrip('/')}/api/v1/projects/{project_id}/areas",
args.api_timeout,
)
area = next((item for item in areas if item.get("name") == scope.area_name), None)
if not area:
raise RuntimeError("Regional scope Area is missing; run provision_geographic_scope.py first")
datasets = list_paginated_items(
session,
f"{args.base_url.rstrip('/')}/api/v1/projects/{project_id}/datasets",
args.api_timeout,
)
existing = next((item for item in datasets if item.get("original_filename") == artifact_path.name), None)
if existing:
persisted_checksum = (existing.get("provenance_metadata") or {}).get("artifact_sha256")
if persisted_checksum and persisted_checksum != manifest["artifact_sha256"]:
raise RuntimeError(
f"Immutable dataset {artifact_path.name} checksum changed; use a new --observed-date for refreshed GRB data"
)
return {"dataset_id": str(existing["id"]), "feature_count": existing.get("feature_count"), "reused": True}
ensure_backend_path()
from app.db.session import SessionLocal
from app.services.dataset_service import DatasetService
metadata_json = {
"feature_count": manifest["feature_count"],
"feature_geometry_count": manifest["feature_count"],
"geometry_types": list(manifest.get("geometry_types") or definition.geometry_types),
"bounds_json": manifest["bounds_json"],
"approximate_area_m2": None,
"invalid_features": 0,
"z_dimension_feature_count": 0,
"canonical_storage_dimension": "2D",
"crs": "EPSG:4326",
"crs_assumed": False,
"extracted_at": manifest["generated_at"],
}
source_metadata = {
"provider": "Digitaal Vlaanderen",
"collections": [item.name for item in definition.collections],
"authority_level": "authoritative",
"theme": definition.key,
"layer_type": f"regional_{definition.key}",
"coverage_scope": scope.key,
"scope_type": scope.scope_type,
"scope_authority": scope.authority_name,
"scope_authority_url": scope.authority_url,
"scope_limitation": scope.limitation_message,
"layer_limitation": definition.limitation_message,
"member_count": len(scope.members),
"member_nis_codes": list(scope.nis_codes),
"feature_count": manifest["feature_count"],
"partition_count": len(partition_paths),
"partition_strategy": manifest["partition_strategy"],
"selection_aggregation": {
"method": "feature_count",
"label": definition.metric_label,
"unit": "objecten",
"is_estimate": False,
},
"attribution": GRB_ATTRIBUTION,
}
provenance_metadata = {
"operator_tool": "provision_regional_grb_context.py",
"operator_explicit_fetch": True,
"manifest_path": str(manifest_path),
"source_urls": manifest["grb_source_urls"],
"artifact_sha256": manifest["artifact_sha256"],
"artifact_size_bytes": manifest["artifact_size_bytes"],
"partition_checksums": {summary["nis_code"]: summary["sha256"] for summary in manifest["partitions"]},
"partition_assignment_rule": manifest["partition_assignment_rule"],
"reference_truncated": False,
}
with SessionLocal() as db:
dataset = DatasetService.import_partitioned_vector_artifact(
db,
project_id=UUID(project_id),
area_id=UUID(str(area["id"])),
artifact_path=artifact_path,
partition_paths=partition_paths,
original_filename=artifact_path.name,
source="operator_official_import",
dataset_role="reference",
source_name="grb",
reference_layer_name=definition.reference_layer_name,
metadata_json=metadata_json,
source_metadata=source_metadata,
provenance_metadata=provenance_metadata,
temporal_series_key=f"grb:{definition.key}:{scope.key}",
observed_at=observed_at(args.observed_date),
temporal_granularity="snapshot",
source_version=args.observed_date.isoformat(),
batch_size=args.batch_size,
)
if dataset.checksum_sha256 != manifest["artifact_sha256"]:
raise RuntimeError("Managed dataset checksum differs from the retained regional artifact")
return {"dataset_id": str(dataset.id), "feature_count": dataset.feature_count, "reused": False}
def main() -> int:
args = parse_args()
scope = GEOGRAPHIC_SCOPES[args.scope]
try:
definitions = selected_definitions(args.layers)
if args.page_limit <= 0 or args.max_features_per_member <= 0 or args.max_total_features <= 0:
raise ValueError("Page and feature limits must be positive")
results: list[dict[str, Any]] = []
for definition in definitions:
artifact_path, partition_paths, manifest_path, manifest = prepare_layer_artifacts(args, scope, definition)
persistence = None if args.fetch_only else provision_dataset(
args,
scope,
definition,
artifact_path,
partition_paths,
manifest_path,
manifest,
)
results.append(
{
"theme": definition.key,
"collections": [item.name for item in definition.collections],
"feature_count": manifest["feature_count"],
"artifact_size_bytes": manifest["artifact_size_bytes"],
"artifact_path": str(artifact_path),
"manifest_path": str(manifest_path),
"reference_truncated": manifest["reference_truncated"],
"persistence": persistence,
}
)
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException) as exc:
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
return 1
print(
json.dumps(
{
"status": "ok",
"mode": "fetch_only" if args.fetch_only else "provisioned",
"scope": scope.key,
"observed_at": args.observed_date.isoformat(),
"member_count": len(scope.members),
"layers": results,
},
ensure_ascii=False,
indent=2,
)
)
return 0
if __name__ == "__main__":
sys.exit(main())
+1
View File
@@ -50,6 +50,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
${PYTHON_BIN} -m py_compile scripts/geographic_scopes.py
${PYTHON_BIN} -m py_compile scripts/provision_geographic_scope.py
${PYTHON_BIN} -m py_compile scripts/provision_regional_grb_buildings.py
${PYTHON_BIN} -m py_compile scripts/provision_regional_grb_context.py
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py