feat: add temporal Mol explorer
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-14 15:19:42 +02:00
parent 0ec1ab4970
commit c0943fe7d4
39 changed files with 3065 additions and 163 deletions
+9
View File
@@ -7,6 +7,15 @@
# Changelog
## Sprint 187 Temporal Mol explorer (2026-07-14)
- Added first-class temporal dataset metadata and immutable dataset-version provenance for uploaded and derived vector/raster datasets.
- Added project temporal-series discovery and bounded snapshot comparison APIs with explicit observation dates, metric deltas, warnings and optional stable-identity object changes.
- Extended bbox selection with source-governed PostGIS aggregations so population is reported as inhabitants and land cover as intersected hectares instead of misleading feature counts.
- Added a calm Latest state/Evolution flow to the map-first explorer, including period selection, metric comparison and added/removed/modified overlays where source identities support them.
- Added explicit operator provisioners for official Statbel Mol population snapshots (2021-2025) and Digitaal Vlaanderen historical land-use snapshots (1778, 1873 and 1969); no source is fetched during application startup.
- Preserved methodological honesty: partial statistical sectors are labelled area-weighted estimates, historical land-use identity changes are not fabricated and all source URLs, versions and processing limitations are persisted.
## Sprint 186 Map-first Mol geographic explorer (2026-07-14)
- Replaced the default dashboard entry with a calm map-first workflow: choose a real data theme, drag a rectangle, query PostGIS automatically and review results.
+30
View File
@@ -935,6 +935,36 @@ the same bbox-selected FeatureCollection as a normal export record with
`export_type="vector_selection_geojson"`. This creates a handoff artifact only;
it does not create a derived dataset.
## Temporal Mol data and evolution
Dataset uploads accept `temporal_series_key`, `observed_at`, `valid_from`,
`valid_to`, `temporal_granularity` and `source_version`. Every new source or
derived dataset also writes dataset version 1 in the same transaction.
After the Mol municipality workspace is available, import the official source
snapshots explicitly:
```bash
docker exec geointel python /app/scripts/provision_mol_population_history.py
docker exec geointel python /app/scripts/provision_mol_historical_landuse.py
```
The first command imports Statbel sector population for 2021-2025. The second
imports Digitaal Vlaanderen historical land use for 1778, 1873 and 1969. Both
are idempotent, use the normal API/DatasetService flow and retain fetched
artifacts in persistent operator storage. They never run on app startup.
Historical land-use work can be bounded explicitly:
```bash
docker exec geointel python /app/scripts/provision_mol_historical_landuse.py --years 1778 1969 --themes forest water
```
`GET /api/v1/projects/{project_id}/temporal/series` discovers the series and
`POST /api/v1/projects/{project_id}/temporal/compare` compares two snapshots
inside one EPSG:4326 bbox. Partial statistical sectors are estimates; old map
editions without stable identities do not produce invented object changes.
## Helpful repository scripts
- `bash scripts/backend_install.sh`
@@ -0,0 +1,72 @@
"""Add temporal dataset metadata and durable dataset-version provenance."""
from alembic import op
import sqlalchemy as sa
revision = "202607140001"
down_revision = "202606120900"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("datasets", sa.Column("temporal_series_key", sa.String(length=255), nullable=True))
op.add_column("datasets", sa.Column("observed_at", sa.DateTime(timezone=True), nullable=True))
op.add_column("datasets", sa.Column("valid_from", sa.DateTime(timezone=True), nullable=True))
op.add_column("datasets", sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True))
op.add_column("datasets", sa.Column("temporal_granularity", sa.String(length=32), nullable=True))
op.add_column("datasets", sa.Column("source_version", sa.String(length=120), nullable=True))
op.add_column("dataset_versions", sa.Column("source_version", sa.String(length=120), nullable=True))
op.add_column("dataset_versions", sa.Column("observed_at", sa.DateTime(timezone=True), nullable=True))
op.add_column("dataset_versions", sa.Column("valid_from", sa.DateTime(timezone=True), nullable=True))
op.add_column("dataset_versions", sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True))
op.add_column("dataset_versions", sa.Column("checksum_sha256", sa.String(length=64), nullable=True))
op.add_column("dataset_versions", sa.Column("source_metadata", sa.JSON(), nullable=True))
op.add_column("dataset_versions", sa.Column("provenance_metadata", sa.JSON(), nullable=True))
op.create_index(
"ix_datasets_project_temporal_series_observed",
"datasets",
["project_id", "temporal_series_key", "observed_at"],
)
op.create_index("ix_dataset_versions_dataset_version", "dataset_versions", ["dataset_id", "version"], unique=True)
op.create_index(
"ix_vector_features_dataset_source_feature",
"vector_features",
["dataset_id", "source_feature_id"],
)
op.create_check_constraint(
"ck_datasets_temporal_valid_range",
"datasets",
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
)
op.create_check_constraint(
"ck_dataset_versions_temporal_valid_range",
"dataset_versions",
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
)
def downgrade() -> None:
op.drop_constraint("ck_dataset_versions_temporal_valid_range", "dataset_versions", type_="check")
op.drop_constraint("ck_datasets_temporal_valid_range", "datasets", type_="check")
op.drop_index("ix_vector_features_dataset_source_feature", table_name="vector_features")
op.drop_index("ix_dataset_versions_dataset_version", table_name="dataset_versions")
op.drop_index("ix_datasets_project_temporal_series_observed", table_name="datasets")
op.drop_column("dataset_versions", "provenance_metadata")
op.drop_column("dataset_versions", "source_metadata")
op.drop_column("dataset_versions", "checksum_sha256")
op.drop_column("dataset_versions", "valid_to")
op.drop_column("dataset_versions", "valid_from")
op.drop_column("dataset_versions", "observed_at")
op.drop_column("dataset_versions", "source_version")
op.drop_column("datasets", "source_version")
op.drop_column("datasets", "temporal_granularity")
op.drop_column("datasets", "valid_to")
op.drop_column("datasets", "valid_from")
op.drop_column("datasets", "observed_at")
op.drop_column("datasets", "temporal_series_key")
+1 -1
View File
@@ -1 +1 @@
__all__ = ["analysis", "areas", "datasets", "health", "projects", "exports", "jobs", "external", "qa"]
__all__ = ["analysis", "areas", "datasets", "health", "projects", "exports", "jobs", "external", "qa", "temporal"]
+48 -1
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
from datetime import datetime
from typing import Any
from uuid import UUID
from uuid import UUID as _UUID
@@ -30,7 +31,7 @@ from app.schemas import (
VectorSelectionResponse,
)
from app.schemas.job import JobCreate
from app.schemas.dataset import DatasetCreateResponse
from app.schemas.dataset import DatasetCreateResponse, DatasetTemporalUpdate
from app.schemas.operations import VectorOperationResult
from app.services.job_service import JobService
from app.services.raster_operations_service import RasterOperationsService
@@ -87,6 +88,12 @@ async def upload_dataset(
reference_layer_name: str | None = Form(None),
source_metadata_json: str | None = Form(None),
provenance_metadata_json: str | None = Form(None),
temporal_series_key: str | None = Form(None),
observed_at: datetime | None = Form(None),
valid_from: datetime | None = Form(None),
valid_to: datetime | None = Form(None),
temporal_granularity: str | None = Form(None),
source_version: str | None = Form(None),
db: Session = Depends(get_db),
):
if area_id is not None:
@@ -108,6 +115,12 @@ async def upload_dataset(
source_metadata=_parse_metadata_json(source_metadata_json, "source_metadata_json"),
provenance_metadata=_parse_metadata_json(provenance_metadata_json, "provenance_metadata_json"),
area_id=area_id,
temporal_series_key=temporal_series_key,
observed_at=observed_at,
valid_from=valid_from,
valid_to=valid_to,
temporal_granularity=temporal_granularity,
source_version=source_version,
)
return envelope(created.model_dump())
@@ -135,6 +148,33 @@ def get_dataset(
return envelope(DatasetCreateResponse.model_validate(dataset).model_dump())
@router.patch("/datasets/{dataset_id}/temporal", response_model=dict)
def update_dataset_temporal_metadata(
project_id: UUID,
dataset_id: UUID,
payload: DatasetTemporalUpdate,
db: Session = Depends(get_db),
):
dataset = DatasetService.get_dataset(db, dataset_id)
if dataset.project_id != project_id:
raise HTTPException(status_code=404, detail="Dataset not found")
updated = DatasetService.update_temporal_metadata(db, dataset_id, payload)
return envelope(updated.model_dump())
@router.get("/datasets/{dataset_id}/versions", response_model=dict)
def list_dataset_versions(
project_id: UUID,
dataset_id: UUID,
db: Session = Depends(get_db),
):
dataset = DatasetService.get_dataset(db, dataset_id)
if dataset.project_id != project_id:
raise HTTPException(status_code=404, detail="Dataset not found")
versions = DatasetService.list_versions(db, dataset_id)
return envelope({"items": [item.model_dump() for item in versions], "total": len(versions)})
@router.post("/datasets/{dataset_id}/metadata/refresh", response_model=dict)
def refresh_dataset_metadata(
project_id: UUID,
@@ -203,6 +243,13 @@ def select_vector_features(
bbox=payload.bbox.model_dump(),
limit=payload.limit,
)
if isinstance(dataset.source_metadata, dict) and dataset.source_metadata.get("selection_aggregation"):
result["summary"] = VectorFeatureService.summarize_features_by_bbox(
db,
dataset=dataset,
bbox=payload.bbox.model_dump(),
total_feature_count=result.get("total_feature_count"),
)
return envelope(VectorSelectionResponse(**result).model_dump(exclude_none=True))
+29
View File
@@ -0,0 +1,29 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from app.db.session import get_db
from app.schemas.temporal import TemporalComparisonRequest
from app.services.temporal_analysis_service import TemporalAnalysisService
from app.utils.response import envelope
router = APIRouter(prefix="/projects/{project_id}/temporal", tags=["temporal"])
@router.get("/series", response_model=dict)
def list_temporal_series(project_id: UUID, db: Session = Depends(get_db)):
series = TemporalAnalysisService.list_series(db, project_id)
return envelope({"items": [item.model_dump() for item in series], "total": len(series)})
@router.post("/compare", response_model=dict)
def compare_temporal_snapshots(
project_id: UUID,
payload: TemporalComparisonRequest,
db: Session = Depends(get_db),
):
return envelope(TemporalAnalysisService.compare(db, project_id=project_id, payload=payload).model_dump())
+2 -1
View File
@@ -5,7 +5,7 @@ from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from app.api.routes import analysis, areas, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation
from app.api.routes import analysis, areas, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, temporal
from app.core.config import get_settings
from app.core.errors import AppError
from app.core.logging import configure_logging
@@ -57,6 +57,7 @@ def create_app() -> FastAPI:
app.include_router(qa.router, prefix=settings.api_prefix)
app.include_router(detection.router, prefix=settings.api_prefix)
app.include_router(segmentation.router, prefix=settings.api_prefix)
app.include_router(temporal.router, prefix=settings.api_prefix)
@app.exception_handler(AppError)
async def app_error(request: Request, exc: AppError): # noqa: ARG001
+34 -1
View File
@@ -4,7 +4,7 @@ import uuid
from datetime import datetime
from geoalchemy2 import Geometry
from sqlalchemy import DateTime, ForeignKey, Float, Index, JSON, String, Text, func
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, func
from sqlalchemy.sql.sqltypes import Integer
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -44,6 +44,18 @@ class Area(Base):
class Dataset(Base):
__tablename__ = "datasets"
__table_args__ = (
CheckConstraint(
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
name="ck_datasets_temporal_valid_range",
),
Index(
"ix_datasets_project_temporal_series_observed",
"project_id",
"temporal_series_key",
"observed_at",
),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
@@ -73,6 +85,12 @@ class Dataset(Base):
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
temporal_granularity: Mapped[str | None] = mapped_column(String(32), nullable=True)
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
status: Mapped[str] = mapped_column(String(32), default="uploaded")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
@@ -92,11 +110,25 @@ class Dataset(Base):
class DatasetVersion(Base):
__tablename__ = "dataset_versions"
__table_args__ = (
CheckConstraint(
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
name="ck_dataset_versions_temporal_valid_range",
),
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
dataset_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False)
version: Mapped[int] = mapped_column(Integer, default=1)
storage_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
@@ -107,6 +139,7 @@ class VectorFeature(Base):
__table_args__ = (
Index("ix_vector_features_dataset_id", "dataset_id"),
Index("ix_vector_features_geometry", "geometry", postgresql_using="gist"),
Index("ix_vector_features_dataset_source_feature", "dataset_id", "source_feature_id"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+2
View File
@@ -77,6 +77,7 @@ from .operations import (
VectorSelectionDeriveRequest,
VectorSelectionRequest,
VectorSelectionResponse,
VectorSelectionSummary,
VectorStatsRequest,
VectorStatsResponse,
)
@@ -134,6 +135,7 @@ __all__ = [
"VectorSelectionDeriveRequest",
"VectorSelectionRequest",
"VectorSelectionResponse",
"VectorSelectionSummary",
"RasterClipRequest",
"RasterStatsResponse",
"RasterReprojectRequest",
+32
View File
@@ -36,6 +36,12 @@ class DatasetCreateResponse(BaseModel):
source_metadata: dict | None = None
provenance_metadata: dict | None = None
imported_at: datetime | None = None
temporal_series_key: str | None = None
observed_at: datetime | None = None
valid_from: datetime | None = None
valid_to: datetime | None = None
temporal_granularity: str | None = None
source_version: str | None = None
project_id: UUID
area_id: UUID | None = None
storage_path: str | None = None
@@ -70,6 +76,32 @@ class DatasetMetadataRefresh(BaseModel):
crs: str | None = None
class DatasetTemporalUpdate(BaseModel):
temporal_series_key: str
observed_at: datetime
valid_from: datetime | None = None
valid_to: datetime | None = None
temporal_granularity: str = "snapshot"
source_version: str | None = None
class DatasetVersionRead(BaseModel):
id: UUID
dataset_id: UUID
version: int
storage_path: str | None = None
source_version: str | None = None
observed_at: datetime | None = None
valid_from: datetime | None = None
valid_to: datetime | None = None
checksum_sha256: str | None = None
source_metadata: dict | None = None
provenance_metadata: dict | None = None
created_at: datetime | None = None
model_config = {"from_attributes": True}
class ExportRequest(BaseModel):
dataset_id: UUID
name: str | None = None
+11
View File
@@ -217,6 +217,16 @@ class VectorSelectionDeriveRequest(VectorSelectionRequest):
output_name: str | None = None
class VectorSelectionSummary(BaseModel):
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
feature_count: int
is_estimate: bool = False
warning: str | None = None
class VectorSelectionResponse(BaseModel):
selection_bbox: VectorSelectionBBox
feature_count: int
@@ -224,3 +234,4 @@ class VectorSelectionResponse(BaseModel):
limit: int
truncated: bool
geojson: dict
summary: VectorSelectionSummary | None = None
+71
View File
@@ -0,0 +1,71 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, Field
from app.schemas.operations import VectorSelectionBBox
class TemporalComparisonRequest(BaseModel):
earlier_dataset_id: UUID
later_dataset_id: UUID
bbox: VectorSelectionBBox
preview_limit: int = Field(default=500, ge=1, le=1000)
class TemporalDatasetRef(BaseModel):
id: UUID
name: str
observed_at: datetime
source_version: str | None = None
class TemporalMetricComparison(BaseModel):
label: str
unit: str
aggregation_method: str
earlier_value: float
later_value: float
absolute_change: float
percent_change: float | None = None
is_estimate: bool = False
class TemporalObjectChanges(BaseModel):
available: bool
added_count: int | None = None
removed_count: int | None = None
modified_count: int | None = None
unchanged_count: int | None = None
class TemporalComparisonResponse(BaseModel):
temporal_series_key: str
earlier: TemporalDatasetRef
later: TemporalDatasetRef
selection_bbox: VectorSelectionBBox
metric: TemporalMetricComparison
object_changes: TemporalObjectChanges
geojson: dict
warnings: list[str]
generated_at: datetime
class TemporalSeriesDataset(BaseModel):
id: UUID
name: str
observed_at: datetime
source_version: str | None = None
feature_count: int | None = None
class TemporalSeriesRead(BaseModel):
temporal_series_key: str
source_name: str | None = None
reference_layer_name: str | None = None
dataset_count: int
first_observed_at: datetime
last_observed_at: datetime
datasets: list[TemporalSeriesDataset]
+184 -95
View File
@@ -12,8 +12,14 @@ from fastapi import UploadFile
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import Dataset, Project
from app.schemas.dataset import DatasetCreateResponse, DatasetStorageResponse, DatasetVectorSummary
from app.models import Dataset, DatasetVersion, Project
from app.schemas.dataset import (
DatasetCreateResponse,
DatasetStorageResponse,
DatasetTemporalUpdate,
DatasetVectorSummary,
DatasetVersionRead,
)
from app.services.geojson_service import parse_geojson_payload, load_dataset_text
from app.services.raster_service import extract_raster_metadata
from app.services.storage_service import StorageService
@@ -26,6 +32,105 @@ class DatasetService:
VECTOR_TYPES = {"vector", "geojson"}
RASTER_TYPES = {"raster", "tif", "tiff", "geotiff"}
VALID_DATASET_ROLES = {"source", "derived", "reference"}
VALID_TEMPORAL_GRANULARITIES = {"snapshot", "day", "month", "year", "period"}
@staticmethod
def _normalize_datetime(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
@staticmethod
def _validate_temporal_metadata(
*,
temporal_series_key: str | None,
observed_at: datetime | None,
valid_from: datetime | None,
valid_to: datetime | None,
temporal_granularity: str | None,
source_version: str | None,
) -> dict[str, Any]:
normalized_key = (temporal_series_key or "").strip() or None
normalized_observed_at = DatasetService._normalize_datetime(observed_at)
normalized_valid_from = DatasetService._normalize_datetime(valid_from)
normalized_valid_to = DatasetService._normalize_datetime(valid_to)
normalized_granularity = (temporal_granularity or "").strip().lower() or None
normalized_source_version = (source_version or "").strip() or None
if normalized_key and len(normalized_key) > 255:
raise AppError(code="INVALID_TEMPORAL_METADATA", message="temporal_series_key is too long", status_code=400)
if normalized_granularity and normalized_granularity not in DatasetService.VALID_TEMPORAL_GRANULARITIES:
raise AppError(
code="INVALID_TEMPORAL_METADATA",
message="temporal_granularity must be snapshot, day, month, year or period",
status_code=400,
)
if normalized_valid_from and normalized_valid_to and normalized_valid_to < normalized_valid_from:
raise AppError(
code="INVALID_TEMPORAL_METADATA",
message="valid_to must be on or after valid_from",
status_code=400,
)
if normalized_key and normalized_observed_at is None:
raise AppError(
code="INVALID_TEMPORAL_METADATA",
message="observed_at is required when temporal_series_key is provided",
status_code=400,
)
if normalized_observed_at and normalized_key is None:
raise AppError(
code="INVALID_TEMPORAL_METADATA",
message="temporal_series_key is required when observed_at is provided",
status_code=400,
)
return {
"temporal_series_key": normalized_key,
"observed_at": normalized_observed_at,
"valid_from": normalized_valid_from,
"valid_to": normalized_valid_to,
"temporal_granularity": normalized_granularity,
"source_version": normalized_source_version,
}
@staticmethod
def _to_response(dataset: Dataset) -> DatasetCreateResponse:
metadata_json = dataset.metadata_json if isinstance(dataset.metadata_json, dict) else {}
return DatasetCreateResponse(
id=dataset.id,
name=dataset.name,
dataset_type=dataset.dataset_type,
source=dataset.source,
dataset_role=dataset.dataset_role,
source_name=dataset.source_name,
reference_layer_name=dataset.reference_layer_name,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
imported_at=dataset.imported_at,
temporal_series_key=dataset.temporal_series_key,
observed_at=dataset.observed_at,
valid_from=dataset.valid_from,
valid_to=dataset.valid_to,
temporal_granularity=dataset.temporal_granularity,
source_version=dataset.source_version,
project_id=dataset.project_id,
area_id=dataset.area_id,
storage_path=dataset.storage_path,
original_filename=dataset.original_filename,
stored_filename=dataset.stored_filename,
content_type=dataset.content_type,
size_bytes=dataset.size_bytes,
checksum_sha256=dataset.checksum_sha256,
crs=dataset.crs,
bounds_json=dataset.bounds_json,
metadata_json=dataset.metadata_json,
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, metadata_json),
status=dataset.status,
derived_from_dataset_id=dataset.derived_from_dataset_id,
created_at=dataset.created_at,
feature_count=metadata_json.get("feature_count"),
)
@staticmethod
def _canonical_dataset_type(dataset_type: str) -> str:
@@ -89,44 +194,7 @@ class DatasetService:
.limit(limit)
.all()
)
response_items = []
for row in rows:
feature_count = None
metadata_json = row.metadata_json or {}
vector_summary = DatasetService._extract_vector_summary(row.dataset_type, metadata_json)
if isinstance(metadata_json, dict):
feature_count = metadata_json.get("feature_count")
response_items.append(
DatasetCreateResponse(
id=row.id,
name=row.name,
dataset_type=row.dataset_type,
source=row.source,
dataset_role=row.dataset_role,
source_name=row.source_name,
reference_layer_name=row.reference_layer_name,
source_metadata=row.source_metadata,
provenance_metadata=row.provenance_metadata,
imported_at=row.imported_at,
project_id=row.project_id,
area_id=row.area_id,
storage_path=row.storage_path,
original_filename=row.original_filename,
stored_filename=row.stored_filename,
content_type=row.content_type,
size_bytes=row.size_bytes,
checksum_sha256=row.checksum_sha256,
crs=row.crs,
bounds_json=row.bounds_json,
metadata_json=row.metadata_json,
vector_summary=vector_summary,
status=row.status,
derived_from_dataset_id=row.derived_from_dataset_id,
created_at=row.created_at,
feature_count=feature_count,
)
)
return response_items, total
return [DatasetService._to_response(row) for row in rows], total
@staticmethod
def _extract_vector_summary(dataset_type: str, metadata_json: dict) -> DatasetVectorSummary | None:
@@ -195,6 +263,12 @@ class DatasetService:
source_metadata: dict | None = None,
provenance_metadata: dict | None = None,
area_id: UUID | None = None,
temporal_series_key: str | None = None,
observed_at: datetime | None = None,
valid_from: datetime | None = None,
valid_to: datetime | None = None,
temporal_granularity: str | None = None,
source_version: str | None = None,
) -> DatasetCreateResponse:
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
@@ -202,6 +276,14 @@ class DatasetService:
filename = DatasetService._validate_upload_filename(file.filename)
canonical_type = DatasetService._canonical_dataset_type(dataset_type)
normalized_role = DatasetService._normalize_dataset_role(dataset_role)
temporal = DatasetService._validate_temporal_metadata(
temporal_series_key=temporal_series_key,
observed_at=observed_at,
valid_from=valid_from,
valid_to=valid_to,
temporal_granularity=temporal_granularity,
source_version=source_version,
)
normalized_source_name = source_name
if normalized_role == "reference" and not normalized_source_name:
normalized_source_name = "manual"
@@ -280,6 +362,7 @@ class DatasetService:
source_metadata=source_metadata,
provenance_metadata=provenance_metadata,
imported_at=datetime.now(timezone.utc),
**temporal,
storage_path=storage_info["storage_path"],
original_filename=storage_info["original_filename"],
stored_filename=storage_info["stored_filename"],
@@ -294,6 +377,20 @@ class DatasetService:
status=status,
)
db.add(dataset)
db.add(
DatasetVersion(
dataset_id=dataset.id,
version=1,
storage_path=dataset.storage_path,
source_version=dataset.source_version,
observed_at=dataset.observed_at,
valid_from=dataset.valid_from,
valid_to=dataset.valid_to,
checksum_sha256=dataset.checksum_sha256,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
)
)
db.commit()
db.refresh(dataset)
@@ -306,34 +403,7 @@ class DatasetService:
feature_class=feature_class,
)
return DatasetCreateResponse(
id=dataset.id,
name=dataset.name,
dataset_type=dataset.dataset_type,
source=dataset.source,
dataset_role=dataset.dataset_role,
source_name=dataset.source_name,
reference_layer_name=dataset.reference_layer_name,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
imported_at=dataset.imported_at,
project_id=dataset.project_id,
area_id=dataset.area_id,
storage_path=dataset.storage_path,
original_filename=dataset.original_filename,
stored_filename=dataset.stored_filename,
content_type=dataset.content_type,
size_bytes=dataset.size_bytes,
checksum_sha256=dataset.checksum_sha256,
crs=dataset.crs,
derived_from_dataset_id=dataset.derived_from_dataset_id,
bounds_json=dataset.bounds_json,
metadata_json=dataset.metadata_json,
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}),
status=dataset.status,
created_at=dataset.created_at,
feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None,
)
return DatasetService._to_response(dataset)
@staticmethod
def refresh_metadata(db: Session, dataset_id: UUID) -> DatasetCreateResponse:
@@ -380,34 +450,53 @@ class DatasetService:
db.commit()
db.refresh(dataset)
return DatasetCreateResponse(
id=dataset.id,
name=dataset.name,
dataset_type=dataset.dataset_type,
source=dataset.source,
dataset_role=dataset.dataset_role,
source_name=dataset.source_name,
reference_layer_name=dataset.reference_layer_name,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
imported_at=dataset.imported_at,
project_id=dataset.project_id,
area_id=dataset.area_id,
storage_path=dataset.storage_path,
original_filename=dataset.original_filename,
stored_filename=dataset.stored_filename,
content_type=dataset.content_type,
size_bytes=dataset.size_bytes,
checksum_sha256=dataset.checksum_sha256,
crs=dataset.crs,
derived_from_dataset_id=dataset.derived_from_dataset_id,
bounds_json=dataset.bounds_json,
metadata_json=dataset.metadata_json,
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}),
status=dataset.status,
created_at=dataset.created_at,
feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None,
return DatasetService._to_response(dataset)
@staticmethod
def update_temporal_metadata(db: Session, dataset_id: UUID, payload: DatasetTemporalUpdate) -> DatasetCreateResponse:
dataset = DatasetService._get_dataset(db, dataset_id)
temporal = DatasetService._validate_temporal_metadata(**payload.model_dump())
if all(getattr(dataset, field) == value for field, value in temporal.items()):
return DatasetService._to_response(dataset)
for field, value in temporal.items():
setattr(dataset, field, value)
latest_version = (
db.query(DatasetVersion)
.filter(DatasetVersion.dataset_id == dataset.id)
.order_by(DatasetVersion.version.desc())
.first()
)
db.add(dataset)
db.add(
DatasetVersion(
dataset_id=dataset.id,
version=(latest_version.version + 1) if latest_version else 1,
storage_path=dataset.storage_path,
source_version=dataset.source_version,
observed_at=dataset.observed_at,
valid_from=dataset.valid_from,
valid_to=dataset.valid_to,
checksum_sha256=dataset.checksum_sha256,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
)
)
db.commit()
db.refresh(dataset)
return DatasetService._to_response(dataset)
@staticmethod
def list_versions(db: Session, dataset_id: UUID) -> list[DatasetVersionRead]:
DatasetService._get_dataset(db, dataset_id)
rows = (
db.query(DatasetVersion)
.filter(DatasetVersion.dataset_id == dataset_id)
.order_by(DatasetVersion.version.desc())
.all()
)
return [DatasetVersionRead.model_validate(row) for row in rows]
@staticmethod
def get_dataset(db: Session, dataset_id: UUID) -> Dataset:
+16 -1
View File
@@ -10,7 +10,7 @@ from uuid import UUID, uuid4
from geoalchemy2.shape import from_shape
from sqlalchemy.orm import Session
from app.models import Area, Dataset, Metric, Project, QualityCheck
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
from app.schemas.demo import DemoWorkflowResponse
from app.services.geojson_service import parse_geojson_payload
from app.services.qa_service import QaService
@@ -29,6 +29,19 @@ class DemoWorkflowService:
RASTER_FILENAME = "demo_context_raster.tif"
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
@staticmethod
def _add_initial_version(db: Session, dataset: Dataset) -> None:
db.add(
DatasetVersion(
dataset_id=dataset.id,
version=1,
storage_path=dataset.storage_path,
checksum_sha256=dataset.checksum_sha256,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
)
)
@staticmethod
def _repo_root() -> Path:
return Path(__file__).resolve().parents[3]
@@ -213,6 +226,7 @@ class DemoWorkflowService:
status="ready",
)
db.add(dataset)
DemoWorkflowService._add_initial_version(db, dataset)
db.commit()
db.refresh(dataset)
VectorFeatureService.persist_geojson_features(
@@ -297,6 +311,7 @@ class DemoWorkflowService:
status="ready",
)
db.add(dataset)
DemoWorkflowService._add_initial_version(db, dataset)
db.commit()
db.refresh(dataset)
return dataset
@@ -12,7 +12,7 @@ from shapely.ops import transform as shapely_transform
from shapely.validation import make_valid
from app.core.errors import AppError
from app.models import Area, Dataset
from app.models import Area, Dataset, DatasetVersion
from app.services.raster_service import extract_raster_metadata
from app.services.storage_service import StorageService
@@ -333,6 +333,21 @@ class RasterOperationsService:
name=output_name,
dataset_type="raster",
source=f"operation:{operation_name}",
dataset_role="derived",
source_name=source_dataset.source_name,
source_metadata=source_dataset.source_metadata,
provenance_metadata=provenance,
imported_at=datetime.now(timezone.utc),
temporal_series_key=(
f"{source_dataset.temporal_series_key}:{operation_name}"
if source_dataset.temporal_series_key
else None
),
observed_at=source_dataset.observed_at,
valid_from=source_dataset.valid_from,
valid_to=source_dataset.valid_to,
temporal_granularity=source_dataset.temporal_granularity,
source_version=source_dataset.source_version,
storage_path=str(output_file),
original_filename=storage_metadata["original_filename"],
stored_filename=storage_metadata["stored_filename"],
@@ -348,6 +363,20 @@ class RasterOperationsService:
status="ready",
)
db.add(derived_dataset)
db.add(
DatasetVersion(
dataset_id=derived_dataset.id,
version=1,
storage_path=derived_dataset.storage_path,
source_version=derived_dataset.source_version,
observed_at=derived_dataset.observed_at,
valid_from=derived_dataset.valid_from,
valid_to=derived_dataset.valid_to,
checksum_sha256=derived_dataset.checksum_sha256,
source_metadata=derived_dataset.source_metadata,
provenance_metadata=derived_dataset.provenance_metadata,
)
)
db.commit()
db.refresh(derived_dataset)
return derived_id
@@ -0,0 +1,310 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from uuid import UUID
from geoalchemy2.functions import ST_Intersects, ST_MakeEnvelope
from geoalchemy2.shape import to_shape
from shapely.geometry import mapping
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import Dataset, VectorFeature
from app.schemas.temporal import (
TemporalComparisonRequest,
TemporalComparisonResponse,
TemporalDatasetRef,
TemporalMetricComparison,
TemporalObjectChanges,
TemporalSeriesDataset,
TemporalSeriesRead,
)
from app.services.vector_feature_service import VectorFeatureService
class TemporalAnalysisService:
IDENTITY_COMPARISON_LIMIT = 5_000
@staticmethod
def list_series(db: Session, project_id: UUID) -> list[TemporalSeriesRead]:
rows = (
db.query(Dataset)
.filter(Dataset.project_id == project_id)
.filter(Dataset.temporal_series_key.isnot(None))
.filter(Dataset.observed_at.isnot(None))
.order_by(Dataset.temporal_series_key.asc(), Dataset.observed_at.asc())
.all()
)
grouped: dict[str, list[Dataset]] = {}
for row in rows:
if row.temporal_series_key:
grouped.setdefault(row.temporal_series_key, []).append(row)
result: list[TemporalSeriesRead] = []
for key, datasets in grouped.items():
observed = [item.observed_at for item in datasets if item.observed_at is not None]
if not observed:
continue
result.append(
TemporalSeriesRead(
temporal_series_key=key,
source_name=datasets[-1].source_name,
reference_layer_name=datasets[-1].reference_layer_name,
dataset_count=len(datasets),
first_observed_at=min(observed),
last_observed_at=max(observed),
datasets=[
TemporalSeriesDataset(
id=item.id,
name=item.name,
observed_at=item.observed_at,
source_version=item.source_version,
feature_count=(item.metadata_json or {}).get("feature_count")
if isinstance(item.metadata_json, dict)
else None,
)
for item in datasets
if item.observed_at is not None
],
)
)
return result
@staticmethod
def compare(
db: Session,
*,
project_id: UUID,
payload: TemporalComparisonRequest,
) -> TemporalComparisonResponse:
if payload.earlier_dataset_id == payload.later_dataset_id:
raise AppError(
code="INVALID_TEMPORAL_COMPARISON",
message="Choose two different dataset snapshots",
status_code=400,
)
earlier = TemporalAnalysisService._get_temporal_dataset(db, project_id, payload.earlier_dataset_id, "Earlier")
later = TemporalAnalysisService._get_temporal_dataset(db, project_id, payload.later_dataset_id, "Later")
if earlier.temporal_series_key != later.temporal_series_key:
raise AppError(
code="INCOMPATIBLE_TEMPORAL_SERIES",
message="Dataset snapshots must belong to the same temporal series",
details={
"earlier_series": earlier.temporal_series_key,
"later_series": later.temporal_series_key,
},
status_code=400,
)
if earlier.observed_at >= later.observed_at:
raise AppError(
code="INVALID_TEMPORAL_ORDER",
message="Earlier snapshot must have an observation date before the later snapshot",
status_code=400,
)
bbox = payload.bbox.model_dump()
earlier_summary = VectorFeatureService.summarize_features_by_bbox(db, dataset=earlier, bbox=bbox)
later_summary = VectorFeatureService.summarize_features_by_bbox(db, dataset=later, bbox=bbox)
if (
earlier_summary["aggregation_method"] != later_summary["aggregation_method"]
or earlier_summary["metric_unit"] != later_summary["metric_unit"]
):
raise AppError(
code="INCOMPATIBLE_TEMPORAL_AGGREGATION",
message="Dataset snapshots use incompatible aggregation semantics",
status_code=400,
)
earlier_value = float(earlier_summary["metric_value"])
later_value = float(later_summary["metric_value"])
absolute_change = later_value - earlier_value
percent_change = (absolute_change / earlier_value * 100.0) if earlier_value else None
warnings = [
warning
for warning in {earlier_summary.get("warning"), later_summary.get("warning")}
if warning
]
object_changes, geojson, identity_warnings = TemporalAnalysisService._compare_identity_features(
db,
earlier=earlier,
later=later,
bbox=bbox,
preview_limit=payload.preview_limit,
)
warnings.extend(identity_warnings)
return TemporalComparisonResponse(
temporal_series_key=earlier.temporal_series_key,
earlier=TemporalDatasetRef(
id=earlier.id,
name=earlier.name,
observed_at=earlier.observed_at,
source_version=earlier.source_version,
),
later=TemporalDatasetRef(
id=later.id,
name=later.name,
observed_at=later.observed_at,
source_version=later.source_version,
),
selection_bbox=payload.bbox,
metric=TemporalMetricComparison(
label=str(later_summary["metric_label"]),
unit=str(later_summary["metric_unit"]),
aggregation_method=str(later_summary["aggregation_method"]),
earlier_value=earlier_value,
later_value=later_value,
absolute_change=absolute_change,
percent_change=percent_change,
is_estimate=bool(earlier_summary["is_estimate"] or later_summary["is_estimate"]),
),
object_changes=object_changes,
geojson=geojson,
warnings=warnings,
generated_at=datetime.now(timezone.utc),
)
@staticmethod
def _get_temporal_dataset(db: Session, project_id: UUID, dataset_id: UUID, label: str) -> Dataset:
dataset = db.get(Dataset, dataset_id)
if not dataset or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message=f"{label} dataset not found", status_code=404)
if dataset.dataset_type not in {"vector", "geojson"}:
raise AppError(
code="DATASET_NOT_VECTOR",
message="Temporal selection comparison currently requires vector datasets",
status_code=400,
)
if not dataset.temporal_series_key or not dataset.observed_at:
raise AppError(
code="TEMPORAL_METADATA_MISSING",
message=f"{label} dataset has no explicit temporal series and observation date",
status_code=400,
)
return dataset
@staticmethod
def _compare_identity_features(
db: Session,
*,
earlier: Dataset,
later: Dataset,
bbox: dict[str, Any],
preview_limit: int,
) -> tuple[TemporalObjectChanges, dict[str, Any], list[str]]:
earlier_config = earlier.source_metadata if isinstance(earlier.source_metadata, dict) else {}
later_config = later.source_metadata if isinstance(later.source_metadata, dict) else {}
if not earlier_config.get("identity_stable") or not later_config.get("identity_stable"):
return (
TemporalObjectChanges(available=False),
{"type": "FeatureCollection", "features": []},
["Object-level changes are unavailable because the source does not guarantee stable feature identifiers."],
)
normalized_bbox = VectorFeatureService._normalize_selection_bbox(bbox)
envelope = ST_MakeEnvelope(
normalized_bbox["min_x"],
normalized_bbox["min_y"],
normalized_bbox["max_x"],
normalized_bbox["max_y"],
4326,
)
def load(dataset_id: UUID) -> list[VectorFeature]:
return (
db.query(VectorFeature)
.filter(VectorFeature.dataset_id == dataset_id)
.filter(ST_Intersects(VectorFeature.geometry, envelope))
.filter(VectorFeature.source_feature_id.isnot(None))
.order_by(VectorFeature.source_feature_id.asc())
.limit(TemporalAnalysisService.IDENTITY_COMPARISON_LIMIT + 1)
.all()
)
earlier_rows = load(earlier.id)
later_rows = load(later.id)
if (
len(earlier_rows) > TemporalAnalysisService.IDENTITY_COMPARISON_LIMIT
or len(later_rows) > TemporalAnalysisService.IDENTITY_COMPARISON_LIMIT
):
return (
TemporalObjectChanges(available=False),
{"type": "FeatureCollection", "features": []},
["Object-level preview was skipped because the selection exceeds the 5,000 feature safety limit."],
)
earlier_by_id = {str(row.source_feature_id): row for row in earlier_rows if row.source_feature_id}
later_by_id = {str(row.source_feature_id): row for row in later_rows if row.source_feature_id}
earlier_ids = set(earlier_by_id)
later_ids = set(later_by_id)
added_ids = sorted(later_ids - earlier_ids)
removed_ids = sorted(earlier_ids - later_ids)
common_ids = sorted(earlier_ids & later_ids)
comparison_property = str(later_config.get("comparison_property") or "").strip() or None
modified_ids: list[str] = []
unchanged_ids: list[str] = []
for feature_id in common_ids:
earlier_row = earlier_by_id[feature_id]
later_row = later_by_id[feature_id]
geometry_changed = not to_shape(earlier_row.geometry).equals(to_shape(later_row.geometry))
value_changed = False
if comparison_property:
value_changed = (earlier_row.properties_json or {}).get(comparison_property) != (
later_row.properties_json or {}
).get(comparison_property)
(modified_ids if geometry_changed or value_changed else unchanged_ids).append(feature_id)
features: list[dict[str, Any]] = []
for change_type, feature_ids, rows in (
("added", added_ids, later_by_id),
("removed", removed_ids, earlier_by_id),
("modified", modified_ids, later_by_id),
):
for feature_id in feature_ids:
if len(features) >= preview_limit:
break
row = rows[feature_id]
properties = dict(row.properties_json or {})
properties.update(
{
"change_type": change_type,
"source_feature_id": feature_id,
"earlier_dataset_id": str(earlier.id),
"later_dataset_id": str(later.id),
}
)
if change_type == "modified" and comparison_property:
before = (earlier_by_id[feature_id].properties_json or {}).get(comparison_property)
after = (later_by_id[feature_id].properties_json or {}).get(comparison_property)
properties.update({"value_before": before, "value_after": after})
if isinstance(before, (int, float)) and isinstance(after, (int, float)):
properties["value_delta"] = after - before
features.append(
{
"type": "Feature",
"id": str(row.id),
"geometry": mapping(to_shape(row.geometry)),
"properties": properties,
}
)
warnings: list[str] = []
total_changes = len(added_ids) + len(removed_ids) + len(modified_ids)
if total_changes > preview_limit:
warnings.append(
f"The map shows the first {preview_limit} of {total_changes} changed features; counts remain complete."
)
return (
TemporalObjectChanges(
available=True,
added_count=len(added_ids),
removed_count=len(removed_ids),
modified_count=len(modified_ids),
unchanged_count=len(unchanged_ids),
),
{"type": "FeatureCollection", "features": features},
warnings,
)
+102 -1
View File
@@ -9,9 +9,10 @@ from geoalchemy2.shape import to_shape
from shapely.geometry import mapping
from shapely.geometry import shape
from shapely.validation import make_valid
from sqlalchemy import Float, cast, func
from app.core.errors import AppError
from app.models import VectorFeature
from app.models import Dataset, VectorFeature
class VectorFeatureService:
@@ -88,6 +89,7 @@ class VectorFeatureService:
dataset_id: UUID,
bbox: dict[str, Any],
limit: int = 100,
dataset: Dataset | None = None,
) -> dict[str, Any]:
normalized_bbox = VectorFeatureService._normalize_selection_bbox(bbox)
safe_limit = max(1, min(int(limit), 1000))
@@ -121,6 +123,14 @@ class VectorFeatureService:
truncated = total_feature_count > safe_limit
selected_rows = rows[:safe_limit]
features = [VectorFeatureService._row_to_geojson_feature(row) for row in selected_rows]
summary = None
if dataset and isinstance(dataset.source_metadata, dict) and dataset.source_metadata.get("selection_aggregation"):
summary = VectorFeatureService.summarize_features_by_bbox(
db,
dataset=dataset,
bbox=normalized_bbox,
total_feature_count=total_feature_count,
)
return {
"selection_bbox": normalized_bbox,
@@ -132,6 +142,97 @@ class VectorFeatureService:
"type": "FeatureCollection",
"features": features,
},
"summary": summary,
}
@staticmethod
def summarize_features_by_bbox(
db,
*,
dataset: Dataset,
bbox: dict[str, Any],
total_feature_count: int | None = None,
) -> dict[str, Any]:
normalized_bbox = VectorFeatureService._normalize_selection_bbox(bbox)
envelope = ST_MakeEnvelope(
normalized_bbox["min_x"],
normalized_bbox["min_y"],
normalized_bbox["max_x"],
normalized_bbox["max_y"],
4326,
)
selection_filter = (
VectorFeature.dataset_id == dataset.id,
ST_Intersects(VectorFeature.geometry, envelope),
)
feature_count = total_feature_count
if feature_count is None:
feature_count = int(db.query(func.count(VectorFeature.id)).filter(*selection_filter).scalar() or 0)
source_metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
config = source_metadata.get("selection_aggregation")
if not isinstance(config, dict):
config = {}
method = str(config.get("method") or "feature_count")
label = str(config.get("label") or "Objecten")
unit = str(config.get("unit") or "objecten")
warning = str(config["warning"]) if config.get("warning") else None
is_estimate = bool(config.get("is_estimate", False))
metric_value = float(feature_count)
if method == "intersection_area":
intersection = func.ST_Intersection(VectorFeature.geometry, envelope)
area_expression = func.ST_Area(func.ST_Transform(intersection, 31370))
area_m2 = db.query(func.coalesce(func.sum(area_expression), 0.0)).filter(*selection_filter).scalar()
divisor = 10_000.0 if unit == "ha" else 1.0
metric_value = float(area_m2 or 0.0) / divisor
elif method == "intersection_length":
intersection = func.ST_Intersection(VectorFeature.geometry, envelope)
length_expression = func.ST_Length(func.ST_Transform(intersection, 31370))
length_m = db.query(func.coalesce(func.sum(length_expression), 0.0)).filter(*selection_filter).scalar()
divisor = 1_000.0 if unit == "km" else 1.0
metric_value = float(length_m or 0.0) / divisor
elif method in {"sum", "area_weighted_sum"}:
property_name = str(config.get("property") or "").strip()
if not property_name:
raise AppError(
code="INVALID_SELECTION_AGGREGATION",
message="Dataset selection aggregation requires a numeric property",
details={"dataset_id": str(dataset.id), "method": method},
status_code=500,
)
numeric_value = cast(VectorFeature.properties_json.op("->>")(property_name), Float)
value_expression = numeric_value
if method == "area_weighted_sum":
source_area = func.ST_Area(func.ST_Transform(VectorFeature.geometry, 31370))
intersection_area = func.ST_Area(
func.ST_Transform(func.ST_Intersection(VectorFeature.geometry, envelope), 31370)
)
value_expression = numeric_value * intersection_area / func.nullif(source_area, 0.0)
is_estimate = True
aggregate_value = (
db.query(func.coalesce(func.sum(value_expression), 0.0))
.filter(*selection_filter)
.filter(VectorFeature.properties_json.op("->>")(property_name).isnot(None))
.scalar()
)
metric_value = float(aggregate_value or 0.0)
elif method != "feature_count":
raise AppError(
code="INVALID_SELECTION_AGGREGATION",
message="Unsupported dataset selection aggregation",
details={"dataset_id": str(dataset.id), "method": method},
status_code=500,
)
return {
"metric_label": label,
"metric_value": metric_value,
"metric_unit": unit,
"aggregation_method": method,
"feature_count": feature_count,
"is_estimate": is_estimate,
"warning": warning,
}
@staticmethod
@@ -15,7 +15,7 @@ from shapely.validation import make_valid
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import Area, Dataset
from app.models import Area, Dataset, DatasetVersion
from app.schemas.dataset import DatasetCreateResponse
from app.schemas.operations import VectorOperationResult
from app.services.geojson_service import parse_geojson_payload
@@ -444,6 +444,16 @@ class VectorOperationsService:
source_metadata=source_metadata,
provenance_metadata=provenance_metadata,
imported_at=datetime.now(timezone.utc),
temporal_series_key=(
f"{source_dataset.temporal_series_key}:{operation}"
if source_dataset.temporal_series_key
else None
),
observed_at=source_dataset.observed_at,
valid_from=source_dataset.valid_from,
valid_to=source_dataset.valid_to,
temporal_granularity=source_dataset.temporal_granularity,
source_version=source_dataset.source_version,
storage_path=storage_info["storage_path"],
original_filename=storage_info["original_filename"],
stored_filename=storage_info["stored_filename"],
@@ -459,6 +469,20 @@ class VectorOperationsService:
status="ready",
)
db.add(derived_dataset)
db.add(
DatasetVersion(
dataset_id=derived_dataset.id,
version=1,
storage_path=derived_dataset.storage_path,
source_version=derived_dataset.source_version,
observed_at=derived_dataset.observed_at,
valid_from=derived_dataset.valid_from,
valid_to=derived_dataset.valid_to,
checksum_sha256=derived_dataset.checksum_sha256,
source_metadata=derived_dataset.source_metadata,
provenance_metadata=derived_dataset.provenance_metadata,
)
)
db.commit()
db.refresh(derived_dataset)
if persist_vector_features:
@@ -7,7 +7,7 @@ import importlib
from geoalchemy2.shape import from_shape
from app.core.errors import AppError
from app.models import Area, Dataset
from app.models import Area, Dataset, DatasetVersion
from app.services.raster_operations_service import RasterOperationsService
from app.api.routes.datasets import _run_job_sync
from shapely.geometry import box
@@ -356,8 +356,12 @@ def test_raster_reproject_returns_persisted_derived_dataset(monkeypatch, tmp_pat
)
assert result_id == output_id
assert len(db.added) == 1
assert len(db.added) == 2
derived = db.added[0]
version = db.added[1]
assert isinstance(version, DatasetVersion)
assert version.dataset_id == output_id
assert version.version == 1
assert derived.id == output_id
assert derived.metadata_json is not None
assert derived.metadata_json["operation"] == "raster.reproject"
@@ -783,9 +787,13 @@ def test_raster_clip_persists_derived_dataset(monkeypatch, tmp_path) -> None:
result_id = RasterOperationsService.clip(db, dataset_id, area_id, "clip-result.tif")
assert result_id == output_id
assert len(db.added) == 1
assert len(db.added) == 2
derived = db.added[0]
version = db.added[1]
assert isinstance(derived, Dataset)
assert isinstance(version, DatasetVersion)
assert version.dataset_id == output_id
assert version.version == 1
assert derived.id == output_id
assert derived.source == "operation:raster.clip"
assert derived.dataset_type == "raster"
@@ -1295,8 +1303,12 @@ def test_raster_index_records_provenance_and_dtype(tmp_path, monkeypatch) -> Non
result_dataset_id = RasterOperationsService.ndvi(db, dataset_id, nir_band=4, red_band=3, output_name="ndvi-test")
assert result_dataset_id == output_dataset_id
assert len(db.added) == 1
assert len(db.added) == 2
derived = db.added[0]
version = db.added[1]
assert isinstance(version, DatasetVersion)
assert version.dataset_id == output_dataset_id
assert version.version == 1
assert derived.id == output_dataset_id
assert derived.metadata_json is not None
assert derived.metadata_json["operation"] == "raster.ndvi"
@@ -0,0 +1,250 @@
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
import pytest
from app.core.errors import AppError
from app.models import Dataset, DatasetVersion
from app.schemas.dataset import DatasetTemporalUpdate
from app.schemas.temporal import TemporalComparisonRequest, TemporalObjectChanges
from app.services.dataset_service import DatasetService
from app.services.temporal_analysis_service import TemporalAnalysisService
from app.services.vector_feature_service import VectorFeatureService
ROOT = Path(__file__).parents[2]
class ScalarQuery:
def __init__(self, value: float):
self.value = value
def filter(self, *args): # noqa: ANN002, ARG002
return self
def scalar(self):
return self.value
class ScalarSession:
def __init__(self, value: float):
self.value = value
def query(self, *args): # noqa: ANN002, ARG002
return ScalarQuery(self.value)
class VersionQuery:
def __init__(self, latest: DatasetVersion | None):
self.latest = latest
def filter(self, *args): # noqa: ANN002, ARG002
return self
def order_by(self, *args): # noqa: ANN002, ARG002
return self
def first(self):
return self.latest
class TemporalUpdateSession:
def __init__(self, dataset: Dataset, latest: DatasetVersion | None):
self.dataset = dataset
self.latest = latest
self.added: list[object] = []
def get(self, model, item_id): # noqa: ANN001
return self.dataset if model is Dataset and item_id == self.dataset.id else None
def query(self, model): # noqa: ANN001
assert model is DatasetVersion
return VersionQuery(self.latest)
def add(self, item): # noqa: ANN001
self.added.append(item)
def commit(self):
return None
def refresh(self, _item):
return None
def temporal_dataset(*, project_id, observed_year: int, metric_method: str = "feature_count") -> Dataset:
return Dataset(
id=uuid4(),
project_id=project_id,
name=f"snapshot-{observed_year}.geojson",
dataset_type="vector",
source="official",
dataset_role="reference",
temporal_series_key="official:test:mol",
observed_at=datetime(observed_year, 1, 1, tzinfo=timezone.utc),
source_version=str(observed_year),
source_metadata={
"selection_aggregation": {
"method": metric_method,
"label": "Objecten",
"unit": "objecten",
}
},
)
def test_temporal_migration_and_models_align() -> None:
migration = (ROOT / "backend/alembic/versions/202607140001_temporal_dataset_foundation.py").read_text(encoding="utf-8")
for field in (
"temporal_series_key",
"observed_at",
"valid_from",
"valid_to",
"temporal_granularity",
"source_version",
):
assert field in migration
assert hasattr(Dataset, field)
assert "ix_vector_features_dataset_source_feature" in migration
assert 'down_revision = "202606120900"' in migration
def test_temporal_metadata_requires_an_explicit_series_and_observation_date() -> None:
with pytest.raises(AppError, match="observed_at is required"):
DatasetService._validate_temporal_metadata(
temporal_series_key="official:test:mol",
observed_at=None,
valid_from=None,
valid_to=None,
temporal_granularity="year",
source_version="2024",
)
with pytest.raises(AppError, match="valid_to must be"):
DatasetService._validate_temporal_metadata(
temporal_series_key="official:test:mol",
observed_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
valid_from=datetime(2024, 12, 31, tzinfo=timezone.utc),
valid_to=datetime(2024, 1, 1, tzinfo=timezone.utc),
temporal_granularity="year",
source_version="2024",
)
def test_temporal_metadata_update_appends_provenance_version_and_is_idempotent() -> None:
project_id = uuid4()
dataset = temporal_dataset(project_id=project_id, observed_year=2024)
dataset.status = "ready"
dataset.metadata_json = {}
latest = DatasetVersion(
dataset_id=dataset.id,
version=3,
observed_at=dataset.observed_at,
source_version="2024",
)
session = TemporalUpdateSession(dataset, latest)
payload = DatasetTemporalUpdate(
temporal_series_key="official:test:mol",
observed_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
temporal_granularity="year",
source_version="2025",
)
updated = DatasetService.update_temporal_metadata(session, dataset.id, payload)
assert updated.observed_at == payload.observed_at
assert latest.version == 3
assert latest.observed_at == datetime(2024, 1, 1, tzinfo=timezone.utc)
assert len(session.added) == 2
appended = session.added[1]
assert isinstance(appended, DatasetVersion)
assert appended.version == 4
assert appended.observed_at == payload.observed_at
session.added.clear()
DatasetService.update_temporal_metadata(session, dataset.id, payload)
assert session.added == []
def test_selection_area_aggregation_returns_hectares_without_loading_all_features() -> None:
project_id = uuid4()
dataset = temporal_dataset(project_id=project_id, observed_year=1969, metric_method="intersection_area")
dataset.source_metadata["selection_aggregation"].update({"label": "Oppervlakte", "unit": "ha"})
result = VectorFeatureService.summarize_features_by_bbox(
ScalarSession(125_000.0),
dataset=dataset,
bbox={"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3, "crs": "EPSG:4326"},
total_feature_count=40,
)
assert result["metric_value"] == 12.5
assert result["metric_unit"] == "ha"
assert result["feature_count"] == 40
def test_temporal_compare_returns_delta_and_canonical_change_payload(monkeypatch) -> None:
project_id = uuid4()
earlier = temporal_dataset(project_id=project_id, observed_year=2021)
later = temporal_dataset(project_id=project_id, observed_year=2024)
def get_dataset(_db, _project_id, dataset_id, _label):
return earlier if dataset_id == earlier.id else later
def summarize(_db, *, dataset, bbox): # noqa: ARG001
value = 100.0 if dataset.id == earlier.id else 115.0
return {
"metric_label": "Inwoners",
"metric_value": value,
"metric_unit": "inwoners",
"aggregation_method": "area_weighted_sum",
"feature_count": 10,
"is_estimate": True,
"warning": "Areal weighting",
}
monkeypatch.setattr(TemporalAnalysisService, "_get_temporal_dataset", staticmethod(get_dataset))
monkeypatch.setattr(VectorFeatureService, "summarize_features_by_bbox", staticmethod(summarize))
monkeypatch.setattr(
TemporalAnalysisService,
"_compare_identity_features",
staticmethod(
lambda *args, **kwargs: (
TemporalObjectChanges(available=True, added_count=1, removed_count=0, modified_count=2, unchanged_count=7),
{"type": "FeatureCollection", "features": []},
[],
)
),
)
result = TemporalAnalysisService.compare(
SimpleNamespace(),
project_id=project_id,
payload=TemporalComparisonRequest(
earlier_dataset_id=earlier.id,
later_dataset_id=later.id,
bbox={"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3},
),
)
assert result.metric.absolute_change == 15.0
assert result.metric.percent_change == 15.0
assert result.metric.is_estimate is True
assert result.object_changes.modified_count == 2
assert result.geojson["type"] == "FeatureCollection"
def test_temporal_frontend_and_official_operator_contracts_exist() -> None:
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
temporal_api = (ROOT / "frontend/src/services/api/temporal.ts").read_text(encoding="utf-8")
population = (ROOT / "scripts/provision_mol_population_history.py").read_text(encoding="utf-8")
landuse = (ROOT / "scripts/provision_mol_historical_landuse.py").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
assert "Laatste toestand" in workspace
assert "Evolutie" in workspace
assert "Vergelijk periode" in workspace
assert "/temporal/compare" in temporal_api
assert "Statbel" in population and "area_weighted_sum" in population
assert "HistLandgebruik" in landuse and "intersection_area" in landuse
assert "provision_mol_population_history.py" in dockerfile
assert "provision_mol_historical_landuse.py" in dockerfile
assert "fake" not in population.lower()
+2
View File
@@ -74,6 +74,8 @@ RUN python scripts/gis_import_smoke.py \
COPY scripts/prepare_operator_real_data_samples.py /app/scripts/prepare_operator_real_data_samples.py
COPY scripts/provision_mol_municipality_workspace.py /app/scripts/provision_mol_municipality_workspace.py
COPY scripts/provision_mol_context_layers.py /app/scripts/provision_mol_context_layers.py
COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py
COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py
COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py
COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
+50
View File
@@ -1393,3 +1393,53 @@ GET /api/v1/exports/{export_id}/download
```
PDF/report-designer functionality can be added after core GeoAI workflows work.
## Temporal datasets and area evolution
Temporal metadata describes the source observation, not API run history. A
snapshot is a normal persisted dataset grouped by `temporal_series_key` and
ordered by `observed_at`. Optional validity uses `valid_from` and `valid_to`;
`temporal_granularity` is `snapshot`, `day`, `month`, `year` or `period`.
Dataset upload accepts those temporal fields plus `source_version`. When a
dataset declares `source_metadata.selection_aggregation`, the vector bbox
selection response also contains a `summary` with metric label/value/unit,
aggregation method, feature count, estimate status and an optional warning.
Supported PostGIS aggregations are feature count, intersection area,
intersection length, numeric sum and area-weighted numeric sum. Area and length
are measured after transformation to EPSG:31370.
### PATCH `/api/v1/projects/{project_id}/datasets/{dataset_id}/temporal`
Updates the temporal provenance of an existing dataset. Series key and
observation date are required together. It does not alter features or
manufacture a historical observation.
### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/versions`
Lists immutable storage/provenance versions. Uploads and derived datasets
create version 1 in the same persistence transaction.
### GET `/api/v1/projects/{project_id}/temporal/series`
Returns dated project series in the canonical envelope. Each item contains its
source/layer identity, first and last observations and ordered datasets.
### POST `/api/v1/projects/{project_id}/temporal/compare`
```json
{
"earlier_dataset_id": "uuid",
"later_dataset_id": "uuid",
"bbox": {"west": 5.0, "south": 51.0, "east": 5.2, "north": 51.2},
"preview_limit": 500
}
```
Both datasets must belong to the project and the same temporal series, with
the earlier observation preceding the later one. The response contains source
snapshot references, selection bbox, earlier/later metric values,
absolute/percentage change, estimate status, warnings and GeoJSON evidence.
Added/removed/modified object changes are calculated only when source
provenance declares stable feature identities; otherwise
`object_changes.available=false` and no object history is inferred.
+26
View File
@@ -7768,3 +7768,29 @@ Limitations:
Next:
- Define authoritative population and land-cover source adapters, then reuse the proven municipality provisioner and bbox analysis flow for the complete Kempen.
## Sprint 187 Temporal Mol explorer (2026-07-14)
Implemented:
- Added observation/validity/source-version fields to datasets and immutable provenance fields to dataset versions, with one Alembic head and indexed temporal/source identity lookups.
- Persisted dataset version 1 atomically for uploads, demo fixtures and derived vector/raster operations.
- Added source-governed PostGIS selection summaries for object count, area, length, numeric sum and area-weighted sum.
- Added project temporal-series discovery and same-series bbox comparison with honest metric deltas and stable-identity-only object changes.
- Added the map-first `Laatste toestand` / `Evolutie` workflow with period selection, automatic rectangle analysis and change overlays.
- Added explicit, idempotent Statbel population (2021-2025) and Digitaal Vlaanderen historical land-use (1778/1873/1969) provisioners for Mol.
- Kept all source fetching operator-triggered; application startup and user queries never fabricate or silently download source data.
Methodology:
- Population totals are source-published per statistical sector. Intersections with partial sectors are labelled area-weighted estimates.
- Historical land-use classes are clipped from official editions and measured in EPSG:31370. They do not claim stable cadastral object identity.
- Every snapshot carries its source URL, observation date, source version, checksum and processing limitations.
Validation before live deployment:
- Script compilation passed.
- New temporal/API/static regression suite passed 6 tests, including append-only/idempotent temporal provenance updates.
- Raster and temporal focused suite passed 27 tests after extending existing assertions to require dataset-version persistence.
- Frontend TypeScript typecheck and production build passed.
- Offline Alembic SQL generation passed with head `202607140001`.
Next:
- Run the complete readiness gate, deploy to Tower/PostGIS, provision the official snapshots and verify current/evolution selection end to end in the internal browser.
+19
View File
@@ -236,3 +236,22 @@ GRB and OSM live imports are intentionally `not_configured` in Sprint 7B. Manual
- Multi-tenant row-level security.
- User accounts.
- Full model registry tables.
## Temporal dataset foundation
Historical observations remain normal `datasets` and `vector_features`; there
is no parallel temporal feature store. Snapshots are grouped by
`datasets.temporal_series_key` and carry `observed_at`, `valid_from`,
`valid_to`, `temporal_granularity` and `source_version`. Observation time is
kept separate from ingestion time (`imported_at`).
`dataset_versions` records immutable storage provenance for every upload and
derived output: dataset-local `version`, storage path, source version,
observation/validity dates, checksum and source/provenance JSON. The
`(dataset_id, version)` pair is unique. Temporal series lookup is indexed by
`(project_id, temporal_series_key, observed_at)` and source-feature lookup by
`(dataset_id, source_feature_id)`.
Time-series comparison is read-only and aggregates persisted geometry inside a
requested bbox. Object-level added/removed/modified evidence is only valid for
sources that explicitly declare stable source feature identifiers.
+17
View File
@@ -138,3 +138,20 @@ V1 dataset strategy is complete when:
- an area can request/cache a reference building layer;
- detection outputs can be compared with that reference layer;
- exports include source metadata.
## Temporal snapshots and evolution
- A historical observation is one persisted dataset snapshot. Existing
datasets are not overwritten and features are not hidden in job JSON.
- Related observations share a stable `temporal_series_key`; `observed_at`
records when the source describes reality, while `imported_at` records
ingestion time.
- The latest-state map uses the latest available observation but must not call
an old source edition current reality.
- Evolution metrics use the same selection geometry, aggregation and units for
both snapshots.
- Partial-sector population is an area-weighted estimate. GeoIntel must not
imply address-level distribution when only sector totals are available.
- Historical cartographic classes can change meaning between editions. Source
classes and processing notes remain provenance, and object changes require
explicit stable source identity.
+26
View File
@@ -29,6 +29,32 @@ start geen verborgen providerfetch. De resulterende GeoJSON-artefacten,
checksums en bron-URL's worden onder persistent operator storage bewaard en
via de bestaande DatasetService/vectorfeature-flow geïmporteerd.
### Mol population history
`scripts/provision_mol_population_history.py` imports official Statbel
population-by-statistical-sector tables and matching sector geometries for
2021 through 2025. It keeps the published sector total as
`population_total`, clips the official geometry to Mol NIS `13025` and uploads
each year as a separate dataset in
`statbel:population-statistical-sector:mol`.
Complete sectors use their published population total. A rectangle that cuts
through a sector uses an explicitly labelled area-weighted estimate; the
source does not justify a more precise intra-sector distribution.
### Mol historical land use
`scripts/provision_mol_historical_landuse.py` uses the official Digitaal
Vlaanderen Historical Land Use WFS for the 1778, 1873 and 1969 collections.
Buildings, forest, water and roads are filtered server-side, clipped to the
official Mol boundary and uploaded through the existing dataset service.
Source classes, request URLs, observation year, simplification tolerance and
methodological limitations remain in provenance.
These historical map editions support exploratory area evolution, not
cadastral object lineage. Their feature identities are declared unstable and
GeoIntel does not fabricate added/removed object counts.
## OSM
- Naam: OpenStreetMap
+13
View File
@@ -4,6 +4,19 @@ React + TypeScript + MapLibre foundation for project/area/dataset workflow.
Mol is the primary operating context. On a fresh session the application opens the map-first geographic explorer, prefers the persisted `Mol Municipality Workbench`, selects the official NIS `13025` municipality boundary and activates the largest available authoritative building layer. Explicit project and dataset selections remain authoritative, and all broader Kempen workflows remain available.
The map-first explorer has two deliberate modes. `Latest state` selects the
latest explicitly dated source snapshot without claiming an old edition is
current, while `Evolution` lets the operator compare an earlier and later
snapshot from the same series over a drawn rectangle. Results show units,
absolute/percentage change, estimate status and source limitations.
Added/removed/modified overlays only appear for stable source identities.
Selection results use dataset-specific PostGIS summaries. Object layers show
intersecting counts, population shows inhabitants with partial-sector
estimates clearly marked and land-cover sources show intersected hectares. The
advanced workbench remains available but is not required for the primary
choose-theme, draw-area, read-result flow.
The primary workflow is deliberately short: choose a data theme, drag a rectangle on the MapLibre map and read the resulting PostGIS evidence. Releasing the drag runs the active theme query and every other available theme query for the same EPSG:4326 bbox. The result panel shows selection area, exact intersection totals, active-theme density, source identity and bounded feature properties. Map rendering remains capped at 1,000 features while `total_feature_count` reports the exact database count.
The theme catalog currently recognizes buildings, population, forest/green, water, roads and parcels from dataset names and canonical `reference_layer_name` metadata. A theme is enabled only when a ready persisted vector dataset exists; otherwise it states `Bron nog niet ingeladen`. This prevents missing population or land-cover sources from appearing as zero-valued observations. The previous technical Map workspace remains available through `Geavanceerde werkbank` for derived datasets, QA/QC evidence and export operations.
+4
View File
@@ -319,6 +319,8 @@ function GeoMap({
'#16a34a',
'removed',
'#dc2626',
'modified',
'#d97706',
'unchanged',
'#2563eb',
'#f97316',
@@ -345,6 +347,8 @@ function GeoMap({
'#15803d',
'removed',
'#b91c1c',
'modified',
'#b45309',
'unchanged',
'#1d4ed8',
'#ea580c',
+285 -55
View File
@@ -3,6 +3,7 @@ import GeoMap from '../GeoMap'
import type { AreaRead, DatasetCreateResponse, MapViewportState, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types'
import { featureCollectionBounds } from '../../lib/geojsonBounds'
import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights'
import { useTemporalComparison } from '../../hooks/useTemporalComparison'
const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson'
const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson'
@@ -90,12 +91,42 @@ function pickThemeDataset(datasets: DatasetCreateResponse[], theme: DataTheme):
(dataset.reference_layer_name && theme.tokens.includes(dataset.reference_layer_name.toLowerCase()) ? 1_000_000 : 0) +
(dataset.source_name === 'grb' ? 100_000 : 0) +
(dataset.dataset_role === 'reference' ? 10_000 : 0) +
(dataset.observed_at ? new Date(dataset.observed_at).getTime() / 100_000_000 : 0) +
(dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0)
return score(right) - score(left)
})
return candidates[0] ?? null
}
function pickThemeTemporalSeries(datasets: DatasetCreateResponse[], theme: DataTheme): DatasetCreateResponse[] {
const groups = new Map<string, DatasetCreateResponse[]>()
for (const dataset of datasets) {
if (!datasetMatchesTheme(dataset, theme) || !dataset.temporal_series_key || !dataset.observed_at) {
continue
}
const items = groups.get(dataset.temporal_series_key) ?? []
items.push(dataset)
groups.set(dataset.temporal_series_key, items)
}
return Array.from(groups.values())
.filter((items) => items.length >= 2)
.sort((left, right) => {
if (right.length !== left.length) {
return right.length - left.length
}
const latest = (items: DatasetCreateResponse[]) => Math.max(...items.map((item) => new Date(item.observed_at ?? 0).getTime()))
return latest(right) - latest(left)
})[0]
?.sort((left, right) => new Date(left.observed_at ?? 0).getTime() - new Date(right.observed_at ?? 0).getTime()) ?? []
}
function formatObservationDate(value: string | null | undefined): string {
if (!value) {
return 'Geen peildatum'
}
return new Intl.DateTimeFormat('nl-BE', { year: 'numeric', month: 'short', day: 'numeric' }).format(new Date(value))
}
function selectionAreaSquareMetres(bbox: VectorSelectionBBox | null): number | null {
if (!bbox) {
return null
@@ -134,6 +165,19 @@ function resultCountLabel(result: VectorSelectionResponse): string {
return result.truncated && result.total_feature_count == null ? `${result.feature_count.toLocaleString('nl-BE')}+` : total.toLocaleString('nl-BE')
}
function resultMetricLabel(result: VectorSelectionResponse): string {
if (!result.summary) {
return resultCountLabel(result)
}
const maximumFractionDigits = result.summary.metric_unit === 'inwoners' ? 0 : 2
return `${result.summary.metric_value.toLocaleString('nl-BE', { maximumFractionDigits })} ${result.summary.metric_unit}`
}
function formatTemporalMetric(value: number, unit: string): string {
const maximumFractionDigits = unit === 'inwoners' || unit === 'objecten' ? 0 : 2
return `${value.toLocaleString('nl-BE', { maximumFractionDigits })} ${unit}`
}
function readablePropertyName(value: string): string {
return value.replace(/_/g, ' ').replace(/\b\w/g, (character) => character.toUpperCase())
}
@@ -444,6 +488,16 @@ export function MapWorkspace({
loadThemeInsights,
clearThemeInsights,
} = useMapThemeSelectionInsights<DataThemeId>(selectedProjectId)
const {
temporalComparison,
temporalComparisonLoading,
temporalComparisonError,
compareTemporalSnapshots,
clearTemporalComparison,
} = useTemporalComparison(selectedProjectId)
const [analysisMode, setAnalysisMode] = useState<'current' | 'evolution'>('current')
const [earlierDatasetId, setEarlierDatasetId] = useState('')
const [laterDatasetId, setLaterDatasetId] = useState('')
const [bboxSelectionMode, setBboxSelectionMode] = useState(false)
const [firstSelectionCorner, setFirstSelectionCorner] = useState<[number, number] | null>(null)
const [bboxInput, setBboxInput] = useState(bboxToInputState(mapSelectionBbox))
@@ -482,6 +536,10 @@ export function MapWorkspace({
)
const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0]
const activeThemeDataset = themeDatasetMap[activeTheme.id]
const activeTemporalSeries = useMemo(
() => pickThemeTemporalSeries(availableMapDatasets, activeTheme),
[activeTheme, availableMapDatasets],
)
const themeResults = useMemo(
() =>
themeInsights.flatMap((insight) => {
@@ -502,6 +560,14 @@ export function MapWorkspace({
const selectedDensity = selectedAreaSquareMetres && selectedAreaSquareMetres > 0
? selectedResultTotal / (selectedAreaSquareMetres / 1_000_000)
: null
const activeMetricValue = activeSelectionResult?.summary?.metric_value ?? selectedResultTotal
const activeMetricUnit = activeSelectionResult?.summary?.metric_unit ?? 'objecten'
const activeMetricLabel = activeSelectionResult?.summary?.metric_label ?? activeTheme.shortLabel
const activeSecondaryMetric = selectedAreaSquareMetres && selectedAreaSquareMetres > 0
? activeMetricUnit === 'ha'
? `${((activeMetricValue * 10_000) / selectedAreaSquareMetres * 100).toLocaleString('nl-BE', { maximumFractionDigits: 1 })}% dekking`
: `${(activeMetricValue / (selectedAreaSquareMetres / 1_000_000)).toLocaleString('nl-BE', { maximumFractionDigits: 1 })} ${activeMetricUnit} / km2`
: null
const selectedResultProperties = useMemo(() => {
const keys = new Map<string, Set<string>>()
for (const feature of activeSelectionResult?.geojson.features ?? []) {
@@ -526,6 +592,14 @@ export function MapWorkspace({
setBboxInput(bboxToInputState(mapSelectionBbox))
}, [mapSelectionBbox])
useEffect(() => {
const first = activeTemporalSeries[0]
const last = activeTemporalSeries[activeTemporalSeries.length - 1]
setEarlierDatasetId(first?.id ?? '')
setLaterDatasetId(last?.id ?? '')
clearTemporalComparison()
}, [activeTemporalSeries])
useEffect(() => {
if (advancedMode || !activeThemeDataset || (selectedMapDataset && datasetMatchesTheme(selectedMapDataset, activeTheme))) {
return
@@ -562,6 +636,7 @@ export function MapWorkspace({
const startBboxSelection = () => {
setFirstSelectionCorner(null)
clearThemeInsights()
clearTemporalComparison()
setBboxSelectionMode(true)
}
@@ -590,6 +665,7 @@ export function MapWorkspace({
setFirstSelectionCorner(null)
setBboxInput(bboxToInputState(null))
clearThemeInsights()
clearTemporalComparison()
onClearMapSelectionExtract()
}
@@ -644,9 +720,15 @@ export function MapWorkspace({
return
}
setActiveThemeId(theme.id)
clearTemporalComparison()
onOpenDatasetInMap(dataset)
}
const setExplorerMode = (mode: 'current' | 'evolution') => {
setAnalysisMode(mode)
clearTemporalComparison()
}
const loadAllThemeResults = async (bbox: VectorSelectionBBox) => {
const availableThemes = DATA_THEMES.flatMap((theme) => {
const dataset = themeDatasetMap[theme.id]
@@ -657,7 +739,18 @@ export function MapWorkspace({
const analyzeSelection = async (bbox: VectorSelectionBBox) => {
setSelectionBbox(bbox)
await Promise.all([onRunMapSelectionExtract(bbox), loadAllThemeResults(bbox)])
const tasks: Array<Promise<unknown>> = [onRunMapSelectionExtract(bbox), loadAllThemeResults(bbox)]
if (analysisMode === 'evolution' && earlierDatasetId && laterDatasetId) {
tasks.push(compareTemporalSnapshots(earlierDatasetId, laterDatasetId, bbox))
}
await Promise.all(tasks)
}
const runTemporalComparison = () => {
if (!mapSelectionBbox || !earlierDatasetId || !laterDatasetId) {
return
}
void compareTemporalSnapshots(earlierDatasetId, laterDatasetId, mapSelectionBbox)
}
const handleMapBboxPreview = (bbox: VectorSelectionBBox) => {
@@ -757,6 +850,26 @@ export function MapWorkspace({
<h2>Wat bevindt zich in dit gebied?</h2>
<p>Kies een datathema, teken een rechthoek en lees de beschikbare gegevens meteen uit.</p>
</div>
<div className="geo-analysis-mode" role="tablist" aria-label="Analyseperiode">
<button
className={analysisMode === 'current' ? 'active' : ''}
type="button"
role="tab"
aria-selected={analysisMode === 'current'}
onClick={() => setExplorerMode('current')}
>
Laatste toestand
</button>
<button
className={analysisMode === 'evolution' ? 'active' : ''}
type="button"
role="tab"
aria-selected={analysisMode === 'evolution'}
onClick={() => setExplorerMode('evolution')}
>
Evolutie
</button>
</div>
<button className="secondary-action geo-explorer-advanced" type="button" onClick={() => setAdvancedMode(true)}>
Geavanceerde werkbank
</button>
@@ -796,15 +909,52 @@ export function MapWorkspace({
</div>
<div className="geo-source-summary">
<span>Actieve bron</span>
<strong>{activeThemeDataset?.name ?? 'Geen databron beschikbaar'}</strong>
<span>{analysisMode === 'evolution' ? 'Tijdreeks' : 'Actieve bron'}</span>
<strong>
{analysisMode === 'evolution'
? activeTemporalSeries[0]?.temporal_series_key ?? 'Geen tijdreeks beschikbaar'
: activeThemeDataset?.name ?? 'Geen databron beschikbaar'}
</strong>
<small>
{activeThemeDataset
? `${activeThemeDataset.source_name ?? activeThemeDataset.source} · ${activeThemeDataset.dataset_role ?? 'source'} · EPSG:4326`
: activeTheme.description}
{analysisMode === 'evolution'
? activeTemporalSeries.length >= 2
? `${activeTemporalSeries.length} officiële meetmomenten · ${formatObservationDate(activeTemporalSeries[0].observed_at)} tot ${formatObservationDate(activeTemporalSeries[activeTemporalSeries.length - 1].observed_at)}`
: 'Minstens twee expliciet gedateerde snapshots zijn vereist.'
: activeThemeDataset
? `${activeThemeDataset.source_name ?? activeThemeDataset.source} · ${activeThemeDataset.dataset_role ?? 'source'} · ${formatObservationDate(activeThemeDataset.observed_at)}`
: activeTheme.description}
</small>
</div>
{analysisMode === 'evolution' ? (
<div className="geo-time-controls" aria-label="Meetmomenten vergelijken">
<label>
Van
<select value={earlierDatasetId} onChange={(event) => { setEarlierDatasetId(event.target.value); clearTemporalComparison() }} disabled={activeTemporalSeries.length < 2}>
{activeTemporalSeries.map((dataset) => (
<option key={dataset.id} value={dataset.id}>{formatObservationDate(dataset.observed_at)}</option>
))}
</select>
</label>
<label>
Naar
<select value={laterDatasetId} onChange={(event) => { setLaterDatasetId(event.target.value); clearTemporalComparison() }} disabled={activeTemporalSeries.length < 2}>
{activeTemporalSeries.map((dataset) => (
<option key={dataset.id} value={dataset.id}>{formatObservationDate(dataset.observed_at)}</option>
))}
</select>
</label>
<button
className="primary-action"
type="button"
disabled={!mapSelectionBbox || !earlierDatasetId || !laterDatasetId || temporalComparisonLoading}
onClick={runTemporalComparison}
>
{temporalComparisonLoading ? 'Vergelijken…' : 'Vergelijk periode'}
</button>
</div>
) : null}
<label className="geo-scope-select">
Werkgebied
<select value={selectedMapAreaId} onChange={(event) => onSelectMapArea(event.target.value)} disabled={areas.length === 0}>
@@ -828,7 +978,7 @@ export function MapWorkspace({
<div className="geo-map-actions">
<button
className={bboxSelectionMode ? 'primary-action geo-draw-active' : 'primary-action'}
disabled={!activeThemeDataset || mapSelectionLoading || themeResultsLoading}
disabled={!activeThemeDataset || (analysisMode === 'evolution' && activeTemporalSeries.length < 2) || mapSelectionLoading || themeResultsLoading}
type="button"
onClick={startBboxSelection}
>
@@ -836,7 +986,7 @@ export function MapWorkspace({
</button>
<button
className="secondary-action"
disabled={!activeThemeDataset || !selectedAreaBbox || mapSelectionLoading || themeResultsLoading}
disabled={!activeThemeDataset || (analysisMode === 'evolution' && activeTemporalSeries.length < 2) || !selectedAreaBbox || mapSelectionLoading || themeResultsLoading}
type="button"
onClick={() => selectedAreaBbox && void analyzeSelection(selectedAreaBbox)}
>
@@ -850,10 +1000,10 @@ export function MapWorkspace({
<div className={bboxSelectionMode ? 'geo-map-canvas geo-map-canvas-drawing' : 'geo-map-canvas'}>
<GeoMap
data={mapFeatureCollection}
data={analysisMode === 'evolution' && temporalComparison?.geojson.features.length ? temporalComparison.geojson : mapFeatureCollection}
areaData={areaFeatureCollection}
selectedFeature={selectedFeature}
selectionData={mapSelectionResult?.geojson ?? null}
selectionData={analysisMode === 'current' ? mapSelectionResult?.geojson ?? null : null}
selectionBbox={mapSelectionBbox}
bboxSelectionMode={bboxSelectionMode}
visible={mapLayerVisible}
@@ -869,8 +1019,18 @@ export function MapWorkspace({
/>
<div className="geo-map-legend" aria-label="Kaartlegende">
<span><i className="geo-legend-area" /> Gemeentegrens</span>
<span><i className="geo-legend-layer" /> {activeTheme.shortLabel}</span>
<span><i className="geo-legend-selection" /> Selectie</span>
{analysisMode === 'evolution' && temporalComparison?.object_changes.available ? (
<>
<span><i className="geo-legend-added" /> Nieuw</span>
<span><i className="geo-legend-removed" /> Verdwenen</span>
<span><i className="geo-legend-modified" /> Gewijzigd</span>
</>
) : (
<>
<span><i className="geo-legend-layer" /> {activeTheme.shortLabel}</span>
<span><i className="geo-legend-selection" /> Selectie</span>
</>
)}
</div>
{bboxSelectionMode ? (
<div className="geo-draw-instruction" role="status">
@@ -900,56 +1060,117 @@ export function MapWorkspace({
<strong>Nog geen gebied geselecteerd</strong>
<p>Teken een rechthoek op de kaart. De analyse start automatisch zodra je loslaat.</p>
</div>
) : mapSelectionLoading || themeResultsLoading ? (
) : mapSelectionLoading || themeResultsLoading || temporalComparisonLoading ? (
<div className="geo-results-loading" role="status">
<span />
<strong>Gegevens worden uit PostGIS gelezen</strong>
</div>
) : (
<>
<div className="geo-primary-metrics">
<div>
<span>Oppervlakte selectie</span>
<strong>{formatArea(selectedAreaSquareMetres)}</strong>
</div>
<div>
<span>{activeTheme.shortLabel}</span>
<strong>{activeSelectionResult ? resultCountLabel(activeSelectionResult) : 'Geen resultaat'}</strong>
</div>
<div>
<span>Dichtheid</span>
<strong>{selectedDensity === null ? 'n.v.t.' : `${selectedDensity.toLocaleString('nl-BE', { maximumFractionDigits: 1 })} / km2`}</strong>
</div>
</div>
<div className="geo-theme-results">
<div className="geo-results-title-row">
<h4>Alle beschikbare themas</h4>
<span>{themeResults.length} bevraagd</span>
</div>
{DATA_THEMES.map((theme) => {
const dataset = themeDatasetMap[theme.id]
const item = themeResults.find((result) => result.theme.id === theme.id)
return (
<div className="geo-theme-result-row" key={theme.id}>
<span className={`geo-theme-symbol geo-theme-symbol-${theme.id}`} aria-hidden="true" />
<span>
<strong>{theme.label}</strong>
<small>{dataset?.source_name ?? dataset?.source ?? 'Geen bron gekoppeld'}</small>
</span>
<b>{item ? resultCountLabel(item.result) : dataset ? 'Niet bevraagd' : 'Bron ontbreekt'}</b>
{analysisMode === 'evolution' ? (
temporalComparison ? (
<>
<div className="geo-primary-metrics geo-temporal-metrics">
<div>
<span>{formatObservationDate(temporalComparison.earlier.observed_at)}</span>
<strong>{formatTemporalMetric(temporalComparison.metric.earlier_value, temporalComparison.metric.unit)}</strong>
</div>
<div>
<span>{formatObservationDate(temporalComparison.later.observed_at)}</span>
<strong>{formatTemporalMetric(temporalComparison.metric.later_value, temporalComparison.metric.unit)}</strong>
</div>
<div className={temporalComparison.metric.absolute_change >= 0 ? 'positive' : 'negative'}>
<span>Verschil</span>
<strong>
{temporalComparison.metric.absolute_change >= 0 ? '+' : ''}
{formatTemporalMetric(temporalComparison.metric.absolute_change, temporalComparison.metric.unit)}
</strong>
<small>
{temporalComparison.metric.percent_change == null
? 'geen percentage bij nulwaarde'
: `${temporalComparison.metric.percent_change >= 0 ? '+' : ''}${temporalComparison.metric.percent_change.toLocaleString('nl-BE', { maximumFractionDigits: 1 })}%`}
</small>
</div>
</div>
)
})}
</div>
<div className="geo-temporal-summary">
<div>
<span>Gebied</span>
<strong>{formatArea(selectedAreaSquareMetres)}</strong>
</div>
<div>
<span>Meting</span>
<strong>{temporalComparison.metric.label}</strong>
</div>
<div>
<span>Methode</span>
<strong>{temporalComparison.metric.is_estimate ? 'Ruimtelijke schatting' : 'Exact'}</strong>
</div>
</div>
{temporalComparison.object_changes.available ? (
<div className="geo-change-counts" aria-label="Objectwijzigingen">
<span><strong>{temporalComparison.object_changes.added_count ?? 0}</strong> nieuw</span>
<span><strong>{temporalComparison.object_changes.removed_count ?? 0}</strong> verdwenen</span>
<span><strong>{temporalComparison.object_changes.modified_count ?? 0}</strong> gewijzigd</span>
</div>
) : null}
{temporalComparison.warnings.map((warning) => (
<p className="geo-data-notice" key={warning}>{warning}</p>
))}
</>
) : (
<div className="geo-results-empty">
<strong>Klaar om te vergelijken</strong>
<p>Kies twee meetmomenten en gebruik Vergelijk periode. Bij een nieuwe rechthoek wordt de vergelijking automatisch herhaald.</p>
</div>
)
) : (
<>
<div className="geo-primary-metrics">
<div>
<span>Oppervlakte selectie</span>
<strong>{formatArea(selectedAreaSquareMetres)}</strong>
</div>
<div>
<span>{activeMetricLabel}</span>
<strong>{activeSelectionResult ? resultMetricLabel(activeSelectionResult) : 'Geen resultaat'}</strong>
</div>
<div>
<span>{activeMetricUnit === 'ha' ? 'Aandeel selectie' : 'Dichtheid'}</span>
<strong>{activeSecondaryMetric ?? (selectedDensity === null ? 'n.v.t.' : `${selectedDensity.toLocaleString('nl-BE', { maximumFractionDigits: 1 })} / km2`)}</strong>
</div>
</div>
{activeSelectionResult?.truncated ? (
<div className="geo-theme-results">
<div className="geo-results-title-row">
<h4>Alle beschikbare themas</h4>
<span>{themeResults.length} bevraagd</span>
</div>
{DATA_THEMES.map((theme) => {
const dataset = themeDatasetMap[theme.id]
const item = themeResults.find((result) => result.theme.id === theme.id)
return (
<div className="geo-theme-result-row" key={theme.id}>
<span className={`geo-theme-symbol geo-theme-symbol-${theme.id}`} aria-hidden="true" />
<span>
<strong>{theme.label}</strong>
<small>{dataset?.source_name ?? dataset?.source ?? 'Geen bron gekoppeld'}</small>
</span>
<b>{item ? resultMetricLabel(item.result) : dataset ? 'Niet bevraagd' : 'Bron ontbreekt'}</b>
</div>
)
})}
</div>
</>
)}
{analysisMode === 'current' && activeSelectionResult?.truncated ? (
<p className="geo-data-notice">De telling is volledig; op de kaart en in de tabel worden maximaal {activeSelectionResult.limit.toLocaleString('nl-BE')} objecten getoond.</p>
) : null}
{mapSelectionError ? <p className="error">{mapSelectionError}</p> : null}
{themeResultsError ? <p className="error">{themeResultsError}</p> : null}
{temporalComparisonError ? <p className="error">{temporalComparisonError}</p> : null}
{selectedResultProperties.length > 0 ? (
{analysisMode === 'current' && selectedResultProperties.length > 0 ? (
<details className="geo-result-details">
<summary>Kenmerken van de gevonden objecten</summary>
<dl>
@@ -963,7 +1184,7 @@ export function MapWorkspace({
</details>
) : null}
{selectedMapFeature ? (
{analysisMode === 'current' && selectedMapFeature ? (
<div className="geo-selected-feature">
<span>Geselecteerd object</span>
<strong>{String(selectedMapFeature.properties?.['name'] ?? selectedMapFeature.properties?.['source_feature_id'] ?? selectedMapFeature.id ?? 'Object')}</strong>
@@ -971,10 +1192,12 @@ export function MapWorkspace({
</div>
) : null}
<div className="geo-result-actions">
<button className="secondary-action" disabled={!activeSelectionResult} type="button" onClick={downloadActiveThemeSelection}>Download GeoJSON</button>
<button className="secondary-action" disabled={!activeSelectionResult} type="button" onClick={copyActiveThemeSelection}>Kopieer gegevens</button>
</div>
{analysisMode === 'current' ? (
<div className="geo-result-actions">
<button className="secondary-action" disabled={!activeSelectionResult} type="button" onClick={downloadActiveThemeSelection}>Download GeoJSON</button>
<button className="secondary-action" disabled={!activeSelectionResult} type="button" onClick={copyActiveThemeSelection}>Kopieer gegevens</button>
</div>
) : null}
</>
)}
</aside>
@@ -982,7 +1205,14 @@ export function MapWorkspace({
<footer className="geo-explorer-footer">
<span><strong>Werkgebied:</strong> {selectedMapArea?.name ?? 'Geen gemeentegrens geselecteerd'}</span>
<span><strong>Bron:</strong> {activeThemeDataset ? `${activeThemeDataset.source_name ?? activeThemeDataset.source} · ${activeThemeDataset.name}` : 'niet beschikbaar'}</span>
<span>
<strong>Bron:</strong>{' '}
{analysisMode === 'evolution'
? activeTemporalSeries[0]?.temporal_series_key ?? 'geen vergelijkbare tijdreeks'
: activeThemeDataset
? `${activeThemeDataset.source_name ?? activeThemeDataset.source} · ${activeThemeDataset.name}`
: 'niet beschikbaar'}
</span>
{usesDefaultOsmBasemap ? <span><strong>Ondergrond:</strong> OpenStreetMap</span> : null}
</footer>
</section>
@@ -0,0 +1,62 @@
import { useEffect, useState } from 'react'
import { formatError } from '../lib/formatError'
import { temporalApi } from '../services/api/temporal'
import type { TemporalComparisonResponse, VectorSelectionBBox } from '../types'
export function useTemporalComparison(selectedProjectId: string | null) {
const [temporalComparison, setTemporalComparison] = useState<TemporalComparisonResponse | null>(null)
const [temporalComparisonLoading, setTemporalComparisonLoading] = useState(false)
const [temporalComparisonError, setTemporalComparisonError] = useState<string | null>(null)
useEffect(() => {
setTemporalComparison(null)
setTemporalComparisonError(null)
}, [selectedProjectId])
const clearTemporalComparison = () => {
setTemporalComparison(null)
setTemporalComparisonError(null)
}
const compareTemporalSnapshots = async (
earlierDatasetId: string,
laterDatasetId: string,
bbox: VectorSelectionBBox,
): Promise<TemporalComparisonResponse | null> => {
if (!selectedProjectId) {
setTemporalComparisonError('Open eerst een project om evoluties te vergelijken.')
return null
}
if (!earlierDatasetId || !laterDatasetId) {
setTemporalComparisonError('Kies twee meetmomenten uit dezelfde tijdreeks.')
return null
}
setTemporalComparisonLoading(true)
setTemporalComparisonError(null)
try {
const result = await temporalApi.compare(selectedProjectId, {
earlier_dataset_id: earlierDatasetId,
later_dataset_id: laterDatasetId,
bbox,
preview_limit: 500,
})
setTemporalComparison(result)
return result
} catch (error) {
setTemporalComparison(null)
setTemporalComparisonError(formatError(error, 'De evolutieanalyse is mislukt.'))
return null
} finally {
setTemporalComparisonLoading(false)
}
}
return {
temporalComparison,
temporalComparisonLoading,
temporalComparisonError,
compareTemporalSnapshots,
clearTemporalComparison,
}
}
+24
View File
@@ -33,6 +33,12 @@ export const datasetsApi = {
sourceMetadataJson?: string
provenanceMetadataJson?: string
areaId?: string
temporalSeriesKey?: string
observedAt?: string
validFrom?: string
validTo?: string
temporalGranularity?: string
sourceVersion?: string
},
): Promise<DatasetCreateResponse> => {
const form = new FormData()
@@ -55,6 +61,24 @@ export const datasetsApi = {
if (payload.areaId) {
form.append('area_id', payload.areaId)
}
if (payload.temporalSeriesKey) {
form.append('temporal_series_key', payload.temporalSeriesKey)
}
if (payload.observedAt) {
form.append('observed_at', payload.observedAt)
}
if (payload.validFrom) {
form.append('valid_from', payload.validFrom)
}
if (payload.validTo) {
form.append('valid_to', payload.validTo)
}
if (payload.temporalGranularity) {
form.append('temporal_granularity', payload.temporalGranularity)
}
if (payload.sourceVersion) {
form.append('source_version', payload.sourceVersion)
}
return apiMultipart<DatasetCreateResponse>(`/api/v1/projects/${projectId}/datasets/upload`, form)
},
refreshMetadata: (projectId: string, datasetId: string): Promise<DatasetCreateResponse> =>
+1
View File
@@ -9,3 +9,4 @@ export { segmentationApi } from './segmentation'
export { exportsApi } from './exports'
export { jobsApi } from './jobs'
export { projectsApi } from './projects'
export { temporalApi } from './temporal'
+13
View File
@@ -0,0 +1,13 @@
import { apiGet, apiPost } from './client'
import type {
TemporalComparisonRequest,
TemporalComparisonResponse,
TemporalSeriesListResponse,
} from '../../types'
export const temporalApi = {
listSeries: (projectId: string): Promise<TemporalSeriesListResponse> =>
apiGet<TemporalSeriesListResponse>(`/api/v1/projects/${projectId}/temporal/series`),
compare: (projectId: string, payload: TemporalComparisonRequest): Promise<TemporalComparisonResponse> =>
apiPost<TemporalComparisonResponse>(`/api/v1/projects/${projectId}/temporal/compare`, payload),
}
+157 -1
View File
@@ -5448,6 +5448,33 @@ section {
flex: 0 0 auto;
}
.geo-analysis-mode {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
flex: 0 0 auto;
border: 1px solid #cdd8d4;
border-radius: 6px;
padding: 0.18rem;
background: #f3f6f5;
}
.geo-analysis-mode button {
min-height: 2.15rem;
border: 0;
border-radius: 4px;
padding: 0.38rem 0.68rem;
background: transparent;
color: #5a6964;
font-size: 0.7rem;
font-weight: 800;
}
.geo-analysis-mode button.active {
background: #ffffff;
color: #174f45;
box-shadow: 0 1px 3px rgba(23, 33, 30, 0.12);
}
.geo-explorer-layout {
display: grid;
grid-template-columns: minmax(15.5rem, 17rem) minmax(30rem, 1fr) minmax(18rem, 20rem);
@@ -5627,6 +5654,32 @@ section {
line-height: 1.35;
}
.geo-time-controls {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 0.45rem;
border-top: 1px solid #e3e9e6;
padding-top: 0.65rem;
}
.geo-time-controls label {
color: #4c5b56;
font-size: 0.66rem;
font-weight: 800;
}
.geo-time-controls select {
min-height: 2.25rem;
margin-top: 0.24rem;
padding: 0.35rem;
font-size: 0.68rem;
}
.geo-time-controls button {
grid-column: 1 / -1;
min-height: 2.25rem;
}
.geo-scope-select {
margin-top: auto;
color: #4c5b56;
@@ -5736,6 +5789,21 @@ section {
background: rgba(107, 74, 170, 0.18);
}
.geo-map-legend .geo-legend-added {
border-color: #15803d;
background: rgba(22, 163, 74, 0.2);
}
.geo-map-legend .geo-legend-removed {
border-color: #b91c1c;
background: rgba(220, 38, 38, 0.18);
}
.geo-map-legend .geo-legend-modified {
border-color: #b45309;
background: rgba(217, 119, 6, 0.2);
}
.geo-draw-instruction,
.geo-viewport-status {
position: absolute;
@@ -5841,6 +5909,78 @@ section {
white-space: nowrap;
}
.geo-temporal-metrics > div.positive {
border-color: #b8d8c5;
background: #f1faf4;
}
.geo-temporal-metrics > div.negative {
border-color: #e2c1bd;
background: #fff6f5;
}
.geo-temporal-metrics small {
color: #64736d;
font-size: 0.62rem;
}
.geo-temporal-summary {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
border: 1px solid #e1e8e5;
border-radius: 5px;
background: #fbfcfc;
}
.geo-temporal-summary > div {
display: grid;
gap: 0.15rem;
min-width: 0;
border-left: 1px solid #e1e8e5;
padding: 0.45rem;
}
.geo-temporal-summary > div:first-child {
border-left: 0;
}
.geo-temporal-summary span {
color: #6a7773;
font-size: 0.6rem;
font-weight: 800;
text-transform: uppercase;
}
.geo-temporal-summary strong {
overflow: hidden;
color: #26332f;
font-size: 0.7rem;
text-overflow: ellipsis;
white-space: nowrap;
}
.geo-change-counts {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 0.35rem;
}
.geo-change-counts span {
display: grid;
gap: 0.1rem;
border: 1px solid #e1e8e5;
border-radius: 5px;
padding: 0.4rem;
color: #687570;
font-size: 0.64rem;
text-align: center;
}
.geo-change-counts strong {
color: #26332f;
font-size: 0.85rem;
}
.geo-theme-results {
display: grid;
gap: 0;
@@ -6034,6 +6174,7 @@ section {
@media (max-width: 920px) {
.geo-explorer-header {
flex-wrap: wrap;
align-items: start;
}
@@ -6075,11 +6216,26 @@ section {
width: 100%;
}
.geo-analysis-mode {
width: 100%;
}
.geo-theme-list,
.geo-primary-metrics {
.geo-primary-metrics,
.geo-temporal-summary,
.geo-change-counts {
grid-template-columns: 1fr;
}
.geo-temporal-summary > div {
border-top: 1px solid #e1e8e5;
border-left: 0;
}
.geo-temporal-summary > div:first-child {
border-top: 0;
}
.geo-map-toolbar {
display: grid;
}
+109
View File
@@ -83,6 +83,12 @@ export interface DatasetCreateResponse {
source_metadata?: Record<string, unknown> | null
provenance_metadata?: Record<string, unknown> | null
imported_at?: string | null
temporal_series_key?: string | null
observed_at?: string | null
valid_from?: string | null
valid_to?: string | null
temporal_granularity?: string | null
source_version?: string | null
project_id: string
area_id?: string | null
storage_path?: string | null
@@ -313,6 +319,109 @@ export interface VectorSelectionResponse {
limit: number
truncated: boolean
geojson: GeoJSON.FeatureCollection
summary?: VectorSelectionSummary | null
}
export interface VectorSelectionSummary {
metric_label: string
metric_value: number
metric_unit: string
aggregation_method: string
feature_count: number
is_estimate: boolean
warning?: string | null
}
export interface DatasetTemporalUpdate {
temporal_series_key: string
observed_at: string
valid_from?: string | null
valid_to?: string | null
temporal_granularity?: string
source_version?: string | null
}
export interface DatasetVersionRead {
id: string
dataset_id: string
version: number
storage_path?: string | null
source_version?: string | null
observed_at?: string | null
valid_from?: string | null
valid_to?: string | null
checksum_sha256?: string | null
source_metadata?: Record<string, unknown> | null
provenance_metadata?: Record<string, unknown> | null
created_at?: string | null
}
export interface TemporalComparisonRequest {
earlier_dataset_id: string
later_dataset_id: string
bbox: VectorSelectionBBox
preview_limit?: number
}
export interface TemporalDatasetRef {
id: string
name: string
observed_at: string
source_version?: string | null
}
export interface TemporalMetricComparison {
label: string
unit: string
aggregation_method: string
earlier_value: number
later_value: number
absolute_change: number
percent_change?: number | null
is_estimate: boolean
}
export interface TemporalObjectChanges {
available: boolean
added_count?: number | null
removed_count?: number | null
modified_count?: number | null
unchanged_count?: number | null
}
export interface TemporalComparisonResponse {
temporal_series_key: string
earlier: TemporalDatasetRef
later: TemporalDatasetRef
selection_bbox: VectorSelectionBBox
metric: TemporalMetricComparison
object_changes: TemporalObjectChanges
geojson: GeoJSON.FeatureCollection
warnings: string[]
generated_at: string
}
export interface TemporalSeriesDataset {
id: string
name: string
observed_at: string
source_version?: string | null
feature_count?: number | null
}
export interface TemporalSeriesRead {
temporal_series_key: string
source_name?: string | null
reference_layer_name?: string | null
dataset_count: number
first_observed_at: string
last_observed_at: string
datasets: TemporalSeriesDataset[]
}
export interface TemporalSeriesListResponse {
items: TemporalSeriesRead[]
total: number
}
export interface DatasetListResponse {
+38
View File
@@ -345,6 +345,10 @@ def upload_layer(
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
"area_id": area_id,
"temporal_series_key": f"grb:{definition.key}:mol",
"observed_at": summary["generated_at"],
"temporal_granularity": "snapshot",
"source_version": summary["generated_at"][:10],
},
files={"file": (path.name, handle, "application/geo+json")},
timeout=timeout,
@@ -352,6 +356,31 @@ def upload_layer(
return response_data(response)
def ensure_temporal_metadata(
session: requests.Session,
base_url: str,
project_id: str,
dataset: dict[str, Any],
definition: LayerDefinition,
generated_at: str,
timeout: int,
) -> dict[str, Any]:
series_key = f"grb:{definition.key}:mol"
if dataset.get("temporal_series_key") == series_key and dataset.get("observed_at"):
return dataset
response = session.patch(
f"{base_url}/api/v1/projects/{project_id}/datasets/{dataset['id']}/temporal",
json={
"temporal_series_key": series_key,
"observed_at": generated_at,
"temporal_granularity": "snapshot",
"source_version": generated_at[:10],
},
timeout=timeout,
)
return response_data(response)
def main() -> int:
args = parse_args()
requested = {item.strip().lower() for item in args.layers.split(",") if item.strip()}
@@ -402,6 +431,15 @@ def main() -> int:
for definition, path, summary in prepared:
dataset = next((item for item in existing if item.get("original_filename") == path.name), None)
if dataset:
dataset = ensure_temporal_metadata(
api_session,
base_url,
project_id,
dataset,
definition,
summary["generated_at"],
args.import_timeout,
)
results.append(
{"layer": definition.key, "dataset_id": dataset["id"], "feature_count": dataset.get("feature_count"), "status": "existing"}
)
+552
View File
@@ -0,0 +1,552 @@
"""Provision official historical land-use theme snapshots for Mol.
The command reads the Digitaal Vlaanderen historical land-use WFS for 1778,
1873 and 1969, clips features to the official Mol boundary, separates the
supported map themes and imports every snapshot through the normal dataset API.
It is an explicit, idempotent operator command and never runs at startup.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable
import requests
from requests.adapters import HTTPAdapter
from shapely.geometry import mapping, shape
from shapely.validation import make_valid
from urllib3.util.retry import Retry
MUNICIPALITY_NAME = "Mol"
MUNICIPALITY_NIS_CODE = "13025"
PROJECT_NAME = "Mol Municipality Workbench"
DEFAULT_API_URL = "http://127.0.0.1:8000"
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/mol-historical-landuse")
DEFAULT_BOUNDARY_PATH = Path("/app/storage/operator-data/mol-municipality/mol_municipality_boundary.geojson")
WFS_URL = "https://geo.api.vlaanderen.be/HistLandgebruik/wfs"
ATTRIBUTION = "Bron: Historisch landgebruik Vlaanderen, Digitaal Vlaanderen"
COLLECTIONS = {1778: "HistLandgebruik:Lgbrk1778", 1873: "HistLandgebruik:Lgbrk1873", 1969: "HistLandgebruik:Lgbrk1969"}
@dataclass(frozen=True)
class ThemeDefinition:
key: str
label: str
matches: Callable[[str], bool]
filter_value: str
filter_mode: str
THEMES = (
ThemeDefinition("buildings", "Historische bebouwing", lambda value: value.startswith("bebouwing"), "bebouwing*", "like"),
ThemeDefinition("forest", "Historisch bos", lambda value: value.startswith("bos-") or value == "bos", "bos*", "like"),
ThemeDefinition("water", "Historisch water", lambda value: value == "water", "water", "equal"),
ThemeDefinition("roads", "Historische wegen", lambda value: value.startswith("weg-"), "weg-*", "like"),
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Provision official historical land-use snapshots for Mol.")
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
parser.add_argument("--project-name", default=PROJECT_NAME)
parser.add_argument("--years", default="1778,1873,1969")
parser.add_argument("--themes", default="buildings,forest,water,roads")
parser.add_argument("--output-dir", type=Path, default=Path(os.environ.get("MOL_HISTORICAL_LANDUSE_OUTPUT_DIR", DEFAULT_OUTPUT_DIR)))
parser.add_argument("--boundary-path", type=Path, default=Path(os.environ.get("MOL_BOUNDARY_PATH", DEFAULT_BOUNDARY_PATH)))
parser.add_argument("--page-size", type=int, default=200)
parser.add_argument("--max-features", type=int, default=100000)
parser.add_argument("--simplify-tolerance-degrees", type=float, default=0.00001)
parser.add_argument("--request-timeout", type=int, default=180)
parser.add_argument("--import-timeout", type=int, default=1800)
parser.add_argument("--force", action="store_true")
parser.add_argument("--fetch-only", action="store_true")
return parser.parse_args()
def build_session() -> requests.Session:
retry = Retry(
total=5,
connect=5,
read=5,
status=5,
backoff_factor=1.0,
status_forcelist=(429, 500, 502, 503, 504),
allowed_methods=frozenset({"GET"}),
raise_on_status=True,
)
session = requests.Session()
session.headers.update({"User-Agent": "GeoIntel-Mol-Historical-Landuse-Operator/1.0"})
adapter = HTTPAdapter(max_retries=retry)
session.mount("https://", adapter)
session.mount("http://", adapter)
return session
def load_boundary(path: Path):
if not path.exists():
raise RuntimeError(f"Mol boundary is missing at {path}; run provision_mol_municipality_workspace.py first")
payload = json.loads(path.read_text(encoding="utf-8"))
features = payload.get("features") or []
if len(features) != 1:
raise RuntimeError("Mol boundary artifact must contain exactly one feature")
boundary = shape(features[0]["geometry"])
if not boundary.is_valid:
boundary = make_valid(boundary)
if boundary.is_empty or not boundary.is_valid:
raise RuntimeError("Mol boundary artifact is invalid")
return boundary
def wfs_filter_xml(definition: ThemeDefinition, bounds: tuple[float, float, float, float]) -> str:
min_x, min_y, max_x, max_y = bounds
comparison = (
f"<fes:PropertyIsLike wildCard='*' singleChar='?' escapeChar='!'>"
f"<fes:ValueReference>KLASSE</fes:ValueReference><fes:Literal>{definition.filter_value}</fes:Literal>"
f"</fes:PropertyIsLike>"
if definition.filter_mode == "like"
else (
"<fes:PropertyIsEqualTo><fes:ValueReference>KLASSE</fes:ValueReference>"
f"<fes:Literal>{definition.filter_value}</fes:Literal></fes:PropertyIsEqualTo>"
)
)
return (
"<fes:Filter xmlns:fes='http://www.opengis.net/fes/2.0' xmlns:gml='http://www.opengis.net/gml/3.2'>"
"<fes:And><fes:BBOX><fes:ValueReference>SHAPE</fes:ValueReference>"
"<gml:Envelope srsName='EPSG:4326'>"
f"<gml:lowerCorner>{min_x:.8f} {min_y:.8f}</gml:lowerCorner>"
f"<gml:upperCorner>{max_x:.8f} {max_y:.8f}</gml:upperCorner>"
"</gml:Envelope></fes:BBOX>"
f"{comparison}</fes:And></fes:Filter>"
)
def fetch_year(
session: requests.Session,
year: int,
boundary,
definitions: list[ThemeDefinition],
*,
page_size: int,
max_features: int,
simplify_tolerance_degrees: float,
timeout: int,
):
collection = COLLECTIONS[year]
features_by_theme: dict[str, list[dict[str, Any]]] = {definition.key: [] for definition in definitions}
for definition in definitions:
filter_xml = wfs_filter_xml(definition, boundary.bounds)
start_index = 0
seen: set[str] = set()
while True:
response = session.get(
WFS_URL,
params={
"service": "WFS",
"version": "2.0.0",
"request": "GetFeature",
"typeNames": collection,
"outputFormat": "application/json",
"srsName": "EPSG:4326",
"FILTER": filter_xml,
"count": page_size,
"startIndex": start_index,
},
timeout=timeout,
)
response.raise_for_status()
page = response.json().get("features") or []
for raw_feature in page:
feature_id = str(raw_feature.get("id") or "")
if not feature_id or feature_id in seen:
continue
seen.add(feature_id)
properties = raw_feature.get("properties") or {}
landuse_class = str(properties.get("KLASSE") or "").strip().lower()
if not definition.matches(landuse_class):
continue
geometry = shape(raw_feature["geometry"])
if not geometry.is_valid:
geometry = make_valid(geometry)
geometry = geometry.intersection(boundary)
if geometry.is_empty:
continue
if not geometry.is_valid:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
continue
if simplify_tolerance_degrees > 0:
geometry = geometry.simplify(simplify_tolerance_degrees, preserve_topology=True)
theme_features = features_by_theme[definition.key]
if len(theme_features) >= max_features:
raise RuntimeError(
f"Historical land use {definition.key} {year} exceeds the {max_features} feature safety limit"
)
theme_features.append({**raw_feature, "id": feature_id, "geometry": mapping(geometry)})
if len(page) < page_size:
break
start_index += len(page)
for definition in definitions:
if not features_by_theme[definition.key]:
raise RuntimeError(f"Historical land-use WFS returned no {definition.key} features for Mol in {year}")
return features_by_theme
def write_theme_snapshot(
session: requests.Session,
*,
year: int,
definition: ThemeDefinition,
boundary,
path: Path,
page_size: int,
max_features: int,
simplify_tolerance_degrees: float,
timeout: int,
) -> int:
collection = COLLECTIONS[year]
filter_xml = wfs_filter_xml(definition, boundary.bounds)
start_index = 0
seen: set[str] = set()
feature_count = 0
first_feature = True
try:
with path.open("w", encoding="utf-8") as output:
output.write(
json.dumps(
{
"type": "FeatureCollection",
"name": f"{definition.label} - Mol {year}",
"municipality": MUNICIPALITY_NAME,
"nis_code": MUNICIPALITY_NIS_CODE,
"observation_year": year,
"attribution": ATTRIBUTION,
},
ensure_ascii=False,
separators=(",", ":"),
)[:-1]
)
output.write(',"features":[')
while True:
response = session.get(
WFS_URL,
params={
"service": "WFS",
"version": "2.0.0",
"request": "GetFeature",
"typeNames": collection,
"outputFormat": "application/json",
"srsName": "EPSG:4326",
"FILTER": filter_xml,
"count": page_size,
"startIndex": start_index,
},
timeout=timeout,
)
response.raise_for_status()
page = response.json().get("features") or []
for raw_feature in page:
feature_id = str(raw_feature.get("id") or "")
if not feature_id or feature_id in seen:
continue
seen.add(feature_id)
properties = dict(raw_feature.get("properties") or {})
landuse_class = str(properties.get("KLASSE") or "").strip().lower()
if not definition.matches(landuse_class):
continue
geometry = shape(raw_feature["geometry"])
if not geometry.is_valid:
geometry = make_valid(geometry)
geometry = geometry.intersection(boundary)
if geometry.is_empty:
continue
if not geometry.is_valid:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
continue
if simplify_tolerance_degrees > 0:
geometry = geometry.simplify(simplify_tolerance_degrees, preserve_topology=True)
properties.update(
{
"source_name": "historical_landuse",
"source_feature_id": feature_id,
"reference_layer_name": definition.key,
"authority_level": "authoritative",
"municipality": MUNICIPALITY_NAME,
"nis_code": MUNICIPALITY_NIS_CODE,
"observation_year": year,
"historical_landuse_class": landuse_class,
"attribution": ATTRIBUTION,
}
)
prepared = {
"type": "Feature",
"id": feature_id,
"geometry": mapping(geometry),
"properties": properties,
}
if not first_feature:
output.write(",")
output.write(json.dumps(prepared, ensure_ascii=False, separators=(",", ":")))
first_feature = False
feature_count += 1
if feature_count > max_features:
raise RuntimeError(
f"Historical land use {definition.key} {year} exceeds the {max_features} feature safety limit"
)
if len(page) < page_size:
break
start_index += len(page)
output.write("]}")
except Exception:
path.unlink(missing_ok=True)
path.with_suffix(".manifest.json").unlink(missing_ok=True)
raise
if feature_count == 0:
path.unlink(missing_ok=True)
raise RuntimeError(f"Historical land-use WFS returned no {definition.key} features for Mol in {year}")
path.with_suffix(".manifest.json").write_text(
json.dumps(
{
"year": year,
"theme": definition.key,
"feature_count": feature_count,
"collection": collection,
"simplify_tolerance_degrees": simplify_tolerance_degrees,
"generated_at": datetime.now(timezone.utc).isoformat(),
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
return feature_count
def build_theme_snapshot(year: int, definition: ThemeDefinition, source_features: list[dict[str, Any]]) -> dict[str, Any]:
features: list[dict[str, Any]] = []
for source_feature in source_features:
properties = dict(source_feature.get("properties") or {})
landuse_class = str(properties.get("KLASSE") or "").strip().lower()
if not definition.matches(landuse_class):
continue
feature_id = str(source_feature["id"])
properties.update(
{
"source_name": "historical_landuse",
"source_feature_id": feature_id,
"reference_layer_name": definition.key,
"authority_level": "authoritative",
"municipality": MUNICIPALITY_NAME,
"nis_code": MUNICIPALITY_NIS_CODE,
"observation_year": year,
"historical_landuse_class": landuse_class,
"attribution": ATTRIBUTION,
}
)
features.append(
{"type": "Feature", "id": feature_id, "geometry": source_feature["geometry"], "properties": properties}
)
if not features:
raise RuntimeError(f"No {definition.key} features were classified for Mol in {year}")
return {
"type": "FeatureCollection",
"name": f"{definition.label} - Mol {year}",
"features": features,
"municipality": MUNICIPALITY_NAME,
"nis_code": MUNICIPALITY_NIS_CODE,
"observation_year": year,
"attribution": ATTRIBUTION,
}
def response_data(response: requests.Response) -> Any:
try:
payload = response.json()
except ValueError as exc:
raise RuntimeError(f"GeoIntel API returned non-JSON ({response.status_code}): {response.text[:300]}") from exc
if not response.ok:
raise RuntimeError(f"GeoIntel API failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:800]}")
if not isinstance(payload, dict) or "data" not in payload:
raise RuntimeError("GeoIntel API response does not use the canonical data envelope")
return payload["data"]
def locate_workspace(session: requests.Session, base_url: str, project_name: str, timeout: int):
projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=timeout))
project = next((item for item in projects.get("items") or [] if item.get("name") == project_name), None)
if not project:
raise RuntimeError(f"Project {project_name!r} is missing")
project_id = str(project["id"])
areas = response_data(session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=timeout))
area = next((item for item in areas.get("items") or [] if "gemeente mol" in str(item.get("name", "")).lower()), None)
if not area:
raise RuntimeError("Official Mol area is missing")
datasets = response_data(session.get(f"{base_url}/api/v1/projects/{project_id}/datasets", params={"limit": 200}, timeout=timeout))
return project_id, str(area["id"]), list(datasets.get("items") or [])
def upload_snapshot(
session: requests.Session,
base_url: str,
project_id: str,
area_id: str,
year: int,
definition: ThemeDefinition,
path: Path,
simplify_tolerance_degrees: float,
timeout: int,
) -> dict[str, Any]:
observed_at = f"{year}-01-01T00:00:00Z"
series_key = f"digitaal-vlaanderen:historical-landuse:{definition.key}:mol"
source_metadata = {
"provider": "Digitaal Vlaanderen",
"collection": COLLECTIONS[year],
"authority_level": "authoritative",
"coverage_scope": "municipality",
"municipality": MUNICIPALITY_NAME,
"nis_code": MUNICIPALITY_NIS_CODE,
"attribution": ATTRIBUTION,
"identity_stable": False,
"geometry_simplification_tolerance_degrees": simplify_tolerance_degrees,
"selection_aggregation": {
"method": "intersection_area",
"label": "Oppervlakte",
"unit": "ha",
"is_estimate": False,
"warning": "Historische kaartklassen en karteermethodes verschillen per bronjaar; interpreteer trends binnen die methodologische context.",
},
}
provenance_metadata = {
"operator_tool": "provision_mol_historical_landuse.py",
"operator_explicit_fetch": True,
"wfs_url": WFS_URL,
"collection": COLLECTIONS[year],
"geometry_simplification_tolerance_degrees": simplify_tolerance_degrees,
"generated_at": datetime.now(timezone.utc).isoformat(),
}
with path.open("rb") as handle:
response = session.post(
f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
data={
"dataset_type": "vector",
"source": "operator_official_import",
"dataset_role": "reference",
"source_name": "historical_landuse",
"reference_layer_name": definition.key,
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
"area_id": area_id,
"temporal_series_key": series_key,
"observed_at": observed_at,
"valid_from": observed_at,
"temporal_granularity": "year",
"source_version": str(year),
},
files={"file": (path.name, handle, "application/geo+json")},
timeout=timeout,
)
return response_data(response)
def main() -> int:
args = parse_args()
try:
years = sorted({int(value.strip()) for value in args.years.split(",") if value.strip()})
except ValueError:
print(json.dumps({"status": "error", "message": "Years must be comma-separated integers"}), file=sys.stderr)
return 2
requested_themes = {value.strip().lower() for value in args.themes.split(",") if value.strip()}
definitions = [definition for definition in THEMES if definition.key in requested_themes]
unsupported_years = [year for year in years if year not in COLLECTIONS]
unsupported_themes = requested_themes - {definition.key for definition in THEMES}
if unsupported_years or unsupported_themes or not years or not definitions:
print(
json.dumps(
{"status": "error", "message": f"Unsupported years={unsupported_years}, themes={sorted(unsupported_themes)}"}
),
file=sys.stderr,
)
return 2
args.output_dir.mkdir(parents=True, exist_ok=True)
results: list[dict[str, Any]] = []
try:
boundary = load_boundary(args.boundary_path)
prepared: list[tuple[int, ThemeDefinition, Path, int]] = []
with build_session() as source_session:
for year in years:
target_paths = {
definition.key: args.output_dir / f"mol_historical_{definition.key}_{year}.geojson"
for definition in definitions
}
for definition in definitions:
path = target_paths[definition.key]
manifest_path = path.with_suffix(".manifest.json")
if args.force or not path.exists() or not manifest_path.exists():
count = write_theme_snapshot(
source_session,
year=year,
definition=definition,
boundary=boundary,
path=path,
page_size=args.page_size,
max_features=args.max_features,
simplify_tolerance_degrees=args.simplify_tolerance_degrees,
timeout=args.request_timeout,
)
else:
count = int(json.loads(manifest_path.read_text(encoding="utf-8"))["feature_count"])
prepared.append((year, definition, path, count))
if args.fetch_only:
results = [
{"year": year, "theme": definition.key, "path": str(path), "feature_count": count, "status": "prepared"}
for year, definition, path, count in prepared
]
else:
base_url = args.base_url.rstrip("/")
with requests.Session() as api_session:
project_id, area_id, existing = locate_workspace(api_session, base_url, args.project_name, args.import_timeout)
for year, definition, path, count in prepared:
series_key = f"digitaal-vlaanderen:historical-landuse:{definition.key}:mol"
dataset = next(
(
item
for item in existing
if item.get("temporal_series_key") == series_key
and str(item.get("observed_at") or "").startswith(str(year))
),
None,
)
if dataset:
results.append({"year": year, "theme": definition.key, "dataset_id": dataset["id"], "feature_count": dataset.get("feature_count"), "status": "existing"})
continue
dataset = upload_snapshot(
api_session,
base_url,
project_id,
area_id,
year,
definition,
path,
args.simplify_tolerance_degrees,
args.import_timeout,
)
results.append({"year": year, "theme": definition.key, "dataset_id": dataset["id"], "feature_count": dataset.get("feature_count"), "status": "imported"})
except (OSError, RuntimeError, requests.RequestException, ValueError, KeyError) as exc:
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
return 1
print(json.dumps({"status": "ok", "municipality": MUNICIPALITY_NAME, "snapshots": results}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -476,6 +476,8 @@ def upload_dataset(
reference_layer_name: str | None,
source_metadata: dict[str, Any],
provenance_metadata: dict[str, Any],
temporal_series_key: str,
observed_at: str,
timeout: int,
) -> dict[str, Any]:
form = {
@@ -486,6 +488,10 @@ def upload_dataset(
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
"area_id": area_id,
"temporal_series_key": temporal_series_key,
"observed_at": observed_at,
"temporal_granularity": "snapshot",
"source_version": observed_at[:10],
}
if reference_layer_name:
form["reference_layer_name"] = reference_layer_name
@@ -499,6 +505,32 @@ def upload_dataset(
return response_data(response)
def ensure_temporal_metadata(
session: requests.Session,
base_url: str,
project_id: str,
dataset: dict[str, Any],
*,
temporal_series_key: str,
observed_at: str,
timeout: int,
) -> dict[str, Any]:
if dataset.get("temporal_series_key") == temporal_series_key and dataset.get("observed_at"):
return dataset
return response_data(
session.patch(
f"{base_url}/api/v1/projects/{project_id}/datasets/{dataset['id']}/temporal",
json={
"temporal_series_key": temporal_series_key,
"observed_at": observed_at,
"temporal_granularity": "snapshot",
"source_version": observed_at[:10],
},
timeout=timeout,
)
)
def provision_workspace(
args: argparse.Namespace,
boundary_path: Path,
@@ -558,6 +590,18 @@ def provision_workspace(
"source_url": GRB_GBG_ITEMS_URL,
"artifact_sha256": manifest["buildings_sha256"],
},
temporal_series_key="grb:buildings:mol",
observed_at=manifest["generated_at"],
timeout=args.import_timeout,
)
else:
building_dataset = ensure_temporal_metadata(
session,
base_url,
project_id,
building_dataset,
temporal_series_key="grb:buildings:mol",
observed_at=manifest["generated_at"],
timeout=args.import_timeout,
)
@@ -587,6 +631,18 @@ def provision_workspace(
"source_url": manifest["boundary_source_url"],
"artifact_sha256": manifest["boundary_sha256"],
},
temporal_series_key="vrbg:municipality-boundary:mol",
observed_at=manifest["generated_at"],
timeout=args.import_timeout,
)
else:
boundary_dataset = ensure_temporal_metadata(
session,
base_url,
project_id,
boundary_dataset,
temporal_series_key="vrbg:municipality-boundary:mol",
observed_at=manifest["generated_at"],
timeout=args.import_timeout,
)
+337
View File
@@ -0,0 +1,337 @@
"""Provision official annual Statbel population snapshots for Mol.
The command joins annual population totals to the matching official
statistical-sector geometries, clips the result to Mol and imports each year
through the existing GeoIntel upload API. It never runs during application
startup and it never synthesizes missing population values.
"""
from __future__ import annotations
import argparse
import csv
import io
import json
import os
import sys
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import requests
from pyproj import Transformer
from requests.adapters import HTTPAdapter
from shapely.geometry import mapping, shape
from shapely.ops import transform
from shapely.validation import make_valid
from urllib3.util.retry import Retry
MUNICIPALITY_NAME = "Mol"
MUNICIPALITY_NIS_CODE = "13025"
PROJECT_NAME = "Mol Municipality Workbench"
SERIES_KEY = "statbel:population-statistical-sector:mol"
ATTRIBUTION = "Bron: Statbel, bevolking per statistische sector, CC BY 4.0"
DEFAULT_API_URL = "http://127.0.0.1:8000"
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/mol-population-history")
DEFAULT_BOUNDARY_PATH = Path("/app/storage/operator-data/mol-municipality/mol_municipality_boundary.geojson")
SECTOR_URL = (
"https://statbel.fgov.be/sites/default/files/files/opendata/Statistische%20sectoren/"
"sh_statbel_statistical_sectors_31370_{year}0101.geojson.zip"
)
POPULATION_URLS = {
2021: "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2021.zip",
2022: "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2022.zip",
2023: "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2023.zip",
2024: "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2024.zip",
2025: "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2025_NEW.zip",
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Provision official annual Statbel population snapshots for Mol.")
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
parser.add_argument("--project-name", default=PROJECT_NAME)
parser.add_argument("--years", default="2021,2022,2023,2024,2025")
parser.add_argument("--output-dir", type=Path, default=Path(os.environ.get("MOL_POPULATION_OUTPUT_DIR", DEFAULT_OUTPUT_DIR)))
parser.add_argument("--boundary-path", type=Path, default=Path(os.environ.get("MOL_BOUNDARY_PATH", DEFAULT_BOUNDARY_PATH)))
parser.add_argument("--request-timeout", type=int, default=180)
parser.add_argument("--import-timeout", type=int, default=900)
parser.add_argument("--force", action="store_true")
parser.add_argument("--fetch-only", action="store_true")
return parser.parse_args()
def build_session() -> requests.Session:
retry = Retry(
total=5,
connect=5,
read=5,
status=5,
backoff_factor=1.0,
status_forcelist=(429, 500, 502, 503, 504),
allowed_methods=frozenset({"GET"}),
raise_on_status=True,
)
session = requests.Session()
session.headers.update({"User-Agent": "GeoIntel-Mol-Population-Operator/1.0"})
adapter = HTTPAdapter(max_retries=retry)
session.mount("https://", adapter)
session.mount("http://", adapter)
return session
def response_data(response: requests.Response) -> Any:
try:
payload = response.json()
except ValueError as exc:
raise RuntimeError(f"GeoIntel API returned non-JSON ({response.status_code}): {response.text[:300]}") from exc
if not response.ok:
raise RuntimeError(f"GeoIntel API failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:800]}")
if not isinstance(payload, dict) or "data" not in payload:
raise RuntimeError("GeoIntel API response does not use the canonical data envelope")
return payload["data"]
def load_boundary(path: Path):
if not path.exists():
raise RuntimeError(f"Mol boundary is missing at {path}; run provision_mol_municipality_workspace.py first")
payload = json.loads(path.read_text(encoding="utf-8"))
features = payload.get("features") or []
if len(features) != 1:
raise RuntimeError("Mol boundary artifact must contain exactly one feature")
boundary = shape(features[0]["geometry"])
if not boundary.is_valid:
boundary = make_valid(boundary)
if boundary.is_empty or not boundary.is_valid:
raise RuntimeError("Mol boundary artifact is invalid")
return boundary
def zip_member_json(content: bytes) -> dict[str, Any]:
with zipfile.ZipFile(io.BytesIO(content)) as archive:
member = next((name for name in archive.namelist() if name.lower().endswith(".geojson")), None)
if not member:
raise RuntimeError("Statbel sector archive contains no GeoJSON file")
return json.loads(archive.read(member).decode("utf-8"))
def population_rows(content: bytes) -> dict[str, dict[str, Any]]:
with zipfile.ZipFile(io.BytesIO(content)) as archive:
member = next((name for name in archive.namelist() if name.lower().endswith((".txt", ".csv"))), None)
if not member:
raise RuntimeError("Statbel population archive contains no text table")
raw = archive.read(member)
try:
text = raw.decode("utf-8-sig")
except UnicodeDecodeError:
text = raw.decode("cp1252")
rows: dict[str, dict[str, Any]] = {}
for row in csv.DictReader(io.StringIO(text), delimiter="|"):
if str(row.get("CD_REFNIS") or "").strip() != MUNICIPALITY_NIS_CODE:
continue
sector_code = str(row.get("CD_SECTOR") or "").strip()
total_raw = str(row.get("TOTAL") or "").strip()
if not sector_code or not total_raw or not total_raw.isdigit():
continue
rows[sector_code] = {
"population_total": int(total_raw),
"sector_name_nl": row.get("TX_DESCR_SECTOR_NL"),
"municipality_name_nl": row.get("TX_DESCR_NL"),
}
if not rows:
raise RuntimeError("Statbel population table contains no usable Mol sectors")
return rows
def build_snapshot(year: int, sector_payload: dict[str, Any], population: dict[str, dict[str, Any]], boundary) -> dict[str, Any]:
transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
features: list[dict[str, Any]] = []
missing_population = 0
for source_feature in sector_payload.get("features") or []:
properties = source_feature.get("properties") or {}
if str(properties.get("cd_munty_refnis") or "") != MUNICIPALITY_NIS_CODE:
continue
sector_code = str(properties.get("cd_sector") or "").strip()
population_values = population.get(sector_code)
if not population_values:
missing_population += 1
continue
geometry = transform(transformer.transform, shape(source_feature["geometry"]))
if not geometry.is_valid:
geometry = make_valid(geometry)
geometry = geometry.intersection(boundary)
if geometry.is_empty:
continue
if not geometry.is_valid:
geometry = make_valid(geometry)
combined = {
**properties,
**population_values,
"source_name": "statbel",
"source_feature_id": sector_code,
"reference_layer_name": "population",
"authority_level": "authoritative",
"municipality": MUNICIPALITY_NAME,
"nis_code": MUNICIPALITY_NIS_CODE,
"observation_year": year,
"attribution": ATTRIBUTION,
}
features.append({"type": "Feature", "id": sector_code, "geometry": mapping(geometry), "properties": combined})
if not features:
raise RuntimeError(f"No joined population sectors were produced for {year}")
return {
"type": "FeatureCollection",
"name": f"Statbel population by statistical sector - Mol {year}",
"features": features,
"municipality": MUNICIPALITY_NAME,
"nis_code": MUNICIPALITY_NIS_CODE,
"observation_year": year,
"missing_population_sector_count": missing_population,
"attribution": ATTRIBUTION,
}
def locate_workspace(session: requests.Session, base_url: str, project_name: str, timeout: int):
projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=timeout))
project = next((item for item in projects.get("items") or [] if item.get("name") == project_name), None)
if not project:
raise RuntimeError(f"Project {project_name!r} is missing")
project_id = str(project["id"])
areas = response_data(session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=timeout))
area = next((item for item in areas.get("items") or [] if "gemeente mol" in str(item.get("name", "")).lower()), None)
if not area:
raise RuntimeError("Official Mol area is missing")
datasets = response_data(session.get(f"{base_url}/api/v1/projects/{project_id}/datasets", params={"limit": 200}, timeout=timeout))
return project_id, str(area["id"]), list(datasets.get("items") or [])
def upload_snapshot(
session: requests.Session,
base_url: str,
project_id: str,
area_id: str,
year: int,
path: Path,
timeout: int,
) -> dict[str, Any]:
observed_at = f"{year}-01-01T00:00:00Z"
source_metadata = {
"provider": "Statbel",
"authority_level": "authoritative",
"coverage_scope": "municipality",
"municipality": MUNICIPALITY_NAME,
"nis_code": MUNICIPALITY_NIS_CODE,
"attribution": ATTRIBUTION,
"license": "CC BY 4.0",
"identity_stable": True,
"comparison_property": "population_total",
"selection_aggregation": {
"method": "area_weighted_sum",
"property": "population_total",
"label": "Inwoners",
"unit": "inwoners",
"is_estimate": True,
"warning": "Bevolking binnen een gedeeltelijke statistische sector is oppervlaktegewogen en blijft een schatting.",
},
}
provenance_metadata = {
"operator_tool": "provision_mol_population_history.py",
"operator_explicit_fetch": True,
"sector_geometry_url": SECTOR_URL.format(year=year),
"population_url": POPULATION_URLS[year],
"generated_at": datetime.now(timezone.utc).isoformat(),
}
with path.open("rb") as handle:
response = session.post(
f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
data={
"dataset_type": "vector",
"source": "operator_official_import",
"dataset_role": "reference",
"source_name": "statbel",
"reference_layer_name": "population",
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
"area_id": area_id,
"temporal_series_key": SERIES_KEY,
"observed_at": observed_at,
"valid_from": observed_at,
"valid_to": f"{year}-12-31T23:59:59Z",
"temporal_granularity": "year",
"source_version": str(year),
},
files={"file": (path.name, handle, "application/geo+json")},
timeout=timeout,
)
return response_data(response)
def main() -> int:
args = parse_args()
try:
years = sorted({int(value.strip()) for value in args.years.split(",") if value.strip()})
except ValueError:
print(json.dumps({"status": "error", "message": "Years must be comma-separated integers"}), file=sys.stderr)
return 2
unsupported = [year for year in years if year not in POPULATION_URLS]
if unsupported or not years:
print(json.dumps({"status": "error", "message": f"Unsupported years: {unsupported}"}), file=sys.stderr)
return 2
args.output_dir.mkdir(parents=True, exist_ok=True)
results: list[dict[str, Any]] = []
try:
boundary = load_boundary(args.boundary_path)
prepared: list[tuple[int, Path, int]] = []
with build_session() as source_session:
for year in years:
path = args.output_dir / f"mol_statbel_population_{year}.geojson"
if args.force or not path.exists():
sectors_response = source_session.get(SECTOR_URL.format(year=year), timeout=args.request_timeout)
sectors_response.raise_for_status()
population_response = source_session.get(POPULATION_URLS[year], timeout=args.request_timeout)
population_response.raise_for_status()
snapshot = build_snapshot(
year,
zip_member_json(sectors_response.content),
population_rows(population_response.content),
boundary,
)
path.write_text(json.dumps(snapshot, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
payload = json.loads(path.read_text(encoding="utf-8"))
prepared.append((year, path, len(payload.get("features") or [])))
if args.fetch_only:
results = [{"year": year, "path": str(path), "feature_count": count, "status": "prepared"} for year, path, count in prepared]
else:
base_url = args.base_url.rstrip("/")
with requests.Session() as api_session:
project_id, area_id, existing = locate_workspace(api_session, base_url, args.project_name, args.import_timeout)
for year, path, count in prepared:
observed_at = f"{year}-01-01T00:00:00+00:00"
dataset = next(
(
item
for item in existing
if item.get("temporal_series_key") == SERIES_KEY
and str(item.get("observed_at") or "").startswith(observed_at[:10])
),
None,
)
if dataset:
results.append({"year": year, "dataset_id": dataset["id"], "feature_count": dataset.get("feature_count"), "status": "existing"})
continue
dataset = upload_snapshot(api_session, base_url, project_id, area_id, year, path, args.import_timeout)
results.append({"year": year, "dataset_id": dataset["id"], "feature_count": dataset.get("feature_count"), "status": "imported"})
except (OSError, RuntimeError, requests.RequestException, ValueError, KeyError, zipfile.BadZipFile) as exc:
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
return 1
print(json.dumps({"status": "ok", "municipality": MUNICIPALITY_NAME, "series": SERIES_KEY, "snapshots": results}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
sys.exit(main())