feat: add temporal Mol explorer
This commit is contained in:
@@ -7,6 +7,15 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 187 Temporal Mol explorer (2026-07-14)
|
||||
|
||||
- Added first-class temporal dataset metadata and immutable dataset-version provenance for uploaded and derived vector/raster datasets.
|
||||
- Added project temporal-series discovery and bounded snapshot comparison APIs with explicit observation dates, metric deltas, warnings and optional stable-identity object changes.
|
||||
- Extended bbox selection with source-governed PostGIS aggregations so population is reported as inhabitants and land cover as intersected hectares instead of misleading feature counts.
|
||||
- Added a calm Latest state/Evolution flow to the map-first explorer, including period selection, metric comparison and added/removed/modified overlays where source identities support them.
|
||||
- Added explicit operator provisioners for official Statbel Mol population snapshots (2021-2025) and Digitaal Vlaanderen historical land-use snapshots (1778, 1873 and 1969); no source is fetched during application startup.
|
||||
- Preserved methodological honesty: partial statistical sectors are labelled area-weighted estimates, historical land-use identity changes are not fabricated and all source URLs, versions and processing limitations are persisted.
|
||||
|
||||
## Sprint 186 Map-first Mol geographic explorer (2026-07-14)
|
||||
|
||||
- Replaced the default dashboard entry with a calm map-first workflow: choose a real data theme, drag a rectangle, query PostGIS automatically and review results.
|
||||
|
||||
@@ -935,6 +935,36 @@ the same bbox-selected FeatureCollection as a normal export record with
|
||||
`export_type="vector_selection_geojson"`. This creates a handoff artifact only;
|
||||
it does not create a derived dataset.
|
||||
|
||||
## Temporal Mol data and evolution
|
||||
|
||||
Dataset uploads accept `temporal_series_key`, `observed_at`, `valid_from`,
|
||||
`valid_to`, `temporal_granularity` and `source_version`. Every new source or
|
||||
derived dataset also writes dataset version 1 in the same transaction.
|
||||
|
||||
After the Mol municipality workspace is available, import the official source
|
||||
snapshots explicitly:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_mol_population_history.py
|
||||
docker exec geointel python /app/scripts/provision_mol_historical_landuse.py
|
||||
```
|
||||
|
||||
The first command imports Statbel sector population for 2021-2025. The second
|
||||
imports Digitaal Vlaanderen historical land use for 1778, 1873 and 1969. Both
|
||||
are idempotent, use the normal API/DatasetService flow and retain fetched
|
||||
artifacts in persistent operator storage. They never run on app startup.
|
||||
|
||||
Historical land-use work can be bounded explicitly:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_mol_historical_landuse.py --years 1778 1969 --themes forest water
|
||||
```
|
||||
|
||||
`GET /api/v1/projects/{project_id}/temporal/series` discovers the series and
|
||||
`POST /api/v1/projects/{project_id}/temporal/compare` compares two snapshots
|
||||
inside one EPSG:4326 bbox. Partial statistical sectors are estimates; old map
|
||||
editions without stable identities do not produce invented object changes.
|
||||
|
||||
## Helpful repository scripts
|
||||
|
||||
- `bash scripts/backend_install.sh`
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Add temporal dataset metadata and durable dataset-version provenance."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "202607140001"
|
||||
down_revision = "202606120900"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("datasets", sa.Column("temporal_series_key", sa.String(length=255), nullable=True))
|
||||
op.add_column("datasets", sa.Column("observed_at", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("datasets", sa.Column("valid_from", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("datasets", sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("datasets", sa.Column("temporal_granularity", sa.String(length=32), nullable=True))
|
||||
op.add_column("datasets", sa.Column("source_version", sa.String(length=120), nullable=True))
|
||||
|
||||
op.add_column("dataset_versions", sa.Column("source_version", sa.String(length=120), nullable=True))
|
||||
op.add_column("dataset_versions", sa.Column("observed_at", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("dataset_versions", sa.Column("valid_from", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("dataset_versions", sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("dataset_versions", sa.Column("checksum_sha256", sa.String(length=64), nullable=True))
|
||||
op.add_column("dataset_versions", sa.Column("source_metadata", sa.JSON(), nullable=True))
|
||||
op.add_column("dataset_versions", sa.Column("provenance_metadata", sa.JSON(), nullable=True))
|
||||
|
||||
op.create_index(
|
||||
"ix_datasets_project_temporal_series_observed",
|
||||
"datasets",
|
||||
["project_id", "temporal_series_key", "observed_at"],
|
||||
)
|
||||
op.create_index("ix_dataset_versions_dataset_version", "dataset_versions", ["dataset_id", "version"], unique=True)
|
||||
op.create_index(
|
||||
"ix_vector_features_dataset_source_feature",
|
||||
"vector_features",
|
||||
["dataset_id", "source_feature_id"],
|
||||
)
|
||||
op.create_check_constraint(
|
||||
"ck_datasets_temporal_valid_range",
|
||||
"datasets",
|
||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||
)
|
||||
op.create_check_constraint(
|
||||
"ck_dataset_versions_temporal_valid_range",
|
||||
"dataset_versions",
|
||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_constraint("ck_dataset_versions_temporal_valid_range", "dataset_versions", type_="check")
|
||||
op.drop_constraint("ck_datasets_temporal_valid_range", "datasets", type_="check")
|
||||
op.drop_index("ix_vector_features_dataset_source_feature", table_name="vector_features")
|
||||
op.drop_index("ix_dataset_versions_dataset_version", table_name="dataset_versions")
|
||||
op.drop_index("ix_datasets_project_temporal_series_observed", table_name="datasets")
|
||||
|
||||
op.drop_column("dataset_versions", "provenance_metadata")
|
||||
op.drop_column("dataset_versions", "source_metadata")
|
||||
op.drop_column("dataset_versions", "checksum_sha256")
|
||||
op.drop_column("dataset_versions", "valid_to")
|
||||
op.drop_column("dataset_versions", "valid_from")
|
||||
op.drop_column("dataset_versions", "observed_at")
|
||||
op.drop_column("dataset_versions", "source_version")
|
||||
|
||||
op.drop_column("datasets", "source_version")
|
||||
op.drop_column("datasets", "temporal_granularity")
|
||||
op.drop_column("datasets", "valid_to")
|
||||
op.drop_column("datasets", "valid_from")
|
||||
op.drop_column("datasets", "observed_at")
|
||||
op.drop_column("datasets", "temporal_series_key")
|
||||
@@ -1 +1 @@
|
||||
__all__ = ["analysis", "areas", "datasets", "health", "projects", "exports", "jobs", "external", "qa"]
|
||||
__all__ = ["analysis", "areas", "datasets", "health", "projects", "exports", "jobs", "external", "qa", "temporal"]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
from uuid import UUID as _UUID
|
||||
@@ -30,7 +31,7 @@ from app.schemas import (
|
||||
VectorSelectionResponse,
|
||||
)
|
||||
from app.schemas.job import JobCreate
|
||||
from app.schemas.dataset import DatasetCreateResponse
|
||||
from app.schemas.dataset import DatasetCreateResponse, DatasetTemporalUpdate
|
||||
from app.schemas.operations import VectorOperationResult
|
||||
from app.services.job_service import JobService
|
||||
from app.services.raster_operations_service import RasterOperationsService
|
||||
@@ -87,6 +88,12 @@ async def upload_dataset(
|
||||
reference_layer_name: str | None = Form(None),
|
||||
source_metadata_json: str | None = Form(None),
|
||||
provenance_metadata_json: str | None = Form(None),
|
||||
temporal_series_key: str | None = Form(None),
|
||||
observed_at: datetime | None = Form(None),
|
||||
valid_from: datetime | None = Form(None),
|
||||
valid_to: datetime | None = Form(None),
|
||||
temporal_granularity: str | None = Form(None),
|
||||
source_version: str | None = Form(None),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
if area_id is not None:
|
||||
@@ -108,6 +115,12 @@ async def upload_dataset(
|
||||
source_metadata=_parse_metadata_json(source_metadata_json, "source_metadata_json"),
|
||||
provenance_metadata=_parse_metadata_json(provenance_metadata_json, "provenance_metadata_json"),
|
||||
area_id=area_id,
|
||||
temporal_series_key=temporal_series_key,
|
||||
observed_at=observed_at,
|
||||
valid_from=valid_from,
|
||||
valid_to=valid_to,
|
||||
temporal_granularity=temporal_granularity,
|
||||
source_version=source_version,
|
||||
)
|
||||
return envelope(created.model_dump())
|
||||
|
||||
@@ -135,6 +148,33 @@ def get_dataset(
|
||||
return envelope(DatasetCreateResponse.model_validate(dataset).model_dump())
|
||||
|
||||
|
||||
@router.patch("/datasets/{dataset_id}/temporal", response_model=dict)
|
||||
def update_dataset_temporal_metadata(
|
||||
project_id: UUID,
|
||||
dataset_id: UUID,
|
||||
payload: DatasetTemporalUpdate,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||
if dataset.project_id != project_id:
|
||||
raise HTTPException(status_code=404, detail="Dataset not found")
|
||||
updated = DatasetService.update_temporal_metadata(db, dataset_id, payload)
|
||||
return envelope(updated.model_dump())
|
||||
|
||||
|
||||
@router.get("/datasets/{dataset_id}/versions", response_model=dict)
|
||||
def list_dataset_versions(
|
||||
project_id: UUID,
|
||||
dataset_id: UUID,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||
if dataset.project_id != project_id:
|
||||
raise HTTPException(status_code=404, detail="Dataset not found")
|
||||
versions = DatasetService.list_versions(db, dataset_id)
|
||||
return envelope({"items": [item.model_dump() for item in versions], "total": len(versions)})
|
||||
|
||||
|
||||
@router.post("/datasets/{dataset_id}/metadata/refresh", response_model=dict)
|
||||
def refresh_dataset_metadata(
|
||||
project_id: UUID,
|
||||
@@ -203,6 +243,13 @@ def select_vector_features(
|
||||
bbox=payload.bbox.model_dump(),
|
||||
limit=payload.limit,
|
||||
)
|
||||
if isinstance(dataset.source_metadata, dict) and dataset.source_metadata.get("selection_aggregation"):
|
||||
result["summary"] = VectorFeatureService.summarize_features_by_bbox(
|
||||
db,
|
||||
dataset=dataset,
|
||||
bbox=payload.bbox.model_dump(),
|
||||
total_feature_count=result.get("total_feature_count"),
|
||||
)
|
||||
return envelope(VectorSelectionResponse(**result).model_dump(exclude_none=True))
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.schemas.temporal import TemporalComparisonRequest
|
||||
from app.services.temporal_analysis_service import TemporalAnalysisService
|
||||
from app.utils.response import envelope
|
||||
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/temporal", tags=["temporal"])
|
||||
|
||||
|
||||
@router.get("/series", response_model=dict)
|
||||
def list_temporal_series(project_id: UUID, db: Session = Depends(get_db)):
|
||||
series = TemporalAnalysisService.list_series(db, project_id)
|
||||
return envelope({"items": [item.model_dump() for item in series], "total": len(series)})
|
||||
|
||||
|
||||
@router.post("/compare", response_model=dict)
|
||||
def compare_temporal_snapshots(
|
||||
project_id: UUID,
|
||||
payload: TemporalComparisonRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
return envelope(TemporalAnalysisService.compare(db, project_id=project_id, payload=payload).model_dump())
|
||||
+2
-1
@@ -5,7 +5,7 @@ from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from app.api.routes import analysis, areas, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation
|
||||
from app.api.routes import analysis, areas, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, temporal
|
||||
from app.core.config import get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.core.logging import configure_logging
|
||||
@@ -57,6 +57,7 @@ def create_app() -> FastAPI:
|
||||
app.include_router(qa.router, prefix=settings.api_prefix)
|
||||
app.include_router(detection.router, prefix=settings.api_prefix)
|
||||
app.include_router(segmentation.router, prefix=settings.api_prefix)
|
||||
app.include_router(temporal.router, prefix=settings.api_prefix)
|
||||
|
||||
@app.exception_handler(AppError)
|
||||
async def app_error(request: Request, exc: AppError): # noqa: ARG001
|
||||
|
||||
@@ -4,7 +4,7 @@ import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from geoalchemy2 import Geometry
|
||||
from sqlalchemy import DateTime, ForeignKey, Float, Index, JSON, String, Text, func
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, func
|
||||
from sqlalchemy.sql.sqltypes import Integer
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
@@ -44,6 +44,18 @@ class Area(Base):
|
||||
|
||||
class Dataset(Base):
|
||||
__tablename__ = "datasets"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||
name="ck_datasets_temporal_valid_range",
|
||||
),
|
||||
Index(
|
||||
"ix_datasets_project_temporal_series_observed",
|
||||
"project_id",
|
||||
"temporal_series_key",
|
||||
"observed_at",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
@@ -73,6 +85,12 @@ class Dataset(Base):
|
||||
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
temporal_granularity: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="uploaded")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
@@ -92,11 +110,25 @@ class Dataset(Base):
|
||||
|
||||
class DatasetVersion(Base):
|
||||
__tablename__ = "dataset_versions"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||
name="ck_dataset_versions_temporal_valid_range",
|
||||
),
|
||||
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
dataset_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False)
|
||||
version: Mapped[int] = mapped_column(Integer, default=1)
|
||||
storage_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
|
||||
@@ -107,6 +139,7 @@ class VectorFeature(Base):
|
||||
__table_args__ = (
|
||||
Index("ix_vector_features_dataset_id", "dataset_id"),
|
||||
Index("ix_vector_features_geometry", "geometry", postgresql_using="gist"),
|
||||
Index("ix_vector_features_dataset_source_feature", "dataset_id", "source_feature_id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
|
||||
@@ -77,6 +77,7 @@ from .operations import (
|
||||
VectorSelectionDeriveRequest,
|
||||
VectorSelectionRequest,
|
||||
VectorSelectionResponse,
|
||||
VectorSelectionSummary,
|
||||
VectorStatsRequest,
|
||||
VectorStatsResponse,
|
||||
)
|
||||
@@ -134,6 +135,7 @@ __all__ = [
|
||||
"VectorSelectionDeriveRequest",
|
||||
"VectorSelectionRequest",
|
||||
"VectorSelectionResponse",
|
||||
"VectorSelectionSummary",
|
||||
"RasterClipRequest",
|
||||
"RasterStatsResponse",
|
||||
"RasterReprojectRequest",
|
||||
|
||||
@@ -36,6 +36,12 @@ class DatasetCreateResponse(BaseModel):
|
||||
source_metadata: dict | None = None
|
||||
provenance_metadata: dict | None = None
|
||||
imported_at: datetime | None = None
|
||||
temporal_series_key: str | None = None
|
||||
observed_at: datetime | None = None
|
||||
valid_from: datetime | None = None
|
||||
valid_to: datetime | None = None
|
||||
temporal_granularity: str | None = None
|
||||
source_version: str | None = None
|
||||
project_id: UUID
|
||||
area_id: UUID | None = None
|
||||
storage_path: str | None = None
|
||||
@@ -70,6 +76,32 @@ class DatasetMetadataRefresh(BaseModel):
|
||||
crs: str | None = None
|
||||
|
||||
|
||||
class DatasetTemporalUpdate(BaseModel):
|
||||
temporal_series_key: str
|
||||
observed_at: datetime
|
||||
valid_from: datetime | None = None
|
||||
valid_to: datetime | None = None
|
||||
temporal_granularity: str = "snapshot"
|
||||
source_version: str | None = None
|
||||
|
||||
|
||||
class DatasetVersionRead(BaseModel):
|
||||
id: UUID
|
||||
dataset_id: UUID
|
||||
version: int
|
||||
storage_path: str | None = None
|
||||
source_version: str | None = None
|
||||
observed_at: datetime | None = None
|
||||
valid_from: datetime | None = None
|
||||
valid_to: datetime | None = None
|
||||
checksum_sha256: str | None = None
|
||||
source_metadata: dict | None = None
|
||||
provenance_metadata: dict | None = None
|
||||
created_at: datetime | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class ExportRequest(BaseModel):
|
||||
dataset_id: UUID
|
||||
name: str | None = None
|
||||
|
||||
@@ -217,6 +217,16 @@ class VectorSelectionDeriveRequest(VectorSelectionRequest):
|
||||
output_name: str | None = None
|
||||
|
||||
|
||||
class VectorSelectionSummary(BaseModel):
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
feature_count: int
|
||||
is_estimate: bool = False
|
||||
warning: str | None = None
|
||||
|
||||
|
||||
class VectorSelectionResponse(BaseModel):
|
||||
selection_bbox: VectorSelectionBBox
|
||||
feature_count: int
|
||||
@@ -224,3 +234,4 @@ class VectorSelectionResponse(BaseModel):
|
||||
limit: int
|
||||
truncated: bool
|
||||
geojson: dict
|
||||
summary: VectorSelectionSummary | None = None
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.schemas.operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class TemporalComparisonRequest(BaseModel):
|
||||
earlier_dataset_id: UUID
|
||||
later_dataset_id: UUID
|
||||
bbox: VectorSelectionBBox
|
||||
preview_limit: int = Field(default=500, ge=1, le=1000)
|
||||
|
||||
|
||||
class TemporalDatasetRef(BaseModel):
|
||||
id: UUID
|
||||
name: str
|
||||
observed_at: datetime
|
||||
source_version: str | None = None
|
||||
|
||||
|
||||
class TemporalMetricComparison(BaseModel):
|
||||
label: str
|
||||
unit: str
|
||||
aggregation_method: str
|
||||
earlier_value: float
|
||||
later_value: float
|
||||
absolute_change: float
|
||||
percent_change: float | None = None
|
||||
is_estimate: bool = False
|
||||
|
||||
|
||||
class TemporalObjectChanges(BaseModel):
|
||||
available: bool
|
||||
added_count: int | None = None
|
||||
removed_count: int | None = None
|
||||
modified_count: int | None = None
|
||||
unchanged_count: int | None = None
|
||||
|
||||
|
||||
class TemporalComparisonResponse(BaseModel):
|
||||
temporal_series_key: str
|
||||
earlier: TemporalDatasetRef
|
||||
later: TemporalDatasetRef
|
||||
selection_bbox: VectorSelectionBBox
|
||||
metric: TemporalMetricComparison
|
||||
object_changes: TemporalObjectChanges
|
||||
geojson: dict
|
||||
warnings: list[str]
|
||||
generated_at: datetime
|
||||
|
||||
|
||||
class TemporalSeriesDataset(BaseModel):
|
||||
id: UUID
|
||||
name: str
|
||||
observed_at: datetime
|
||||
source_version: str | None = None
|
||||
feature_count: int | None = None
|
||||
|
||||
|
||||
class TemporalSeriesRead(BaseModel):
|
||||
temporal_series_key: str
|
||||
source_name: str | None = None
|
||||
reference_layer_name: str | None = None
|
||||
dataset_count: int
|
||||
first_observed_at: datetime
|
||||
last_observed_at: datetime
|
||||
datasets: list[TemporalSeriesDataset]
|
||||
@@ -12,8 +12,14 @@ from fastapi import UploadFile
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, Project
|
||||
from app.schemas.dataset import DatasetCreateResponse, DatasetStorageResponse, DatasetVectorSummary
|
||||
from app.models import Dataset, DatasetVersion, Project
|
||||
from app.schemas.dataset import (
|
||||
DatasetCreateResponse,
|
||||
DatasetStorageResponse,
|
||||
DatasetTemporalUpdate,
|
||||
DatasetVectorSummary,
|
||||
DatasetVersionRead,
|
||||
)
|
||||
from app.services.geojson_service import parse_geojson_payload, load_dataset_text
|
||||
from app.services.raster_service import extract_raster_metadata
|
||||
from app.services.storage_service import StorageService
|
||||
@@ -26,6 +32,105 @@ class DatasetService:
|
||||
VECTOR_TYPES = {"vector", "geojson"}
|
||||
RASTER_TYPES = {"raster", "tif", "tiff", "geotiff"}
|
||||
VALID_DATASET_ROLES = {"source", "derived", "reference"}
|
||||
VALID_TEMPORAL_GRANULARITIES = {"snapshot", "day", "month", "year", "period"}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_datetime(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
@staticmethod
|
||||
def _validate_temporal_metadata(
|
||||
*,
|
||||
temporal_series_key: str | None,
|
||||
observed_at: datetime | None,
|
||||
valid_from: datetime | None,
|
||||
valid_to: datetime | None,
|
||||
temporal_granularity: str | None,
|
||||
source_version: str | None,
|
||||
) -> dict[str, Any]:
|
||||
normalized_key = (temporal_series_key or "").strip() or None
|
||||
normalized_observed_at = DatasetService._normalize_datetime(observed_at)
|
||||
normalized_valid_from = DatasetService._normalize_datetime(valid_from)
|
||||
normalized_valid_to = DatasetService._normalize_datetime(valid_to)
|
||||
normalized_granularity = (temporal_granularity or "").strip().lower() or None
|
||||
normalized_source_version = (source_version or "").strip() or None
|
||||
|
||||
if normalized_key and len(normalized_key) > 255:
|
||||
raise AppError(code="INVALID_TEMPORAL_METADATA", message="temporal_series_key is too long", status_code=400)
|
||||
if normalized_granularity and normalized_granularity not in DatasetService.VALID_TEMPORAL_GRANULARITIES:
|
||||
raise AppError(
|
||||
code="INVALID_TEMPORAL_METADATA",
|
||||
message="temporal_granularity must be snapshot, day, month, year or period",
|
||||
status_code=400,
|
||||
)
|
||||
if normalized_valid_from and normalized_valid_to and normalized_valid_to < normalized_valid_from:
|
||||
raise AppError(
|
||||
code="INVALID_TEMPORAL_METADATA",
|
||||
message="valid_to must be on or after valid_from",
|
||||
status_code=400,
|
||||
)
|
||||
if normalized_key and normalized_observed_at is None:
|
||||
raise AppError(
|
||||
code="INVALID_TEMPORAL_METADATA",
|
||||
message="observed_at is required when temporal_series_key is provided",
|
||||
status_code=400,
|
||||
)
|
||||
if normalized_observed_at and normalized_key is None:
|
||||
raise AppError(
|
||||
code="INVALID_TEMPORAL_METADATA",
|
||||
message="temporal_series_key is required when observed_at is provided",
|
||||
status_code=400,
|
||||
)
|
||||
return {
|
||||
"temporal_series_key": normalized_key,
|
||||
"observed_at": normalized_observed_at,
|
||||
"valid_from": normalized_valid_from,
|
||||
"valid_to": normalized_valid_to,
|
||||
"temporal_granularity": normalized_granularity,
|
||||
"source_version": normalized_source_version,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _to_response(dataset: Dataset) -> DatasetCreateResponse:
|
||||
metadata_json = dataset.metadata_json if isinstance(dataset.metadata_json, dict) else {}
|
||||
return DatasetCreateResponse(
|
||||
id=dataset.id,
|
||||
name=dataset.name,
|
||||
dataset_type=dataset.dataset_type,
|
||||
source=dataset.source,
|
||||
dataset_role=dataset.dataset_role,
|
||||
source_name=dataset.source_name,
|
||||
reference_layer_name=dataset.reference_layer_name,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
imported_at=dataset.imported_at,
|
||||
temporal_series_key=dataset.temporal_series_key,
|
||||
observed_at=dataset.observed_at,
|
||||
valid_from=dataset.valid_from,
|
||||
valid_to=dataset.valid_to,
|
||||
temporal_granularity=dataset.temporal_granularity,
|
||||
source_version=dataset.source_version,
|
||||
project_id=dataset.project_id,
|
||||
area_id=dataset.area_id,
|
||||
storage_path=dataset.storage_path,
|
||||
original_filename=dataset.original_filename,
|
||||
stored_filename=dataset.stored_filename,
|
||||
content_type=dataset.content_type,
|
||||
size_bytes=dataset.size_bytes,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
crs=dataset.crs,
|
||||
bounds_json=dataset.bounds_json,
|
||||
metadata_json=dataset.metadata_json,
|
||||
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, metadata_json),
|
||||
status=dataset.status,
|
||||
derived_from_dataset_id=dataset.derived_from_dataset_id,
|
||||
created_at=dataset.created_at,
|
||||
feature_count=metadata_json.get("feature_count"),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _canonical_dataset_type(dataset_type: str) -> str:
|
||||
@@ -89,44 +194,7 @@ class DatasetService:
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
response_items = []
|
||||
for row in rows:
|
||||
feature_count = None
|
||||
metadata_json = row.metadata_json or {}
|
||||
vector_summary = DatasetService._extract_vector_summary(row.dataset_type, metadata_json)
|
||||
if isinstance(metadata_json, dict):
|
||||
feature_count = metadata_json.get("feature_count")
|
||||
response_items.append(
|
||||
DatasetCreateResponse(
|
||||
id=row.id,
|
||||
name=row.name,
|
||||
dataset_type=row.dataset_type,
|
||||
source=row.source,
|
||||
dataset_role=row.dataset_role,
|
||||
source_name=row.source_name,
|
||||
reference_layer_name=row.reference_layer_name,
|
||||
source_metadata=row.source_metadata,
|
||||
provenance_metadata=row.provenance_metadata,
|
||||
imported_at=row.imported_at,
|
||||
project_id=row.project_id,
|
||||
area_id=row.area_id,
|
||||
storage_path=row.storage_path,
|
||||
original_filename=row.original_filename,
|
||||
stored_filename=row.stored_filename,
|
||||
content_type=row.content_type,
|
||||
size_bytes=row.size_bytes,
|
||||
checksum_sha256=row.checksum_sha256,
|
||||
crs=row.crs,
|
||||
bounds_json=row.bounds_json,
|
||||
metadata_json=row.metadata_json,
|
||||
vector_summary=vector_summary,
|
||||
status=row.status,
|
||||
derived_from_dataset_id=row.derived_from_dataset_id,
|
||||
created_at=row.created_at,
|
||||
feature_count=feature_count,
|
||||
)
|
||||
)
|
||||
return response_items, total
|
||||
return [DatasetService._to_response(row) for row in rows], total
|
||||
|
||||
@staticmethod
|
||||
def _extract_vector_summary(dataset_type: str, metadata_json: dict) -> DatasetVectorSummary | None:
|
||||
@@ -195,6 +263,12 @@ class DatasetService:
|
||||
source_metadata: dict | None = None,
|
||||
provenance_metadata: dict | None = None,
|
||||
area_id: UUID | None = None,
|
||||
temporal_series_key: str | None = None,
|
||||
observed_at: datetime | None = None,
|
||||
valid_from: datetime | None = None,
|
||||
valid_to: datetime | None = None,
|
||||
temporal_granularity: str | None = None,
|
||||
source_version: str | None = None,
|
||||
) -> DatasetCreateResponse:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
@@ -202,6 +276,14 @@ class DatasetService:
|
||||
filename = DatasetService._validate_upload_filename(file.filename)
|
||||
canonical_type = DatasetService._canonical_dataset_type(dataset_type)
|
||||
normalized_role = DatasetService._normalize_dataset_role(dataset_role)
|
||||
temporal = DatasetService._validate_temporal_metadata(
|
||||
temporal_series_key=temporal_series_key,
|
||||
observed_at=observed_at,
|
||||
valid_from=valid_from,
|
||||
valid_to=valid_to,
|
||||
temporal_granularity=temporal_granularity,
|
||||
source_version=source_version,
|
||||
)
|
||||
normalized_source_name = source_name
|
||||
if normalized_role == "reference" and not normalized_source_name:
|
||||
normalized_source_name = "manual"
|
||||
@@ -280,6 +362,7 @@ class DatasetService:
|
||||
source_metadata=source_metadata,
|
||||
provenance_metadata=provenance_metadata,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
**temporal,
|
||||
storage_path=storage_info["storage_path"],
|
||||
original_filename=storage_info["original_filename"],
|
||||
stored_filename=storage_info["stored_filename"],
|
||||
@@ -294,6 +377,20 @@ class DatasetService:
|
||||
status=status,
|
||||
)
|
||||
db.add(dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
storage_path=dataset.storage_path,
|
||||
source_version=dataset.source_version,
|
||||
observed_at=dataset.observed_at,
|
||||
valid_from=dataset.valid_from,
|
||||
valid_to=dataset.valid_to,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
|
||||
@@ -306,34 +403,7 @@ class DatasetService:
|
||||
feature_class=feature_class,
|
||||
)
|
||||
|
||||
return DatasetCreateResponse(
|
||||
id=dataset.id,
|
||||
name=dataset.name,
|
||||
dataset_type=dataset.dataset_type,
|
||||
source=dataset.source,
|
||||
dataset_role=dataset.dataset_role,
|
||||
source_name=dataset.source_name,
|
||||
reference_layer_name=dataset.reference_layer_name,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
imported_at=dataset.imported_at,
|
||||
project_id=dataset.project_id,
|
||||
area_id=dataset.area_id,
|
||||
storage_path=dataset.storage_path,
|
||||
original_filename=dataset.original_filename,
|
||||
stored_filename=dataset.stored_filename,
|
||||
content_type=dataset.content_type,
|
||||
size_bytes=dataset.size_bytes,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
crs=dataset.crs,
|
||||
derived_from_dataset_id=dataset.derived_from_dataset_id,
|
||||
bounds_json=dataset.bounds_json,
|
||||
metadata_json=dataset.metadata_json,
|
||||
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}),
|
||||
status=dataset.status,
|
||||
created_at=dataset.created_at,
|
||||
feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None,
|
||||
)
|
||||
return DatasetService._to_response(dataset)
|
||||
|
||||
@staticmethod
|
||||
def refresh_metadata(db: Session, dataset_id: UUID) -> DatasetCreateResponse:
|
||||
@@ -380,34 +450,53 @@ class DatasetService:
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
|
||||
return DatasetCreateResponse(
|
||||
id=dataset.id,
|
||||
name=dataset.name,
|
||||
dataset_type=dataset.dataset_type,
|
||||
source=dataset.source,
|
||||
dataset_role=dataset.dataset_role,
|
||||
source_name=dataset.source_name,
|
||||
reference_layer_name=dataset.reference_layer_name,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
imported_at=dataset.imported_at,
|
||||
project_id=dataset.project_id,
|
||||
area_id=dataset.area_id,
|
||||
storage_path=dataset.storage_path,
|
||||
original_filename=dataset.original_filename,
|
||||
stored_filename=dataset.stored_filename,
|
||||
content_type=dataset.content_type,
|
||||
size_bytes=dataset.size_bytes,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
crs=dataset.crs,
|
||||
derived_from_dataset_id=dataset.derived_from_dataset_id,
|
||||
bounds_json=dataset.bounds_json,
|
||||
metadata_json=dataset.metadata_json,
|
||||
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}),
|
||||
status=dataset.status,
|
||||
created_at=dataset.created_at,
|
||||
feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None,
|
||||
return DatasetService._to_response(dataset)
|
||||
|
||||
@staticmethod
|
||||
def update_temporal_metadata(db: Session, dataset_id: UUID, payload: DatasetTemporalUpdate) -> DatasetCreateResponse:
|
||||
dataset = DatasetService._get_dataset(db, dataset_id)
|
||||
temporal = DatasetService._validate_temporal_metadata(**payload.model_dump())
|
||||
if all(getattr(dataset, field) == value for field, value in temporal.items()):
|
||||
return DatasetService._to_response(dataset)
|
||||
|
||||
for field, value in temporal.items():
|
||||
setattr(dataset, field, value)
|
||||
|
||||
latest_version = (
|
||||
db.query(DatasetVersion)
|
||||
.filter(DatasetVersion.dataset_id == dataset.id)
|
||||
.order_by(DatasetVersion.version.desc())
|
||||
.first()
|
||||
)
|
||||
db.add(dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=(latest_version.version + 1) if latest_version else 1,
|
||||
storage_path=dataset.storage_path,
|
||||
source_version=dataset.source_version,
|
||||
observed_at=dataset.observed_at,
|
||||
valid_from=dataset.valid_from,
|
||||
valid_to=dataset.valid_to,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
return DatasetService._to_response(dataset)
|
||||
|
||||
@staticmethod
|
||||
def list_versions(db: Session, dataset_id: UUID) -> list[DatasetVersionRead]:
|
||||
DatasetService._get_dataset(db, dataset_id)
|
||||
rows = (
|
||||
db.query(DatasetVersion)
|
||||
.filter(DatasetVersion.dataset_id == dataset_id)
|
||||
.order_by(DatasetVersion.version.desc())
|
||||
.all()
|
||||
)
|
||||
return [DatasetVersionRead.model_validate(row) for row in rows]
|
||||
|
||||
@staticmethod
|
||||
def get_dataset(db: Session, dataset_id: UUID) -> Dataset:
|
||||
|
||||
@@ -10,7 +10,7 @@ from uuid import UUID, uuid4
|
||||
from geoalchemy2.shape import from_shape
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Area, Dataset, Metric, Project, QualityCheck
|
||||
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
|
||||
from app.schemas.demo import DemoWorkflowResponse
|
||||
from app.services.geojson_service import parse_geojson_payload
|
||||
from app.services.qa_service import QaService
|
||||
@@ -29,6 +29,19 @@ class DemoWorkflowService:
|
||||
RASTER_FILENAME = "demo_context_raster.tif"
|
||||
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
|
||||
|
||||
@staticmethod
|
||||
def _add_initial_version(db: Session, dataset: Dataset) -> None:
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
storage_path=dataset.storage_path,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _repo_root() -> Path:
|
||||
return Path(__file__).resolve().parents[3]
|
||||
@@ -213,6 +226,7 @@ class DemoWorkflowService:
|
||||
status="ready",
|
||||
)
|
||||
db.add(dataset)
|
||||
DemoWorkflowService._add_initial_version(db, dataset)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
VectorFeatureService.persist_geojson_features(
|
||||
@@ -297,6 +311,7 @@ class DemoWorkflowService:
|
||||
status="ready",
|
||||
)
|
||||
db.add(dataset)
|
||||
DemoWorkflowService._add_initial_version(db, dataset)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
return dataset
|
||||
|
||||
@@ -12,7 +12,7 @@ from shapely.ops import transform as shapely_transform
|
||||
from shapely.validation import make_valid
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset
|
||||
from app.models import Area, Dataset, DatasetVersion
|
||||
from app.services.raster_service import extract_raster_metadata
|
||||
from app.services.storage_service import StorageService
|
||||
|
||||
@@ -333,6 +333,21 @@ class RasterOperationsService:
|
||||
name=output_name,
|
||||
dataset_type="raster",
|
||||
source=f"operation:{operation_name}",
|
||||
dataset_role="derived",
|
||||
source_name=source_dataset.source_name,
|
||||
source_metadata=source_dataset.source_metadata,
|
||||
provenance_metadata=provenance,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
temporal_series_key=(
|
||||
f"{source_dataset.temporal_series_key}:{operation_name}"
|
||||
if source_dataset.temporal_series_key
|
||||
else None
|
||||
),
|
||||
observed_at=source_dataset.observed_at,
|
||||
valid_from=source_dataset.valid_from,
|
||||
valid_to=source_dataset.valid_to,
|
||||
temporal_granularity=source_dataset.temporal_granularity,
|
||||
source_version=source_dataset.source_version,
|
||||
storage_path=str(output_file),
|
||||
original_filename=storage_metadata["original_filename"],
|
||||
stored_filename=storage_metadata["stored_filename"],
|
||||
@@ -348,6 +363,20 @@ class RasterOperationsService:
|
||||
status="ready",
|
||||
)
|
||||
db.add(derived_dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=derived_dataset.id,
|
||||
version=1,
|
||||
storage_path=derived_dataset.storage_path,
|
||||
source_version=derived_dataset.source_version,
|
||||
observed_at=derived_dataset.observed_at,
|
||||
valid_from=derived_dataset.valid_from,
|
||||
valid_to=derived_dataset.valid_to,
|
||||
checksum_sha256=derived_dataset.checksum_sha256,
|
||||
source_metadata=derived_dataset.source_metadata,
|
||||
provenance_metadata=derived_dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(derived_dataset)
|
||||
return derived_id
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.functions import ST_Intersects, ST_MakeEnvelope
|
||||
from geoalchemy2.shape import to_shape
|
||||
from shapely.geometry import mapping
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, VectorFeature
|
||||
from app.schemas.temporal import (
|
||||
TemporalComparisonRequest,
|
||||
TemporalComparisonResponse,
|
||||
TemporalDatasetRef,
|
||||
TemporalMetricComparison,
|
||||
TemporalObjectChanges,
|
||||
TemporalSeriesDataset,
|
||||
TemporalSeriesRead,
|
||||
)
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
class TemporalAnalysisService:
|
||||
IDENTITY_COMPARISON_LIMIT = 5_000
|
||||
|
||||
@staticmethod
|
||||
def list_series(db: Session, project_id: UUID) -> list[TemporalSeriesRead]:
|
||||
rows = (
|
||||
db.query(Dataset)
|
||||
.filter(Dataset.project_id == project_id)
|
||||
.filter(Dataset.temporal_series_key.isnot(None))
|
||||
.filter(Dataset.observed_at.isnot(None))
|
||||
.order_by(Dataset.temporal_series_key.asc(), Dataset.observed_at.asc())
|
||||
.all()
|
||||
)
|
||||
grouped: dict[str, list[Dataset]] = {}
|
||||
for row in rows:
|
||||
if row.temporal_series_key:
|
||||
grouped.setdefault(row.temporal_series_key, []).append(row)
|
||||
|
||||
result: list[TemporalSeriesRead] = []
|
||||
for key, datasets in grouped.items():
|
||||
observed = [item.observed_at for item in datasets if item.observed_at is not None]
|
||||
if not observed:
|
||||
continue
|
||||
result.append(
|
||||
TemporalSeriesRead(
|
||||
temporal_series_key=key,
|
||||
source_name=datasets[-1].source_name,
|
||||
reference_layer_name=datasets[-1].reference_layer_name,
|
||||
dataset_count=len(datasets),
|
||||
first_observed_at=min(observed),
|
||||
last_observed_at=max(observed),
|
||||
datasets=[
|
||||
TemporalSeriesDataset(
|
||||
id=item.id,
|
||||
name=item.name,
|
||||
observed_at=item.observed_at,
|
||||
source_version=item.source_version,
|
||||
feature_count=(item.metadata_json or {}).get("feature_count")
|
||||
if isinstance(item.metadata_json, dict)
|
||||
else None,
|
||||
)
|
||||
for item in datasets
|
||||
if item.observed_at is not None
|
||||
],
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def compare(
|
||||
db: Session,
|
||||
*,
|
||||
project_id: UUID,
|
||||
payload: TemporalComparisonRequest,
|
||||
) -> TemporalComparisonResponse:
|
||||
if payload.earlier_dataset_id == payload.later_dataset_id:
|
||||
raise AppError(
|
||||
code="INVALID_TEMPORAL_COMPARISON",
|
||||
message="Choose two different dataset snapshots",
|
||||
status_code=400,
|
||||
)
|
||||
earlier = TemporalAnalysisService._get_temporal_dataset(db, project_id, payload.earlier_dataset_id, "Earlier")
|
||||
later = TemporalAnalysisService._get_temporal_dataset(db, project_id, payload.later_dataset_id, "Later")
|
||||
if earlier.temporal_series_key != later.temporal_series_key:
|
||||
raise AppError(
|
||||
code="INCOMPATIBLE_TEMPORAL_SERIES",
|
||||
message="Dataset snapshots must belong to the same temporal series",
|
||||
details={
|
||||
"earlier_series": earlier.temporal_series_key,
|
||||
"later_series": later.temporal_series_key,
|
||||
},
|
||||
status_code=400,
|
||||
)
|
||||
if earlier.observed_at >= later.observed_at:
|
||||
raise AppError(
|
||||
code="INVALID_TEMPORAL_ORDER",
|
||||
message="Earlier snapshot must have an observation date before the later snapshot",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
bbox = payload.bbox.model_dump()
|
||||
earlier_summary = VectorFeatureService.summarize_features_by_bbox(db, dataset=earlier, bbox=bbox)
|
||||
later_summary = VectorFeatureService.summarize_features_by_bbox(db, dataset=later, bbox=bbox)
|
||||
if (
|
||||
earlier_summary["aggregation_method"] != later_summary["aggregation_method"]
|
||||
or earlier_summary["metric_unit"] != later_summary["metric_unit"]
|
||||
):
|
||||
raise AppError(
|
||||
code="INCOMPATIBLE_TEMPORAL_AGGREGATION",
|
||||
message="Dataset snapshots use incompatible aggregation semantics",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
earlier_value = float(earlier_summary["metric_value"])
|
||||
later_value = float(later_summary["metric_value"])
|
||||
absolute_change = later_value - earlier_value
|
||||
percent_change = (absolute_change / earlier_value * 100.0) if earlier_value else None
|
||||
warnings = [
|
||||
warning
|
||||
for warning in {earlier_summary.get("warning"), later_summary.get("warning")}
|
||||
if warning
|
||||
]
|
||||
|
||||
object_changes, geojson, identity_warnings = TemporalAnalysisService._compare_identity_features(
|
||||
db,
|
||||
earlier=earlier,
|
||||
later=later,
|
||||
bbox=bbox,
|
||||
preview_limit=payload.preview_limit,
|
||||
)
|
||||
warnings.extend(identity_warnings)
|
||||
|
||||
return TemporalComparisonResponse(
|
||||
temporal_series_key=earlier.temporal_series_key,
|
||||
earlier=TemporalDatasetRef(
|
||||
id=earlier.id,
|
||||
name=earlier.name,
|
||||
observed_at=earlier.observed_at,
|
||||
source_version=earlier.source_version,
|
||||
),
|
||||
later=TemporalDatasetRef(
|
||||
id=later.id,
|
||||
name=later.name,
|
||||
observed_at=later.observed_at,
|
||||
source_version=later.source_version,
|
||||
),
|
||||
selection_bbox=payload.bbox,
|
||||
metric=TemporalMetricComparison(
|
||||
label=str(later_summary["metric_label"]),
|
||||
unit=str(later_summary["metric_unit"]),
|
||||
aggregation_method=str(later_summary["aggregation_method"]),
|
||||
earlier_value=earlier_value,
|
||||
later_value=later_value,
|
||||
absolute_change=absolute_change,
|
||||
percent_change=percent_change,
|
||||
is_estimate=bool(earlier_summary["is_estimate"] or later_summary["is_estimate"]),
|
||||
),
|
||||
object_changes=object_changes,
|
||||
geojson=geojson,
|
||||
warnings=warnings,
|
||||
generated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_temporal_dataset(db: Session, project_id: UUID, dataset_id: UUID, label: str) -> Dataset:
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if not dataset or dataset.project_id != project_id:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message=f"{label} dataset not found", status_code=404)
|
||||
if dataset.dataset_type not in {"vector", "geojson"}:
|
||||
raise AppError(
|
||||
code="DATASET_NOT_VECTOR",
|
||||
message="Temporal selection comparison currently requires vector datasets",
|
||||
status_code=400,
|
||||
)
|
||||
if not dataset.temporal_series_key or not dataset.observed_at:
|
||||
raise AppError(
|
||||
code="TEMPORAL_METADATA_MISSING",
|
||||
message=f"{label} dataset has no explicit temporal series and observation date",
|
||||
status_code=400,
|
||||
)
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def _compare_identity_features(
|
||||
db: Session,
|
||||
*,
|
||||
earlier: Dataset,
|
||||
later: Dataset,
|
||||
bbox: dict[str, Any],
|
||||
preview_limit: int,
|
||||
) -> tuple[TemporalObjectChanges, dict[str, Any], list[str]]:
|
||||
earlier_config = earlier.source_metadata if isinstance(earlier.source_metadata, dict) else {}
|
||||
later_config = later.source_metadata if isinstance(later.source_metadata, dict) else {}
|
||||
if not earlier_config.get("identity_stable") or not later_config.get("identity_stable"):
|
||||
return (
|
||||
TemporalObjectChanges(available=False),
|
||||
{"type": "FeatureCollection", "features": []},
|
||||
["Object-level changes are unavailable because the source does not guarantee stable feature identifiers."],
|
||||
)
|
||||
|
||||
normalized_bbox = VectorFeatureService._normalize_selection_bbox(bbox)
|
||||
envelope = ST_MakeEnvelope(
|
||||
normalized_bbox["min_x"],
|
||||
normalized_bbox["min_y"],
|
||||
normalized_bbox["max_x"],
|
||||
normalized_bbox["max_y"],
|
||||
4326,
|
||||
)
|
||||
|
||||
def load(dataset_id: UUID) -> list[VectorFeature]:
|
||||
return (
|
||||
db.query(VectorFeature)
|
||||
.filter(VectorFeature.dataset_id == dataset_id)
|
||||
.filter(ST_Intersects(VectorFeature.geometry, envelope))
|
||||
.filter(VectorFeature.source_feature_id.isnot(None))
|
||||
.order_by(VectorFeature.source_feature_id.asc())
|
||||
.limit(TemporalAnalysisService.IDENTITY_COMPARISON_LIMIT + 1)
|
||||
.all()
|
||||
)
|
||||
|
||||
earlier_rows = load(earlier.id)
|
||||
later_rows = load(later.id)
|
||||
if (
|
||||
len(earlier_rows) > TemporalAnalysisService.IDENTITY_COMPARISON_LIMIT
|
||||
or len(later_rows) > TemporalAnalysisService.IDENTITY_COMPARISON_LIMIT
|
||||
):
|
||||
return (
|
||||
TemporalObjectChanges(available=False),
|
||||
{"type": "FeatureCollection", "features": []},
|
||||
["Object-level preview was skipped because the selection exceeds the 5,000 feature safety limit."],
|
||||
)
|
||||
|
||||
earlier_by_id = {str(row.source_feature_id): row for row in earlier_rows if row.source_feature_id}
|
||||
later_by_id = {str(row.source_feature_id): row for row in later_rows if row.source_feature_id}
|
||||
earlier_ids = set(earlier_by_id)
|
||||
later_ids = set(later_by_id)
|
||||
added_ids = sorted(later_ids - earlier_ids)
|
||||
removed_ids = sorted(earlier_ids - later_ids)
|
||||
common_ids = sorted(earlier_ids & later_ids)
|
||||
comparison_property = str(later_config.get("comparison_property") or "").strip() or None
|
||||
modified_ids: list[str] = []
|
||||
unchanged_ids: list[str] = []
|
||||
|
||||
for feature_id in common_ids:
|
||||
earlier_row = earlier_by_id[feature_id]
|
||||
later_row = later_by_id[feature_id]
|
||||
geometry_changed = not to_shape(earlier_row.geometry).equals(to_shape(later_row.geometry))
|
||||
value_changed = False
|
||||
if comparison_property:
|
||||
value_changed = (earlier_row.properties_json or {}).get(comparison_property) != (
|
||||
later_row.properties_json or {}
|
||||
).get(comparison_property)
|
||||
(modified_ids if geometry_changed or value_changed else unchanged_ids).append(feature_id)
|
||||
|
||||
features: list[dict[str, Any]] = []
|
||||
for change_type, feature_ids, rows in (
|
||||
("added", added_ids, later_by_id),
|
||||
("removed", removed_ids, earlier_by_id),
|
||||
("modified", modified_ids, later_by_id),
|
||||
):
|
||||
for feature_id in feature_ids:
|
||||
if len(features) >= preview_limit:
|
||||
break
|
||||
row = rows[feature_id]
|
||||
properties = dict(row.properties_json or {})
|
||||
properties.update(
|
||||
{
|
||||
"change_type": change_type,
|
||||
"source_feature_id": feature_id,
|
||||
"earlier_dataset_id": str(earlier.id),
|
||||
"later_dataset_id": str(later.id),
|
||||
}
|
||||
)
|
||||
if change_type == "modified" and comparison_property:
|
||||
before = (earlier_by_id[feature_id].properties_json or {}).get(comparison_property)
|
||||
after = (later_by_id[feature_id].properties_json or {}).get(comparison_property)
|
||||
properties.update({"value_before": before, "value_after": after})
|
||||
if isinstance(before, (int, float)) and isinstance(after, (int, float)):
|
||||
properties["value_delta"] = after - before
|
||||
features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": str(row.id),
|
||||
"geometry": mapping(to_shape(row.geometry)),
|
||||
"properties": properties,
|
||||
}
|
||||
)
|
||||
|
||||
warnings: list[str] = []
|
||||
total_changes = len(added_ids) + len(removed_ids) + len(modified_ids)
|
||||
if total_changes > preview_limit:
|
||||
warnings.append(
|
||||
f"The map shows the first {preview_limit} of {total_changes} changed features; counts remain complete."
|
||||
)
|
||||
return (
|
||||
TemporalObjectChanges(
|
||||
available=True,
|
||||
added_count=len(added_ids),
|
||||
removed_count=len(removed_ids),
|
||||
modified_count=len(modified_ids),
|
||||
unchanged_count=len(unchanged_ids),
|
||||
),
|
||||
{"type": "FeatureCollection", "features": features},
|
||||
warnings,
|
||||
)
|
||||
@@ -9,9 +9,10 @@ from geoalchemy2.shape import to_shape
|
||||
from shapely.geometry import mapping
|
||||
from shapely.geometry import shape
|
||||
from shapely.validation import make_valid
|
||||
from sqlalchemy import Float, cast, func
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import VectorFeature
|
||||
from app.models import Dataset, VectorFeature
|
||||
|
||||
|
||||
class VectorFeatureService:
|
||||
@@ -88,6 +89,7 @@ class VectorFeatureService:
|
||||
dataset_id: UUID,
|
||||
bbox: dict[str, Any],
|
||||
limit: int = 100,
|
||||
dataset: Dataset | None = None,
|
||||
) -> dict[str, Any]:
|
||||
normalized_bbox = VectorFeatureService._normalize_selection_bbox(bbox)
|
||||
safe_limit = max(1, min(int(limit), 1000))
|
||||
@@ -121,6 +123,14 @@ class VectorFeatureService:
|
||||
truncated = total_feature_count > safe_limit
|
||||
selected_rows = rows[:safe_limit]
|
||||
features = [VectorFeatureService._row_to_geojson_feature(row) for row in selected_rows]
|
||||
summary = None
|
||||
if dataset and isinstance(dataset.source_metadata, dict) and dataset.source_metadata.get("selection_aggregation"):
|
||||
summary = VectorFeatureService.summarize_features_by_bbox(
|
||||
db,
|
||||
dataset=dataset,
|
||||
bbox=normalized_bbox,
|
||||
total_feature_count=total_feature_count,
|
||||
)
|
||||
|
||||
return {
|
||||
"selection_bbox": normalized_bbox,
|
||||
@@ -132,6 +142,97 @@ class VectorFeatureService:
|
||||
"type": "FeatureCollection",
|
||||
"features": features,
|
||||
},
|
||||
"summary": summary,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def summarize_features_by_bbox(
|
||||
db,
|
||||
*,
|
||||
dataset: Dataset,
|
||||
bbox: dict[str, Any],
|
||||
total_feature_count: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
normalized_bbox = VectorFeatureService._normalize_selection_bbox(bbox)
|
||||
envelope = ST_MakeEnvelope(
|
||||
normalized_bbox["min_x"],
|
||||
normalized_bbox["min_y"],
|
||||
normalized_bbox["max_x"],
|
||||
normalized_bbox["max_y"],
|
||||
4326,
|
||||
)
|
||||
selection_filter = (
|
||||
VectorFeature.dataset_id == dataset.id,
|
||||
ST_Intersects(VectorFeature.geometry, envelope),
|
||||
)
|
||||
feature_count = total_feature_count
|
||||
if feature_count is None:
|
||||
feature_count = int(db.query(func.count(VectorFeature.id)).filter(*selection_filter).scalar() or 0)
|
||||
|
||||
source_metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||
config = source_metadata.get("selection_aggregation")
|
||||
if not isinstance(config, dict):
|
||||
config = {}
|
||||
method = str(config.get("method") or "feature_count")
|
||||
label = str(config.get("label") or "Objecten")
|
||||
unit = str(config.get("unit") or "objecten")
|
||||
warning = str(config["warning"]) if config.get("warning") else None
|
||||
is_estimate = bool(config.get("is_estimate", False))
|
||||
|
||||
metric_value = float(feature_count)
|
||||
if method == "intersection_area":
|
||||
intersection = func.ST_Intersection(VectorFeature.geometry, envelope)
|
||||
area_expression = func.ST_Area(func.ST_Transform(intersection, 31370))
|
||||
area_m2 = db.query(func.coalesce(func.sum(area_expression), 0.0)).filter(*selection_filter).scalar()
|
||||
divisor = 10_000.0 if unit == "ha" else 1.0
|
||||
metric_value = float(area_m2 or 0.0) / divisor
|
||||
elif method == "intersection_length":
|
||||
intersection = func.ST_Intersection(VectorFeature.geometry, envelope)
|
||||
length_expression = func.ST_Length(func.ST_Transform(intersection, 31370))
|
||||
length_m = db.query(func.coalesce(func.sum(length_expression), 0.0)).filter(*selection_filter).scalar()
|
||||
divisor = 1_000.0 if unit == "km" else 1.0
|
||||
metric_value = float(length_m or 0.0) / divisor
|
||||
elif method in {"sum", "area_weighted_sum"}:
|
||||
property_name = str(config.get("property") or "").strip()
|
||||
if not property_name:
|
||||
raise AppError(
|
||||
code="INVALID_SELECTION_AGGREGATION",
|
||||
message="Dataset selection aggregation requires a numeric property",
|
||||
details={"dataset_id": str(dataset.id), "method": method},
|
||||
status_code=500,
|
||||
)
|
||||
numeric_value = cast(VectorFeature.properties_json.op("->>")(property_name), Float)
|
||||
value_expression = numeric_value
|
||||
if method == "area_weighted_sum":
|
||||
source_area = func.ST_Area(func.ST_Transform(VectorFeature.geometry, 31370))
|
||||
intersection_area = func.ST_Area(
|
||||
func.ST_Transform(func.ST_Intersection(VectorFeature.geometry, envelope), 31370)
|
||||
)
|
||||
value_expression = numeric_value * intersection_area / func.nullif(source_area, 0.0)
|
||||
is_estimate = True
|
||||
aggregate_value = (
|
||||
db.query(func.coalesce(func.sum(value_expression), 0.0))
|
||||
.filter(*selection_filter)
|
||||
.filter(VectorFeature.properties_json.op("->>")(property_name).isnot(None))
|
||||
.scalar()
|
||||
)
|
||||
metric_value = float(aggregate_value or 0.0)
|
||||
elif method != "feature_count":
|
||||
raise AppError(
|
||||
code="INVALID_SELECTION_AGGREGATION",
|
||||
message="Unsupported dataset selection aggregation",
|
||||
details={"dataset_id": str(dataset.id), "method": method},
|
||||
status_code=500,
|
||||
)
|
||||
|
||||
return {
|
||||
"metric_label": label,
|
||||
"metric_value": metric_value,
|
||||
"metric_unit": unit,
|
||||
"aggregation_method": method,
|
||||
"feature_count": feature_count,
|
||||
"is_estimate": is_estimate,
|
||||
"warning": warning,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -15,7 +15,7 @@ from shapely.validation import make_valid
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset
|
||||
from app.models import Area, Dataset, DatasetVersion
|
||||
from app.schemas.dataset import DatasetCreateResponse
|
||||
from app.schemas.operations import VectorOperationResult
|
||||
from app.services.geojson_service import parse_geojson_payload
|
||||
@@ -444,6 +444,16 @@ class VectorOperationsService:
|
||||
source_metadata=source_metadata,
|
||||
provenance_metadata=provenance_metadata,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
temporal_series_key=(
|
||||
f"{source_dataset.temporal_series_key}:{operation}"
|
||||
if source_dataset.temporal_series_key
|
||||
else None
|
||||
),
|
||||
observed_at=source_dataset.observed_at,
|
||||
valid_from=source_dataset.valid_from,
|
||||
valid_to=source_dataset.valid_to,
|
||||
temporal_granularity=source_dataset.temporal_granularity,
|
||||
source_version=source_dataset.source_version,
|
||||
storage_path=storage_info["storage_path"],
|
||||
original_filename=storage_info["original_filename"],
|
||||
stored_filename=storage_info["stored_filename"],
|
||||
@@ -459,6 +469,20 @@ class VectorOperationsService:
|
||||
status="ready",
|
||||
)
|
||||
db.add(derived_dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=derived_dataset.id,
|
||||
version=1,
|
||||
storage_path=derived_dataset.storage_path,
|
||||
source_version=derived_dataset.source_version,
|
||||
observed_at=derived_dataset.observed_at,
|
||||
valid_from=derived_dataset.valid_from,
|
||||
valid_to=derived_dataset.valid_to,
|
||||
checksum_sha256=derived_dataset.checksum_sha256,
|
||||
source_metadata=derived_dataset.source_metadata,
|
||||
provenance_metadata=derived_dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(derived_dataset)
|
||||
if persist_vector_features:
|
||||
|
||||
@@ -7,7 +7,7 @@ import importlib
|
||||
|
||||
from geoalchemy2.shape import from_shape
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset
|
||||
from app.models import Area, Dataset, DatasetVersion
|
||||
from app.services.raster_operations_service import RasterOperationsService
|
||||
from app.api.routes.datasets import _run_job_sync
|
||||
from shapely.geometry import box
|
||||
@@ -356,8 +356,12 @@ def test_raster_reproject_returns_persisted_derived_dataset(monkeypatch, tmp_pat
|
||||
)
|
||||
|
||||
assert result_id == output_id
|
||||
assert len(db.added) == 1
|
||||
assert len(db.added) == 2
|
||||
derived = db.added[0]
|
||||
version = db.added[1]
|
||||
assert isinstance(version, DatasetVersion)
|
||||
assert version.dataset_id == output_id
|
||||
assert version.version == 1
|
||||
assert derived.id == output_id
|
||||
assert derived.metadata_json is not None
|
||||
assert derived.metadata_json["operation"] == "raster.reproject"
|
||||
@@ -783,9 +787,13 @@ def test_raster_clip_persists_derived_dataset(monkeypatch, tmp_path) -> None:
|
||||
result_id = RasterOperationsService.clip(db, dataset_id, area_id, "clip-result.tif")
|
||||
|
||||
assert result_id == output_id
|
||||
assert len(db.added) == 1
|
||||
assert len(db.added) == 2
|
||||
derived = db.added[0]
|
||||
version = db.added[1]
|
||||
assert isinstance(derived, Dataset)
|
||||
assert isinstance(version, DatasetVersion)
|
||||
assert version.dataset_id == output_id
|
||||
assert version.version == 1
|
||||
assert derived.id == output_id
|
||||
assert derived.source == "operation:raster.clip"
|
||||
assert derived.dataset_type == "raster"
|
||||
@@ -1295,8 +1303,12 @@ def test_raster_index_records_provenance_and_dtype(tmp_path, monkeypatch) -> Non
|
||||
|
||||
result_dataset_id = RasterOperationsService.ndvi(db, dataset_id, nir_band=4, red_band=3, output_name="ndvi-test")
|
||||
assert result_dataset_id == output_dataset_id
|
||||
assert len(db.added) == 1
|
||||
assert len(db.added) == 2
|
||||
derived = db.added[0]
|
||||
version = db.added[1]
|
||||
assert isinstance(version, DatasetVersion)
|
||||
assert version.dataset_id == output_dataset_id
|
||||
assert version.version == 1
|
||||
assert derived.id == output_dataset_id
|
||||
assert derived.metadata_json is not None
|
||||
assert derived.metadata_json["operation"] == "raster.ndvi"
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, DatasetVersion
|
||||
from app.schemas.dataset import DatasetTemporalUpdate
|
||||
from app.schemas.temporal import TemporalComparisonRequest, TemporalObjectChanges
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.temporal_analysis_service import TemporalAnalysisService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
|
||||
|
||||
class ScalarQuery:
|
||||
def __init__(self, value: float):
|
||||
self.value = value
|
||||
|
||||
def filter(self, *args): # noqa: ANN002, ARG002
|
||||
return self
|
||||
|
||||
def scalar(self):
|
||||
return self.value
|
||||
|
||||
|
||||
class ScalarSession:
|
||||
def __init__(self, value: float):
|
||||
self.value = value
|
||||
|
||||
def query(self, *args): # noqa: ANN002, ARG002
|
||||
return ScalarQuery(self.value)
|
||||
|
||||
|
||||
class VersionQuery:
|
||||
def __init__(self, latest: DatasetVersion | None):
|
||||
self.latest = latest
|
||||
|
||||
def filter(self, *args): # noqa: ANN002, ARG002
|
||||
return self
|
||||
|
||||
def order_by(self, *args): # noqa: ANN002, ARG002
|
||||
return self
|
||||
|
||||
def first(self):
|
||||
return self.latest
|
||||
|
||||
|
||||
class TemporalUpdateSession:
|
||||
def __init__(self, dataset: Dataset, latest: DatasetVersion | None):
|
||||
self.dataset = dataset
|
||||
self.latest = latest
|
||||
self.added: list[object] = []
|
||||
|
||||
def get(self, model, item_id): # noqa: ANN001
|
||||
return self.dataset if model is Dataset and item_id == self.dataset.id else None
|
||||
|
||||
def query(self, model): # noqa: ANN001
|
||||
assert model is DatasetVersion
|
||||
return VersionQuery(self.latest)
|
||||
|
||||
def add(self, item): # noqa: ANN001
|
||||
self.added.append(item)
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
def refresh(self, _item):
|
||||
return None
|
||||
|
||||
|
||||
def temporal_dataset(*, project_id, observed_year: int, metric_method: str = "feature_count") -> Dataset:
|
||||
return Dataset(
|
||||
id=uuid4(),
|
||||
project_id=project_id,
|
||||
name=f"snapshot-{observed_year}.geojson",
|
||||
dataset_type="vector",
|
||||
source="official",
|
||||
dataset_role="reference",
|
||||
temporal_series_key="official:test:mol",
|
||||
observed_at=datetime(observed_year, 1, 1, tzinfo=timezone.utc),
|
||||
source_version=str(observed_year),
|
||||
source_metadata={
|
||||
"selection_aggregation": {
|
||||
"method": metric_method,
|
||||
"label": "Objecten",
|
||||
"unit": "objecten",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_temporal_migration_and_models_align() -> None:
|
||||
migration = (ROOT / "backend/alembic/versions/202607140001_temporal_dataset_foundation.py").read_text(encoding="utf-8")
|
||||
for field in (
|
||||
"temporal_series_key",
|
||||
"observed_at",
|
||||
"valid_from",
|
||||
"valid_to",
|
||||
"temporal_granularity",
|
||||
"source_version",
|
||||
):
|
||||
assert field in migration
|
||||
assert hasattr(Dataset, field)
|
||||
assert "ix_vector_features_dataset_source_feature" in migration
|
||||
assert 'down_revision = "202606120900"' in migration
|
||||
|
||||
|
||||
def test_temporal_metadata_requires_an_explicit_series_and_observation_date() -> None:
|
||||
with pytest.raises(AppError, match="observed_at is required"):
|
||||
DatasetService._validate_temporal_metadata(
|
||||
temporal_series_key="official:test:mol",
|
||||
observed_at=None,
|
||||
valid_from=None,
|
||||
valid_to=None,
|
||||
temporal_granularity="year",
|
||||
source_version="2024",
|
||||
)
|
||||
with pytest.raises(AppError, match="valid_to must be"):
|
||||
DatasetService._validate_temporal_metadata(
|
||||
temporal_series_key="official:test:mol",
|
||||
observed_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
valid_from=datetime(2024, 12, 31, tzinfo=timezone.utc),
|
||||
valid_to=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
temporal_granularity="year",
|
||||
source_version="2024",
|
||||
)
|
||||
|
||||
|
||||
def test_temporal_metadata_update_appends_provenance_version_and_is_idempotent() -> None:
|
||||
project_id = uuid4()
|
||||
dataset = temporal_dataset(project_id=project_id, observed_year=2024)
|
||||
dataset.status = "ready"
|
||||
dataset.metadata_json = {}
|
||||
latest = DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=3,
|
||||
observed_at=dataset.observed_at,
|
||||
source_version="2024",
|
||||
)
|
||||
session = TemporalUpdateSession(dataset, latest)
|
||||
payload = DatasetTemporalUpdate(
|
||||
temporal_series_key="official:test:mol",
|
||||
observed_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||
temporal_granularity="year",
|
||||
source_version="2025",
|
||||
)
|
||||
|
||||
updated = DatasetService.update_temporal_metadata(session, dataset.id, payload)
|
||||
|
||||
assert updated.observed_at == payload.observed_at
|
||||
assert latest.version == 3
|
||||
assert latest.observed_at == datetime(2024, 1, 1, tzinfo=timezone.utc)
|
||||
assert len(session.added) == 2
|
||||
appended = session.added[1]
|
||||
assert isinstance(appended, DatasetVersion)
|
||||
assert appended.version == 4
|
||||
assert appended.observed_at == payload.observed_at
|
||||
|
||||
session.added.clear()
|
||||
DatasetService.update_temporal_metadata(session, dataset.id, payload)
|
||||
assert session.added == []
|
||||
|
||||
|
||||
def test_selection_area_aggregation_returns_hectares_without_loading_all_features() -> None:
|
||||
project_id = uuid4()
|
||||
dataset = temporal_dataset(project_id=project_id, observed_year=1969, metric_method="intersection_area")
|
||||
dataset.source_metadata["selection_aggregation"].update({"label": "Oppervlakte", "unit": "ha"})
|
||||
result = VectorFeatureService.summarize_features_by_bbox(
|
||||
ScalarSession(125_000.0),
|
||||
dataset=dataset,
|
||||
bbox={"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3, "crs": "EPSG:4326"},
|
||||
total_feature_count=40,
|
||||
)
|
||||
assert result["metric_value"] == 12.5
|
||||
assert result["metric_unit"] == "ha"
|
||||
assert result["feature_count"] == 40
|
||||
|
||||
|
||||
def test_temporal_compare_returns_delta_and_canonical_change_payload(monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
earlier = temporal_dataset(project_id=project_id, observed_year=2021)
|
||||
later = temporal_dataset(project_id=project_id, observed_year=2024)
|
||||
|
||||
def get_dataset(_db, _project_id, dataset_id, _label):
|
||||
return earlier if dataset_id == earlier.id else later
|
||||
|
||||
def summarize(_db, *, dataset, bbox): # noqa: ARG001
|
||||
value = 100.0 if dataset.id == earlier.id else 115.0
|
||||
return {
|
||||
"metric_label": "Inwoners",
|
||||
"metric_value": value,
|
||||
"metric_unit": "inwoners",
|
||||
"aggregation_method": "area_weighted_sum",
|
||||
"feature_count": 10,
|
||||
"is_estimate": True,
|
||||
"warning": "Areal weighting",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(TemporalAnalysisService, "_get_temporal_dataset", staticmethod(get_dataset))
|
||||
monkeypatch.setattr(VectorFeatureService, "summarize_features_by_bbox", staticmethod(summarize))
|
||||
monkeypatch.setattr(
|
||||
TemporalAnalysisService,
|
||||
"_compare_identity_features",
|
||||
staticmethod(
|
||||
lambda *args, **kwargs: (
|
||||
TemporalObjectChanges(available=True, added_count=1, removed_count=0, modified_count=2, unchanged_count=7),
|
||||
{"type": "FeatureCollection", "features": []},
|
||||
[],
|
||||
)
|
||||
),
|
||||
)
|
||||
result = TemporalAnalysisService.compare(
|
||||
SimpleNamespace(),
|
||||
project_id=project_id,
|
||||
payload=TemporalComparisonRequest(
|
||||
earlier_dataset_id=earlier.id,
|
||||
later_dataset_id=later.id,
|
||||
bbox={"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3},
|
||||
),
|
||||
)
|
||||
assert result.metric.absolute_change == 15.0
|
||||
assert result.metric.percent_change == 15.0
|
||||
assert result.metric.is_estimate is True
|
||||
assert result.object_changes.modified_count == 2
|
||||
assert result.geojson["type"] == "FeatureCollection"
|
||||
|
||||
|
||||
def test_temporal_frontend_and_official_operator_contracts_exist() -> None:
|
||||
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
|
||||
temporal_api = (ROOT / "frontend/src/services/api/temporal.ts").read_text(encoding="utf-8")
|
||||
population = (ROOT / "scripts/provision_mol_population_history.py").read_text(encoding="utf-8")
|
||||
landuse = (ROOT / "scripts/provision_mol_historical_landuse.py").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
|
||||
assert "Laatste toestand" in workspace
|
||||
assert "Evolutie" in workspace
|
||||
assert "Vergelijk periode" in workspace
|
||||
assert "/temporal/compare" in temporal_api
|
||||
assert "Statbel" in population and "area_weighted_sum" in population
|
||||
assert "HistLandgebruik" in landuse and "intersection_area" in landuse
|
||||
assert "provision_mol_population_history.py" in dockerfile
|
||||
assert "provision_mol_historical_landuse.py" in dockerfile
|
||||
assert "fake" not in population.lower()
|
||||
@@ -74,6 +74,8 @@ RUN python scripts/gis_import_smoke.py \
|
||||
COPY scripts/prepare_operator_real_data_samples.py /app/scripts/prepare_operator_real_data_samples.py
|
||||
COPY scripts/provision_mol_municipality_workspace.py /app/scripts/provision_mol_municipality_workspace.py
|
||||
COPY scripts/provision_mol_context_layers.py /app/scripts/provision_mol_context_layers.py
|
||||
COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py
|
||||
COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py
|
||||
COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py
|
||||
COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
|
||||
COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
|
||||
|
||||
@@ -1393,3 +1393,53 @@ GET /api/v1/exports/{export_id}/download
|
||||
```
|
||||
|
||||
PDF/report-designer functionality can be added after core GeoAI workflows work.
|
||||
|
||||
## Temporal datasets and area evolution
|
||||
|
||||
Temporal metadata describes the source observation, not API run history. A
|
||||
snapshot is a normal persisted dataset grouped by `temporal_series_key` and
|
||||
ordered by `observed_at`. Optional validity uses `valid_from` and `valid_to`;
|
||||
`temporal_granularity` is `snapshot`, `day`, `month`, `year` or `period`.
|
||||
|
||||
Dataset upload accepts those temporal fields plus `source_version`. When a
|
||||
dataset declares `source_metadata.selection_aggregation`, the vector bbox
|
||||
selection response also contains a `summary` with metric label/value/unit,
|
||||
aggregation method, feature count, estimate status and an optional warning.
|
||||
Supported PostGIS aggregations are feature count, intersection area,
|
||||
intersection length, numeric sum and area-weighted numeric sum. Area and length
|
||||
are measured after transformation to EPSG:31370.
|
||||
|
||||
### PATCH `/api/v1/projects/{project_id}/datasets/{dataset_id}/temporal`
|
||||
|
||||
Updates the temporal provenance of an existing dataset. Series key and
|
||||
observation date are required together. It does not alter features or
|
||||
manufacture a historical observation.
|
||||
|
||||
### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/versions`
|
||||
|
||||
Lists immutable storage/provenance versions. Uploads and derived datasets
|
||||
create version 1 in the same persistence transaction.
|
||||
|
||||
### GET `/api/v1/projects/{project_id}/temporal/series`
|
||||
|
||||
Returns dated project series in the canonical envelope. Each item contains its
|
||||
source/layer identity, first and last observations and ordered datasets.
|
||||
|
||||
### POST `/api/v1/projects/{project_id}/temporal/compare`
|
||||
|
||||
```json
|
||||
{
|
||||
"earlier_dataset_id": "uuid",
|
||||
"later_dataset_id": "uuid",
|
||||
"bbox": {"west": 5.0, "south": 51.0, "east": 5.2, "north": 51.2},
|
||||
"preview_limit": 500
|
||||
}
|
||||
```
|
||||
|
||||
Both datasets must belong to the project and the same temporal series, with
|
||||
the earlier observation preceding the later one. The response contains source
|
||||
snapshot references, selection bbox, earlier/later metric values,
|
||||
absolute/percentage change, estimate status, warnings and GeoJSON evidence.
|
||||
Added/removed/modified object changes are calculated only when source
|
||||
provenance declares stable feature identities; otherwise
|
||||
`object_changes.available=false` and no object history is inferred.
|
||||
|
||||
@@ -7768,3 +7768,29 @@ Limitations:
|
||||
|
||||
Next:
|
||||
- Define authoritative population and land-cover source adapters, then reuse the proven municipality provisioner and bbox analysis flow for the complete Kempen.
|
||||
|
||||
## Sprint 187 Temporal Mol explorer (2026-07-14)
|
||||
|
||||
Implemented:
|
||||
- Added observation/validity/source-version fields to datasets and immutable provenance fields to dataset versions, with one Alembic head and indexed temporal/source identity lookups.
|
||||
- Persisted dataset version 1 atomically for uploads, demo fixtures and derived vector/raster operations.
|
||||
- Added source-governed PostGIS selection summaries for object count, area, length, numeric sum and area-weighted sum.
|
||||
- Added project temporal-series discovery and same-series bbox comparison with honest metric deltas and stable-identity-only object changes.
|
||||
- Added the map-first `Laatste toestand` / `Evolutie` workflow with period selection, automatic rectangle analysis and change overlays.
|
||||
- Added explicit, idempotent Statbel population (2021-2025) and Digitaal Vlaanderen historical land-use (1778/1873/1969) provisioners for Mol.
|
||||
- Kept all source fetching operator-triggered; application startup and user queries never fabricate or silently download source data.
|
||||
|
||||
Methodology:
|
||||
- Population totals are source-published per statistical sector. Intersections with partial sectors are labelled area-weighted estimates.
|
||||
- Historical land-use classes are clipped from official editions and measured in EPSG:31370. They do not claim stable cadastral object identity.
|
||||
- Every snapshot carries its source URL, observation date, source version, checksum and processing limitations.
|
||||
|
||||
Validation before live deployment:
|
||||
- Script compilation passed.
|
||||
- New temporal/API/static regression suite passed 6 tests, including append-only/idempotent temporal provenance updates.
|
||||
- Raster and temporal focused suite passed 27 tests after extending existing assertions to require dataset-version persistence.
|
||||
- Frontend TypeScript typecheck and production build passed.
|
||||
- Offline Alembic SQL generation passed with head `202607140001`.
|
||||
|
||||
Next:
|
||||
- Run the complete readiness gate, deploy to Tower/PostGIS, provision the official snapshots and verify current/evolution selection end to end in the internal browser.
|
||||
|
||||
@@ -236,3 +236,22 @@ GRB and OSM live imports are intentionally `not_configured` in Sprint 7B. Manual
|
||||
- Multi-tenant row-level security.
|
||||
- User accounts.
|
||||
- Full model registry tables.
|
||||
|
||||
## Temporal dataset foundation
|
||||
|
||||
Historical observations remain normal `datasets` and `vector_features`; there
|
||||
is no parallel temporal feature store. Snapshots are grouped by
|
||||
`datasets.temporal_series_key` and carry `observed_at`, `valid_from`,
|
||||
`valid_to`, `temporal_granularity` and `source_version`. Observation time is
|
||||
kept separate from ingestion time (`imported_at`).
|
||||
|
||||
`dataset_versions` records immutable storage provenance for every upload and
|
||||
derived output: dataset-local `version`, storage path, source version,
|
||||
observation/validity dates, checksum and source/provenance JSON. The
|
||||
`(dataset_id, version)` pair is unique. Temporal series lookup is indexed by
|
||||
`(project_id, temporal_series_key, observed_at)` and source-feature lookup by
|
||||
`(dataset_id, source_feature_id)`.
|
||||
|
||||
Time-series comparison is read-only and aggregates persisted geometry inside a
|
||||
requested bbox. Object-level added/removed/modified evidence is only valid for
|
||||
sources that explicitly declare stable source feature identifiers.
|
||||
|
||||
@@ -138,3 +138,20 @@ V1 dataset strategy is complete when:
|
||||
- an area can request/cache a reference building layer;
|
||||
- detection outputs can be compared with that reference layer;
|
||||
- exports include source metadata.
|
||||
|
||||
## Temporal snapshots and evolution
|
||||
|
||||
- A historical observation is one persisted dataset snapshot. Existing
|
||||
datasets are not overwritten and features are not hidden in job JSON.
|
||||
- Related observations share a stable `temporal_series_key`; `observed_at`
|
||||
records when the source describes reality, while `imported_at` records
|
||||
ingestion time.
|
||||
- The latest-state map uses the latest available observation but must not call
|
||||
an old source edition current reality.
|
||||
- Evolution metrics use the same selection geometry, aggregation and units for
|
||||
both snapshots.
|
||||
- Partial-sector population is an area-weighted estimate. GeoIntel must not
|
||||
imply address-level distribution when only sector totals are available.
|
||||
- Historical cartographic classes can change meaning between editions. Source
|
||||
classes and processing notes remain provenance, and object changes require
|
||||
explicit stable source identity.
|
||||
|
||||
@@ -29,6 +29,32 @@ start geen verborgen providerfetch. De resulterende GeoJSON-artefacten,
|
||||
checksums en bron-URL's worden onder persistent operator storage bewaard en
|
||||
via de bestaande DatasetService/vectorfeature-flow geïmporteerd.
|
||||
|
||||
### Mol population history
|
||||
|
||||
`scripts/provision_mol_population_history.py` imports official Statbel
|
||||
population-by-statistical-sector tables and matching sector geometries for
|
||||
2021 through 2025. It keeps the published sector total as
|
||||
`population_total`, clips the official geometry to Mol NIS `13025` and uploads
|
||||
each year as a separate dataset in
|
||||
`statbel:population-statistical-sector:mol`.
|
||||
|
||||
Complete sectors use their published population total. A rectangle that cuts
|
||||
through a sector uses an explicitly labelled area-weighted estimate; the
|
||||
source does not justify a more precise intra-sector distribution.
|
||||
|
||||
### Mol historical land use
|
||||
|
||||
`scripts/provision_mol_historical_landuse.py` uses the official Digitaal
|
||||
Vlaanderen Historical Land Use WFS for the 1778, 1873 and 1969 collections.
|
||||
Buildings, forest, water and roads are filtered server-side, clipped to the
|
||||
official Mol boundary and uploaded through the existing dataset service.
|
||||
Source classes, request URLs, observation year, simplification tolerance and
|
||||
methodological limitations remain in provenance.
|
||||
|
||||
These historical map editions support exploratory area evolution, not
|
||||
cadastral object lineage. Their feature identities are declared unstable and
|
||||
GeoIntel does not fabricate added/removed object counts.
|
||||
|
||||
## OSM
|
||||
|
||||
- Naam: OpenStreetMap
|
||||
|
||||
@@ -4,6 +4,19 @@ React + TypeScript + MapLibre foundation for project/area/dataset workflow.
|
||||
|
||||
Mol is the primary operating context. On a fresh session the application opens the map-first geographic explorer, prefers the persisted `Mol Municipality Workbench`, selects the official NIS `13025` municipality boundary and activates the largest available authoritative building layer. Explicit project and dataset selections remain authoritative, and all broader Kempen workflows remain available.
|
||||
|
||||
The map-first explorer has two deliberate modes. `Latest state` selects the
|
||||
latest explicitly dated source snapshot without claiming an old edition is
|
||||
current, while `Evolution` lets the operator compare an earlier and later
|
||||
snapshot from the same series over a drawn rectangle. Results show units,
|
||||
absolute/percentage change, estimate status and source limitations.
|
||||
Added/removed/modified overlays only appear for stable source identities.
|
||||
|
||||
Selection results use dataset-specific PostGIS summaries. Object layers show
|
||||
intersecting counts, population shows inhabitants with partial-sector
|
||||
estimates clearly marked and land-cover sources show intersected hectares. The
|
||||
advanced workbench remains available but is not required for the primary
|
||||
choose-theme, draw-area, read-result flow.
|
||||
|
||||
The primary workflow is deliberately short: choose a data theme, drag a rectangle on the MapLibre map and read the resulting PostGIS evidence. Releasing the drag runs the active theme query and every other available theme query for the same EPSG:4326 bbox. The result panel shows selection area, exact intersection totals, active-theme density, source identity and bounded feature properties. Map rendering remains capped at 1,000 features while `total_feature_count` reports the exact database count.
|
||||
|
||||
The theme catalog currently recognizes buildings, population, forest/green, water, roads and parcels from dataset names and canonical `reference_layer_name` metadata. A theme is enabled only when a ready persisted vector dataset exists; otherwise it states `Bron nog niet ingeladen`. This prevents missing population or land-cover sources from appearing as zero-valued observations. The previous technical Map workspace remains available through `Geavanceerde werkbank` for derived datasets, QA/QC evidence and export operations.
|
||||
|
||||
@@ -319,6 +319,8 @@ function GeoMap({
|
||||
'#16a34a',
|
||||
'removed',
|
||||
'#dc2626',
|
||||
'modified',
|
||||
'#d97706',
|
||||
'unchanged',
|
||||
'#2563eb',
|
||||
'#f97316',
|
||||
@@ -345,6 +347,8 @@ function GeoMap({
|
||||
'#15803d',
|
||||
'removed',
|
||||
'#b91c1c',
|
||||
'modified',
|
||||
'#b45309',
|
||||
'unchanged',
|
||||
'#1d4ed8',
|
||||
'#ea580c',
|
||||
|
||||
@@ -3,6 +3,7 @@ import GeoMap from '../GeoMap'
|
||||
import type { AreaRead, DatasetCreateResponse, MapViewportState, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types'
|
||||
import { featureCollectionBounds } from '../../lib/geojsonBounds'
|
||||
import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights'
|
||||
import { useTemporalComparison } from '../../hooks/useTemporalComparison'
|
||||
|
||||
const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson'
|
||||
const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson'
|
||||
@@ -90,12 +91,42 @@ function pickThemeDataset(datasets: DatasetCreateResponse[], theme: DataTheme):
|
||||
(dataset.reference_layer_name && theme.tokens.includes(dataset.reference_layer_name.toLowerCase()) ? 1_000_000 : 0) +
|
||||
(dataset.source_name === 'grb' ? 100_000 : 0) +
|
||||
(dataset.dataset_role === 'reference' ? 10_000 : 0) +
|
||||
(dataset.observed_at ? new Date(dataset.observed_at).getTime() / 100_000_000 : 0) +
|
||||
(dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0)
|
||||
return score(right) - score(left)
|
||||
})
|
||||
return candidates[0] ?? null
|
||||
}
|
||||
|
||||
function pickThemeTemporalSeries(datasets: DatasetCreateResponse[], theme: DataTheme): DatasetCreateResponse[] {
|
||||
const groups = new Map<string, DatasetCreateResponse[]>()
|
||||
for (const dataset of datasets) {
|
||||
if (!datasetMatchesTheme(dataset, theme) || !dataset.temporal_series_key || !dataset.observed_at) {
|
||||
continue
|
||||
}
|
||||
const items = groups.get(dataset.temporal_series_key) ?? []
|
||||
items.push(dataset)
|
||||
groups.set(dataset.temporal_series_key, items)
|
||||
}
|
||||
return Array.from(groups.values())
|
||||
.filter((items) => items.length >= 2)
|
||||
.sort((left, right) => {
|
||||
if (right.length !== left.length) {
|
||||
return right.length - left.length
|
||||
}
|
||||
const latest = (items: DatasetCreateResponse[]) => Math.max(...items.map((item) => new Date(item.observed_at ?? 0).getTime()))
|
||||
return latest(right) - latest(left)
|
||||
})[0]
|
||||
?.sort((left, right) => new Date(left.observed_at ?? 0).getTime() - new Date(right.observed_at ?? 0).getTime()) ?? []
|
||||
}
|
||||
|
||||
function formatObservationDate(value: string | null | undefined): string {
|
||||
if (!value) {
|
||||
return 'Geen peildatum'
|
||||
}
|
||||
return new Intl.DateTimeFormat('nl-BE', { year: 'numeric', month: 'short', day: 'numeric' }).format(new Date(value))
|
||||
}
|
||||
|
||||
function selectionAreaSquareMetres(bbox: VectorSelectionBBox | null): number | null {
|
||||
if (!bbox) {
|
||||
return null
|
||||
@@ -134,6 +165,19 @@ function resultCountLabel(result: VectorSelectionResponse): string {
|
||||
return result.truncated && result.total_feature_count == null ? `${result.feature_count.toLocaleString('nl-BE')}+` : total.toLocaleString('nl-BE')
|
||||
}
|
||||
|
||||
function resultMetricLabel(result: VectorSelectionResponse): string {
|
||||
if (!result.summary) {
|
||||
return resultCountLabel(result)
|
||||
}
|
||||
const maximumFractionDigits = result.summary.metric_unit === 'inwoners' ? 0 : 2
|
||||
return `${result.summary.metric_value.toLocaleString('nl-BE', { maximumFractionDigits })} ${result.summary.metric_unit}`
|
||||
}
|
||||
|
||||
function formatTemporalMetric(value: number, unit: string): string {
|
||||
const maximumFractionDigits = unit === 'inwoners' || unit === 'objecten' ? 0 : 2
|
||||
return `${value.toLocaleString('nl-BE', { maximumFractionDigits })} ${unit}`
|
||||
}
|
||||
|
||||
function readablePropertyName(value: string): string {
|
||||
return value.replace(/_/g, ' ').replace(/\b\w/g, (character) => character.toUpperCase())
|
||||
}
|
||||
@@ -444,6 +488,16 @@ export function MapWorkspace({
|
||||
loadThemeInsights,
|
||||
clearThemeInsights,
|
||||
} = useMapThemeSelectionInsights<DataThemeId>(selectedProjectId)
|
||||
const {
|
||||
temporalComparison,
|
||||
temporalComparisonLoading,
|
||||
temporalComparisonError,
|
||||
compareTemporalSnapshots,
|
||||
clearTemporalComparison,
|
||||
} = useTemporalComparison(selectedProjectId)
|
||||
const [analysisMode, setAnalysisMode] = useState<'current' | 'evolution'>('current')
|
||||
const [earlierDatasetId, setEarlierDatasetId] = useState('')
|
||||
const [laterDatasetId, setLaterDatasetId] = useState('')
|
||||
const [bboxSelectionMode, setBboxSelectionMode] = useState(false)
|
||||
const [firstSelectionCorner, setFirstSelectionCorner] = useState<[number, number] | null>(null)
|
||||
const [bboxInput, setBboxInput] = useState(bboxToInputState(mapSelectionBbox))
|
||||
@@ -482,6 +536,10 @@ export function MapWorkspace({
|
||||
)
|
||||
const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0]
|
||||
const activeThemeDataset = themeDatasetMap[activeTheme.id]
|
||||
const activeTemporalSeries = useMemo(
|
||||
() => pickThemeTemporalSeries(availableMapDatasets, activeTheme),
|
||||
[activeTheme, availableMapDatasets],
|
||||
)
|
||||
const themeResults = useMemo(
|
||||
() =>
|
||||
themeInsights.flatMap((insight) => {
|
||||
@@ -502,6 +560,14 @@ export function MapWorkspace({
|
||||
const selectedDensity = selectedAreaSquareMetres && selectedAreaSquareMetres > 0
|
||||
? selectedResultTotal / (selectedAreaSquareMetres / 1_000_000)
|
||||
: null
|
||||
const activeMetricValue = activeSelectionResult?.summary?.metric_value ?? selectedResultTotal
|
||||
const activeMetricUnit = activeSelectionResult?.summary?.metric_unit ?? 'objecten'
|
||||
const activeMetricLabel = activeSelectionResult?.summary?.metric_label ?? activeTheme.shortLabel
|
||||
const activeSecondaryMetric = selectedAreaSquareMetres && selectedAreaSquareMetres > 0
|
||||
? activeMetricUnit === 'ha'
|
||||
? `${((activeMetricValue * 10_000) / selectedAreaSquareMetres * 100).toLocaleString('nl-BE', { maximumFractionDigits: 1 })}% dekking`
|
||||
: `${(activeMetricValue / (selectedAreaSquareMetres / 1_000_000)).toLocaleString('nl-BE', { maximumFractionDigits: 1 })} ${activeMetricUnit} / km2`
|
||||
: null
|
||||
const selectedResultProperties = useMemo(() => {
|
||||
const keys = new Map<string, Set<string>>()
|
||||
for (const feature of activeSelectionResult?.geojson.features ?? []) {
|
||||
@@ -526,6 +592,14 @@ export function MapWorkspace({
|
||||
setBboxInput(bboxToInputState(mapSelectionBbox))
|
||||
}, [mapSelectionBbox])
|
||||
|
||||
useEffect(() => {
|
||||
const first = activeTemporalSeries[0]
|
||||
const last = activeTemporalSeries[activeTemporalSeries.length - 1]
|
||||
setEarlierDatasetId(first?.id ?? '')
|
||||
setLaterDatasetId(last?.id ?? '')
|
||||
clearTemporalComparison()
|
||||
}, [activeTemporalSeries])
|
||||
|
||||
useEffect(() => {
|
||||
if (advancedMode || !activeThemeDataset || (selectedMapDataset && datasetMatchesTheme(selectedMapDataset, activeTheme))) {
|
||||
return
|
||||
@@ -562,6 +636,7 @@ export function MapWorkspace({
|
||||
const startBboxSelection = () => {
|
||||
setFirstSelectionCorner(null)
|
||||
clearThemeInsights()
|
||||
clearTemporalComparison()
|
||||
setBboxSelectionMode(true)
|
||||
}
|
||||
|
||||
@@ -590,6 +665,7 @@ export function MapWorkspace({
|
||||
setFirstSelectionCorner(null)
|
||||
setBboxInput(bboxToInputState(null))
|
||||
clearThemeInsights()
|
||||
clearTemporalComparison()
|
||||
onClearMapSelectionExtract()
|
||||
}
|
||||
|
||||
@@ -644,9 +720,15 @@ export function MapWorkspace({
|
||||
return
|
||||
}
|
||||
setActiveThemeId(theme.id)
|
||||
clearTemporalComparison()
|
||||
onOpenDatasetInMap(dataset)
|
||||
}
|
||||
|
||||
const setExplorerMode = (mode: 'current' | 'evolution') => {
|
||||
setAnalysisMode(mode)
|
||||
clearTemporalComparison()
|
||||
}
|
||||
|
||||
const loadAllThemeResults = async (bbox: VectorSelectionBBox) => {
|
||||
const availableThemes = DATA_THEMES.flatMap((theme) => {
|
||||
const dataset = themeDatasetMap[theme.id]
|
||||
@@ -657,7 +739,18 @@ export function MapWorkspace({
|
||||
|
||||
const analyzeSelection = async (bbox: VectorSelectionBBox) => {
|
||||
setSelectionBbox(bbox)
|
||||
await Promise.all([onRunMapSelectionExtract(bbox), loadAllThemeResults(bbox)])
|
||||
const tasks: Array<Promise<unknown>> = [onRunMapSelectionExtract(bbox), loadAllThemeResults(bbox)]
|
||||
if (analysisMode === 'evolution' && earlierDatasetId && laterDatasetId) {
|
||||
tasks.push(compareTemporalSnapshots(earlierDatasetId, laterDatasetId, bbox))
|
||||
}
|
||||
await Promise.all(tasks)
|
||||
}
|
||||
|
||||
const runTemporalComparison = () => {
|
||||
if (!mapSelectionBbox || !earlierDatasetId || !laterDatasetId) {
|
||||
return
|
||||
}
|
||||
void compareTemporalSnapshots(earlierDatasetId, laterDatasetId, mapSelectionBbox)
|
||||
}
|
||||
|
||||
const handleMapBboxPreview = (bbox: VectorSelectionBBox) => {
|
||||
@@ -757,6 +850,26 @@ export function MapWorkspace({
|
||||
<h2>Wat bevindt zich in dit gebied?</h2>
|
||||
<p>Kies een datathema, teken een rechthoek en lees de beschikbare gegevens meteen uit.</p>
|
||||
</div>
|
||||
<div className="geo-analysis-mode" role="tablist" aria-label="Analyseperiode">
|
||||
<button
|
||||
className={analysisMode === 'current' ? 'active' : ''}
|
||||
type="button"
|
||||
role="tab"
|
||||
aria-selected={analysisMode === 'current'}
|
||||
onClick={() => setExplorerMode('current')}
|
||||
>
|
||||
Laatste toestand
|
||||
</button>
|
||||
<button
|
||||
className={analysisMode === 'evolution' ? 'active' : ''}
|
||||
type="button"
|
||||
role="tab"
|
||||
aria-selected={analysisMode === 'evolution'}
|
||||
onClick={() => setExplorerMode('evolution')}
|
||||
>
|
||||
Evolutie
|
||||
</button>
|
||||
</div>
|
||||
<button className="secondary-action geo-explorer-advanced" type="button" onClick={() => setAdvancedMode(true)}>
|
||||
Geavanceerde werkbank
|
||||
</button>
|
||||
@@ -796,15 +909,52 @@ export function MapWorkspace({
|
||||
</div>
|
||||
|
||||
<div className="geo-source-summary">
|
||||
<span>Actieve bron</span>
|
||||
<strong>{activeThemeDataset?.name ?? 'Geen databron beschikbaar'}</strong>
|
||||
<span>{analysisMode === 'evolution' ? 'Tijdreeks' : 'Actieve bron'}</span>
|
||||
<strong>
|
||||
{analysisMode === 'evolution'
|
||||
? activeTemporalSeries[0]?.temporal_series_key ?? 'Geen tijdreeks beschikbaar'
|
||||
: activeThemeDataset?.name ?? 'Geen databron beschikbaar'}
|
||||
</strong>
|
||||
<small>
|
||||
{activeThemeDataset
|
||||
? `${activeThemeDataset.source_name ?? activeThemeDataset.source} · ${activeThemeDataset.dataset_role ?? 'source'} · EPSG:4326`
|
||||
: activeTheme.description}
|
||||
{analysisMode === 'evolution'
|
||||
? activeTemporalSeries.length >= 2
|
||||
? `${activeTemporalSeries.length} officiële meetmomenten · ${formatObservationDate(activeTemporalSeries[0].observed_at)} tot ${formatObservationDate(activeTemporalSeries[activeTemporalSeries.length - 1].observed_at)}`
|
||||
: 'Minstens twee expliciet gedateerde snapshots zijn vereist.'
|
||||
: activeThemeDataset
|
||||
? `${activeThemeDataset.source_name ?? activeThemeDataset.source} · ${activeThemeDataset.dataset_role ?? 'source'} · ${formatObservationDate(activeThemeDataset.observed_at)}`
|
||||
: activeTheme.description}
|
||||
</small>
|
||||
</div>
|
||||
|
||||
{analysisMode === 'evolution' ? (
|
||||
<div className="geo-time-controls" aria-label="Meetmomenten vergelijken">
|
||||
<label>
|
||||
Van
|
||||
<select value={earlierDatasetId} onChange={(event) => { setEarlierDatasetId(event.target.value); clearTemporalComparison() }} disabled={activeTemporalSeries.length < 2}>
|
||||
{activeTemporalSeries.map((dataset) => (
|
||||
<option key={dataset.id} value={dataset.id}>{formatObservationDate(dataset.observed_at)}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
Naar
|
||||
<select value={laterDatasetId} onChange={(event) => { setLaterDatasetId(event.target.value); clearTemporalComparison() }} disabled={activeTemporalSeries.length < 2}>
|
||||
{activeTemporalSeries.map((dataset) => (
|
||||
<option key={dataset.id} value={dataset.id}>{formatObservationDate(dataset.observed_at)}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<button
|
||||
className="primary-action"
|
||||
type="button"
|
||||
disabled={!mapSelectionBbox || !earlierDatasetId || !laterDatasetId || temporalComparisonLoading}
|
||||
onClick={runTemporalComparison}
|
||||
>
|
||||
{temporalComparisonLoading ? 'Vergelijken…' : 'Vergelijk periode'}
|
||||
</button>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<label className="geo-scope-select">
|
||||
Werkgebied
|
||||
<select value={selectedMapAreaId} onChange={(event) => onSelectMapArea(event.target.value)} disabled={areas.length === 0}>
|
||||
@@ -828,7 +978,7 @@ export function MapWorkspace({
|
||||
<div className="geo-map-actions">
|
||||
<button
|
||||
className={bboxSelectionMode ? 'primary-action geo-draw-active' : 'primary-action'}
|
||||
disabled={!activeThemeDataset || mapSelectionLoading || themeResultsLoading}
|
||||
disabled={!activeThemeDataset || (analysisMode === 'evolution' && activeTemporalSeries.length < 2) || mapSelectionLoading || themeResultsLoading}
|
||||
type="button"
|
||||
onClick={startBboxSelection}
|
||||
>
|
||||
@@ -836,7 +986,7 @@ export function MapWorkspace({
|
||||
</button>
|
||||
<button
|
||||
className="secondary-action"
|
||||
disabled={!activeThemeDataset || !selectedAreaBbox || mapSelectionLoading || themeResultsLoading}
|
||||
disabled={!activeThemeDataset || (analysisMode === 'evolution' && activeTemporalSeries.length < 2) || !selectedAreaBbox || mapSelectionLoading || themeResultsLoading}
|
||||
type="button"
|
||||
onClick={() => selectedAreaBbox && void analyzeSelection(selectedAreaBbox)}
|
||||
>
|
||||
@@ -850,10 +1000,10 @@ export function MapWorkspace({
|
||||
|
||||
<div className={bboxSelectionMode ? 'geo-map-canvas geo-map-canvas-drawing' : 'geo-map-canvas'}>
|
||||
<GeoMap
|
||||
data={mapFeatureCollection}
|
||||
data={analysisMode === 'evolution' && temporalComparison?.geojson.features.length ? temporalComparison.geojson : mapFeatureCollection}
|
||||
areaData={areaFeatureCollection}
|
||||
selectedFeature={selectedFeature}
|
||||
selectionData={mapSelectionResult?.geojson ?? null}
|
||||
selectionData={analysisMode === 'current' ? mapSelectionResult?.geojson ?? null : null}
|
||||
selectionBbox={mapSelectionBbox}
|
||||
bboxSelectionMode={bboxSelectionMode}
|
||||
visible={mapLayerVisible}
|
||||
@@ -869,8 +1019,18 @@ export function MapWorkspace({
|
||||
/>
|
||||
<div className="geo-map-legend" aria-label="Kaartlegende">
|
||||
<span><i className="geo-legend-area" /> Gemeentegrens</span>
|
||||
<span><i className="geo-legend-layer" /> {activeTheme.shortLabel}</span>
|
||||
<span><i className="geo-legend-selection" /> Selectie</span>
|
||||
{analysisMode === 'evolution' && temporalComparison?.object_changes.available ? (
|
||||
<>
|
||||
<span><i className="geo-legend-added" /> Nieuw</span>
|
||||
<span><i className="geo-legend-removed" /> Verdwenen</span>
|
||||
<span><i className="geo-legend-modified" /> Gewijzigd</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<span><i className="geo-legend-layer" /> {activeTheme.shortLabel}</span>
|
||||
<span><i className="geo-legend-selection" /> Selectie</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
{bboxSelectionMode ? (
|
||||
<div className="geo-draw-instruction" role="status">
|
||||
@@ -900,56 +1060,117 @@ export function MapWorkspace({
|
||||
<strong>Nog geen gebied geselecteerd</strong>
|
||||
<p>Teken een rechthoek op de kaart. De analyse start automatisch zodra je loslaat.</p>
|
||||
</div>
|
||||
) : mapSelectionLoading || themeResultsLoading ? (
|
||||
) : mapSelectionLoading || themeResultsLoading || temporalComparisonLoading ? (
|
||||
<div className="geo-results-loading" role="status">
|
||||
<span />
|
||||
<strong>Gegevens worden uit PostGIS gelezen…</strong>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<div className="geo-primary-metrics">
|
||||
<div>
|
||||
<span>Oppervlakte selectie</span>
|
||||
<strong>{formatArea(selectedAreaSquareMetres)}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span>{activeTheme.shortLabel}</span>
|
||||
<strong>{activeSelectionResult ? resultCountLabel(activeSelectionResult) : 'Geen resultaat'}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span>Dichtheid</span>
|
||||
<strong>{selectedDensity === null ? 'n.v.t.' : `${selectedDensity.toLocaleString('nl-BE', { maximumFractionDigits: 1 })} / km2`}</strong>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="geo-theme-results">
|
||||
<div className="geo-results-title-row">
|
||||
<h4>Alle beschikbare thema’s</h4>
|
||||
<span>{themeResults.length} bevraagd</span>
|
||||
</div>
|
||||
{DATA_THEMES.map((theme) => {
|
||||
const dataset = themeDatasetMap[theme.id]
|
||||
const item = themeResults.find((result) => result.theme.id === theme.id)
|
||||
return (
|
||||
<div className="geo-theme-result-row" key={theme.id}>
|
||||
<span className={`geo-theme-symbol geo-theme-symbol-${theme.id}`} aria-hidden="true" />
|
||||
<span>
|
||||
<strong>{theme.label}</strong>
|
||||
<small>{dataset?.source_name ?? dataset?.source ?? 'Geen bron gekoppeld'}</small>
|
||||
</span>
|
||||
<b>{item ? resultCountLabel(item.result) : dataset ? 'Niet bevraagd' : 'Bron ontbreekt'}</b>
|
||||
{analysisMode === 'evolution' ? (
|
||||
temporalComparison ? (
|
||||
<>
|
||||
<div className="geo-primary-metrics geo-temporal-metrics">
|
||||
<div>
|
||||
<span>{formatObservationDate(temporalComparison.earlier.observed_at)}</span>
|
||||
<strong>{formatTemporalMetric(temporalComparison.metric.earlier_value, temporalComparison.metric.unit)}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span>{formatObservationDate(temporalComparison.later.observed_at)}</span>
|
||||
<strong>{formatTemporalMetric(temporalComparison.metric.later_value, temporalComparison.metric.unit)}</strong>
|
||||
</div>
|
||||
<div className={temporalComparison.metric.absolute_change >= 0 ? 'positive' : 'negative'}>
|
||||
<span>Verschil</span>
|
||||
<strong>
|
||||
{temporalComparison.metric.absolute_change >= 0 ? '+' : ''}
|
||||
{formatTemporalMetric(temporalComparison.metric.absolute_change, temporalComparison.metric.unit)}
|
||||
</strong>
|
||||
<small>
|
||||
{temporalComparison.metric.percent_change == null
|
||||
? 'geen percentage bij nulwaarde'
|
||||
: `${temporalComparison.metric.percent_change >= 0 ? '+' : ''}${temporalComparison.metric.percent_change.toLocaleString('nl-BE', { maximumFractionDigits: 1 })}%`}
|
||||
</small>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
<div className="geo-temporal-summary">
|
||||
<div>
|
||||
<span>Gebied</span>
|
||||
<strong>{formatArea(selectedAreaSquareMetres)}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span>Meting</span>
|
||||
<strong>{temporalComparison.metric.label}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span>Methode</span>
|
||||
<strong>{temporalComparison.metric.is_estimate ? 'Ruimtelijke schatting' : 'Exact'}</strong>
|
||||
</div>
|
||||
</div>
|
||||
{temporalComparison.object_changes.available ? (
|
||||
<div className="geo-change-counts" aria-label="Objectwijzigingen">
|
||||
<span><strong>{temporalComparison.object_changes.added_count ?? 0}</strong> nieuw</span>
|
||||
<span><strong>{temporalComparison.object_changes.removed_count ?? 0}</strong> verdwenen</span>
|
||||
<span><strong>{temporalComparison.object_changes.modified_count ?? 0}</strong> gewijzigd</span>
|
||||
</div>
|
||||
) : null}
|
||||
{temporalComparison.warnings.map((warning) => (
|
||||
<p className="geo-data-notice" key={warning}>{warning}</p>
|
||||
))}
|
||||
</>
|
||||
) : (
|
||||
<div className="geo-results-empty">
|
||||
<strong>Klaar om te vergelijken</strong>
|
||||
<p>Kies twee meetmomenten en gebruik “Vergelijk periode”. Bij een nieuwe rechthoek wordt de vergelijking automatisch herhaald.</p>
|
||||
</div>
|
||||
)
|
||||
) : (
|
||||
<>
|
||||
<div className="geo-primary-metrics">
|
||||
<div>
|
||||
<span>Oppervlakte selectie</span>
|
||||
<strong>{formatArea(selectedAreaSquareMetres)}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span>{activeMetricLabel}</span>
|
||||
<strong>{activeSelectionResult ? resultMetricLabel(activeSelectionResult) : 'Geen resultaat'}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span>{activeMetricUnit === 'ha' ? 'Aandeel selectie' : 'Dichtheid'}</span>
|
||||
<strong>{activeSecondaryMetric ?? (selectedDensity === null ? 'n.v.t.' : `${selectedDensity.toLocaleString('nl-BE', { maximumFractionDigits: 1 })} / km2`)}</strong>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{activeSelectionResult?.truncated ? (
|
||||
<div className="geo-theme-results">
|
||||
<div className="geo-results-title-row">
|
||||
<h4>Alle beschikbare thema’s</h4>
|
||||
<span>{themeResults.length} bevraagd</span>
|
||||
</div>
|
||||
{DATA_THEMES.map((theme) => {
|
||||
const dataset = themeDatasetMap[theme.id]
|
||||
const item = themeResults.find((result) => result.theme.id === theme.id)
|
||||
return (
|
||||
<div className="geo-theme-result-row" key={theme.id}>
|
||||
<span className={`geo-theme-symbol geo-theme-symbol-${theme.id}`} aria-hidden="true" />
|
||||
<span>
|
||||
<strong>{theme.label}</strong>
|
||||
<small>{dataset?.source_name ?? dataset?.source ?? 'Geen bron gekoppeld'}</small>
|
||||
</span>
|
||||
<b>{item ? resultMetricLabel(item.result) : dataset ? 'Niet bevraagd' : 'Bron ontbreekt'}</b>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{analysisMode === 'current' && activeSelectionResult?.truncated ? (
|
||||
<p className="geo-data-notice">De telling is volledig; op de kaart en in de tabel worden maximaal {activeSelectionResult.limit.toLocaleString('nl-BE')} objecten getoond.</p>
|
||||
) : null}
|
||||
{mapSelectionError ? <p className="error">{mapSelectionError}</p> : null}
|
||||
{themeResultsError ? <p className="error">{themeResultsError}</p> : null}
|
||||
{temporalComparisonError ? <p className="error">{temporalComparisonError}</p> : null}
|
||||
|
||||
{selectedResultProperties.length > 0 ? (
|
||||
{analysisMode === 'current' && selectedResultProperties.length > 0 ? (
|
||||
<details className="geo-result-details">
|
||||
<summary>Kenmerken van de gevonden objecten</summary>
|
||||
<dl>
|
||||
@@ -963,7 +1184,7 @@ export function MapWorkspace({
|
||||
</details>
|
||||
) : null}
|
||||
|
||||
{selectedMapFeature ? (
|
||||
{analysisMode === 'current' && selectedMapFeature ? (
|
||||
<div className="geo-selected-feature">
|
||||
<span>Geselecteerd object</span>
|
||||
<strong>{String(selectedMapFeature.properties?.['name'] ?? selectedMapFeature.properties?.['source_feature_id'] ?? selectedMapFeature.id ?? 'Object')}</strong>
|
||||
@@ -971,10 +1192,12 @@ export function MapWorkspace({
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="geo-result-actions">
|
||||
<button className="secondary-action" disabled={!activeSelectionResult} type="button" onClick={downloadActiveThemeSelection}>Download GeoJSON</button>
|
||||
<button className="secondary-action" disabled={!activeSelectionResult} type="button" onClick={copyActiveThemeSelection}>Kopieer gegevens</button>
|
||||
</div>
|
||||
{analysisMode === 'current' ? (
|
||||
<div className="geo-result-actions">
|
||||
<button className="secondary-action" disabled={!activeSelectionResult} type="button" onClick={downloadActiveThemeSelection}>Download GeoJSON</button>
|
||||
<button className="secondary-action" disabled={!activeSelectionResult} type="button" onClick={copyActiveThemeSelection}>Kopieer gegevens</button>
|
||||
</div>
|
||||
) : null}
|
||||
</>
|
||||
)}
|
||||
</aside>
|
||||
@@ -982,7 +1205,14 @@ export function MapWorkspace({
|
||||
|
||||
<footer className="geo-explorer-footer">
|
||||
<span><strong>Werkgebied:</strong> {selectedMapArea?.name ?? 'Geen gemeentegrens geselecteerd'}</span>
|
||||
<span><strong>Bron:</strong> {activeThemeDataset ? `${activeThemeDataset.source_name ?? activeThemeDataset.source} · ${activeThemeDataset.name}` : 'niet beschikbaar'}</span>
|
||||
<span>
|
||||
<strong>Bron:</strong>{' '}
|
||||
{analysisMode === 'evolution'
|
||||
? activeTemporalSeries[0]?.temporal_series_key ?? 'geen vergelijkbare tijdreeks'
|
||||
: activeThemeDataset
|
||||
? `${activeThemeDataset.source_name ?? activeThemeDataset.source} · ${activeThemeDataset.name}`
|
||||
: 'niet beschikbaar'}
|
||||
</span>
|
||||
{usesDefaultOsmBasemap ? <span><strong>Ondergrond:</strong> OpenStreetMap</span> : null}
|
||||
</footer>
|
||||
</section>
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { formatError } from '../lib/formatError'
|
||||
import { temporalApi } from '../services/api/temporal'
|
||||
import type { TemporalComparisonResponse, VectorSelectionBBox } from '../types'
|
||||
|
||||
export function useTemporalComparison(selectedProjectId: string | null) {
|
||||
const [temporalComparison, setTemporalComparison] = useState<TemporalComparisonResponse | null>(null)
|
||||
const [temporalComparisonLoading, setTemporalComparisonLoading] = useState(false)
|
||||
const [temporalComparisonError, setTemporalComparisonError] = useState<string | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
setTemporalComparison(null)
|
||||
setTemporalComparisonError(null)
|
||||
}, [selectedProjectId])
|
||||
|
||||
const clearTemporalComparison = () => {
|
||||
setTemporalComparison(null)
|
||||
setTemporalComparisonError(null)
|
||||
}
|
||||
|
||||
const compareTemporalSnapshots = async (
|
||||
earlierDatasetId: string,
|
||||
laterDatasetId: string,
|
||||
bbox: VectorSelectionBBox,
|
||||
): Promise<TemporalComparisonResponse | null> => {
|
||||
if (!selectedProjectId) {
|
||||
setTemporalComparisonError('Open eerst een project om evoluties te vergelijken.')
|
||||
return null
|
||||
}
|
||||
if (!earlierDatasetId || !laterDatasetId) {
|
||||
setTemporalComparisonError('Kies twee meetmomenten uit dezelfde tijdreeks.')
|
||||
return null
|
||||
}
|
||||
|
||||
setTemporalComparisonLoading(true)
|
||||
setTemporalComparisonError(null)
|
||||
try {
|
||||
const result = await temporalApi.compare(selectedProjectId, {
|
||||
earlier_dataset_id: earlierDatasetId,
|
||||
later_dataset_id: laterDatasetId,
|
||||
bbox,
|
||||
preview_limit: 500,
|
||||
})
|
||||
setTemporalComparison(result)
|
||||
return result
|
||||
} catch (error) {
|
||||
setTemporalComparison(null)
|
||||
setTemporalComparisonError(formatError(error, 'De evolutieanalyse is mislukt.'))
|
||||
return null
|
||||
} finally {
|
||||
setTemporalComparisonLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
temporalComparison,
|
||||
temporalComparisonLoading,
|
||||
temporalComparisonError,
|
||||
compareTemporalSnapshots,
|
||||
clearTemporalComparison,
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,12 @@ export const datasetsApi = {
|
||||
sourceMetadataJson?: string
|
||||
provenanceMetadataJson?: string
|
||||
areaId?: string
|
||||
temporalSeriesKey?: string
|
||||
observedAt?: string
|
||||
validFrom?: string
|
||||
validTo?: string
|
||||
temporalGranularity?: string
|
||||
sourceVersion?: string
|
||||
},
|
||||
): Promise<DatasetCreateResponse> => {
|
||||
const form = new FormData()
|
||||
@@ -55,6 +61,24 @@ export const datasetsApi = {
|
||||
if (payload.areaId) {
|
||||
form.append('area_id', payload.areaId)
|
||||
}
|
||||
if (payload.temporalSeriesKey) {
|
||||
form.append('temporal_series_key', payload.temporalSeriesKey)
|
||||
}
|
||||
if (payload.observedAt) {
|
||||
form.append('observed_at', payload.observedAt)
|
||||
}
|
||||
if (payload.validFrom) {
|
||||
form.append('valid_from', payload.validFrom)
|
||||
}
|
||||
if (payload.validTo) {
|
||||
form.append('valid_to', payload.validTo)
|
||||
}
|
||||
if (payload.temporalGranularity) {
|
||||
form.append('temporal_granularity', payload.temporalGranularity)
|
||||
}
|
||||
if (payload.sourceVersion) {
|
||||
form.append('source_version', payload.sourceVersion)
|
||||
}
|
||||
return apiMultipart<DatasetCreateResponse>(`/api/v1/projects/${projectId}/datasets/upload`, form)
|
||||
},
|
||||
refreshMetadata: (projectId: string, datasetId: string): Promise<DatasetCreateResponse> =>
|
||||
|
||||
@@ -9,3 +9,4 @@ export { segmentationApi } from './segmentation'
|
||||
export { exportsApi } from './exports'
|
||||
export { jobsApi } from './jobs'
|
||||
export { projectsApi } from './projects'
|
||||
export { temporalApi } from './temporal'
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import { apiGet, apiPost } from './client'
|
||||
import type {
|
||||
TemporalComparisonRequest,
|
||||
TemporalComparisonResponse,
|
||||
TemporalSeriesListResponse,
|
||||
} from '../../types'
|
||||
|
||||
export const temporalApi = {
|
||||
listSeries: (projectId: string): Promise<TemporalSeriesListResponse> =>
|
||||
apiGet<TemporalSeriesListResponse>(`/api/v1/projects/${projectId}/temporal/series`),
|
||||
compare: (projectId: string, payload: TemporalComparisonRequest): Promise<TemporalComparisonResponse> =>
|
||||
apiPost<TemporalComparisonResponse>(`/api/v1/projects/${projectId}/temporal/compare`, payload),
|
||||
}
|
||||
+157
-1
@@ -5448,6 +5448,33 @@ section {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.geo-analysis-mode {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
flex: 0 0 auto;
|
||||
border: 1px solid #cdd8d4;
|
||||
border-radius: 6px;
|
||||
padding: 0.18rem;
|
||||
background: #f3f6f5;
|
||||
}
|
||||
|
||||
.geo-analysis-mode button {
|
||||
min-height: 2.15rem;
|
||||
border: 0;
|
||||
border-radius: 4px;
|
||||
padding: 0.38rem 0.68rem;
|
||||
background: transparent;
|
||||
color: #5a6964;
|
||||
font-size: 0.7rem;
|
||||
font-weight: 800;
|
||||
}
|
||||
|
||||
.geo-analysis-mode button.active {
|
||||
background: #ffffff;
|
||||
color: #174f45;
|
||||
box-shadow: 0 1px 3px rgba(23, 33, 30, 0.12);
|
||||
}
|
||||
|
||||
.geo-explorer-layout {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(15.5rem, 17rem) minmax(30rem, 1fr) minmax(18rem, 20rem);
|
||||
@@ -5627,6 +5654,32 @@ section {
|
||||
line-height: 1.35;
|
||||
}
|
||||
|
||||
.geo-time-controls {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 0.45rem;
|
||||
border-top: 1px solid #e3e9e6;
|
||||
padding-top: 0.65rem;
|
||||
}
|
||||
|
||||
.geo-time-controls label {
|
||||
color: #4c5b56;
|
||||
font-size: 0.66rem;
|
||||
font-weight: 800;
|
||||
}
|
||||
|
||||
.geo-time-controls select {
|
||||
min-height: 2.25rem;
|
||||
margin-top: 0.24rem;
|
||||
padding: 0.35rem;
|
||||
font-size: 0.68rem;
|
||||
}
|
||||
|
||||
.geo-time-controls button {
|
||||
grid-column: 1 / -1;
|
||||
min-height: 2.25rem;
|
||||
}
|
||||
|
||||
.geo-scope-select {
|
||||
margin-top: auto;
|
||||
color: #4c5b56;
|
||||
@@ -5736,6 +5789,21 @@ section {
|
||||
background: rgba(107, 74, 170, 0.18);
|
||||
}
|
||||
|
||||
.geo-map-legend .geo-legend-added {
|
||||
border-color: #15803d;
|
||||
background: rgba(22, 163, 74, 0.2);
|
||||
}
|
||||
|
||||
.geo-map-legend .geo-legend-removed {
|
||||
border-color: #b91c1c;
|
||||
background: rgba(220, 38, 38, 0.18);
|
||||
}
|
||||
|
||||
.geo-map-legend .geo-legend-modified {
|
||||
border-color: #b45309;
|
||||
background: rgba(217, 119, 6, 0.2);
|
||||
}
|
||||
|
||||
.geo-draw-instruction,
|
||||
.geo-viewport-status {
|
||||
position: absolute;
|
||||
@@ -5841,6 +5909,78 @@ section {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.geo-temporal-metrics > div.positive {
|
||||
border-color: #b8d8c5;
|
||||
background: #f1faf4;
|
||||
}
|
||||
|
||||
.geo-temporal-metrics > div.negative {
|
||||
border-color: #e2c1bd;
|
||||
background: #fff6f5;
|
||||
}
|
||||
|
||||
.geo-temporal-metrics small {
|
||||
color: #64736d;
|
||||
font-size: 0.62rem;
|
||||
}
|
||||
|
||||
.geo-temporal-summary {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
border: 1px solid #e1e8e5;
|
||||
border-radius: 5px;
|
||||
background: #fbfcfc;
|
||||
}
|
||||
|
||||
.geo-temporal-summary > div {
|
||||
display: grid;
|
||||
gap: 0.15rem;
|
||||
min-width: 0;
|
||||
border-left: 1px solid #e1e8e5;
|
||||
padding: 0.45rem;
|
||||
}
|
||||
|
||||
.geo-temporal-summary > div:first-child {
|
||||
border-left: 0;
|
||||
}
|
||||
|
||||
.geo-temporal-summary span {
|
||||
color: #6a7773;
|
||||
font-size: 0.6rem;
|
||||
font-weight: 800;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.geo-temporal-summary strong {
|
||||
overflow: hidden;
|
||||
color: #26332f;
|
||||
font-size: 0.7rem;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.geo-change-counts {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 0.35rem;
|
||||
}
|
||||
|
||||
.geo-change-counts span {
|
||||
display: grid;
|
||||
gap: 0.1rem;
|
||||
border: 1px solid #e1e8e5;
|
||||
border-radius: 5px;
|
||||
padding: 0.4rem;
|
||||
color: #687570;
|
||||
font-size: 0.64rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.geo-change-counts strong {
|
||||
color: #26332f;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
.geo-theme-results {
|
||||
display: grid;
|
||||
gap: 0;
|
||||
@@ -6034,6 +6174,7 @@ section {
|
||||
|
||||
@media (max-width: 920px) {
|
||||
.geo-explorer-header {
|
||||
flex-wrap: wrap;
|
||||
align-items: start;
|
||||
}
|
||||
|
||||
@@ -6075,11 +6216,26 @@ section {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.geo-analysis-mode {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.geo-theme-list,
|
||||
.geo-primary-metrics {
|
||||
.geo-primary-metrics,
|
||||
.geo-temporal-summary,
|
||||
.geo-change-counts {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.geo-temporal-summary > div {
|
||||
border-top: 1px solid #e1e8e5;
|
||||
border-left: 0;
|
||||
}
|
||||
|
||||
.geo-temporal-summary > div:first-child {
|
||||
border-top: 0;
|
||||
}
|
||||
|
||||
.geo-map-toolbar {
|
||||
display: grid;
|
||||
}
|
||||
|
||||
@@ -83,6 +83,12 @@ export interface DatasetCreateResponse {
|
||||
source_metadata?: Record<string, unknown> | null
|
||||
provenance_metadata?: Record<string, unknown> | null
|
||||
imported_at?: string | null
|
||||
temporal_series_key?: string | null
|
||||
observed_at?: string | null
|
||||
valid_from?: string | null
|
||||
valid_to?: string | null
|
||||
temporal_granularity?: string | null
|
||||
source_version?: string | null
|
||||
project_id: string
|
||||
area_id?: string | null
|
||||
storage_path?: string | null
|
||||
@@ -313,6 +319,109 @@ export interface VectorSelectionResponse {
|
||||
limit: number
|
||||
truncated: boolean
|
||||
geojson: GeoJSON.FeatureCollection
|
||||
summary?: VectorSelectionSummary | null
|
||||
}
|
||||
|
||||
export interface VectorSelectionSummary {
|
||||
metric_label: string
|
||||
metric_value: number
|
||||
metric_unit: string
|
||||
aggregation_method: string
|
||||
feature_count: number
|
||||
is_estimate: boolean
|
||||
warning?: string | null
|
||||
}
|
||||
|
||||
export interface DatasetTemporalUpdate {
|
||||
temporal_series_key: string
|
||||
observed_at: string
|
||||
valid_from?: string | null
|
||||
valid_to?: string | null
|
||||
temporal_granularity?: string
|
||||
source_version?: string | null
|
||||
}
|
||||
|
||||
export interface DatasetVersionRead {
|
||||
id: string
|
||||
dataset_id: string
|
||||
version: number
|
||||
storage_path?: string | null
|
||||
source_version?: string | null
|
||||
observed_at?: string | null
|
||||
valid_from?: string | null
|
||||
valid_to?: string | null
|
||||
checksum_sha256?: string | null
|
||||
source_metadata?: Record<string, unknown> | null
|
||||
provenance_metadata?: Record<string, unknown> | null
|
||||
created_at?: string | null
|
||||
}
|
||||
|
||||
export interface TemporalComparisonRequest {
|
||||
earlier_dataset_id: string
|
||||
later_dataset_id: string
|
||||
bbox: VectorSelectionBBox
|
||||
preview_limit?: number
|
||||
}
|
||||
|
||||
export interface TemporalDatasetRef {
|
||||
id: string
|
||||
name: string
|
||||
observed_at: string
|
||||
source_version?: string | null
|
||||
}
|
||||
|
||||
export interface TemporalMetricComparison {
|
||||
label: string
|
||||
unit: string
|
||||
aggregation_method: string
|
||||
earlier_value: number
|
||||
later_value: number
|
||||
absolute_change: number
|
||||
percent_change?: number | null
|
||||
is_estimate: boolean
|
||||
}
|
||||
|
||||
export interface TemporalObjectChanges {
|
||||
available: boolean
|
||||
added_count?: number | null
|
||||
removed_count?: number | null
|
||||
modified_count?: number | null
|
||||
unchanged_count?: number | null
|
||||
}
|
||||
|
||||
export interface TemporalComparisonResponse {
|
||||
temporal_series_key: string
|
||||
earlier: TemporalDatasetRef
|
||||
later: TemporalDatasetRef
|
||||
selection_bbox: VectorSelectionBBox
|
||||
metric: TemporalMetricComparison
|
||||
object_changes: TemporalObjectChanges
|
||||
geojson: GeoJSON.FeatureCollection
|
||||
warnings: string[]
|
||||
generated_at: string
|
||||
}
|
||||
|
||||
export interface TemporalSeriesDataset {
|
||||
id: string
|
||||
name: string
|
||||
observed_at: string
|
||||
source_version?: string | null
|
||||
feature_count?: number | null
|
||||
}
|
||||
|
||||
export interface TemporalSeriesRead {
|
||||
temporal_series_key: string
|
||||
source_name?: string | null
|
||||
reference_layer_name?: string | null
|
||||
dataset_count: number
|
||||
first_observed_at: string
|
||||
last_observed_at: string
|
||||
datasets: TemporalSeriesDataset[]
|
||||
}
|
||||
|
||||
export interface TemporalSeriesListResponse {
|
||||
items: TemporalSeriesRead[]
|
||||
total: number
|
||||
}
|
||||
|
||||
export interface DatasetListResponse {
|
||||
|
||||
@@ -345,6 +345,10 @@ def upload_layer(
|
||||
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
|
||||
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
|
||||
"area_id": area_id,
|
||||
"temporal_series_key": f"grb:{definition.key}:mol",
|
||||
"observed_at": summary["generated_at"],
|
||||
"temporal_granularity": "snapshot",
|
||||
"source_version": summary["generated_at"][:10],
|
||||
},
|
||||
files={"file": (path.name, handle, "application/geo+json")},
|
||||
timeout=timeout,
|
||||
@@ -352,6 +356,31 @@ def upload_layer(
|
||||
return response_data(response)
|
||||
|
||||
|
||||
def ensure_temporal_metadata(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
dataset: dict[str, Any],
|
||||
definition: LayerDefinition,
|
||||
generated_at: str,
|
||||
timeout: int,
|
||||
) -> dict[str, Any]:
|
||||
series_key = f"grb:{definition.key}:mol"
|
||||
if dataset.get("temporal_series_key") == series_key and dataset.get("observed_at"):
|
||||
return dataset
|
||||
response = session.patch(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/{dataset['id']}/temporal",
|
||||
json={
|
||||
"temporal_series_key": series_key,
|
||||
"observed_at": generated_at,
|
||||
"temporal_granularity": "snapshot",
|
||||
"source_version": generated_at[:10],
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
return response_data(response)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
requested = {item.strip().lower() for item in args.layers.split(",") if item.strip()}
|
||||
@@ -402,6 +431,15 @@ def main() -> int:
|
||||
for definition, path, summary in prepared:
|
||||
dataset = next((item for item in existing if item.get("original_filename") == path.name), None)
|
||||
if dataset:
|
||||
dataset = ensure_temporal_metadata(
|
||||
api_session,
|
||||
base_url,
|
||||
project_id,
|
||||
dataset,
|
||||
definition,
|
||||
summary["generated_at"],
|
||||
args.import_timeout,
|
||||
)
|
||||
results.append(
|
||||
{"layer": definition.key, "dataset_id": dataset["id"], "feature_count": dataset.get("feature_count"), "status": "existing"}
|
||||
)
|
||||
|
||||
@@ -0,0 +1,552 @@
|
||||
"""Provision official historical land-use theme snapshots for Mol.
|
||||
|
||||
The command reads the Digitaal Vlaanderen historical land-use WFS for 1778,
|
||||
1873 and 1969, clips features to the official Mol boundary, separates the
|
||||
supported map themes and imports every snapshot through the normal dataset API.
|
||||
It is an explicit, idempotent operator command and never runs at startup.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from shapely.geometry import mapping, shape
|
||||
from shapely.validation import make_valid
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
|
||||
MUNICIPALITY_NAME = "Mol"
|
||||
MUNICIPALITY_NIS_CODE = "13025"
|
||||
PROJECT_NAME = "Mol Municipality Workbench"
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/mol-historical-landuse")
|
||||
DEFAULT_BOUNDARY_PATH = Path("/app/storage/operator-data/mol-municipality/mol_municipality_boundary.geojson")
|
||||
WFS_URL = "https://geo.api.vlaanderen.be/HistLandgebruik/wfs"
|
||||
ATTRIBUTION = "Bron: Historisch landgebruik Vlaanderen, Digitaal Vlaanderen"
|
||||
COLLECTIONS = {1778: "HistLandgebruik:Lgbrk1778", 1873: "HistLandgebruik:Lgbrk1873", 1969: "HistLandgebruik:Lgbrk1969"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ThemeDefinition:
|
||||
key: str
|
||||
label: str
|
||||
matches: Callable[[str], bool]
|
||||
filter_value: str
|
||||
filter_mode: str
|
||||
|
||||
|
||||
THEMES = (
|
||||
ThemeDefinition("buildings", "Historische bebouwing", lambda value: value.startswith("bebouwing"), "bebouwing*", "like"),
|
||||
ThemeDefinition("forest", "Historisch bos", lambda value: value.startswith("bos-") or value == "bos", "bos*", "like"),
|
||||
ThemeDefinition("water", "Historisch water", lambda value: value == "water", "water", "equal"),
|
||||
ThemeDefinition("roads", "Historische wegen", lambda value: value.startswith("weg-"), "weg-*", "like"),
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Provision official historical land-use snapshots for Mol.")
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument("--project-name", default=PROJECT_NAME)
|
||||
parser.add_argument("--years", default="1778,1873,1969")
|
||||
parser.add_argument("--themes", default="buildings,forest,water,roads")
|
||||
parser.add_argument("--output-dir", type=Path, default=Path(os.environ.get("MOL_HISTORICAL_LANDUSE_OUTPUT_DIR", DEFAULT_OUTPUT_DIR)))
|
||||
parser.add_argument("--boundary-path", type=Path, default=Path(os.environ.get("MOL_BOUNDARY_PATH", DEFAULT_BOUNDARY_PATH)))
|
||||
parser.add_argument("--page-size", type=int, default=200)
|
||||
parser.add_argument("--max-features", type=int, default=100000)
|
||||
parser.add_argument("--simplify-tolerance-degrees", type=float, default=0.00001)
|
||||
parser.add_argument("--request-timeout", type=int, default=180)
|
||||
parser.add_argument("--import-timeout", type=int, default=1800)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument("--fetch-only", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_session() -> requests.Session:
|
||||
retry = Retry(
|
||||
total=5,
|
||||
connect=5,
|
||||
read=5,
|
||||
status=5,
|
||||
backoff_factor=1.0,
|
||||
status_forcelist=(429, 500, 502, 503, 504),
|
||||
allowed_methods=frozenset({"GET"}),
|
||||
raise_on_status=True,
|
||||
)
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": "GeoIntel-Mol-Historical-Landuse-Operator/1.0"})
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
return session
|
||||
|
||||
|
||||
def load_boundary(path: Path):
|
||||
if not path.exists():
|
||||
raise RuntimeError(f"Mol boundary is missing at {path}; run provision_mol_municipality_workspace.py first")
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
features = payload.get("features") or []
|
||||
if len(features) != 1:
|
||||
raise RuntimeError("Mol boundary artifact must contain exactly one feature")
|
||||
boundary = shape(features[0]["geometry"])
|
||||
if not boundary.is_valid:
|
||||
boundary = make_valid(boundary)
|
||||
if boundary.is_empty or not boundary.is_valid:
|
||||
raise RuntimeError("Mol boundary artifact is invalid")
|
||||
return boundary
|
||||
|
||||
|
||||
def wfs_filter_xml(definition: ThemeDefinition, bounds: tuple[float, float, float, float]) -> str:
|
||||
min_x, min_y, max_x, max_y = bounds
|
||||
comparison = (
|
||||
f"<fes:PropertyIsLike wildCard='*' singleChar='?' escapeChar='!'>"
|
||||
f"<fes:ValueReference>KLASSE</fes:ValueReference><fes:Literal>{definition.filter_value}</fes:Literal>"
|
||||
f"</fes:PropertyIsLike>"
|
||||
if definition.filter_mode == "like"
|
||||
else (
|
||||
"<fes:PropertyIsEqualTo><fes:ValueReference>KLASSE</fes:ValueReference>"
|
||||
f"<fes:Literal>{definition.filter_value}</fes:Literal></fes:PropertyIsEqualTo>"
|
||||
)
|
||||
)
|
||||
return (
|
||||
"<fes:Filter xmlns:fes='http://www.opengis.net/fes/2.0' xmlns:gml='http://www.opengis.net/gml/3.2'>"
|
||||
"<fes:And><fes:BBOX><fes:ValueReference>SHAPE</fes:ValueReference>"
|
||||
"<gml:Envelope srsName='EPSG:4326'>"
|
||||
f"<gml:lowerCorner>{min_x:.8f} {min_y:.8f}</gml:lowerCorner>"
|
||||
f"<gml:upperCorner>{max_x:.8f} {max_y:.8f}</gml:upperCorner>"
|
||||
"</gml:Envelope></fes:BBOX>"
|
||||
f"{comparison}</fes:And></fes:Filter>"
|
||||
)
|
||||
|
||||
|
||||
def fetch_year(
|
||||
session: requests.Session,
|
||||
year: int,
|
||||
boundary,
|
||||
definitions: list[ThemeDefinition],
|
||||
*,
|
||||
page_size: int,
|
||||
max_features: int,
|
||||
simplify_tolerance_degrees: float,
|
||||
timeout: int,
|
||||
):
|
||||
collection = COLLECTIONS[year]
|
||||
features_by_theme: dict[str, list[dict[str, Any]]] = {definition.key: [] for definition in definitions}
|
||||
for definition in definitions:
|
||||
filter_xml = wfs_filter_xml(definition, boundary.bounds)
|
||||
start_index = 0
|
||||
seen: set[str] = set()
|
||||
while True:
|
||||
response = session.get(
|
||||
WFS_URL,
|
||||
params={
|
||||
"service": "WFS",
|
||||
"version": "2.0.0",
|
||||
"request": "GetFeature",
|
||||
"typeNames": collection,
|
||||
"outputFormat": "application/json",
|
||||
"srsName": "EPSG:4326",
|
||||
"FILTER": filter_xml,
|
||||
"count": page_size,
|
||||
"startIndex": start_index,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
page = response.json().get("features") or []
|
||||
for raw_feature in page:
|
||||
feature_id = str(raw_feature.get("id") or "")
|
||||
if not feature_id or feature_id in seen:
|
||||
continue
|
||||
seen.add(feature_id)
|
||||
properties = raw_feature.get("properties") or {}
|
||||
landuse_class = str(properties.get("KLASSE") or "").strip().lower()
|
||||
if not definition.matches(landuse_class):
|
||||
continue
|
||||
geometry = shape(raw_feature["geometry"])
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
geometry = geometry.intersection(boundary)
|
||||
if geometry.is_empty:
|
||||
continue
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
if geometry.is_empty or not geometry.is_valid:
|
||||
continue
|
||||
if simplify_tolerance_degrees > 0:
|
||||
geometry = geometry.simplify(simplify_tolerance_degrees, preserve_topology=True)
|
||||
theme_features = features_by_theme[definition.key]
|
||||
if len(theme_features) >= max_features:
|
||||
raise RuntimeError(
|
||||
f"Historical land use {definition.key} {year} exceeds the {max_features} feature safety limit"
|
||||
)
|
||||
theme_features.append({**raw_feature, "id": feature_id, "geometry": mapping(geometry)})
|
||||
if len(page) < page_size:
|
||||
break
|
||||
start_index += len(page)
|
||||
for definition in definitions:
|
||||
if not features_by_theme[definition.key]:
|
||||
raise RuntimeError(f"Historical land-use WFS returned no {definition.key} features for Mol in {year}")
|
||||
return features_by_theme
|
||||
|
||||
|
||||
def write_theme_snapshot(
|
||||
session: requests.Session,
|
||||
*,
|
||||
year: int,
|
||||
definition: ThemeDefinition,
|
||||
boundary,
|
||||
path: Path,
|
||||
page_size: int,
|
||||
max_features: int,
|
||||
simplify_tolerance_degrees: float,
|
||||
timeout: int,
|
||||
) -> int:
|
||||
collection = COLLECTIONS[year]
|
||||
filter_xml = wfs_filter_xml(definition, boundary.bounds)
|
||||
start_index = 0
|
||||
seen: set[str] = set()
|
||||
feature_count = 0
|
||||
first_feature = True
|
||||
try:
|
||||
with path.open("w", encoding="utf-8") as output:
|
||||
output.write(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"name": f"{definition.label} - Mol {year}",
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"observation_year": year,
|
||||
"attribution": ATTRIBUTION,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
)[:-1]
|
||||
)
|
||||
output.write(',"features":[')
|
||||
while True:
|
||||
response = session.get(
|
||||
WFS_URL,
|
||||
params={
|
||||
"service": "WFS",
|
||||
"version": "2.0.0",
|
||||
"request": "GetFeature",
|
||||
"typeNames": collection,
|
||||
"outputFormat": "application/json",
|
||||
"srsName": "EPSG:4326",
|
||||
"FILTER": filter_xml,
|
||||
"count": page_size,
|
||||
"startIndex": start_index,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
page = response.json().get("features") or []
|
||||
for raw_feature in page:
|
||||
feature_id = str(raw_feature.get("id") or "")
|
||||
if not feature_id or feature_id in seen:
|
||||
continue
|
||||
seen.add(feature_id)
|
||||
properties = dict(raw_feature.get("properties") or {})
|
||||
landuse_class = str(properties.get("KLASSE") or "").strip().lower()
|
||||
if not definition.matches(landuse_class):
|
||||
continue
|
||||
geometry = shape(raw_feature["geometry"])
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
geometry = geometry.intersection(boundary)
|
||||
if geometry.is_empty:
|
||||
continue
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
if geometry.is_empty or not geometry.is_valid:
|
||||
continue
|
||||
if simplify_tolerance_degrees > 0:
|
||||
geometry = geometry.simplify(simplify_tolerance_degrees, preserve_topology=True)
|
||||
properties.update(
|
||||
{
|
||||
"source_name": "historical_landuse",
|
||||
"source_feature_id": feature_id,
|
||||
"reference_layer_name": definition.key,
|
||||
"authority_level": "authoritative",
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"observation_year": year,
|
||||
"historical_landuse_class": landuse_class,
|
||||
"attribution": ATTRIBUTION,
|
||||
}
|
||||
)
|
||||
prepared = {
|
||||
"type": "Feature",
|
||||
"id": feature_id,
|
||||
"geometry": mapping(geometry),
|
||||
"properties": properties,
|
||||
}
|
||||
if not first_feature:
|
||||
output.write(",")
|
||||
output.write(json.dumps(prepared, ensure_ascii=False, separators=(",", ":")))
|
||||
first_feature = False
|
||||
feature_count += 1
|
||||
if feature_count > max_features:
|
||||
raise RuntimeError(
|
||||
f"Historical land use {definition.key} {year} exceeds the {max_features} feature safety limit"
|
||||
)
|
||||
if len(page) < page_size:
|
||||
break
|
||||
start_index += len(page)
|
||||
output.write("]}")
|
||||
except Exception:
|
||||
path.unlink(missing_ok=True)
|
||||
path.with_suffix(".manifest.json").unlink(missing_ok=True)
|
||||
raise
|
||||
if feature_count == 0:
|
||||
path.unlink(missing_ok=True)
|
||||
raise RuntimeError(f"Historical land-use WFS returned no {definition.key} features for Mol in {year}")
|
||||
path.with_suffix(".manifest.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"year": year,
|
||||
"theme": definition.key,
|
||||
"feature_count": feature_count,
|
||||
"collection": collection,
|
||||
"simplify_tolerance_degrees": simplify_tolerance_degrees,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return feature_count
|
||||
|
||||
|
||||
def build_theme_snapshot(year: int, definition: ThemeDefinition, source_features: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
features: list[dict[str, Any]] = []
|
||||
for source_feature in source_features:
|
||||
properties = dict(source_feature.get("properties") or {})
|
||||
landuse_class = str(properties.get("KLASSE") or "").strip().lower()
|
||||
if not definition.matches(landuse_class):
|
||||
continue
|
||||
feature_id = str(source_feature["id"])
|
||||
properties.update(
|
||||
{
|
||||
"source_name": "historical_landuse",
|
||||
"source_feature_id": feature_id,
|
||||
"reference_layer_name": definition.key,
|
||||
"authority_level": "authoritative",
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"observation_year": year,
|
||||
"historical_landuse_class": landuse_class,
|
||||
"attribution": ATTRIBUTION,
|
||||
}
|
||||
)
|
||||
features.append(
|
||||
{"type": "Feature", "id": feature_id, "geometry": source_feature["geometry"], "properties": properties}
|
||||
)
|
||||
if not features:
|
||||
raise RuntimeError(f"No {definition.key} features were classified for Mol in {year}")
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"name": f"{definition.label} - Mol {year}",
|
||||
"features": features,
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"observation_year": year,
|
||||
"attribution": ATTRIBUTION,
|
||||
}
|
||||
|
||||
|
||||
def response_data(response: requests.Response) -> Any:
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"GeoIntel API returned non-JSON ({response.status_code}): {response.text[:300]}") from exc
|
||||
if not response.ok:
|
||||
raise RuntimeError(f"GeoIntel API failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:800]}")
|
||||
if not isinstance(payload, dict) or "data" not in payload:
|
||||
raise RuntimeError("GeoIntel API response does not use the canonical data envelope")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def locate_workspace(session: requests.Session, base_url: str, project_name: str, timeout: int):
|
||||
projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=timeout))
|
||||
project = next((item for item in projects.get("items") or [] if item.get("name") == project_name), None)
|
||||
if not project:
|
||||
raise RuntimeError(f"Project {project_name!r} is missing")
|
||||
project_id = str(project["id"])
|
||||
areas = response_data(session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=timeout))
|
||||
area = next((item for item in areas.get("items") or [] if "gemeente mol" in str(item.get("name", "")).lower()), None)
|
||||
if not area:
|
||||
raise RuntimeError("Official Mol area is missing")
|
||||
datasets = response_data(session.get(f"{base_url}/api/v1/projects/{project_id}/datasets", params={"limit": 200}, timeout=timeout))
|
||||
return project_id, str(area["id"]), list(datasets.get("items") or [])
|
||||
|
||||
|
||||
def upload_snapshot(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
area_id: str,
|
||||
year: int,
|
||||
definition: ThemeDefinition,
|
||||
path: Path,
|
||||
simplify_tolerance_degrees: float,
|
||||
timeout: int,
|
||||
) -> dict[str, Any]:
|
||||
observed_at = f"{year}-01-01T00:00:00Z"
|
||||
series_key = f"digitaal-vlaanderen:historical-landuse:{definition.key}:mol"
|
||||
source_metadata = {
|
||||
"provider": "Digitaal Vlaanderen",
|
||||
"collection": COLLECTIONS[year],
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": "municipality",
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"attribution": ATTRIBUTION,
|
||||
"identity_stable": False,
|
||||
"geometry_simplification_tolerance_degrees": simplify_tolerance_degrees,
|
||||
"selection_aggregation": {
|
||||
"method": "intersection_area",
|
||||
"label": "Oppervlakte",
|
||||
"unit": "ha",
|
||||
"is_estimate": False,
|
||||
"warning": "Historische kaartklassen en karteermethodes verschillen per bronjaar; interpreteer trends binnen die methodologische context.",
|
||||
},
|
||||
}
|
||||
provenance_metadata = {
|
||||
"operator_tool": "provision_mol_historical_landuse.py",
|
||||
"operator_explicit_fetch": True,
|
||||
"wfs_url": WFS_URL,
|
||||
"collection": COLLECTIONS[year],
|
||||
"geometry_simplification_tolerance_degrees": simplify_tolerance_degrees,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
with path.open("rb") as handle:
|
||||
response = session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
|
||||
data={
|
||||
"dataset_type": "vector",
|
||||
"source": "operator_official_import",
|
||||
"dataset_role": "reference",
|
||||
"source_name": "historical_landuse",
|
||||
"reference_layer_name": definition.key,
|
||||
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
|
||||
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
|
||||
"area_id": area_id,
|
||||
"temporal_series_key": series_key,
|
||||
"observed_at": observed_at,
|
||||
"valid_from": observed_at,
|
||||
"temporal_granularity": "year",
|
||||
"source_version": str(year),
|
||||
},
|
||||
files={"file": (path.name, handle, "application/geo+json")},
|
||||
timeout=timeout,
|
||||
)
|
||||
return response_data(response)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
years = sorted({int(value.strip()) for value in args.years.split(",") if value.strip()})
|
||||
except ValueError:
|
||||
print(json.dumps({"status": "error", "message": "Years must be comma-separated integers"}), file=sys.stderr)
|
||||
return 2
|
||||
requested_themes = {value.strip().lower() for value in args.themes.split(",") if value.strip()}
|
||||
definitions = [definition for definition in THEMES if definition.key in requested_themes]
|
||||
unsupported_years = [year for year in years if year not in COLLECTIONS]
|
||||
unsupported_themes = requested_themes - {definition.key for definition in THEMES}
|
||||
if unsupported_years or unsupported_themes or not years or not definitions:
|
||||
print(
|
||||
json.dumps(
|
||||
{"status": "error", "message": f"Unsupported years={unsupported_years}, themes={sorted(unsupported_themes)}"}
|
||||
),
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
results: list[dict[str, Any]] = []
|
||||
try:
|
||||
boundary = load_boundary(args.boundary_path)
|
||||
prepared: list[tuple[int, ThemeDefinition, Path, int]] = []
|
||||
with build_session() as source_session:
|
||||
for year in years:
|
||||
target_paths = {
|
||||
definition.key: args.output_dir / f"mol_historical_{definition.key}_{year}.geojson"
|
||||
for definition in definitions
|
||||
}
|
||||
for definition in definitions:
|
||||
path = target_paths[definition.key]
|
||||
manifest_path = path.with_suffix(".manifest.json")
|
||||
if args.force or not path.exists() or not manifest_path.exists():
|
||||
count = write_theme_snapshot(
|
||||
source_session,
|
||||
year=year,
|
||||
definition=definition,
|
||||
boundary=boundary,
|
||||
path=path,
|
||||
page_size=args.page_size,
|
||||
max_features=args.max_features,
|
||||
simplify_tolerance_degrees=args.simplify_tolerance_degrees,
|
||||
timeout=args.request_timeout,
|
||||
)
|
||||
else:
|
||||
count = int(json.loads(manifest_path.read_text(encoding="utf-8"))["feature_count"])
|
||||
prepared.append((year, definition, path, count))
|
||||
|
||||
if args.fetch_only:
|
||||
results = [
|
||||
{"year": year, "theme": definition.key, "path": str(path), "feature_count": count, "status": "prepared"}
|
||||
for year, definition, path, count in prepared
|
||||
]
|
||||
else:
|
||||
base_url = args.base_url.rstrip("/")
|
||||
with requests.Session() as api_session:
|
||||
project_id, area_id, existing = locate_workspace(api_session, base_url, args.project_name, args.import_timeout)
|
||||
for year, definition, path, count in prepared:
|
||||
series_key = f"digitaal-vlaanderen:historical-landuse:{definition.key}:mol"
|
||||
dataset = next(
|
||||
(
|
||||
item
|
||||
for item in existing
|
||||
if item.get("temporal_series_key") == series_key
|
||||
and str(item.get("observed_at") or "").startswith(str(year))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if dataset:
|
||||
results.append({"year": year, "theme": definition.key, "dataset_id": dataset["id"], "feature_count": dataset.get("feature_count"), "status": "existing"})
|
||||
continue
|
||||
dataset = upload_snapshot(
|
||||
api_session,
|
||||
base_url,
|
||||
project_id,
|
||||
area_id,
|
||||
year,
|
||||
definition,
|
||||
path,
|
||||
args.simplify_tolerance_degrees,
|
||||
args.import_timeout,
|
||||
)
|
||||
results.append({"year": year, "theme": definition.key, "dataset_id": dataset["id"], "feature_count": dataset.get("feature_count"), "status": "imported"})
|
||||
except (OSError, RuntimeError, requests.RequestException, ValueError, KeyError) as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(json.dumps({"status": "ok", "municipality": MUNICIPALITY_NAME, "snapshots": results}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -476,6 +476,8 @@ def upload_dataset(
|
||||
reference_layer_name: str | None,
|
||||
source_metadata: dict[str, Any],
|
||||
provenance_metadata: dict[str, Any],
|
||||
temporal_series_key: str,
|
||||
observed_at: str,
|
||||
timeout: int,
|
||||
) -> dict[str, Any]:
|
||||
form = {
|
||||
@@ -486,6 +488,10 @@ def upload_dataset(
|
||||
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
|
||||
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
|
||||
"area_id": area_id,
|
||||
"temporal_series_key": temporal_series_key,
|
||||
"observed_at": observed_at,
|
||||
"temporal_granularity": "snapshot",
|
||||
"source_version": observed_at[:10],
|
||||
}
|
||||
if reference_layer_name:
|
||||
form["reference_layer_name"] = reference_layer_name
|
||||
@@ -499,6 +505,32 @@ def upload_dataset(
|
||||
return response_data(response)
|
||||
|
||||
|
||||
def ensure_temporal_metadata(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
dataset: dict[str, Any],
|
||||
*,
|
||||
temporal_series_key: str,
|
||||
observed_at: str,
|
||||
timeout: int,
|
||||
) -> dict[str, Any]:
|
||||
if dataset.get("temporal_series_key") == temporal_series_key and dataset.get("observed_at"):
|
||||
return dataset
|
||||
return response_data(
|
||||
session.patch(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/{dataset['id']}/temporal",
|
||||
json={
|
||||
"temporal_series_key": temporal_series_key,
|
||||
"observed_at": observed_at,
|
||||
"temporal_granularity": "snapshot",
|
||||
"source_version": observed_at[:10],
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def provision_workspace(
|
||||
args: argparse.Namespace,
|
||||
boundary_path: Path,
|
||||
@@ -558,6 +590,18 @@ def provision_workspace(
|
||||
"source_url": GRB_GBG_ITEMS_URL,
|
||||
"artifact_sha256": manifest["buildings_sha256"],
|
||||
},
|
||||
temporal_series_key="grb:buildings:mol",
|
||||
observed_at=manifest["generated_at"],
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
else:
|
||||
building_dataset = ensure_temporal_metadata(
|
||||
session,
|
||||
base_url,
|
||||
project_id,
|
||||
building_dataset,
|
||||
temporal_series_key="grb:buildings:mol",
|
||||
observed_at=manifest["generated_at"],
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
|
||||
@@ -587,6 +631,18 @@ def provision_workspace(
|
||||
"source_url": manifest["boundary_source_url"],
|
||||
"artifact_sha256": manifest["boundary_sha256"],
|
||||
},
|
||||
temporal_series_key="vrbg:municipality-boundary:mol",
|
||||
observed_at=manifest["generated_at"],
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
else:
|
||||
boundary_dataset = ensure_temporal_metadata(
|
||||
session,
|
||||
base_url,
|
||||
project_id,
|
||||
boundary_dataset,
|
||||
temporal_series_key="vrbg:municipality-boundary:mol",
|
||||
observed_at=manifest["generated_at"],
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
"""Provision official annual Statbel population snapshots for Mol.
|
||||
|
||||
The command joins annual population totals to the matching official
|
||||
statistical-sector geometries, clips the result to Mol and imports each year
|
||||
through the existing GeoIntel upload API. It never runs during application
|
||||
startup and it never synthesizes missing population values.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyproj import Transformer
|
||||
from requests.adapters import HTTPAdapter
|
||||
from shapely.geometry import mapping, shape
|
||||
from shapely.ops import transform
|
||||
from shapely.validation import make_valid
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
|
||||
MUNICIPALITY_NAME = "Mol"
|
||||
MUNICIPALITY_NIS_CODE = "13025"
|
||||
PROJECT_NAME = "Mol Municipality Workbench"
|
||||
SERIES_KEY = "statbel:population-statistical-sector:mol"
|
||||
ATTRIBUTION = "Bron: Statbel, bevolking per statistische sector, CC BY 4.0"
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/mol-population-history")
|
||||
DEFAULT_BOUNDARY_PATH = Path("/app/storage/operator-data/mol-municipality/mol_municipality_boundary.geojson")
|
||||
SECTOR_URL = (
|
||||
"https://statbel.fgov.be/sites/default/files/files/opendata/Statistische%20sectoren/"
|
||||
"sh_statbel_statistical_sectors_31370_{year}0101.geojson.zip"
|
||||
)
|
||||
POPULATION_URLS = {
|
||||
2021: "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2021.zip",
|
||||
2022: "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2022.zip",
|
||||
2023: "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2023.zip",
|
||||
2024: "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2024.zip",
|
||||
2025: "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2025_NEW.zip",
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Provision official annual Statbel population snapshots for Mol.")
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument("--project-name", default=PROJECT_NAME)
|
||||
parser.add_argument("--years", default="2021,2022,2023,2024,2025")
|
||||
parser.add_argument("--output-dir", type=Path, default=Path(os.environ.get("MOL_POPULATION_OUTPUT_DIR", DEFAULT_OUTPUT_DIR)))
|
||||
parser.add_argument("--boundary-path", type=Path, default=Path(os.environ.get("MOL_BOUNDARY_PATH", DEFAULT_BOUNDARY_PATH)))
|
||||
parser.add_argument("--request-timeout", type=int, default=180)
|
||||
parser.add_argument("--import-timeout", type=int, default=900)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument("--fetch-only", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_session() -> requests.Session:
|
||||
retry = Retry(
|
||||
total=5,
|
||||
connect=5,
|
||||
read=5,
|
||||
status=5,
|
||||
backoff_factor=1.0,
|
||||
status_forcelist=(429, 500, 502, 503, 504),
|
||||
allowed_methods=frozenset({"GET"}),
|
||||
raise_on_status=True,
|
||||
)
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": "GeoIntel-Mol-Population-Operator/1.0"})
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
return session
|
||||
|
||||
|
||||
def response_data(response: requests.Response) -> Any:
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"GeoIntel API returned non-JSON ({response.status_code}): {response.text[:300]}") from exc
|
||||
if not response.ok:
|
||||
raise RuntimeError(f"GeoIntel API failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:800]}")
|
||||
if not isinstance(payload, dict) or "data" not in payload:
|
||||
raise RuntimeError("GeoIntel API response does not use the canonical data envelope")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def load_boundary(path: Path):
|
||||
if not path.exists():
|
||||
raise RuntimeError(f"Mol boundary is missing at {path}; run provision_mol_municipality_workspace.py first")
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
features = payload.get("features") or []
|
||||
if len(features) != 1:
|
||||
raise RuntimeError("Mol boundary artifact must contain exactly one feature")
|
||||
boundary = shape(features[0]["geometry"])
|
||||
if not boundary.is_valid:
|
||||
boundary = make_valid(boundary)
|
||||
if boundary.is_empty or not boundary.is_valid:
|
||||
raise RuntimeError("Mol boundary artifact is invalid")
|
||||
return boundary
|
||||
|
||||
|
||||
def zip_member_json(content: bytes) -> dict[str, Any]:
|
||||
with zipfile.ZipFile(io.BytesIO(content)) as archive:
|
||||
member = next((name for name in archive.namelist() if name.lower().endswith(".geojson")), None)
|
||||
if not member:
|
||||
raise RuntimeError("Statbel sector archive contains no GeoJSON file")
|
||||
return json.loads(archive.read(member).decode("utf-8"))
|
||||
|
||||
|
||||
def population_rows(content: bytes) -> dict[str, dict[str, Any]]:
|
||||
with zipfile.ZipFile(io.BytesIO(content)) as archive:
|
||||
member = next((name for name in archive.namelist() if name.lower().endswith((".txt", ".csv"))), None)
|
||||
if not member:
|
||||
raise RuntimeError("Statbel population archive contains no text table")
|
||||
raw = archive.read(member)
|
||||
try:
|
||||
text = raw.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
text = raw.decode("cp1252")
|
||||
rows: dict[str, dict[str, Any]] = {}
|
||||
for row in csv.DictReader(io.StringIO(text), delimiter="|"):
|
||||
if str(row.get("CD_REFNIS") or "").strip() != MUNICIPALITY_NIS_CODE:
|
||||
continue
|
||||
sector_code = str(row.get("CD_SECTOR") or "").strip()
|
||||
total_raw = str(row.get("TOTAL") or "").strip()
|
||||
if not sector_code or not total_raw or not total_raw.isdigit():
|
||||
continue
|
||||
rows[sector_code] = {
|
||||
"population_total": int(total_raw),
|
||||
"sector_name_nl": row.get("TX_DESCR_SECTOR_NL"),
|
||||
"municipality_name_nl": row.get("TX_DESCR_NL"),
|
||||
}
|
||||
if not rows:
|
||||
raise RuntimeError("Statbel population table contains no usable Mol sectors")
|
||||
return rows
|
||||
|
||||
|
||||
def build_snapshot(year: int, sector_payload: dict[str, Any], population: dict[str, dict[str, Any]], boundary) -> dict[str, Any]:
|
||||
transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
||||
features: list[dict[str, Any]] = []
|
||||
missing_population = 0
|
||||
for source_feature in sector_payload.get("features") or []:
|
||||
properties = source_feature.get("properties") or {}
|
||||
if str(properties.get("cd_munty_refnis") or "") != MUNICIPALITY_NIS_CODE:
|
||||
continue
|
||||
sector_code = str(properties.get("cd_sector") or "").strip()
|
||||
population_values = population.get(sector_code)
|
||||
if not population_values:
|
||||
missing_population += 1
|
||||
continue
|
||||
geometry = transform(transformer.transform, shape(source_feature["geometry"]))
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
geometry = geometry.intersection(boundary)
|
||||
if geometry.is_empty:
|
||||
continue
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
combined = {
|
||||
**properties,
|
||||
**population_values,
|
||||
"source_name": "statbel",
|
||||
"source_feature_id": sector_code,
|
||||
"reference_layer_name": "population",
|
||||
"authority_level": "authoritative",
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"observation_year": year,
|
||||
"attribution": ATTRIBUTION,
|
||||
}
|
||||
features.append({"type": "Feature", "id": sector_code, "geometry": mapping(geometry), "properties": combined})
|
||||
if not features:
|
||||
raise RuntimeError(f"No joined population sectors were produced for {year}")
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"name": f"Statbel population by statistical sector - Mol {year}",
|
||||
"features": features,
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"observation_year": year,
|
||||
"missing_population_sector_count": missing_population,
|
||||
"attribution": ATTRIBUTION,
|
||||
}
|
||||
|
||||
|
||||
def locate_workspace(session: requests.Session, base_url: str, project_name: str, timeout: int):
|
||||
projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=timeout))
|
||||
project = next((item for item in projects.get("items") or [] if item.get("name") == project_name), None)
|
||||
if not project:
|
||||
raise RuntimeError(f"Project {project_name!r} is missing")
|
||||
project_id = str(project["id"])
|
||||
areas = response_data(session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=timeout))
|
||||
area = next((item for item in areas.get("items") or [] if "gemeente mol" in str(item.get("name", "")).lower()), None)
|
||||
if not area:
|
||||
raise RuntimeError("Official Mol area is missing")
|
||||
datasets = response_data(session.get(f"{base_url}/api/v1/projects/{project_id}/datasets", params={"limit": 200}, timeout=timeout))
|
||||
return project_id, str(area["id"]), list(datasets.get("items") or [])
|
||||
|
||||
|
||||
def upload_snapshot(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
area_id: str,
|
||||
year: int,
|
||||
path: Path,
|
||||
timeout: int,
|
||||
) -> dict[str, Any]:
|
||||
observed_at = f"{year}-01-01T00:00:00Z"
|
||||
source_metadata = {
|
||||
"provider": "Statbel",
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": "municipality",
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"attribution": ATTRIBUTION,
|
||||
"license": "CC BY 4.0",
|
||||
"identity_stable": True,
|
||||
"comparison_property": "population_total",
|
||||
"selection_aggregation": {
|
||||
"method": "area_weighted_sum",
|
||||
"property": "population_total",
|
||||
"label": "Inwoners",
|
||||
"unit": "inwoners",
|
||||
"is_estimate": True,
|
||||
"warning": "Bevolking binnen een gedeeltelijke statistische sector is oppervlaktegewogen en blijft een schatting.",
|
||||
},
|
||||
}
|
||||
provenance_metadata = {
|
||||
"operator_tool": "provision_mol_population_history.py",
|
||||
"operator_explicit_fetch": True,
|
||||
"sector_geometry_url": SECTOR_URL.format(year=year),
|
||||
"population_url": POPULATION_URLS[year],
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
with path.open("rb") as handle:
|
||||
response = session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
|
||||
data={
|
||||
"dataset_type": "vector",
|
||||
"source": "operator_official_import",
|
||||
"dataset_role": "reference",
|
||||
"source_name": "statbel",
|
||||
"reference_layer_name": "population",
|
||||
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
|
||||
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
|
||||
"area_id": area_id,
|
||||
"temporal_series_key": SERIES_KEY,
|
||||
"observed_at": observed_at,
|
||||
"valid_from": observed_at,
|
||||
"valid_to": f"{year}-12-31T23:59:59Z",
|
||||
"temporal_granularity": "year",
|
||||
"source_version": str(year),
|
||||
},
|
||||
files={"file": (path.name, handle, "application/geo+json")},
|
||||
timeout=timeout,
|
||||
)
|
||||
return response_data(response)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
years = sorted({int(value.strip()) for value in args.years.split(",") if value.strip()})
|
||||
except ValueError:
|
||||
print(json.dumps({"status": "error", "message": "Years must be comma-separated integers"}), file=sys.stderr)
|
||||
return 2
|
||||
unsupported = [year for year in years if year not in POPULATION_URLS]
|
||||
if unsupported or not years:
|
||||
print(json.dumps({"status": "error", "message": f"Unsupported years: {unsupported}"}), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
results: list[dict[str, Any]] = []
|
||||
try:
|
||||
boundary = load_boundary(args.boundary_path)
|
||||
prepared: list[tuple[int, Path, int]] = []
|
||||
with build_session() as source_session:
|
||||
for year in years:
|
||||
path = args.output_dir / f"mol_statbel_population_{year}.geojson"
|
||||
if args.force or not path.exists():
|
||||
sectors_response = source_session.get(SECTOR_URL.format(year=year), timeout=args.request_timeout)
|
||||
sectors_response.raise_for_status()
|
||||
population_response = source_session.get(POPULATION_URLS[year], timeout=args.request_timeout)
|
||||
population_response.raise_for_status()
|
||||
snapshot = build_snapshot(
|
||||
year,
|
||||
zip_member_json(sectors_response.content),
|
||||
population_rows(population_response.content),
|
||||
boundary,
|
||||
)
|
||||
path.write_text(json.dumps(snapshot, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
prepared.append((year, path, len(payload.get("features") or [])))
|
||||
|
||||
if args.fetch_only:
|
||||
results = [{"year": year, "path": str(path), "feature_count": count, "status": "prepared"} for year, path, count in prepared]
|
||||
else:
|
||||
base_url = args.base_url.rstrip("/")
|
||||
with requests.Session() as api_session:
|
||||
project_id, area_id, existing = locate_workspace(api_session, base_url, args.project_name, args.import_timeout)
|
||||
for year, path, count in prepared:
|
||||
observed_at = f"{year}-01-01T00:00:00+00:00"
|
||||
dataset = next(
|
||||
(
|
||||
item
|
||||
for item in existing
|
||||
if item.get("temporal_series_key") == SERIES_KEY
|
||||
and str(item.get("observed_at") or "").startswith(observed_at[:10])
|
||||
),
|
||||
None,
|
||||
)
|
||||
if dataset:
|
||||
results.append({"year": year, "dataset_id": dataset["id"], "feature_count": dataset.get("feature_count"), "status": "existing"})
|
||||
continue
|
||||
dataset = upload_snapshot(api_session, base_url, project_id, area_id, year, path, args.import_timeout)
|
||||
results.append({"year": year, "dataset_id": dataset["id"], "feature_count": dataset.get("feature_count"), "status": "imported"})
|
||||
except (OSError, RuntimeError, requests.RequestException, ValueError, KeyError, zipfile.BadZipFile) as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(json.dumps({"status": "ok", "municipality": MUNICIPALITY_NAME, "series": SERIES_KEY, "snapshots": results}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user