feat: add temporal Mol explorer
This commit is contained in:
@@ -935,6 +935,36 @@ the same bbox-selected FeatureCollection as a normal export record with
|
||||
`export_type="vector_selection_geojson"`. This creates a handoff artifact only;
|
||||
it does not create a derived dataset.
|
||||
|
||||
## Temporal Mol data and evolution
|
||||
|
||||
Dataset uploads accept `temporal_series_key`, `observed_at`, `valid_from`,
|
||||
`valid_to`, `temporal_granularity` and `source_version`. Every new source or
|
||||
derived dataset also writes dataset version 1 in the same transaction.
|
||||
|
||||
After the Mol municipality workspace is available, import the official source
|
||||
snapshots explicitly:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_mol_population_history.py
|
||||
docker exec geointel python /app/scripts/provision_mol_historical_landuse.py
|
||||
```
|
||||
|
||||
The first command imports Statbel sector population for 2021-2025. The second
|
||||
imports Digitaal Vlaanderen historical land use for 1778, 1873 and 1969. Both
|
||||
are idempotent, use the normal API/DatasetService flow and retain fetched
|
||||
artifacts in persistent operator storage. They never run on app startup.
|
||||
|
||||
Historical land-use work can be bounded explicitly:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_mol_historical_landuse.py --years 1778 1969 --themes forest water
|
||||
```
|
||||
|
||||
`GET /api/v1/projects/{project_id}/temporal/series` discovers the series and
|
||||
`POST /api/v1/projects/{project_id}/temporal/compare` compares two snapshots
|
||||
inside one EPSG:4326 bbox. Partial statistical sectors are estimates; old map
|
||||
editions without stable identities do not produce invented object changes.
|
||||
|
||||
## Helpful repository scripts
|
||||
|
||||
- `bash scripts/backend_install.sh`
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Add temporal dataset metadata and durable dataset-version provenance."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "202607140001"
|
||||
down_revision = "202606120900"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("datasets", sa.Column("temporal_series_key", sa.String(length=255), nullable=True))
|
||||
op.add_column("datasets", sa.Column("observed_at", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("datasets", sa.Column("valid_from", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("datasets", sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("datasets", sa.Column("temporal_granularity", sa.String(length=32), nullable=True))
|
||||
op.add_column("datasets", sa.Column("source_version", sa.String(length=120), nullable=True))
|
||||
|
||||
op.add_column("dataset_versions", sa.Column("source_version", sa.String(length=120), nullable=True))
|
||||
op.add_column("dataset_versions", sa.Column("observed_at", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("dataset_versions", sa.Column("valid_from", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("dataset_versions", sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("dataset_versions", sa.Column("checksum_sha256", sa.String(length=64), nullable=True))
|
||||
op.add_column("dataset_versions", sa.Column("source_metadata", sa.JSON(), nullable=True))
|
||||
op.add_column("dataset_versions", sa.Column("provenance_metadata", sa.JSON(), nullable=True))
|
||||
|
||||
op.create_index(
|
||||
"ix_datasets_project_temporal_series_observed",
|
||||
"datasets",
|
||||
["project_id", "temporal_series_key", "observed_at"],
|
||||
)
|
||||
op.create_index("ix_dataset_versions_dataset_version", "dataset_versions", ["dataset_id", "version"], unique=True)
|
||||
op.create_index(
|
||||
"ix_vector_features_dataset_source_feature",
|
||||
"vector_features",
|
||||
["dataset_id", "source_feature_id"],
|
||||
)
|
||||
op.create_check_constraint(
|
||||
"ck_datasets_temporal_valid_range",
|
||||
"datasets",
|
||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||
)
|
||||
op.create_check_constraint(
|
||||
"ck_dataset_versions_temporal_valid_range",
|
||||
"dataset_versions",
|
||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_constraint("ck_dataset_versions_temporal_valid_range", "dataset_versions", type_="check")
|
||||
op.drop_constraint("ck_datasets_temporal_valid_range", "datasets", type_="check")
|
||||
op.drop_index("ix_vector_features_dataset_source_feature", table_name="vector_features")
|
||||
op.drop_index("ix_dataset_versions_dataset_version", table_name="dataset_versions")
|
||||
op.drop_index("ix_datasets_project_temporal_series_observed", table_name="datasets")
|
||||
|
||||
op.drop_column("dataset_versions", "provenance_metadata")
|
||||
op.drop_column("dataset_versions", "source_metadata")
|
||||
op.drop_column("dataset_versions", "checksum_sha256")
|
||||
op.drop_column("dataset_versions", "valid_to")
|
||||
op.drop_column("dataset_versions", "valid_from")
|
||||
op.drop_column("dataset_versions", "observed_at")
|
||||
op.drop_column("dataset_versions", "source_version")
|
||||
|
||||
op.drop_column("datasets", "source_version")
|
||||
op.drop_column("datasets", "temporal_granularity")
|
||||
op.drop_column("datasets", "valid_to")
|
||||
op.drop_column("datasets", "valid_from")
|
||||
op.drop_column("datasets", "observed_at")
|
||||
op.drop_column("datasets", "temporal_series_key")
|
||||
@@ -1 +1 @@
|
||||
__all__ = ["analysis", "areas", "datasets", "health", "projects", "exports", "jobs", "external", "qa"]
|
||||
__all__ = ["analysis", "areas", "datasets", "health", "projects", "exports", "jobs", "external", "qa", "temporal"]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
from uuid import UUID as _UUID
|
||||
@@ -30,7 +31,7 @@ from app.schemas import (
|
||||
VectorSelectionResponse,
|
||||
)
|
||||
from app.schemas.job import JobCreate
|
||||
from app.schemas.dataset import DatasetCreateResponse
|
||||
from app.schemas.dataset import DatasetCreateResponse, DatasetTemporalUpdate
|
||||
from app.schemas.operations import VectorOperationResult
|
||||
from app.services.job_service import JobService
|
||||
from app.services.raster_operations_service import RasterOperationsService
|
||||
@@ -87,6 +88,12 @@ async def upload_dataset(
|
||||
reference_layer_name: str | None = Form(None),
|
||||
source_metadata_json: str | None = Form(None),
|
||||
provenance_metadata_json: str | None = Form(None),
|
||||
temporal_series_key: str | None = Form(None),
|
||||
observed_at: datetime | None = Form(None),
|
||||
valid_from: datetime | None = Form(None),
|
||||
valid_to: datetime | None = Form(None),
|
||||
temporal_granularity: str | None = Form(None),
|
||||
source_version: str | None = Form(None),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
if area_id is not None:
|
||||
@@ -108,6 +115,12 @@ async def upload_dataset(
|
||||
source_metadata=_parse_metadata_json(source_metadata_json, "source_metadata_json"),
|
||||
provenance_metadata=_parse_metadata_json(provenance_metadata_json, "provenance_metadata_json"),
|
||||
area_id=area_id,
|
||||
temporal_series_key=temporal_series_key,
|
||||
observed_at=observed_at,
|
||||
valid_from=valid_from,
|
||||
valid_to=valid_to,
|
||||
temporal_granularity=temporal_granularity,
|
||||
source_version=source_version,
|
||||
)
|
||||
return envelope(created.model_dump())
|
||||
|
||||
@@ -135,6 +148,33 @@ def get_dataset(
|
||||
return envelope(DatasetCreateResponse.model_validate(dataset).model_dump())
|
||||
|
||||
|
||||
@router.patch("/datasets/{dataset_id}/temporal", response_model=dict)
|
||||
def update_dataset_temporal_metadata(
|
||||
project_id: UUID,
|
||||
dataset_id: UUID,
|
||||
payload: DatasetTemporalUpdate,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||
if dataset.project_id != project_id:
|
||||
raise HTTPException(status_code=404, detail="Dataset not found")
|
||||
updated = DatasetService.update_temporal_metadata(db, dataset_id, payload)
|
||||
return envelope(updated.model_dump())
|
||||
|
||||
|
||||
@router.get("/datasets/{dataset_id}/versions", response_model=dict)
|
||||
def list_dataset_versions(
|
||||
project_id: UUID,
|
||||
dataset_id: UUID,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||
if dataset.project_id != project_id:
|
||||
raise HTTPException(status_code=404, detail="Dataset not found")
|
||||
versions = DatasetService.list_versions(db, dataset_id)
|
||||
return envelope({"items": [item.model_dump() for item in versions], "total": len(versions)})
|
||||
|
||||
|
||||
@router.post("/datasets/{dataset_id}/metadata/refresh", response_model=dict)
|
||||
def refresh_dataset_metadata(
|
||||
project_id: UUID,
|
||||
@@ -203,6 +243,13 @@ def select_vector_features(
|
||||
bbox=payload.bbox.model_dump(),
|
||||
limit=payload.limit,
|
||||
)
|
||||
if isinstance(dataset.source_metadata, dict) and dataset.source_metadata.get("selection_aggregation"):
|
||||
result["summary"] = VectorFeatureService.summarize_features_by_bbox(
|
||||
db,
|
||||
dataset=dataset,
|
||||
bbox=payload.bbox.model_dump(),
|
||||
total_feature_count=result.get("total_feature_count"),
|
||||
)
|
||||
return envelope(VectorSelectionResponse(**result).model_dump(exclude_none=True))
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.schemas.temporal import TemporalComparisonRequest
|
||||
from app.services.temporal_analysis_service import TemporalAnalysisService
|
||||
from app.utils.response import envelope
|
||||
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/temporal", tags=["temporal"])
|
||||
|
||||
|
||||
@router.get("/series", response_model=dict)
|
||||
def list_temporal_series(project_id: UUID, db: Session = Depends(get_db)):
|
||||
series = TemporalAnalysisService.list_series(db, project_id)
|
||||
return envelope({"items": [item.model_dump() for item in series], "total": len(series)})
|
||||
|
||||
|
||||
@router.post("/compare", response_model=dict)
|
||||
def compare_temporal_snapshots(
|
||||
project_id: UUID,
|
||||
payload: TemporalComparisonRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
return envelope(TemporalAnalysisService.compare(db, project_id=project_id, payload=payload).model_dump())
|
||||
+2
-1
@@ -5,7 +5,7 @@ from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from app.api.routes import analysis, areas, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation
|
||||
from app.api.routes import analysis, areas, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, temporal
|
||||
from app.core.config import get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.core.logging import configure_logging
|
||||
@@ -57,6 +57,7 @@ def create_app() -> FastAPI:
|
||||
app.include_router(qa.router, prefix=settings.api_prefix)
|
||||
app.include_router(detection.router, prefix=settings.api_prefix)
|
||||
app.include_router(segmentation.router, prefix=settings.api_prefix)
|
||||
app.include_router(temporal.router, prefix=settings.api_prefix)
|
||||
|
||||
@app.exception_handler(AppError)
|
||||
async def app_error(request: Request, exc: AppError): # noqa: ARG001
|
||||
|
||||
@@ -4,7 +4,7 @@ import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from geoalchemy2 import Geometry
|
||||
from sqlalchemy import DateTime, ForeignKey, Float, Index, JSON, String, Text, func
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, func
|
||||
from sqlalchemy.sql.sqltypes import Integer
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
@@ -44,6 +44,18 @@ class Area(Base):
|
||||
|
||||
class Dataset(Base):
|
||||
__tablename__ = "datasets"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||
name="ck_datasets_temporal_valid_range",
|
||||
),
|
||||
Index(
|
||||
"ix_datasets_project_temporal_series_observed",
|
||||
"project_id",
|
||||
"temporal_series_key",
|
||||
"observed_at",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
@@ -73,6 +85,12 @@ class Dataset(Base):
|
||||
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
temporal_granularity: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="uploaded")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
@@ -92,11 +110,25 @@ class Dataset(Base):
|
||||
|
||||
class DatasetVersion(Base):
|
||||
__tablename__ = "dataset_versions"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||
name="ck_dataset_versions_temporal_valid_range",
|
||||
),
|
||||
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
dataset_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False)
|
||||
version: Mapped[int] = mapped_column(Integer, default=1)
|
||||
storage_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
|
||||
@@ -107,6 +139,7 @@ class VectorFeature(Base):
|
||||
__table_args__ = (
|
||||
Index("ix_vector_features_dataset_id", "dataset_id"),
|
||||
Index("ix_vector_features_geometry", "geometry", postgresql_using="gist"),
|
||||
Index("ix_vector_features_dataset_source_feature", "dataset_id", "source_feature_id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
|
||||
@@ -77,6 +77,7 @@ from .operations import (
|
||||
VectorSelectionDeriveRequest,
|
||||
VectorSelectionRequest,
|
||||
VectorSelectionResponse,
|
||||
VectorSelectionSummary,
|
||||
VectorStatsRequest,
|
||||
VectorStatsResponse,
|
||||
)
|
||||
@@ -134,6 +135,7 @@ __all__ = [
|
||||
"VectorSelectionDeriveRequest",
|
||||
"VectorSelectionRequest",
|
||||
"VectorSelectionResponse",
|
||||
"VectorSelectionSummary",
|
||||
"RasterClipRequest",
|
||||
"RasterStatsResponse",
|
||||
"RasterReprojectRequest",
|
||||
|
||||
@@ -36,6 +36,12 @@ class DatasetCreateResponse(BaseModel):
|
||||
source_metadata: dict | None = None
|
||||
provenance_metadata: dict | None = None
|
||||
imported_at: datetime | None = None
|
||||
temporal_series_key: str | None = None
|
||||
observed_at: datetime | None = None
|
||||
valid_from: datetime | None = None
|
||||
valid_to: datetime | None = None
|
||||
temporal_granularity: str | None = None
|
||||
source_version: str | None = None
|
||||
project_id: UUID
|
||||
area_id: UUID | None = None
|
||||
storage_path: str | None = None
|
||||
@@ -70,6 +76,32 @@ class DatasetMetadataRefresh(BaseModel):
|
||||
crs: str | None = None
|
||||
|
||||
|
||||
class DatasetTemporalUpdate(BaseModel):
|
||||
temporal_series_key: str
|
||||
observed_at: datetime
|
||||
valid_from: datetime | None = None
|
||||
valid_to: datetime | None = None
|
||||
temporal_granularity: str = "snapshot"
|
||||
source_version: str | None = None
|
||||
|
||||
|
||||
class DatasetVersionRead(BaseModel):
|
||||
id: UUID
|
||||
dataset_id: UUID
|
||||
version: int
|
||||
storage_path: str | None = None
|
||||
source_version: str | None = None
|
||||
observed_at: datetime | None = None
|
||||
valid_from: datetime | None = None
|
||||
valid_to: datetime | None = None
|
||||
checksum_sha256: str | None = None
|
||||
source_metadata: dict | None = None
|
||||
provenance_metadata: dict | None = None
|
||||
created_at: datetime | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class ExportRequest(BaseModel):
|
||||
dataset_id: UUID
|
||||
name: str | None = None
|
||||
|
||||
@@ -217,6 +217,16 @@ class VectorSelectionDeriveRequest(VectorSelectionRequest):
|
||||
output_name: str | None = None
|
||||
|
||||
|
||||
class VectorSelectionSummary(BaseModel):
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
feature_count: int
|
||||
is_estimate: bool = False
|
||||
warning: str | None = None
|
||||
|
||||
|
||||
class VectorSelectionResponse(BaseModel):
|
||||
selection_bbox: VectorSelectionBBox
|
||||
feature_count: int
|
||||
@@ -224,3 +234,4 @@ class VectorSelectionResponse(BaseModel):
|
||||
limit: int
|
||||
truncated: bool
|
||||
geojson: dict
|
||||
summary: VectorSelectionSummary | None = None
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.schemas.operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class TemporalComparisonRequest(BaseModel):
|
||||
earlier_dataset_id: UUID
|
||||
later_dataset_id: UUID
|
||||
bbox: VectorSelectionBBox
|
||||
preview_limit: int = Field(default=500, ge=1, le=1000)
|
||||
|
||||
|
||||
class TemporalDatasetRef(BaseModel):
|
||||
id: UUID
|
||||
name: str
|
||||
observed_at: datetime
|
||||
source_version: str | None = None
|
||||
|
||||
|
||||
class TemporalMetricComparison(BaseModel):
|
||||
label: str
|
||||
unit: str
|
||||
aggregation_method: str
|
||||
earlier_value: float
|
||||
later_value: float
|
||||
absolute_change: float
|
||||
percent_change: float | None = None
|
||||
is_estimate: bool = False
|
||||
|
||||
|
||||
class TemporalObjectChanges(BaseModel):
|
||||
available: bool
|
||||
added_count: int | None = None
|
||||
removed_count: int | None = None
|
||||
modified_count: int | None = None
|
||||
unchanged_count: int | None = None
|
||||
|
||||
|
||||
class TemporalComparisonResponse(BaseModel):
|
||||
temporal_series_key: str
|
||||
earlier: TemporalDatasetRef
|
||||
later: TemporalDatasetRef
|
||||
selection_bbox: VectorSelectionBBox
|
||||
metric: TemporalMetricComparison
|
||||
object_changes: TemporalObjectChanges
|
||||
geojson: dict
|
||||
warnings: list[str]
|
||||
generated_at: datetime
|
||||
|
||||
|
||||
class TemporalSeriesDataset(BaseModel):
|
||||
id: UUID
|
||||
name: str
|
||||
observed_at: datetime
|
||||
source_version: str | None = None
|
||||
feature_count: int | None = None
|
||||
|
||||
|
||||
class TemporalSeriesRead(BaseModel):
|
||||
temporal_series_key: str
|
||||
source_name: str | None = None
|
||||
reference_layer_name: str | None = None
|
||||
dataset_count: int
|
||||
first_observed_at: datetime
|
||||
last_observed_at: datetime
|
||||
datasets: list[TemporalSeriesDataset]
|
||||
@@ -12,8 +12,14 @@ from fastapi import UploadFile
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, Project
|
||||
from app.schemas.dataset import DatasetCreateResponse, DatasetStorageResponse, DatasetVectorSummary
|
||||
from app.models import Dataset, DatasetVersion, Project
|
||||
from app.schemas.dataset import (
|
||||
DatasetCreateResponse,
|
||||
DatasetStorageResponse,
|
||||
DatasetTemporalUpdate,
|
||||
DatasetVectorSummary,
|
||||
DatasetVersionRead,
|
||||
)
|
||||
from app.services.geojson_service import parse_geojson_payload, load_dataset_text
|
||||
from app.services.raster_service import extract_raster_metadata
|
||||
from app.services.storage_service import StorageService
|
||||
@@ -26,6 +32,105 @@ class DatasetService:
|
||||
VECTOR_TYPES = {"vector", "geojson"}
|
||||
RASTER_TYPES = {"raster", "tif", "tiff", "geotiff"}
|
||||
VALID_DATASET_ROLES = {"source", "derived", "reference"}
|
||||
VALID_TEMPORAL_GRANULARITIES = {"snapshot", "day", "month", "year", "period"}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_datetime(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
@staticmethod
|
||||
def _validate_temporal_metadata(
|
||||
*,
|
||||
temporal_series_key: str | None,
|
||||
observed_at: datetime | None,
|
||||
valid_from: datetime | None,
|
||||
valid_to: datetime | None,
|
||||
temporal_granularity: str | None,
|
||||
source_version: str | None,
|
||||
) -> dict[str, Any]:
|
||||
normalized_key = (temporal_series_key or "").strip() or None
|
||||
normalized_observed_at = DatasetService._normalize_datetime(observed_at)
|
||||
normalized_valid_from = DatasetService._normalize_datetime(valid_from)
|
||||
normalized_valid_to = DatasetService._normalize_datetime(valid_to)
|
||||
normalized_granularity = (temporal_granularity or "").strip().lower() or None
|
||||
normalized_source_version = (source_version or "").strip() or None
|
||||
|
||||
if normalized_key and len(normalized_key) > 255:
|
||||
raise AppError(code="INVALID_TEMPORAL_METADATA", message="temporal_series_key is too long", status_code=400)
|
||||
if normalized_granularity and normalized_granularity not in DatasetService.VALID_TEMPORAL_GRANULARITIES:
|
||||
raise AppError(
|
||||
code="INVALID_TEMPORAL_METADATA",
|
||||
message="temporal_granularity must be snapshot, day, month, year or period",
|
||||
status_code=400,
|
||||
)
|
||||
if normalized_valid_from and normalized_valid_to and normalized_valid_to < normalized_valid_from:
|
||||
raise AppError(
|
||||
code="INVALID_TEMPORAL_METADATA",
|
||||
message="valid_to must be on or after valid_from",
|
||||
status_code=400,
|
||||
)
|
||||
if normalized_key and normalized_observed_at is None:
|
||||
raise AppError(
|
||||
code="INVALID_TEMPORAL_METADATA",
|
||||
message="observed_at is required when temporal_series_key is provided",
|
||||
status_code=400,
|
||||
)
|
||||
if normalized_observed_at and normalized_key is None:
|
||||
raise AppError(
|
||||
code="INVALID_TEMPORAL_METADATA",
|
||||
message="temporal_series_key is required when observed_at is provided",
|
||||
status_code=400,
|
||||
)
|
||||
return {
|
||||
"temporal_series_key": normalized_key,
|
||||
"observed_at": normalized_observed_at,
|
||||
"valid_from": normalized_valid_from,
|
||||
"valid_to": normalized_valid_to,
|
||||
"temporal_granularity": normalized_granularity,
|
||||
"source_version": normalized_source_version,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _to_response(dataset: Dataset) -> DatasetCreateResponse:
|
||||
metadata_json = dataset.metadata_json if isinstance(dataset.metadata_json, dict) else {}
|
||||
return DatasetCreateResponse(
|
||||
id=dataset.id,
|
||||
name=dataset.name,
|
||||
dataset_type=dataset.dataset_type,
|
||||
source=dataset.source,
|
||||
dataset_role=dataset.dataset_role,
|
||||
source_name=dataset.source_name,
|
||||
reference_layer_name=dataset.reference_layer_name,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
imported_at=dataset.imported_at,
|
||||
temporal_series_key=dataset.temporal_series_key,
|
||||
observed_at=dataset.observed_at,
|
||||
valid_from=dataset.valid_from,
|
||||
valid_to=dataset.valid_to,
|
||||
temporal_granularity=dataset.temporal_granularity,
|
||||
source_version=dataset.source_version,
|
||||
project_id=dataset.project_id,
|
||||
area_id=dataset.area_id,
|
||||
storage_path=dataset.storage_path,
|
||||
original_filename=dataset.original_filename,
|
||||
stored_filename=dataset.stored_filename,
|
||||
content_type=dataset.content_type,
|
||||
size_bytes=dataset.size_bytes,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
crs=dataset.crs,
|
||||
bounds_json=dataset.bounds_json,
|
||||
metadata_json=dataset.metadata_json,
|
||||
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, metadata_json),
|
||||
status=dataset.status,
|
||||
derived_from_dataset_id=dataset.derived_from_dataset_id,
|
||||
created_at=dataset.created_at,
|
||||
feature_count=metadata_json.get("feature_count"),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _canonical_dataset_type(dataset_type: str) -> str:
|
||||
@@ -89,44 +194,7 @@ class DatasetService:
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
response_items = []
|
||||
for row in rows:
|
||||
feature_count = None
|
||||
metadata_json = row.metadata_json or {}
|
||||
vector_summary = DatasetService._extract_vector_summary(row.dataset_type, metadata_json)
|
||||
if isinstance(metadata_json, dict):
|
||||
feature_count = metadata_json.get("feature_count")
|
||||
response_items.append(
|
||||
DatasetCreateResponse(
|
||||
id=row.id,
|
||||
name=row.name,
|
||||
dataset_type=row.dataset_type,
|
||||
source=row.source,
|
||||
dataset_role=row.dataset_role,
|
||||
source_name=row.source_name,
|
||||
reference_layer_name=row.reference_layer_name,
|
||||
source_metadata=row.source_metadata,
|
||||
provenance_metadata=row.provenance_metadata,
|
||||
imported_at=row.imported_at,
|
||||
project_id=row.project_id,
|
||||
area_id=row.area_id,
|
||||
storage_path=row.storage_path,
|
||||
original_filename=row.original_filename,
|
||||
stored_filename=row.stored_filename,
|
||||
content_type=row.content_type,
|
||||
size_bytes=row.size_bytes,
|
||||
checksum_sha256=row.checksum_sha256,
|
||||
crs=row.crs,
|
||||
bounds_json=row.bounds_json,
|
||||
metadata_json=row.metadata_json,
|
||||
vector_summary=vector_summary,
|
||||
status=row.status,
|
||||
derived_from_dataset_id=row.derived_from_dataset_id,
|
||||
created_at=row.created_at,
|
||||
feature_count=feature_count,
|
||||
)
|
||||
)
|
||||
return response_items, total
|
||||
return [DatasetService._to_response(row) for row in rows], total
|
||||
|
||||
@staticmethod
|
||||
def _extract_vector_summary(dataset_type: str, metadata_json: dict) -> DatasetVectorSummary | None:
|
||||
@@ -195,6 +263,12 @@ class DatasetService:
|
||||
source_metadata: dict | None = None,
|
||||
provenance_metadata: dict | None = None,
|
||||
area_id: UUID | None = None,
|
||||
temporal_series_key: str | None = None,
|
||||
observed_at: datetime | None = None,
|
||||
valid_from: datetime | None = None,
|
||||
valid_to: datetime | None = None,
|
||||
temporal_granularity: str | None = None,
|
||||
source_version: str | None = None,
|
||||
) -> DatasetCreateResponse:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
@@ -202,6 +276,14 @@ class DatasetService:
|
||||
filename = DatasetService._validate_upload_filename(file.filename)
|
||||
canonical_type = DatasetService._canonical_dataset_type(dataset_type)
|
||||
normalized_role = DatasetService._normalize_dataset_role(dataset_role)
|
||||
temporal = DatasetService._validate_temporal_metadata(
|
||||
temporal_series_key=temporal_series_key,
|
||||
observed_at=observed_at,
|
||||
valid_from=valid_from,
|
||||
valid_to=valid_to,
|
||||
temporal_granularity=temporal_granularity,
|
||||
source_version=source_version,
|
||||
)
|
||||
normalized_source_name = source_name
|
||||
if normalized_role == "reference" and not normalized_source_name:
|
||||
normalized_source_name = "manual"
|
||||
@@ -280,6 +362,7 @@ class DatasetService:
|
||||
source_metadata=source_metadata,
|
||||
provenance_metadata=provenance_metadata,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
**temporal,
|
||||
storage_path=storage_info["storage_path"],
|
||||
original_filename=storage_info["original_filename"],
|
||||
stored_filename=storage_info["stored_filename"],
|
||||
@@ -294,6 +377,20 @@ class DatasetService:
|
||||
status=status,
|
||||
)
|
||||
db.add(dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
storage_path=dataset.storage_path,
|
||||
source_version=dataset.source_version,
|
||||
observed_at=dataset.observed_at,
|
||||
valid_from=dataset.valid_from,
|
||||
valid_to=dataset.valid_to,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
|
||||
@@ -306,34 +403,7 @@ class DatasetService:
|
||||
feature_class=feature_class,
|
||||
)
|
||||
|
||||
return DatasetCreateResponse(
|
||||
id=dataset.id,
|
||||
name=dataset.name,
|
||||
dataset_type=dataset.dataset_type,
|
||||
source=dataset.source,
|
||||
dataset_role=dataset.dataset_role,
|
||||
source_name=dataset.source_name,
|
||||
reference_layer_name=dataset.reference_layer_name,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
imported_at=dataset.imported_at,
|
||||
project_id=dataset.project_id,
|
||||
area_id=dataset.area_id,
|
||||
storage_path=dataset.storage_path,
|
||||
original_filename=dataset.original_filename,
|
||||
stored_filename=dataset.stored_filename,
|
||||
content_type=dataset.content_type,
|
||||
size_bytes=dataset.size_bytes,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
crs=dataset.crs,
|
||||
derived_from_dataset_id=dataset.derived_from_dataset_id,
|
||||
bounds_json=dataset.bounds_json,
|
||||
metadata_json=dataset.metadata_json,
|
||||
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}),
|
||||
status=dataset.status,
|
||||
created_at=dataset.created_at,
|
||||
feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None,
|
||||
)
|
||||
return DatasetService._to_response(dataset)
|
||||
|
||||
@staticmethod
|
||||
def refresh_metadata(db: Session, dataset_id: UUID) -> DatasetCreateResponse:
|
||||
@@ -380,34 +450,53 @@ class DatasetService:
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
|
||||
return DatasetCreateResponse(
|
||||
id=dataset.id,
|
||||
name=dataset.name,
|
||||
dataset_type=dataset.dataset_type,
|
||||
source=dataset.source,
|
||||
dataset_role=dataset.dataset_role,
|
||||
source_name=dataset.source_name,
|
||||
reference_layer_name=dataset.reference_layer_name,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
imported_at=dataset.imported_at,
|
||||
project_id=dataset.project_id,
|
||||
area_id=dataset.area_id,
|
||||
storage_path=dataset.storage_path,
|
||||
original_filename=dataset.original_filename,
|
||||
stored_filename=dataset.stored_filename,
|
||||
content_type=dataset.content_type,
|
||||
size_bytes=dataset.size_bytes,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
crs=dataset.crs,
|
||||
derived_from_dataset_id=dataset.derived_from_dataset_id,
|
||||
bounds_json=dataset.bounds_json,
|
||||
metadata_json=dataset.metadata_json,
|
||||
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}),
|
||||
status=dataset.status,
|
||||
created_at=dataset.created_at,
|
||||
feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None,
|
||||
return DatasetService._to_response(dataset)
|
||||
|
||||
@staticmethod
|
||||
def update_temporal_metadata(db: Session, dataset_id: UUID, payload: DatasetTemporalUpdate) -> DatasetCreateResponse:
|
||||
dataset = DatasetService._get_dataset(db, dataset_id)
|
||||
temporal = DatasetService._validate_temporal_metadata(**payload.model_dump())
|
||||
if all(getattr(dataset, field) == value for field, value in temporal.items()):
|
||||
return DatasetService._to_response(dataset)
|
||||
|
||||
for field, value in temporal.items():
|
||||
setattr(dataset, field, value)
|
||||
|
||||
latest_version = (
|
||||
db.query(DatasetVersion)
|
||||
.filter(DatasetVersion.dataset_id == dataset.id)
|
||||
.order_by(DatasetVersion.version.desc())
|
||||
.first()
|
||||
)
|
||||
db.add(dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=(latest_version.version + 1) if latest_version else 1,
|
||||
storage_path=dataset.storage_path,
|
||||
source_version=dataset.source_version,
|
||||
observed_at=dataset.observed_at,
|
||||
valid_from=dataset.valid_from,
|
||||
valid_to=dataset.valid_to,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
return DatasetService._to_response(dataset)
|
||||
|
||||
@staticmethod
|
||||
def list_versions(db: Session, dataset_id: UUID) -> list[DatasetVersionRead]:
|
||||
DatasetService._get_dataset(db, dataset_id)
|
||||
rows = (
|
||||
db.query(DatasetVersion)
|
||||
.filter(DatasetVersion.dataset_id == dataset_id)
|
||||
.order_by(DatasetVersion.version.desc())
|
||||
.all()
|
||||
)
|
||||
return [DatasetVersionRead.model_validate(row) for row in rows]
|
||||
|
||||
@staticmethod
|
||||
def get_dataset(db: Session, dataset_id: UUID) -> Dataset:
|
||||
|
||||
@@ -10,7 +10,7 @@ from uuid import UUID, uuid4
|
||||
from geoalchemy2.shape import from_shape
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Area, Dataset, Metric, Project, QualityCheck
|
||||
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
|
||||
from app.schemas.demo import DemoWorkflowResponse
|
||||
from app.services.geojson_service import parse_geojson_payload
|
||||
from app.services.qa_service import QaService
|
||||
@@ -29,6 +29,19 @@ class DemoWorkflowService:
|
||||
RASTER_FILENAME = "demo_context_raster.tif"
|
||||
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
|
||||
|
||||
@staticmethod
|
||||
def _add_initial_version(db: Session, dataset: Dataset) -> None:
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
storage_path=dataset.storage_path,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _repo_root() -> Path:
|
||||
return Path(__file__).resolve().parents[3]
|
||||
@@ -213,6 +226,7 @@ class DemoWorkflowService:
|
||||
status="ready",
|
||||
)
|
||||
db.add(dataset)
|
||||
DemoWorkflowService._add_initial_version(db, dataset)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
VectorFeatureService.persist_geojson_features(
|
||||
@@ -297,6 +311,7 @@ class DemoWorkflowService:
|
||||
status="ready",
|
||||
)
|
||||
db.add(dataset)
|
||||
DemoWorkflowService._add_initial_version(db, dataset)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
return dataset
|
||||
|
||||
@@ -12,7 +12,7 @@ from shapely.ops import transform as shapely_transform
|
||||
from shapely.validation import make_valid
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset
|
||||
from app.models import Area, Dataset, DatasetVersion
|
||||
from app.services.raster_service import extract_raster_metadata
|
||||
from app.services.storage_service import StorageService
|
||||
|
||||
@@ -333,6 +333,21 @@ class RasterOperationsService:
|
||||
name=output_name,
|
||||
dataset_type="raster",
|
||||
source=f"operation:{operation_name}",
|
||||
dataset_role="derived",
|
||||
source_name=source_dataset.source_name,
|
||||
source_metadata=source_dataset.source_metadata,
|
||||
provenance_metadata=provenance,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
temporal_series_key=(
|
||||
f"{source_dataset.temporal_series_key}:{operation_name}"
|
||||
if source_dataset.temporal_series_key
|
||||
else None
|
||||
),
|
||||
observed_at=source_dataset.observed_at,
|
||||
valid_from=source_dataset.valid_from,
|
||||
valid_to=source_dataset.valid_to,
|
||||
temporal_granularity=source_dataset.temporal_granularity,
|
||||
source_version=source_dataset.source_version,
|
||||
storage_path=str(output_file),
|
||||
original_filename=storage_metadata["original_filename"],
|
||||
stored_filename=storage_metadata["stored_filename"],
|
||||
@@ -348,6 +363,20 @@ class RasterOperationsService:
|
||||
status="ready",
|
||||
)
|
||||
db.add(derived_dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=derived_dataset.id,
|
||||
version=1,
|
||||
storage_path=derived_dataset.storage_path,
|
||||
source_version=derived_dataset.source_version,
|
||||
observed_at=derived_dataset.observed_at,
|
||||
valid_from=derived_dataset.valid_from,
|
||||
valid_to=derived_dataset.valid_to,
|
||||
checksum_sha256=derived_dataset.checksum_sha256,
|
||||
source_metadata=derived_dataset.source_metadata,
|
||||
provenance_metadata=derived_dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(derived_dataset)
|
||||
return derived_id
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.functions import ST_Intersects, ST_MakeEnvelope
|
||||
from geoalchemy2.shape import to_shape
|
||||
from shapely.geometry import mapping
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, VectorFeature
|
||||
from app.schemas.temporal import (
|
||||
TemporalComparisonRequest,
|
||||
TemporalComparisonResponse,
|
||||
TemporalDatasetRef,
|
||||
TemporalMetricComparison,
|
||||
TemporalObjectChanges,
|
||||
TemporalSeriesDataset,
|
||||
TemporalSeriesRead,
|
||||
)
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
class TemporalAnalysisService:
|
||||
IDENTITY_COMPARISON_LIMIT = 5_000
|
||||
|
||||
@staticmethod
|
||||
def list_series(db: Session, project_id: UUID) -> list[TemporalSeriesRead]:
|
||||
rows = (
|
||||
db.query(Dataset)
|
||||
.filter(Dataset.project_id == project_id)
|
||||
.filter(Dataset.temporal_series_key.isnot(None))
|
||||
.filter(Dataset.observed_at.isnot(None))
|
||||
.order_by(Dataset.temporal_series_key.asc(), Dataset.observed_at.asc())
|
||||
.all()
|
||||
)
|
||||
grouped: dict[str, list[Dataset]] = {}
|
||||
for row in rows:
|
||||
if row.temporal_series_key:
|
||||
grouped.setdefault(row.temporal_series_key, []).append(row)
|
||||
|
||||
result: list[TemporalSeriesRead] = []
|
||||
for key, datasets in grouped.items():
|
||||
observed = [item.observed_at for item in datasets if item.observed_at is not None]
|
||||
if not observed:
|
||||
continue
|
||||
result.append(
|
||||
TemporalSeriesRead(
|
||||
temporal_series_key=key,
|
||||
source_name=datasets[-1].source_name,
|
||||
reference_layer_name=datasets[-1].reference_layer_name,
|
||||
dataset_count=len(datasets),
|
||||
first_observed_at=min(observed),
|
||||
last_observed_at=max(observed),
|
||||
datasets=[
|
||||
TemporalSeriesDataset(
|
||||
id=item.id,
|
||||
name=item.name,
|
||||
observed_at=item.observed_at,
|
||||
source_version=item.source_version,
|
||||
feature_count=(item.metadata_json or {}).get("feature_count")
|
||||
if isinstance(item.metadata_json, dict)
|
||||
else None,
|
||||
)
|
||||
for item in datasets
|
||||
if item.observed_at is not None
|
||||
],
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def compare(
|
||||
db: Session,
|
||||
*,
|
||||
project_id: UUID,
|
||||
payload: TemporalComparisonRequest,
|
||||
) -> TemporalComparisonResponse:
|
||||
if payload.earlier_dataset_id == payload.later_dataset_id:
|
||||
raise AppError(
|
||||
code="INVALID_TEMPORAL_COMPARISON",
|
||||
message="Choose two different dataset snapshots",
|
||||
status_code=400,
|
||||
)
|
||||
earlier = TemporalAnalysisService._get_temporal_dataset(db, project_id, payload.earlier_dataset_id, "Earlier")
|
||||
later = TemporalAnalysisService._get_temporal_dataset(db, project_id, payload.later_dataset_id, "Later")
|
||||
if earlier.temporal_series_key != later.temporal_series_key:
|
||||
raise AppError(
|
||||
code="INCOMPATIBLE_TEMPORAL_SERIES",
|
||||
message="Dataset snapshots must belong to the same temporal series",
|
||||
details={
|
||||
"earlier_series": earlier.temporal_series_key,
|
||||
"later_series": later.temporal_series_key,
|
||||
},
|
||||
status_code=400,
|
||||
)
|
||||
if earlier.observed_at >= later.observed_at:
|
||||
raise AppError(
|
||||
code="INVALID_TEMPORAL_ORDER",
|
||||
message="Earlier snapshot must have an observation date before the later snapshot",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
bbox = payload.bbox.model_dump()
|
||||
earlier_summary = VectorFeatureService.summarize_features_by_bbox(db, dataset=earlier, bbox=bbox)
|
||||
later_summary = VectorFeatureService.summarize_features_by_bbox(db, dataset=later, bbox=bbox)
|
||||
if (
|
||||
earlier_summary["aggregation_method"] != later_summary["aggregation_method"]
|
||||
or earlier_summary["metric_unit"] != later_summary["metric_unit"]
|
||||
):
|
||||
raise AppError(
|
||||
code="INCOMPATIBLE_TEMPORAL_AGGREGATION",
|
||||
message="Dataset snapshots use incompatible aggregation semantics",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
earlier_value = float(earlier_summary["metric_value"])
|
||||
later_value = float(later_summary["metric_value"])
|
||||
absolute_change = later_value - earlier_value
|
||||
percent_change = (absolute_change / earlier_value * 100.0) if earlier_value else None
|
||||
warnings = [
|
||||
warning
|
||||
for warning in {earlier_summary.get("warning"), later_summary.get("warning")}
|
||||
if warning
|
||||
]
|
||||
|
||||
object_changes, geojson, identity_warnings = TemporalAnalysisService._compare_identity_features(
|
||||
db,
|
||||
earlier=earlier,
|
||||
later=later,
|
||||
bbox=bbox,
|
||||
preview_limit=payload.preview_limit,
|
||||
)
|
||||
warnings.extend(identity_warnings)
|
||||
|
||||
return TemporalComparisonResponse(
|
||||
temporal_series_key=earlier.temporal_series_key,
|
||||
earlier=TemporalDatasetRef(
|
||||
id=earlier.id,
|
||||
name=earlier.name,
|
||||
observed_at=earlier.observed_at,
|
||||
source_version=earlier.source_version,
|
||||
),
|
||||
later=TemporalDatasetRef(
|
||||
id=later.id,
|
||||
name=later.name,
|
||||
observed_at=later.observed_at,
|
||||
source_version=later.source_version,
|
||||
),
|
||||
selection_bbox=payload.bbox,
|
||||
metric=TemporalMetricComparison(
|
||||
label=str(later_summary["metric_label"]),
|
||||
unit=str(later_summary["metric_unit"]),
|
||||
aggregation_method=str(later_summary["aggregation_method"]),
|
||||
earlier_value=earlier_value,
|
||||
later_value=later_value,
|
||||
absolute_change=absolute_change,
|
||||
percent_change=percent_change,
|
||||
is_estimate=bool(earlier_summary["is_estimate"] or later_summary["is_estimate"]),
|
||||
),
|
||||
object_changes=object_changes,
|
||||
geojson=geojson,
|
||||
warnings=warnings,
|
||||
generated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_temporal_dataset(db: Session, project_id: UUID, dataset_id: UUID, label: str) -> Dataset:
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if not dataset or dataset.project_id != project_id:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message=f"{label} dataset not found", status_code=404)
|
||||
if dataset.dataset_type not in {"vector", "geojson"}:
|
||||
raise AppError(
|
||||
code="DATASET_NOT_VECTOR",
|
||||
message="Temporal selection comparison currently requires vector datasets",
|
||||
status_code=400,
|
||||
)
|
||||
if not dataset.temporal_series_key or not dataset.observed_at:
|
||||
raise AppError(
|
||||
code="TEMPORAL_METADATA_MISSING",
|
||||
message=f"{label} dataset has no explicit temporal series and observation date",
|
||||
status_code=400,
|
||||
)
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def _compare_identity_features(
|
||||
db: Session,
|
||||
*,
|
||||
earlier: Dataset,
|
||||
later: Dataset,
|
||||
bbox: dict[str, Any],
|
||||
preview_limit: int,
|
||||
) -> tuple[TemporalObjectChanges, dict[str, Any], list[str]]:
|
||||
earlier_config = earlier.source_metadata if isinstance(earlier.source_metadata, dict) else {}
|
||||
later_config = later.source_metadata if isinstance(later.source_metadata, dict) else {}
|
||||
if not earlier_config.get("identity_stable") or not later_config.get("identity_stable"):
|
||||
return (
|
||||
TemporalObjectChanges(available=False),
|
||||
{"type": "FeatureCollection", "features": []},
|
||||
["Object-level changes are unavailable because the source does not guarantee stable feature identifiers."],
|
||||
)
|
||||
|
||||
normalized_bbox = VectorFeatureService._normalize_selection_bbox(bbox)
|
||||
envelope = ST_MakeEnvelope(
|
||||
normalized_bbox["min_x"],
|
||||
normalized_bbox["min_y"],
|
||||
normalized_bbox["max_x"],
|
||||
normalized_bbox["max_y"],
|
||||
4326,
|
||||
)
|
||||
|
||||
def load(dataset_id: UUID) -> list[VectorFeature]:
|
||||
return (
|
||||
db.query(VectorFeature)
|
||||
.filter(VectorFeature.dataset_id == dataset_id)
|
||||
.filter(ST_Intersects(VectorFeature.geometry, envelope))
|
||||
.filter(VectorFeature.source_feature_id.isnot(None))
|
||||
.order_by(VectorFeature.source_feature_id.asc())
|
||||
.limit(TemporalAnalysisService.IDENTITY_COMPARISON_LIMIT + 1)
|
||||
.all()
|
||||
)
|
||||
|
||||
earlier_rows = load(earlier.id)
|
||||
later_rows = load(later.id)
|
||||
if (
|
||||
len(earlier_rows) > TemporalAnalysisService.IDENTITY_COMPARISON_LIMIT
|
||||
or len(later_rows) > TemporalAnalysisService.IDENTITY_COMPARISON_LIMIT
|
||||
):
|
||||
return (
|
||||
TemporalObjectChanges(available=False),
|
||||
{"type": "FeatureCollection", "features": []},
|
||||
["Object-level preview was skipped because the selection exceeds the 5,000 feature safety limit."],
|
||||
)
|
||||
|
||||
earlier_by_id = {str(row.source_feature_id): row for row in earlier_rows if row.source_feature_id}
|
||||
later_by_id = {str(row.source_feature_id): row for row in later_rows if row.source_feature_id}
|
||||
earlier_ids = set(earlier_by_id)
|
||||
later_ids = set(later_by_id)
|
||||
added_ids = sorted(later_ids - earlier_ids)
|
||||
removed_ids = sorted(earlier_ids - later_ids)
|
||||
common_ids = sorted(earlier_ids & later_ids)
|
||||
comparison_property = str(later_config.get("comparison_property") or "").strip() or None
|
||||
modified_ids: list[str] = []
|
||||
unchanged_ids: list[str] = []
|
||||
|
||||
for feature_id in common_ids:
|
||||
earlier_row = earlier_by_id[feature_id]
|
||||
later_row = later_by_id[feature_id]
|
||||
geometry_changed = not to_shape(earlier_row.geometry).equals(to_shape(later_row.geometry))
|
||||
value_changed = False
|
||||
if comparison_property:
|
||||
value_changed = (earlier_row.properties_json or {}).get(comparison_property) != (
|
||||
later_row.properties_json or {}
|
||||
).get(comparison_property)
|
||||
(modified_ids if geometry_changed or value_changed else unchanged_ids).append(feature_id)
|
||||
|
||||
features: list[dict[str, Any]] = []
|
||||
for change_type, feature_ids, rows in (
|
||||
("added", added_ids, later_by_id),
|
||||
("removed", removed_ids, earlier_by_id),
|
||||
("modified", modified_ids, later_by_id),
|
||||
):
|
||||
for feature_id in feature_ids:
|
||||
if len(features) >= preview_limit:
|
||||
break
|
||||
row = rows[feature_id]
|
||||
properties = dict(row.properties_json or {})
|
||||
properties.update(
|
||||
{
|
||||
"change_type": change_type,
|
||||
"source_feature_id": feature_id,
|
||||
"earlier_dataset_id": str(earlier.id),
|
||||
"later_dataset_id": str(later.id),
|
||||
}
|
||||
)
|
||||
if change_type == "modified" and comparison_property:
|
||||
before = (earlier_by_id[feature_id].properties_json or {}).get(comparison_property)
|
||||
after = (later_by_id[feature_id].properties_json or {}).get(comparison_property)
|
||||
properties.update({"value_before": before, "value_after": after})
|
||||
if isinstance(before, (int, float)) and isinstance(after, (int, float)):
|
||||
properties["value_delta"] = after - before
|
||||
features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": str(row.id),
|
||||
"geometry": mapping(to_shape(row.geometry)),
|
||||
"properties": properties,
|
||||
}
|
||||
)
|
||||
|
||||
warnings: list[str] = []
|
||||
total_changes = len(added_ids) + len(removed_ids) + len(modified_ids)
|
||||
if total_changes > preview_limit:
|
||||
warnings.append(
|
||||
f"The map shows the first {preview_limit} of {total_changes} changed features; counts remain complete."
|
||||
)
|
||||
return (
|
||||
TemporalObjectChanges(
|
||||
available=True,
|
||||
added_count=len(added_ids),
|
||||
removed_count=len(removed_ids),
|
||||
modified_count=len(modified_ids),
|
||||
unchanged_count=len(unchanged_ids),
|
||||
),
|
||||
{"type": "FeatureCollection", "features": features},
|
||||
warnings,
|
||||
)
|
||||
@@ -9,9 +9,10 @@ from geoalchemy2.shape import to_shape
|
||||
from shapely.geometry import mapping
|
||||
from shapely.geometry import shape
|
||||
from shapely.validation import make_valid
|
||||
from sqlalchemy import Float, cast, func
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import VectorFeature
|
||||
from app.models import Dataset, VectorFeature
|
||||
|
||||
|
||||
class VectorFeatureService:
|
||||
@@ -88,6 +89,7 @@ class VectorFeatureService:
|
||||
dataset_id: UUID,
|
||||
bbox: dict[str, Any],
|
||||
limit: int = 100,
|
||||
dataset: Dataset | None = None,
|
||||
) -> dict[str, Any]:
|
||||
normalized_bbox = VectorFeatureService._normalize_selection_bbox(bbox)
|
||||
safe_limit = max(1, min(int(limit), 1000))
|
||||
@@ -121,6 +123,14 @@ class VectorFeatureService:
|
||||
truncated = total_feature_count > safe_limit
|
||||
selected_rows = rows[:safe_limit]
|
||||
features = [VectorFeatureService._row_to_geojson_feature(row) for row in selected_rows]
|
||||
summary = None
|
||||
if dataset and isinstance(dataset.source_metadata, dict) and dataset.source_metadata.get("selection_aggregation"):
|
||||
summary = VectorFeatureService.summarize_features_by_bbox(
|
||||
db,
|
||||
dataset=dataset,
|
||||
bbox=normalized_bbox,
|
||||
total_feature_count=total_feature_count,
|
||||
)
|
||||
|
||||
return {
|
||||
"selection_bbox": normalized_bbox,
|
||||
@@ -132,6 +142,97 @@ class VectorFeatureService:
|
||||
"type": "FeatureCollection",
|
||||
"features": features,
|
||||
},
|
||||
"summary": summary,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def summarize_features_by_bbox(
|
||||
db,
|
||||
*,
|
||||
dataset: Dataset,
|
||||
bbox: dict[str, Any],
|
||||
total_feature_count: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
normalized_bbox = VectorFeatureService._normalize_selection_bbox(bbox)
|
||||
envelope = ST_MakeEnvelope(
|
||||
normalized_bbox["min_x"],
|
||||
normalized_bbox["min_y"],
|
||||
normalized_bbox["max_x"],
|
||||
normalized_bbox["max_y"],
|
||||
4326,
|
||||
)
|
||||
selection_filter = (
|
||||
VectorFeature.dataset_id == dataset.id,
|
||||
ST_Intersects(VectorFeature.geometry, envelope),
|
||||
)
|
||||
feature_count = total_feature_count
|
||||
if feature_count is None:
|
||||
feature_count = int(db.query(func.count(VectorFeature.id)).filter(*selection_filter).scalar() or 0)
|
||||
|
||||
source_metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||
config = source_metadata.get("selection_aggregation")
|
||||
if not isinstance(config, dict):
|
||||
config = {}
|
||||
method = str(config.get("method") or "feature_count")
|
||||
label = str(config.get("label") or "Objecten")
|
||||
unit = str(config.get("unit") or "objecten")
|
||||
warning = str(config["warning"]) if config.get("warning") else None
|
||||
is_estimate = bool(config.get("is_estimate", False))
|
||||
|
||||
metric_value = float(feature_count)
|
||||
if method == "intersection_area":
|
||||
intersection = func.ST_Intersection(VectorFeature.geometry, envelope)
|
||||
area_expression = func.ST_Area(func.ST_Transform(intersection, 31370))
|
||||
area_m2 = db.query(func.coalesce(func.sum(area_expression), 0.0)).filter(*selection_filter).scalar()
|
||||
divisor = 10_000.0 if unit == "ha" else 1.0
|
||||
metric_value = float(area_m2 or 0.0) / divisor
|
||||
elif method == "intersection_length":
|
||||
intersection = func.ST_Intersection(VectorFeature.geometry, envelope)
|
||||
length_expression = func.ST_Length(func.ST_Transform(intersection, 31370))
|
||||
length_m = db.query(func.coalesce(func.sum(length_expression), 0.0)).filter(*selection_filter).scalar()
|
||||
divisor = 1_000.0 if unit == "km" else 1.0
|
||||
metric_value = float(length_m or 0.0) / divisor
|
||||
elif method in {"sum", "area_weighted_sum"}:
|
||||
property_name = str(config.get("property") or "").strip()
|
||||
if not property_name:
|
||||
raise AppError(
|
||||
code="INVALID_SELECTION_AGGREGATION",
|
||||
message="Dataset selection aggregation requires a numeric property",
|
||||
details={"dataset_id": str(dataset.id), "method": method},
|
||||
status_code=500,
|
||||
)
|
||||
numeric_value = cast(VectorFeature.properties_json.op("->>")(property_name), Float)
|
||||
value_expression = numeric_value
|
||||
if method == "area_weighted_sum":
|
||||
source_area = func.ST_Area(func.ST_Transform(VectorFeature.geometry, 31370))
|
||||
intersection_area = func.ST_Area(
|
||||
func.ST_Transform(func.ST_Intersection(VectorFeature.geometry, envelope), 31370)
|
||||
)
|
||||
value_expression = numeric_value * intersection_area / func.nullif(source_area, 0.0)
|
||||
is_estimate = True
|
||||
aggregate_value = (
|
||||
db.query(func.coalesce(func.sum(value_expression), 0.0))
|
||||
.filter(*selection_filter)
|
||||
.filter(VectorFeature.properties_json.op("->>")(property_name).isnot(None))
|
||||
.scalar()
|
||||
)
|
||||
metric_value = float(aggregate_value or 0.0)
|
||||
elif method != "feature_count":
|
||||
raise AppError(
|
||||
code="INVALID_SELECTION_AGGREGATION",
|
||||
message="Unsupported dataset selection aggregation",
|
||||
details={"dataset_id": str(dataset.id), "method": method},
|
||||
status_code=500,
|
||||
)
|
||||
|
||||
return {
|
||||
"metric_label": label,
|
||||
"metric_value": metric_value,
|
||||
"metric_unit": unit,
|
||||
"aggregation_method": method,
|
||||
"feature_count": feature_count,
|
||||
"is_estimate": is_estimate,
|
||||
"warning": warning,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -15,7 +15,7 @@ from shapely.validation import make_valid
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset
|
||||
from app.models import Area, Dataset, DatasetVersion
|
||||
from app.schemas.dataset import DatasetCreateResponse
|
||||
from app.schemas.operations import VectorOperationResult
|
||||
from app.services.geojson_service import parse_geojson_payload
|
||||
@@ -444,6 +444,16 @@ class VectorOperationsService:
|
||||
source_metadata=source_metadata,
|
||||
provenance_metadata=provenance_metadata,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
temporal_series_key=(
|
||||
f"{source_dataset.temporal_series_key}:{operation}"
|
||||
if source_dataset.temporal_series_key
|
||||
else None
|
||||
),
|
||||
observed_at=source_dataset.observed_at,
|
||||
valid_from=source_dataset.valid_from,
|
||||
valid_to=source_dataset.valid_to,
|
||||
temporal_granularity=source_dataset.temporal_granularity,
|
||||
source_version=source_dataset.source_version,
|
||||
storage_path=storage_info["storage_path"],
|
||||
original_filename=storage_info["original_filename"],
|
||||
stored_filename=storage_info["stored_filename"],
|
||||
@@ -459,6 +469,20 @@ class VectorOperationsService:
|
||||
status="ready",
|
||||
)
|
||||
db.add(derived_dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=derived_dataset.id,
|
||||
version=1,
|
||||
storage_path=derived_dataset.storage_path,
|
||||
source_version=derived_dataset.source_version,
|
||||
observed_at=derived_dataset.observed_at,
|
||||
valid_from=derived_dataset.valid_from,
|
||||
valid_to=derived_dataset.valid_to,
|
||||
checksum_sha256=derived_dataset.checksum_sha256,
|
||||
source_metadata=derived_dataset.source_metadata,
|
||||
provenance_metadata=derived_dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(derived_dataset)
|
||||
if persist_vector_features:
|
||||
|
||||
@@ -7,7 +7,7 @@ import importlib
|
||||
|
||||
from geoalchemy2.shape import from_shape
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset
|
||||
from app.models import Area, Dataset, DatasetVersion
|
||||
from app.services.raster_operations_service import RasterOperationsService
|
||||
from app.api.routes.datasets import _run_job_sync
|
||||
from shapely.geometry import box
|
||||
@@ -356,8 +356,12 @@ def test_raster_reproject_returns_persisted_derived_dataset(monkeypatch, tmp_pat
|
||||
)
|
||||
|
||||
assert result_id == output_id
|
||||
assert len(db.added) == 1
|
||||
assert len(db.added) == 2
|
||||
derived = db.added[0]
|
||||
version = db.added[1]
|
||||
assert isinstance(version, DatasetVersion)
|
||||
assert version.dataset_id == output_id
|
||||
assert version.version == 1
|
||||
assert derived.id == output_id
|
||||
assert derived.metadata_json is not None
|
||||
assert derived.metadata_json["operation"] == "raster.reproject"
|
||||
@@ -783,9 +787,13 @@ def test_raster_clip_persists_derived_dataset(monkeypatch, tmp_path) -> None:
|
||||
result_id = RasterOperationsService.clip(db, dataset_id, area_id, "clip-result.tif")
|
||||
|
||||
assert result_id == output_id
|
||||
assert len(db.added) == 1
|
||||
assert len(db.added) == 2
|
||||
derived = db.added[0]
|
||||
version = db.added[1]
|
||||
assert isinstance(derived, Dataset)
|
||||
assert isinstance(version, DatasetVersion)
|
||||
assert version.dataset_id == output_id
|
||||
assert version.version == 1
|
||||
assert derived.id == output_id
|
||||
assert derived.source == "operation:raster.clip"
|
||||
assert derived.dataset_type == "raster"
|
||||
@@ -1295,8 +1303,12 @@ def test_raster_index_records_provenance_and_dtype(tmp_path, monkeypatch) -> Non
|
||||
|
||||
result_dataset_id = RasterOperationsService.ndvi(db, dataset_id, nir_band=4, red_band=3, output_name="ndvi-test")
|
||||
assert result_dataset_id == output_dataset_id
|
||||
assert len(db.added) == 1
|
||||
assert len(db.added) == 2
|
||||
derived = db.added[0]
|
||||
version = db.added[1]
|
||||
assert isinstance(version, DatasetVersion)
|
||||
assert version.dataset_id == output_dataset_id
|
||||
assert version.version == 1
|
||||
assert derived.id == output_dataset_id
|
||||
assert derived.metadata_json is not None
|
||||
assert derived.metadata_json["operation"] == "raster.ndvi"
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, DatasetVersion
|
||||
from app.schemas.dataset import DatasetTemporalUpdate
|
||||
from app.schemas.temporal import TemporalComparisonRequest, TemporalObjectChanges
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.temporal_analysis_service import TemporalAnalysisService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
|
||||
|
||||
class ScalarQuery:
|
||||
def __init__(self, value: float):
|
||||
self.value = value
|
||||
|
||||
def filter(self, *args): # noqa: ANN002, ARG002
|
||||
return self
|
||||
|
||||
def scalar(self):
|
||||
return self.value
|
||||
|
||||
|
||||
class ScalarSession:
|
||||
def __init__(self, value: float):
|
||||
self.value = value
|
||||
|
||||
def query(self, *args): # noqa: ANN002, ARG002
|
||||
return ScalarQuery(self.value)
|
||||
|
||||
|
||||
class VersionQuery:
|
||||
def __init__(self, latest: DatasetVersion | None):
|
||||
self.latest = latest
|
||||
|
||||
def filter(self, *args): # noqa: ANN002, ARG002
|
||||
return self
|
||||
|
||||
def order_by(self, *args): # noqa: ANN002, ARG002
|
||||
return self
|
||||
|
||||
def first(self):
|
||||
return self.latest
|
||||
|
||||
|
||||
class TemporalUpdateSession:
|
||||
def __init__(self, dataset: Dataset, latest: DatasetVersion | None):
|
||||
self.dataset = dataset
|
||||
self.latest = latest
|
||||
self.added: list[object] = []
|
||||
|
||||
def get(self, model, item_id): # noqa: ANN001
|
||||
return self.dataset if model is Dataset and item_id == self.dataset.id else None
|
||||
|
||||
def query(self, model): # noqa: ANN001
|
||||
assert model is DatasetVersion
|
||||
return VersionQuery(self.latest)
|
||||
|
||||
def add(self, item): # noqa: ANN001
|
||||
self.added.append(item)
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
def refresh(self, _item):
|
||||
return None
|
||||
|
||||
|
||||
def temporal_dataset(*, project_id, observed_year: int, metric_method: str = "feature_count") -> Dataset:
|
||||
return Dataset(
|
||||
id=uuid4(),
|
||||
project_id=project_id,
|
||||
name=f"snapshot-{observed_year}.geojson",
|
||||
dataset_type="vector",
|
||||
source="official",
|
||||
dataset_role="reference",
|
||||
temporal_series_key="official:test:mol",
|
||||
observed_at=datetime(observed_year, 1, 1, tzinfo=timezone.utc),
|
||||
source_version=str(observed_year),
|
||||
source_metadata={
|
||||
"selection_aggregation": {
|
||||
"method": metric_method,
|
||||
"label": "Objecten",
|
||||
"unit": "objecten",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_temporal_migration_and_models_align() -> None:
|
||||
migration = (ROOT / "backend/alembic/versions/202607140001_temporal_dataset_foundation.py").read_text(encoding="utf-8")
|
||||
for field in (
|
||||
"temporal_series_key",
|
||||
"observed_at",
|
||||
"valid_from",
|
||||
"valid_to",
|
||||
"temporal_granularity",
|
||||
"source_version",
|
||||
):
|
||||
assert field in migration
|
||||
assert hasattr(Dataset, field)
|
||||
assert "ix_vector_features_dataset_source_feature" in migration
|
||||
assert 'down_revision = "202606120900"' in migration
|
||||
|
||||
|
||||
def test_temporal_metadata_requires_an_explicit_series_and_observation_date() -> None:
|
||||
with pytest.raises(AppError, match="observed_at is required"):
|
||||
DatasetService._validate_temporal_metadata(
|
||||
temporal_series_key="official:test:mol",
|
||||
observed_at=None,
|
||||
valid_from=None,
|
||||
valid_to=None,
|
||||
temporal_granularity="year",
|
||||
source_version="2024",
|
||||
)
|
||||
with pytest.raises(AppError, match="valid_to must be"):
|
||||
DatasetService._validate_temporal_metadata(
|
||||
temporal_series_key="official:test:mol",
|
||||
observed_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
valid_from=datetime(2024, 12, 31, tzinfo=timezone.utc),
|
||||
valid_to=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
temporal_granularity="year",
|
||||
source_version="2024",
|
||||
)
|
||||
|
||||
|
||||
def test_temporal_metadata_update_appends_provenance_version_and_is_idempotent() -> None:
|
||||
project_id = uuid4()
|
||||
dataset = temporal_dataset(project_id=project_id, observed_year=2024)
|
||||
dataset.status = "ready"
|
||||
dataset.metadata_json = {}
|
||||
latest = DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=3,
|
||||
observed_at=dataset.observed_at,
|
||||
source_version="2024",
|
||||
)
|
||||
session = TemporalUpdateSession(dataset, latest)
|
||||
payload = DatasetTemporalUpdate(
|
||||
temporal_series_key="official:test:mol",
|
||||
observed_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||
temporal_granularity="year",
|
||||
source_version="2025",
|
||||
)
|
||||
|
||||
updated = DatasetService.update_temporal_metadata(session, dataset.id, payload)
|
||||
|
||||
assert updated.observed_at == payload.observed_at
|
||||
assert latest.version == 3
|
||||
assert latest.observed_at == datetime(2024, 1, 1, tzinfo=timezone.utc)
|
||||
assert len(session.added) == 2
|
||||
appended = session.added[1]
|
||||
assert isinstance(appended, DatasetVersion)
|
||||
assert appended.version == 4
|
||||
assert appended.observed_at == payload.observed_at
|
||||
|
||||
session.added.clear()
|
||||
DatasetService.update_temporal_metadata(session, dataset.id, payload)
|
||||
assert session.added == []
|
||||
|
||||
|
||||
def test_selection_area_aggregation_returns_hectares_without_loading_all_features() -> None:
|
||||
project_id = uuid4()
|
||||
dataset = temporal_dataset(project_id=project_id, observed_year=1969, metric_method="intersection_area")
|
||||
dataset.source_metadata["selection_aggregation"].update({"label": "Oppervlakte", "unit": "ha"})
|
||||
result = VectorFeatureService.summarize_features_by_bbox(
|
||||
ScalarSession(125_000.0),
|
||||
dataset=dataset,
|
||||
bbox={"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3, "crs": "EPSG:4326"},
|
||||
total_feature_count=40,
|
||||
)
|
||||
assert result["metric_value"] == 12.5
|
||||
assert result["metric_unit"] == "ha"
|
||||
assert result["feature_count"] == 40
|
||||
|
||||
|
||||
def test_temporal_compare_returns_delta_and_canonical_change_payload(monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
earlier = temporal_dataset(project_id=project_id, observed_year=2021)
|
||||
later = temporal_dataset(project_id=project_id, observed_year=2024)
|
||||
|
||||
def get_dataset(_db, _project_id, dataset_id, _label):
|
||||
return earlier if dataset_id == earlier.id else later
|
||||
|
||||
def summarize(_db, *, dataset, bbox): # noqa: ARG001
|
||||
value = 100.0 if dataset.id == earlier.id else 115.0
|
||||
return {
|
||||
"metric_label": "Inwoners",
|
||||
"metric_value": value,
|
||||
"metric_unit": "inwoners",
|
||||
"aggregation_method": "area_weighted_sum",
|
||||
"feature_count": 10,
|
||||
"is_estimate": True,
|
||||
"warning": "Areal weighting",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(TemporalAnalysisService, "_get_temporal_dataset", staticmethod(get_dataset))
|
||||
monkeypatch.setattr(VectorFeatureService, "summarize_features_by_bbox", staticmethod(summarize))
|
||||
monkeypatch.setattr(
|
||||
TemporalAnalysisService,
|
||||
"_compare_identity_features",
|
||||
staticmethod(
|
||||
lambda *args, **kwargs: (
|
||||
TemporalObjectChanges(available=True, added_count=1, removed_count=0, modified_count=2, unchanged_count=7),
|
||||
{"type": "FeatureCollection", "features": []},
|
||||
[],
|
||||
)
|
||||
),
|
||||
)
|
||||
result = TemporalAnalysisService.compare(
|
||||
SimpleNamespace(),
|
||||
project_id=project_id,
|
||||
payload=TemporalComparisonRequest(
|
||||
earlier_dataset_id=earlier.id,
|
||||
later_dataset_id=later.id,
|
||||
bbox={"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3},
|
||||
),
|
||||
)
|
||||
assert result.metric.absolute_change == 15.0
|
||||
assert result.metric.percent_change == 15.0
|
||||
assert result.metric.is_estimate is True
|
||||
assert result.object_changes.modified_count == 2
|
||||
assert result.geojson["type"] == "FeatureCollection"
|
||||
|
||||
|
||||
def test_temporal_frontend_and_official_operator_contracts_exist() -> None:
|
||||
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
|
||||
temporal_api = (ROOT / "frontend/src/services/api/temporal.ts").read_text(encoding="utf-8")
|
||||
population = (ROOT / "scripts/provision_mol_population_history.py").read_text(encoding="utf-8")
|
||||
landuse = (ROOT / "scripts/provision_mol_historical_landuse.py").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
|
||||
assert "Laatste toestand" in workspace
|
||||
assert "Evolutie" in workspace
|
||||
assert "Vergelijk periode" in workspace
|
||||
assert "/temporal/compare" in temporal_api
|
||||
assert "Statbel" in population and "area_weighted_sum" in population
|
||||
assert "HistLandgebruik" in landuse and "intersection_area" in landuse
|
||||
assert "provision_mol_population_history.py" in dockerfile
|
||||
assert "provision_mol_historical_landuse.py" in dockerfile
|
||||
assert "fake" not in population.lower()
|
||||
Reference in New Issue
Block a user