diff --git a/backend/app/services/temporal_analysis_service.py b/backend/app/services/temporal_analysis_service.py index 09768650..5ea40472 100644 --- a/backend/app/services/temporal_analysis_service.py +++ b/backend/app/services/temporal_analysis_service.py @@ -32,6 +32,37 @@ class TemporalAnalysisService: "provision_regional_grb_context.py", } + @staticmethod + def _canonical_observation_snapshots(datasets: list[Dataset]) -> list[Dataset]: + by_observation: dict[datetime, Dataset] = {} + for dataset in datasets: + if dataset.observed_at is None: + continue + current = by_observation.get(dataset.observed_at) + dataset_recency = max( + ( + value.timestamp() + for value in (dataset.imported_at, dataset.updated_at, dataset.created_at) + if value is not None + ), + default=0.0, + ) + current_recency = max( + ( + value.timestamp() + for value in ( + getattr(current, "imported_at", None), + getattr(current, "updated_at", None), + getattr(current, "created_at", None), + ) + if value is not None + ), + default=0.0, + ) + if current is None or (dataset_recency, str(dataset.id)) > (current_recency, str(current.id)): + by_observation[dataset.observed_at] = dataset + return sorted(by_observation.values(), key=lambda item: item.observed_at) + @staticmethod def list_series(db: Session, project_id: UUID) -> list[TemporalSeriesRead]: rows = ( @@ -49,6 +80,7 @@ class TemporalAnalysisService: result: list[TemporalSeriesRead] = [] for key, datasets in grouped.items(): + datasets = TemporalAnalysisService._canonical_observation_snapshots(datasets) observed = [item.observed_at for item in datasets if item.observed_at is not None] if not observed: continue @@ -118,6 +150,13 @@ class TemporalAnalysisService: bbox, selection_area.geometry, ) + + def is_preclipped_to_selection_area(dataset: Dataset) -> bool: + return bool( + selection_area + and VectorFeatureService.can_use_full_area_fast_path(dataset, selection_area.id) + ) + summaries: dict[UUID, dict[str, Any]] = {} def summarize(dataset: Dataset) -> dict[str, Any]: @@ -126,14 +165,9 @@ class TemporalAnalysisService: return cached kwargs: dict[str, Any] = {"dataset": dataset, "bbox": bbox} if selection_area is not None: - kwargs["selection_geometry"] = selection_geometry - kwargs["full_dataset_area"] = ( - selection_covers_full_area - and VectorFeatureService.can_use_full_area_fast_path( - dataset, - selection_area.id, - ) - ) + dataset_is_preclipped = is_preclipped_to_selection_area(dataset) + kwargs["selection_geometry"] = None if dataset_is_preclipped else selection_geometry + kwargs["full_dataset_area"] = selection_covers_full_area and dataset_is_preclipped summary = VectorFeatureService.summarize_features_by_bbox(db, **kwargs) summaries[dataset.id] = summary return summary @@ -164,16 +198,20 @@ class TemporalAnalysisService: later=later, bbox=bbox, preview_limit=payload.preview_limit, - selection_geometry=selection_geometry, + selection_geometry=( + None + if is_preclipped_to_selection_area(earlier) and is_preclipped_to_selection_area(later) + else selection_geometry + ), earlier_full_dataset_area=( selection_covers_full_area - and VectorFeatureService.can_use_full_area_fast_path(earlier, selection_area.id) + and is_preclipped_to_selection_area(earlier) if selection_area is not None else False ), later_full_dataset_area=( selection_covers_full_area - and VectorFeatureService.can_use_full_area_fast_path(later, selection_area.id) + and is_preclipped_to_selection_area(later) if selection_area is not None else False ), @@ -298,8 +336,7 @@ class TemporalAnalysisService: ) else: datasets = fallback_datasets - unique = {dataset.id: dataset for dataset in datasets} - ordered = sorted(unique.values(), key=lambda item: item.observed_at or datetime.min.replace(tzinfo=timezone.utc)) + ordered = TemporalAnalysisService._canonical_observation_snapshots(datasets) observations: list[TemporalObservation] = [] for dataset in ordered: if dataset.observed_at is None: diff --git a/backend/tests/test_sprint187_temporal_map_foundation.py b/backend/tests/test_sprint187_temporal_map_foundation.py index c19afd64..962473f6 100644 --- a/backend/tests/test_sprint187_temporal_map_foundation.py +++ b/backend/tests/test_sprint187_temporal_map_foundation.py @@ -133,6 +133,20 @@ def temporal_dataset(*, project_id, observed_year: int, metric_method: str = "fe ) +def test_temporal_series_keeps_only_latest_snapshot_per_observation_date() -> None: + project_id = uuid4() + old = temporal_dataset(project_id=project_id, observed_year=2025) + old.imported_at = datetime(2026, 7, 19, tzinfo=timezone.utc) + latest = temporal_dataset(project_id=project_id, observed_year=2025) + latest.imported_at = datetime(2026, 7, 21, tzinfo=timezone.utc) + earlier = temporal_dataset(project_id=project_id, observed_year=2022) + earlier.imported_at = datetime(2026, 7, 21, tzinfo=timezone.utc) + + canonical = TemporalAnalysisService._canonical_observation_snapshots([old, latest, earlier]) + + assert [dataset.id for dataset in canonical] == [earlier.id, latest.id] + + def governed_grb_dataset(*, project_id, observed_day: int) -> Dataset: dataset = temporal_dataset(project_id=project_id, observed_year=2026, metric_method="intersection_area") dataset.observed_at = datetime(2026, 7, observed_day, tzinfo=timezone.utc) diff --git a/backend/tests/test_sprint237_flanders_thematic_on_demand.py b/backend/tests/test_sprint237_flanders_thematic_on_demand.py index f7f5108d..d2c1766b 100644 --- a/backend/tests/test_sprint237_flanders_thematic_on_demand.py +++ b/backend/tests/test_sprint237_flanders_thematic_on_demand.py @@ -16,7 +16,7 @@ def test_flanders_workspace_exposes_governed_thematic_products_on_demand() -> No assert "activeScopeProject?.name === FLANDERS_WORKSPACE_PROJECT_NAME" in workspace assert "new Map" in workspace - assert "'Op aanvraag'" in workspace + assert "'Automatisch'" in workspace assert "theme.id === 'space_occupation'" in workspace assert "setActiveThemeId(fallbackTheme.id)" in workspace assert "return `referentiejaar ${observationYear}`" in workspace diff --git a/backend/tests/test_sprint239_bounded_grb_acquisition.py b/backend/tests/test_sprint239_bounded_grb_acquisition.py index 32a41ffc..848df6f1 100644 --- a/backend/tests/test_sprint239_bounded_grb_acquisition.py +++ b/backend/tests/test_sprint239_bounded_grb_acquisition.py @@ -391,6 +391,6 @@ def test_grb_frontend_and_contracts_use_only_the_governed_backend_path() -> None assert "result[product.key] = null" in workspace assert ": onDemandThemeActive\n ? null\n : mapFeatureCollection" in workspace assert "onSetContextLayerLabel" in workspace - assert "'Op aanvraag'" in workspace + assert "'Automatisch'" in workspace assert "/datasets/grb/acquire" in contracts assert "geo.api.vlaanderen.be" not in workspace diff --git a/docs/DATA_COVERAGE_STATUS.md b/docs/DATA_COVERAGE_STATUS.md new file mode 100644 index 00000000..57c5b373 --- /dev/null +++ b/docs/DATA_COVERAGE_STATUS.md @@ -0,0 +1,74 @@ +# GeoIntel data coverage status + +Status date: 2026-07-21 + +This document is the operational interpretation of the source registry. It +does not replace the legal/source provenance stored with each Dataset. + +## Status labels in the workbench + +- `Beschikbaar`: a persisted Dataset covers the selection and can be queried + immediately. +- `Automatisch`: an audited official source contract exists. GeoIntel obtains + and persists only the bounded selection when analysis starts. +- `Alleen huidig`: one official observation exists, so no honest evolution can + be calculated. +- `Ontbreekt`: no operational source contract covers that theme and zone. + +`Automatisch` therefore does not mean unavailable or simulated. It means that +the source is materialized on first use and reused afterwards. AI training +cannot replace missing official GIS observations and is not used to fabricate +coverage or historical dates. + +## Current operational coverage + +| Zone | Persisted or bounded operational sources | Historical comparison | +| --- | --- | --- | +| Belgium | NGI administrative boundaries; Statbel population/statistical sectors | Statbel population 2021-2025 | +| Flanders | GRB buildings, roads, water and parcels; DHMV terrain/surface; VMM flood scenarios; BWK/Natura 2000; DOV soil; policy rasters for space, open space, accessibility and services; agriculture and orthophoto where governed | Population 2021-2025; land-use/land-cover series where retained; agriculture editions; historical maps/orthophotos where the selected product has a real observation date | +| Wallonia | Bounded PICC buildings, roads and hydrography; governed SPW bed-elevation/bathymetry products | No general cross-theme regional history yet | +| Brussels | Bounded UrbIS buildings, street axes and cadastral parcels | No general cross-theme regional history yet | +| Belgian North Sea | RBINS reporting units; Marine Spatial Plan 2026-2034; governed MDK bathymetry only when runtime acquisition is explicitly configured | No multi-epoch bathymetry or marine-plan trend yet | + +Mol and the Kempen are golden regression areas. Their persisted partitions are +not national coverage. A partition is only selected when its recorded +`bbox_epsg4326` intersects the drawn rectangle; otherwise GeoIntel uses an +applicable bounded official source or reports the theme as unsupported. + +## Priority coverage gaps + +1. Govern the public Walloon WALOUS land-cover editions (including the + published 2018/2020 change product) as a real regional time series. A class + crosswalk is required because the older COSW 2005/2007 methodology differs. + Official catalogue: + `https://geoportail.wallonie.be/catalogue-donnees?search-text=occupation+du+sol`. +2. Govern the current public Walloon flood-hazard vector/raster products and + retain their model scenario semantics separately from observed floods. + Official record: + `https://geoportail.wallonie.be/catalogue/14084108-2c7b-4091-b62d-ff0fc235213a.html`. +3. Add the public UrbIS Land Cover product (regional situation 2024) for + Brussels through its official WFS/download contract. Keep it separate from + cadastral parcels and buildings. Product specification: + `https://urbisdownload.datastore.brussels/UrbIS/TechSpec/LandCover_TechSpec_FR20240401.pdf`. +4. Add a common Belgium-wide topographic baseline with normalized theme + semantics across NGI, Flanders, Wallonia and Brussels. +5. Govern comparable Walloon and Brussels historical editions before exposing + evolution for buildings, roads, land cover, soil, elevation or flood risk. +6. Add nationally comparable land-cover history with explicit class crosswalks + and uncertainty; never compare incompatible legends silently. +7. Add multi-epoch marine bathymetry and survey-footprint metadata before + presenting seabed evolution. +8. Expand persisted raster partition manifests beyond the regression regions + only where repeated use justifies caching; bounded acquisition remains the + default for one-off selections. +9. Add source freshness probes only for publishers with stable official edition + contracts. Do not infer a new observation from an import or HTTP date. + +## Acceptance rules for a new source + +A source is visible as operational only after its licence, authority, +observation time, CRS, spatial coverage, schema, units and limitations are +validated. It must persist through DatasetService and the canonical feature or +raster flow, retain provenance/checksums, return semantic metrics, and have a +selection-level regression test. Historical support additionally requires at +least two distinct, methodologically comparable official observation dates. diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index 107a994e..b9efd2d8 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -21,6 +21,8 @@ import { bboxToInputState, bboxesEqual, copyText, + datasetIntersectsSelection, + deduplicateTemporalSnapshots, downloadJsonFile, formatArea, formatBboxLabel, @@ -571,13 +573,11 @@ function listThemeTemporalSeries(datasets: DatasetCreateResponse[], theme: DataT groups.set(dataset.temporal_series_key, items) } return Array.from(groups.entries()) - .filter(([, items]) => items.length >= 2) .map(([key, items]) => { - const ordered = [...items].sort( - (left, right) => new Date(left.observed_at ?? 0).getTime() - new Date(right.observed_at ?? 0).getTime(), - ) + const ordered = deduplicateTemporalSnapshots(items) return { key, label: temporalSeriesLabel(ordered), items: ordered } }) + .filter((group) => group.items.length >= 2) .sort((left, right) => { if (right.items.length !== left.items.length) { return right.items.length - left.items.length @@ -1048,7 +1048,7 @@ export function MapWorkspace({ productKey: product.key, displayName: product.display_name, theme: product.theme, - availabilityLabel: `${product.native_resolution_m} m · ${product.observation_year} · laad bij selectie`, + availabilityLabel: `${product.native_resolution_m} m · ${product.observation_year} · automatisch bij selectie`, attribution: product.attribution, limitationMessage: product.limitation_message, }) @@ -1059,7 +1059,7 @@ export function MapWorkspace({ productKey: product.key, displayName: product.display_name, theme: product.key, - availabilityLabel: 'officiële vectorbron · laad bij selectie', + availabilityLabel: 'officiële vectorbron · automatisch bij selectie', attribution: product.attribution, limitationMessage: product.limitation_message, }) @@ -1075,7 +1075,7 @@ export function MapWorkspace({ productKey: source.key, displayName: source.display_name, theme: 'bathymetry', - availabilityLabel: 'historische profielpunten · laad bij selectie', + availabilityLabel: 'historische profielpunten · automatisch bij selectie', attribution: source.attribution, limitationMessage: source.limitation_message, }) @@ -1089,7 +1089,7 @@ export function MapWorkspace({ productKey: product.key, displayName: product.display_name, theme: product.theme, - availabilityLabel: `${product.observation_label} · officiële vectorbron · laad bij selectie`, + availabilityLabel: `${product.observation_label} · officiële vectorbron · automatisch bij selectie`, attribution: product.attribution, limitationMessage: product.limitation_message, }) @@ -1103,7 +1103,7 @@ export function MapWorkspace({ productKey: dhmvProduct.key, displayName: dhmvProduct.display_name, theme: 'elevation', - availabilityLabel: `${dhmvProduct.native_resolution_m} m · ${dhmvProduct.acquisition_period} · laad bij selectie`, + availabilityLabel: `${dhmvProduct.native_resolution_m} m · ${dhmvProduct.acquisition_period} · automatisch bij selectie`, attribution: dhmvProduct.attribution, limitationMessage: dhmvProduct.limitation_message, }) @@ -1119,7 +1119,7 @@ export function MapWorkspace({ productKey: floodProduct.key, displayName: floodProduct.display_name, theme: 'flood_hazard', - availabilityLabel: `${floodProduct.native_resolution_m} m · ${floodProduct.climate_context} · T${floodProduct.return_period_years} · laad bij selectie`, + availabilityLabel: `${floodProduct.native_resolution_m} m · ${floodProduct.climate_context} · T${floodProduct.return_period_years} · automatisch bij selectie`, attribution: floodProduct.attribution, limitationMessage: floodProduct.limitation_message, }) @@ -1329,6 +1329,19 @@ export function MapWorkspace({ const activeTemporalSeriesGroup = activeTemporalSeriesGroups.find((group) => group.key === selectedTemporalSeriesKey) ?? activeTemporalSeriesGroups[0] const activeTemporalSeries = activeTemporalSeriesGroup?.items ?? EMPTY_TEMPORAL_SERIES + const earlierTemporalOptions = activeTemporalSeries.slice(0, -1) + const selectedEarlierSnapshot = activeTemporalSeries.find((dataset) => dataset.id === earlierDatasetId) + const selectedLaterSnapshot = activeTemporalSeries.find((dataset) => dataset.id === laterDatasetId) + const selectedEarlierTime = new Date(selectedEarlierSnapshot?.observed_at ?? 0).getTime() + const laterTemporalOptions = activeTemporalSeries.filter( + (dataset) => new Date(dataset.observed_at ?? 0).getTime() > selectedEarlierTime, + ) + const temporalSelectionValid = Boolean( + selectedEarlierSnapshot + && selectedLaterSnapshot + && selectedEarlierSnapshot.id !== selectedLaterSnapshot.id + && selectedEarlierTime < new Date(selectedLaterSnapshot.observed_at ?? 0).getTime(), + ) const activeSeriesIsDailyGrb = activeTemporalSeries.length >= 2 && activeTemporalSeries.every((dataset) => dataset.source_name === 'grb') && new Date(activeTemporalSeries[activeTemporalSeries.length - 1].observed_at ?? 0).getTime() @@ -1378,7 +1391,7 @@ export function MapWorkspace({ activeSelectionResult.total_feature_count ?? activeSelectionResult.feature_count ).toLocaleString('nl-BE')} objecten gemeten` - : 'Op aanvraag' + : 'Automatisch bij selectie' : null onSetContextLayerLabel(contextLayerLabel) return () => onSetContextLayerLabel(null) @@ -1841,13 +1854,26 @@ export function MapWorkspace({ const resultFeatureLimit = selectionFeatureLimit(bbox) for (const theme of DATA_THEMES) { const dataset = themeDatasetMap[theme.id] - if (dataset && persistedDatasetSupportsSelection(dataset, bbox)) { + const partitioned = Boolean( + dataset + && regionalScopeSelected + && (isPartitionedRaster(dataset) || isPartitionedBathymetry(dataset)), + ) + const coveringPartitions = partitioned + ? themePartitionMap[theme.id].filter((partition) => datasetIntersectsSelection(partition, bbox)) + : [] + const persistedCoverageAvailable = Boolean( + dataset + && persistedDatasetSupportsSelection(dataset, bbox) + && (!partitioned || coveringPartitions.length > 0), + ) + if (dataset && persistedCoverageAvailable) { availableThemes.push({ themeId: theme.id, dataset, + datasetIds: coveringPartitions.map((partition) => partition.id), featureLimit: resultFeatureLimit, - partitioned: regionalScopeSelected - && (isPartitionedRaster(dataset) || isPartitionedBathymetry(dataset)), + partitioned, }) continue } @@ -1877,16 +1903,20 @@ export function MapWorkspace({ const startedAt = Date.now() setMapAnalysisDurationMs(null) setSelectionBbox(bbox) - const tasks: Array> = [loadAllThemeResults(bbox, areaId)] + const tasks: Array> = analysisMode === 'current' + ? [loadAllThemeResults(bbox, areaId)] + : [] const activeDatasetSupportsSelection = !activeThemeDataset || persistedDatasetSupportsSelection(activeThemeDataset, bbox) if ( - activeThemeAvailable && !regionalPartitionedThemeActive && !onDemandThemeActive + analysisMode === 'current' + && advancedMode + && activeThemeAvailable && !regionalPartitionedThemeActive && !onDemandThemeActive && activeDatasetSupportsSelection ) { tasks.push(onRunMapSelectionExtract(bbox, areaId)) } - if (analysisMode === 'evolution' && earlierDatasetId && laterDatasetId) { + if (analysisMode === 'evolution' && temporalSelectionValid) { tasks.push(compareTemporalSnapshots(earlierDatasetId, laterDatasetId, bbox, areaId)) } try { @@ -1899,7 +1929,7 @@ export function MapWorkspace({ } const runTemporalComparison = () => { - if (!mapSelectionBbox || !earlierDatasetId || !laterDatasetId) { + if (!mapSelectionBbox || !temporalSelectionValid) { return } void compareTemporalSnapshots( @@ -2127,7 +2157,7 @@ export function MapWorkspace({ ? 'Alleen huidige toestand' : 'Bron nog niet ingeladen' : dataset - ? datasetAvailabilityLabel(dataset, partitions) + ? `${datasetAvailabilityLabel(dataset, partitions)}${onDemandProduct ? ' · zo nodig automatisch aangevuld' : ''}` : onDemandProduct ? onDemandProduct.availabilityLabel : 'Bron nog niet ingeladen'} @@ -2138,7 +2168,7 @@ export function MapWorkspace({ ? 'Laden' : analysisMode === 'evolution' ? evolutionAvailable ? 'Tijdreeks' : dataset ? 'Alleen huidig' : 'Ontbreekt' - : dataset ? 'Beschikbaar' : onDemandProduct ? 'Op aanvraag' : 'Ontbreekt'} + : dataset ? 'Beschikbaar' : onDemandProduct ? 'Automatisch' : 'Ontbreekt'} ) @@ -2172,7 +2202,7 @@ export function MapWorkspace({ : regionalBathymetryThemeActive ? `${activeThemePartitions.length} gecontroleerde gemeentepartities · selectie wordt ruimtelijk samengevoegd` : activeThemeDataset - ? `${getDatasetSourceDisplayName(activeThemeDataset)} · ${formatDatasetObservation(activeThemeDataset)}` + ? `${getDatasetSourceDisplayName(activeThemeDataset)} · ${formatDatasetObservation(activeThemeDataset)}${onDemandProductMap.get(activeTheme.id) ? ' · ontbrekende lokale dekking wordt automatisch aangevuld' : ''}` : activeOnDemandMapProduct ? `${activeOnDemandMapProduct.attribution} · wordt alleen voor de gekozen selectie ingeladen` : activeTheme.description} @@ -2286,16 +2316,32 @@ export function MapWorkspace({ ) : null}