Add governed source freshness audit

backend/app/api/routes/datasets.py

@@ -44,6 +44,7 @@ from app.services.raster_operations_service import RasterOperationsService
 from app.services.vector_operations_service import VectorOperationsService
 from app.services.vector_feature_service import VectorFeatureService
 from app.services.dataset_service import DatasetService
+from app.services.source_freshness_service import SourceFreshnessService
 from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
 from app.services.dhmv_acquisition_service import DhmvAcquisitionService
 from app.services.terrain_analysis_service import TerrainAnalysisService
@@ -245,6 +246,15 @@ def list_datasets(
     return envelope({"items": [item.model_dump() for item in datasets], "total": total, "limit": limit, "offset": offset})
 
 
+@router.get("/datasets/source-freshness", response_model=dict)
+def audit_dataset_source_freshness(
+    project_id: UUID,
+    db: Session = Depends(get_db),
+):
+    report = SourceFreshnessService.audit_project(db, project_id)
+    return envelope(report.model_dump())
+
+
 @router.get("/datasets/{dataset_id}", response_model=dict)
 def get_dataset(
     project_id: UUID,

backend/app/schemas/__init__.py

@@ -5,6 +5,12 @@ from .project import ProjectCreate, ProjectList, ProjectRead, ProjectUpdate
 from .area import AreaCreate, AreaList, AreaRead, AreaUpdate
 from .analysis import ChangeDetectionRequest, ChangeDetectionSummary
 from .dataset import DatasetCreateResponse, DatasetList
+from .source_freshness import (
+    SourceFreshnessItem,
+    SourceFreshnessReport,
+    SourceFreshnessSummary,
+    SourceIntegritySummary,
+)
 from .detection import (
     DetectionListResponse,
     DetectionModelCapability,
@@ -131,6 +137,10 @@ __all__ = [
     "ChangeDetectionSummary",
     "DatasetCreateResponse",
     "DatasetList",
+    "SourceFreshnessItem",
+    "SourceFreshnessReport",
+    "SourceFreshnessSummary",
+    "SourceIntegritySummary",
     "DetectionListResponse",
     "DetectionModelCapability",
     "DetectionModelsResponse",

backend/app/schemas/source_freshness.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel
+
+
+SourceFreshnessStatus = Literal["current", "due", "review_required", "local"]
+SourceRefreshPolicy = Literal["rolling_snapshot", "annual_release", "edition", "scenario", "archive", "local"]
+
+
+class SourceIntegritySummary(BaseModel):
+    missing_version_count: int = 0
+    checksum_mismatch_count: int = 0
+    missing_storage_file_count: int = 0
+    size_mismatch_count: int = 0
+
+    @property
+    def issue_count(self) -> int:
+        return (
+            self.missing_version_count
+            + self.checksum_mismatch_count
+            + self.missing_storage_file_count
+            + self.size_mismatch_count
+        )
+
+
+class SourceFreshnessItem(BaseModel):
+    source_name: str
+    display_name: str
+    dataset_count: int
+    ready_count: int
+    version_count: int
+    latest_imported_at: datetime | None = None
+    latest_observed_at: datetime | None = None
+    latest_source_version: str | None = None
+    refresh_policy: SourceRefreshPolicy
+    review_interval_days: int | None = None
+    next_review_at: datetime | None = None
+    status: SourceFreshnessStatus
+    historical_series: bool
+    auto_refresh_supported: bool = False
+    reason: str
+    recommended_action: str
+    integrity: SourceIntegritySummary
+
+
+class SourceFreshnessSummary(BaseModel):
+    source_count: int
+    dataset_count: int
+    current_count: int
+    due_count: int
+    review_required_count: int
+    local_count: int
+    sources_with_integrity_issues: int
+    integrity_issue_count: int
+
+
+class SourceFreshnessReport(BaseModel):
+    project_id: UUID
+    generated_at: datetime
+    summary: SourceFreshnessSummary
+    items: list[SourceFreshnessItem]
+    limitations: list[str]

backend/app/services/source_freshness_service.py

@@ -0,0 +1,302 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Iterable
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.core.errors import AppError
+from app.models import Dataset, DatasetVersion, Project
+from app.schemas.source_freshness import (
+    SourceFreshnessItem,
+    SourceFreshnessReport,
+    SourceFreshnessSummary,
+    SourceIntegritySummary,
+)
+
+
+@dataclass(frozen=True)
+class SourcePolicy:
+    display_name: str
+    refresh_policy: str
+    review_interval_days: int | None = None
+
+
+SOURCE_POLICIES: dict[str, SourcePolicy] = {
+    "grb": SourcePolicy("GRB gebouwen en context", "rolling_snapshot", 90),
+    "vrbg": SourcePolicy("VRBG wegenregister", "rolling_snapshot", 90),
+    "digitaal_vlaanderen_buildings_addresses_register": SourcePolicy(
+        "Gebouwen- en adressenregister", "rolling_snapshot", 90
+    ),
+    "digitaal_vlaanderen_orthophoto": SourcePolicy("Orthofoto Vlaanderen", "rolling_snapshot", 180),
+    "agentschap_landbouw_zeevisserij_agricultural_parcels": SourcePolicy(
+        "Landbouwgebruikspercelen", "annual_release"
+    ),
+    "department_omgeving_land_use": SourcePolicy("Landgebruik Vlaanderen", "annual_release"),
+    "inbo_bwk_natura2000": SourcePolicy("BWK en Natura 2000", "annual_release"),
+    "statbel": SourcePolicy("Statbel bevolking", "annual_release"),
+    "waterinfo": SourcePolicy("Waterinfo meetreeksen", "annual_release"),
+    "digitaal_vlaanderen_dhmv": SourcePolicy("Digitaal Hoogtemodel Vlaanderen", "edition"),
+    "department_omgeving_thematic_raster": SourcePolicy("Omgeving thematische rasters", "edition"),
+    "dov_soil_map": SourcePolicy("DOV bodemkaart", "edition"),
+    "vmm_flood_hazard": SourcePolicy("VMM overstromingskaarten", "scenario"),
+    "historical_landuse": SourcePolicy("Historisch landgebruik", "archive"),
+    "manual": SourcePolicy("Handmatig ingeladen gegevens", "local"),
+    "fixture": SourcePolicy("Test- en demonstratiegegevens", "local"),
+    "map_selection": SourcePolicy("Bewaarde kaartselecties", "local"),
+}
+
+DEFAULT_POLICY = SourcePolicy("Niet-geclassificeerde bron", "edition")
+
+
+def _as_utc(value: datetime | None) -> datetime | None:
+    if value is None:
+        return None
+    if value.tzinfo is None:
+        return value.replace(tzinfo=timezone.utc)
+    return value.astimezone(timezone.utc)
+
+
+def _source_key(dataset: Dataset) -> str:
+    return (dataset.source_name or dataset.source or "unknown").strip().lower() or "unknown"
+
+
+def _latest_datetime(values: Iterable[datetime | None]) -> datetime | None:
+    normalized = [_as_utc(value) for value in values if value is not None]
+    return max(normalized) if normalized else None
+
+
+def _latest_dataset(datasets: list[Dataset]) -> Dataset:
+    return max(
+        datasets,
+        key=lambda item: (
+            _as_utc(item.observed_at) or datetime.min.replace(tzinfo=timezone.utc),
+            _as_utc(item.imported_at) or datetime.min.replace(tzinfo=timezone.utc),
+            str(item.id),
+        ),
+    )
+
+
+def _is_local_storage_path(storage_path: str) -> bool:
+    normalized = storage_path.strip().lower()
+    return bool(normalized) and "://" not in normalized and not normalized.startswith("/vsi")
+
+
+def _integrity_summary(datasets: list[Dataset], versions_by_dataset: dict[UUID, list[DatasetVersion]]) -> SourceIntegritySummary:
+    summary = SourceIntegritySummary()
+    for dataset in datasets:
+        versions = versions_by_dataset.get(dataset.id, [])
+        latest_version = max(versions, key=lambda item: item.version) if versions else None
+        if dataset.status == "ready" and latest_version is None:
+            summary.missing_version_count += 1
+        if (
+            latest_version is not None
+            and dataset.checksum_sha256
+            and latest_version.checksum_sha256
+            and dataset.checksum_sha256 != latest_version.checksum_sha256
+        ):
+            summary.checksum_mismatch_count += 1
+        if dataset.storage_path and _is_local_storage_path(dataset.storage_path):
+            path = Path(dataset.storage_path)
+            try:
+                if not path.is_file():
+                    summary.missing_storage_file_count += 1
+                elif dataset.size_bytes is not None and path.stat().st_size != dataset.size_bytes:
+                    summary.size_mismatch_count += 1
+            except OSError:
+                summary.missing_storage_file_count += 1
+    return summary
+
+
+def _classify_source(
+    policy: SourcePolicy,
+    datasets: list[Dataset],
+    integrity: SourceIntegritySummary,
+    now: datetime,
+) -> tuple[str, datetime | None, str, str]:
+    latest_imported = _latest_datetime(item.imported_at for item in datasets)
+    latest_observed = _latest_datetime(item.observed_at for item in datasets)
+    has_source_version = any(bool((item.source_version or "").strip()) for item in datasets)
+
+    if integrity.issue_count:
+        return (
+            "review_required",
+            None,
+            "De bewaarde dataset- en versie-evidentie bevat een integriteitsafwijking.",
+            "Controleer opslag, checksum en datasetversies voordat deze bron opnieuw wordt gebruikt.",
+        )
+    if policy.refresh_policy == "local":
+        return (
+            "local",
+            None,
+            "Deze bron is lokaal aangemaakt en heeft geen externe publicatiecyclus.",
+            "Geen bronverversing nodig; beheer de lokale dataset via de bestaande werkstroom.",
+        )
+    if policy.refresh_policy == "rolling_snapshot":
+        if latest_imported is None:
+            return (
+                "review_required",
+                None,
+                "De importdatum voor deze rollende bron ontbreekt.",
+                "Controleer de provenance voordat een nieuwe begrensde import wordt gestart.",
+            )
+        next_review = latest_imported + timedelta(days=policy.review_interval_days or 90)
+        if next_review <= now:
+            return (
+                "due",
+                next_review,
+                "De lokale snapshot heeft zijn geplande controledatum bereikt.",
+                "Vergelijk de broncatalogus en voer alleen daarna een begrensde, expliciete verversing uit.",
+            )
+        return (
+            "current",
+            next_review,
+            "De lokale snapshot valt binnen de afgesproken controleperiode.",
+            "Geen actie nodig tot de volgende controledatum.",
+        )
+    if policy.refresh_policy == "annual_release":
+        if latest_observed is None:
+            return (
+                "review_required",
+                None,
+                "De recentste waarnemings- of editieperiode ontbreekt.",
+                "Vul eerst officiële tijds- en versieprovenance aan; download niets automatisch.",
+            )
+        next_review = datetime(latest_observed.year + 2, 1, 1, tzinfo=timezone.utc)
+        if latest_observed.year < now.year - 1:
+            return (
+                "due",
+                next_review,
+                f"De recentste bewaarde jaargang is {latest_observed.year}.",
+                "Controleer of de officiële bron een recentere definitieve jaargang publiceerde.",
+            )
+        return (
+            "current",
+            next_review,
+            f"De recentste bewaarde jaargang is {latest_observed.year}.",
+            "Controleer bij de volgende publicatiecyclus of een nieuwe definitieve jaargang beschikbaar is.",
+        )
+    if not has_source_version:
+        return (
+            "review_required",
+            None,
+            "Deze vaste publicatie heeft geen herkenbare bronversie.",
+            "Leg de officiële editie of scenarioversie vast voordat de bron als gecontroleerd geldt.",
+        )
+    return (
+        "current",
+        None,
+        "Dit is een vaste editie, scenario- of archiefpublicatie met vastgelegde bronversie.",
+        "Vervang deze editie niet automatisch; voeg een nieuwe officiële editie als afzonderlijke versie toe.",
+    )
+
+
+class SourceFreshnessService:
+    @staticmethod
+    def build_report(
+        project_id: UUID,
+        datasets: list[Dataset],
+        versions: list[DatasetVersion],
+        *,
+        now: datetime | None = None,
+    ) -> SourceFreshnessReport:
+        generated_at = _as_utc(now) or datetime.now(timezone.utc)
+        versions_by_dataset: dict[UUID, list[DatasetVersion]] = defaultdict(list)
+        for version in versions:
+            versions_by_dataset[version.dataset_id].append(version)
+
+        datasets_by_source: dict[str, list[Dataset]] = defaultdict(list)
+        for dataset in datasets:
+            datasets_by_source[_source_key(dataset)].append(dataset)
+
+        items: list[SourceFreshnessItem] = []
+        for source_name, source_datasets in datasets_by_source.items():
+            policy = SOURCE_POLICIES.get(source_name, DEFAULT_POLICY)
+            integrity = _integrity_summary(source_datasets, versions_by_dataset)
+            status, next_review_at, reason, recommended_action = _classify_source(
+                policy, source_datasets, integrity, generated_at
+            )
+            if policy is DEFAULT_POLICY and not integrity.issue_count:
+                status = "review_required"
+                next_review_at = None
+                reason = "Voor deze bron is nog geen expliciete publicatie- of controlecyclus vastgelegd."
+                recommended_action = "Classificeer de bron eerst als snapshot, jaargang, vaste editie, scenario, archief of lokaal."
+            latest = _latest_dataset(source_datasets)
+            observed_values = {value for item in source_datasets if (value := _as_utc(item.observed_at)) is not None}
+            temporal_keys = {item.temporal_series_key for item in source_datasets if item.temporal_series_key}
+            items.append(
+                SourceFreshnessItem(
+                    source_name=source_name,
+                    display_name=policy.display_name if policy is not DEFAULT_POLICY else source_name.replace("_", " ").title(),
+                    dataset_count=len(source_datasets),
+                    ready_count=sum(item.status == "ready" for item in source_datasets),
+                    version_count=sum(len(versions_by_dataset.get(item.id, [])) for item in source_datasets),
+                    latest_imported_at=_latest_datetime(item.imported_at for item in source_datasets),
+                    latest_observed_at=_latest_datetime(item.observed_at for item in source_datasets),
+                    latest_source_version=next(
+                        (
+                            item.source_version
+                            for item in sorted(
+                                source_datasets,
+                                key=lambda value: (
+                                    _as_utc(value.observed_at) or datetime.min.replace(tzinfo=timezone.utc),
+                                    _as_utc(value.imported_at) or datetime.min.replace(tzinfo=timezone.utc),
+                                ),
+                                reverse=True,
+                            )
+                            if item.source_version
+                        ),
+                        latest.source_version,
+                    ),
+                    refresh_policy=policy.refresh_policy,
+                    review_interval_days=policy.review_interval_days,
+                    next_review_at=next_review_at,
+                    status=status,
+                    historical_series=len(observed_values) > 1 or len(temporal_keys) > 1,
+                    reason=reason,
+                    recommended_action=recommended_action,
+                    integrity=integrity,
+                )
+            )
+
+        status_rank = {"review_required": 0, "due": 1, "current": 2, "local": 3}
+        items.sort(key=lambda item: (status_rank[item.status], item.display_name.lower()))
+        integrity_issue_count = sum(item.integrity.issue_count for item in items)
+        summary = SourceFreshnessSummary(
+            source_count=len(items),
+            dataset_count=len(datasets),
+            current_count=sum(item.status == "current" for item in items),
+            due_count=sum(item.status == "due" for item in items),
+            review_required_count=sum(item.status == "review_required" for item in items),
+            local_count=sum(item.status == "local" for item in items),
+            sources_with_integrity_issues=sum(item.integrity.issue_count > 0 for item in items),
+            integrity_issue_count=integrity_issue_count,
+        )
+        return SourceFreshnessReport(
+            project_id=project_id,
+            generated_at=generated_at,
+            summary=summary,
+            items=items,
+            limitations=[
+                "Deze controle leest uitsluitend lokale dataset-, versie- en opslaggegevens.",
+                "Er worden geen externe catalogi bevraagd, bestanden gedownload of datasets overschreven.",
+                "Een vaste editie of scenario-publicatie wordt niet verouderd genoemd alleen omdat de publicatiedatum oud is.",
+            ],
+        )
+
+    @staticmethod
+    def audit_project(db: Session, project_id: UUID, *, now: datetime | None = None) -> SourceFreshnessReport:
+        if not db.get(Project, project_id):
+            raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
+        datasets = db.query(Dataset).filter(Dataset.project_id == project_id).all()
+        dataset_ids = [dataset.id for dataset in datasets]
+        versions = (
+            db.query(DatasetVersion).filter(DatasetVersion.dataset_id.in_(dataset_ids)).all()
+            if dataset_ids
+            else []
+        )
+        return SourceFreshnessService.build_report(project_id, datasets, versions, now=now)