Add governed source freshness audit
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-16 15:05:42 +02:00
parent 2723043e3d
commit 1e0e7b6499
22 changed files with 1148 additions and 0 deletions
+24
View File
@@ -1419,3 +1419,27 @@ evidence and existing Dataset. `--force` explicitly refetches the WFS but still
fails closed if a different state-2025 checksum is already persisted. The
regional output uses the same selection-summary API as Mol; no new endpoint or
direct PostGIS write is introduced.
## Source freshness and version audit
`GET /api/v1/projects/{project_id}/datasets/source-freshness` derives a
read-only source status from persisted Dataset, DatasetVersion and storage
evidence. It distinguishes rolling snapshots and annual publications from
fixed editions, scenarios, historical archives and local artifacts. Fixed
source editions are never called stale solely because they are old.
The endpoint checks missing versions, checksum disagreement, missing local
files and stored-size disagreement. It performs no provider request and no
database write. The packaged operator command is suitable for an explicit
Unraid cron entry:
```bash
docker exec geointel python /app/scripts/audit_source_freshness.py \
--project-id 82a85913-c522-45d7-84a1-02b393d89e55 \
--api-url http://127.0.0.1/api/v1 \
--fail-on integrity \
--output /app/storage/operator-evidence/source-freshness/latest.json
```
Use `--fail-on due` to make a planned review date fail automation, or
`--fail-on never` for reporting only. The command never starts a refresh.
+10
View File
@@ -44,6 +44,7 @@ from app.services.raster_operations_service import RasterOperationsService
from app.services.vector_operations_service import VectorOperationsService
from app.services.vector_feature_service import VectorFeatureService
from app.services.dataset_service import DatasetService
from app.services.source_freshness_service import SourceFreshnessService
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.terrain_analysis_service import TerrainAnalysisService
@@ -245,6 +246,15 @@ def list_datasets(
return envelope({"items": [item.model_dump() for item in datasets], "total": total, "limit": limit, "offset": offset})
@router.get("/datasets/source-freshness", response_model=dict)
def audit_dataset_source_freshness(
project_id: UUID,
db: Session = Depends(get_db),
):
report = SourceFreshnessService.audit_project(db, project_id)
return envelope(report.model_dump())
@router.get("/datasets/{dataset_id}", response_model=dict)
def get_dataset(
project_id: UUID,
+10
View File
@@ -5,6 +5,12 @@ from .project import ProjectCreate, ProjectList, ProjectRead, ProjectUpdate
from .area import AreaCreate, AreaList, AreaRead, AreaUpdate
from .analysis import ChangeDetectionRequest, ChangeDetectionSummary
from .dataset import DatasetCreateResponse, DatasetList
from .source_freshness import (
SourceFreshnessItem,
SourceFreshnessReport,
SourceFreshnessSummary,
SourceIntegritySummary,
)
from .detection import (
DetectionListResponse,
DetectionModelCapability,
@@ -131,6 +137,10 @@ __all__ = [
"ChangeDetectionSummary",
"DatasetCreateResponse",
"DatasetList",
"SourceFreshnessItem",
"SourceFreshnessReport",
"SourceFreshnessSummary",
"SourceIntegritySummary",
"DetectionListResponse",
"DetectionModelCapability",
"DetectionModelsResponse",
+66
View File
@@ -0,0 +1,66 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel
SourceFreshnessStatus = Literal["current", "due", "review_required", "local"]
SourceRefreshPolicy = Literal["rolling_snapshot", "annual_release", "edition", "scenario", "archive", "local"]
class SourceIntegritySummary(BaseModel):
missing_version_count: int = 0
checksum_mismatch_count: int = 0
missing_storage_file_count: int = 0
size_mismatch_count: int = 0
@property
def issue_count(self) -> int:
return (
self.missing_version_count
+ self.checksum_mismatch_count
+ self.missing_storage_file_count
+ self.size_mismatch_count
)
class SourceFreshnessItem(BaseModel):
source_name: str
display_name: str
dataset_count: int
ready_count: int
version_count: int
latest_imported_at: datetime | None = None
latest_observed_at: datetime | None = None
latest_source_version: str | None = None
refresh_policy: SourceRefreshPolicy
review_interval_days: int | None = None
next_review_at: datetime | None = None
status: SourceFreshnessStatus
historical_series: bool
auto_refresh_supported: bool = False
reason: str
recommended_action: str
integrity: SourceIntegritySummary
class SourceFreshnessSummary(BaseModel):
source_count: int
dataset_count: int
current_count: int
due_count: int
review_required_count: int
local_count: int
sources_with_integrity_issues: int
integrity_issue_count: int
class SourceFreshnessReport(BaseModel):
project_id: UUID
generated_at: datetime
summary: SourceFreshnessSummary
items: list[SourceFreshnessItem]
limitations: list[str]
@@ -0,0 +1,302 @@
from __future__ import annotations
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Iterable
from uuid import UUID
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import Dataset, DatasetVersion, Project
from app.schemas.source_freshness import (
SourceFreshnessItem,
SourceFreshnessReport,
SourceFreshnessSummary,
SourceIntegritySummary,
)
@dataclass(frozen=True)
class SourcePolicy:
display_name: str
refresh_policy: str
review_interval_days: int | None = None
SOURCE_POLICIES: dict[str, SourcePolicy] = {
"grb": SourcePolicy("GRB gebouwen en context", "rolling_snapshot", 90),
"vrbg": SourcePolicy("VRBG wegenregister", "rolling_snapshot", 90),
"digitaal_vlaanderen_buildings_addresses_register": SourcePolicy(
"Gebouwen- en adressenregister", "rolling_snapshot", 90
),
"digitaal_vlaanderen_orthophoto": SourcePolicy("Orthofoto Vlaanderen", "rolling_snapshot", 180),
"agentschap_landbouw_zeevisserij_agricultural_parcels": SourcePolicy(
"Landbouwgebruikspercelen", "annual_release"
),
"department_omgeving_land_use": SourcePolicy("Landgebruik Vlaanderen", "annual_release"),
"inbo_bwk_natura2000": SourcePolicy("BWK en Natura 2000", "annual_release"),
"statbel": SourcePolicy("Statbel bevolking", "annual_release"),
"waterinfo": SourcePolicy("Waterinfo meetreeksen", "annual_release"),
"digitaal_vlaanderen_dhmv": SourcePolicy("Digitaal Hoogtemodel Vlaanderen", "edition"),
"department_omgeving_thematic_raster": SourcePolicy("Omgeving thematische rasters", "edition"),
"dov_soil_map": SourcePolicy("DOV bodemkaart", "edition"),
"vmm_flood_hazard": SourcePolicy("VMM overstromingskaarten", "scenario"),
"historical_landuse": SourcePolicy("Historisch landgebruik", "archive"),
"manual": SourcePolicy("Handmatig ingeladen gegevens", "local"),
"fixture": SourcePolicy("Test- en demonstratiegegevens", "local"),
"map_selection": SourcePolicy("Bewaarde kaartselecties", "local"),
}
DEFAULT_POLICY = SourcePolicy("Niet-geclassificeerde bron", "edition")
def _as_utc(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def _source_key(dataset: Dataset) -> str:
return (dataset.source_name or dataset.source or "unknown").strip().lower() or "unknown"
def _latest_datetime(values: Iterable[datetime | None]) -> datetime | None:
normalized = [_as_utc(value) for value in values if value is not None]
return max(normalized) if normalized else None
def _latest_dataset(datasets: list[Dataset]) -> Dataset:
return max(
datasets,
key=lambda item: (
_as_utc(item.observed_at) or datetime.min.replace(tzinfo=timezone.utc),
_as_utc(item.imported_at) or datetime.min.replace(tzinfo=timezone.utc),
str(item.id),
),
)
def _is_local_storage_path(storage_path: str) -> bool:
normalized = storage_path.strip().lower()
return bool(normalized) and "://" not in normalized and not normalized.startswith("/vsi")
def _integrity_summary(datasets: list[Dataset], versions_by_dataset: dict[UUID, list[DatasetVersion]]) -> SourceIntegritySummary:
summary = SourceIntegritySummary()
for dataset in datasets:
versions = versions_by_dataset.get(dataset.id, [])
latest_version = max(versions, key=lambda item: item.version) if versions else None
if dataset.status == "ready" and latest_version is None:
summary.missing_version_count += 1
if (
latest_version is not None
and dataset.checksum_sha256
and latest_version.checksum_sha256
and dataset.checksum_sha256 != latest_version.checksum_sha256
):
summary.checksum_mismatch_count += 1
if dataset.storage_path and _is_local_storage_path(dataset.storage_path):
path = Path(dataset.storage_path)
try:
if not path.is_file():
summary.missing_storage_file_count += 1
elif dataset.size_bytes is not None and path.stat().st_size != dataset.size_bytes:
summary.size_mismatch_count += 1
except OSError:
summary.missing_storage_file_count += 1
return summary
def _classify_source(
policy: SourcePolicy,
datasets: list[Dataset],
integrity: SourceIntegritySummary,
now: datetime,
) -> tuple[str, datetime | None, str, str]:
latest_imported = _latest_datetime(item.imported_at for item in datasets)
latest_observed = _latest_datetime(item.observed_at for item in datasets)
has_source_version = any(bool((item.source_version or "").strip()) for item in datasets)
if integrity.issue_count:
return (
"review_required",
None,
"De bewaarde dataset- en versie-evidentie bevat een integriteitsafwijking.",
"Controleer opslag, checksum en datasetversies voordat deze bron opnieuw wordt gebruikt.",
)
if policy.refresh_policy == "local":
return (
"local",
None,
"Deze bron is lokaal aangemaakt en heeft geen externe publicatiecyclus.",
"Geen bronverversing nodig; beheer de lokale dataset via de bestaande werkstroom.",
)
if policy.refresh_policy == "rolling_snapshot":
if latest_imported is None:
return (
"review_required",
None,
"De importdatum voor deze rollende bron ontbreekt.",
"Controleer de provenance voordat een nieuwe begrensde import wordt gestart.",
)
next_review = latest_imported + timedelta(days=policy.review_interval_days or 90)
if next_review <= now:
return (
"due",
next_review,
"De lokale snapshot heeft zijn geplande controledatum bereikt.",
"Vergelijk de broncatalogus en voer alleen daarna een begrensde, expliciete verversing uit.",
)
return (
"current",
next_review,
"De lokale snapshot valt binnen de afgesproken controleperiode.",
"Geen actie nodig tot de volgende controledatum.",
)
if policy.refresh_policy == "annual_release":
if latest_observed is None:
return (
"review_required",
None,
"De recentste waarnemings- of editieperiode ontbreekt.",
"Vul eerst officiële tijds- en versieprovenance aan; download niets automatisch.",
)
next_review = datetime(latest_observed.year + 2, 1, 1, tzinfo=timezone.utc)
if latest_observed.year < now.year - 1:
return (
"due",
next_review,
f"De recentste bewaarde jaargang is {latest_observed.year}.",
"Controleer of de officiële bron een recentere definitieve jaargang publiceerde.",
)
return (
"current",
next_review,
f"De recentste bewaarde jaargang is {latest_observed.year}.",
"Controleer bij de volgende publicatiecyclus of een nieuwe definitieve jaargang beschikbaar is.",
)
if not has_source_version:
return (
"review_required",
None,
"Deze vaste publicatie heeft geen herkenbare bronversie.",
"Leg de officiële editie of scenarioversie vast voordat de bron als gecontroleerd geldt.",
)
return (
"current",
None,
"Dit is een vaste editie, scenario- of archiefpublicatie met vastgelegde bronversie.",
"Vervang deze editie niet automatisch; voeg een nieuwe officiële editie als afzonderlijke versie toe.",
)
class SourceFreshnessService:
@staticmethod
def build_report(
project_id: UUID,
datasets: list[Dataset],
versions: list[DatasetVersion],
*,
now: datetime | None = None,
) -> SourceFreshnessReport:
generated_at = _as_utc(now) or datetime.now(timezone.utc)
versions_by_dataset: dict[UUID, list[DatasetVersion]] = defaultdict(list)
for version in versions:
versions_by_dataset[version.dataset_id].append(version)
datasets_by_source: dict[str, list[Dataset]] = defaultdict(list)
for dataset in datasets:
datasets_by_source[_source_key(dataset)].append(dataset)
items: list[SourceFreshnessItem] = []
for source_name, source_datasets in datasets_by_source.items():
policy = SOURCE_POLICIES.get(source_name, DEFAULT_POLICY)
integrity = _integrity_summary(source_datasets, versions_by_dataset)
status, next_review_at, reason, recommended_action = _classify_source(
policy, source_datasets, integrity, generated_at
)
if policy is DEFAULT_POLICY and not integrity.issue_count:
status = "review_required"
next_review_at = None
reason = "Voor deze bron is nog geen expliciete publicatie- of controlecyclus vastgelegd."
recommended_action = "Classificeer de bron eerst als snapshot, jaargang, vaste editie, scenario, archief of lokaal."
latest = _latest_dataset(source_datasets)
observed_values = {value for item in source_datasets if (value := _as_utc(item.observed_at)) is not None}
temporal_keys = {item.temporal_series_key for item in source_datasets if item.temporal_series_key}
items.append(
SourceFreshnessItem(
source_name=source_name,
display_name=policy.display_name if policy is not DEFAULT_POLICY else source_name.replace("_", " ").title(),
dataset_count=len(source_datasets),
ready_count=sum(item.status == "ready" for item in source_datasets),
version_count=sum(len(versions_by_dataset.get(item.id, [])) for item in source_datasets),
latest_imported_at=_latest_datetime(item.imported_at for item in source_datasets),
latest_observed_at=_latest_datetime(item.observed_at for item in source_datasets),
latest_source_version=next(
(
item.source_version
for item in sorted(
source_datasets,
key=lambda value: (
_as_utc(value.observed_at) or datetime.min.replace(tzinfo=timezone.utc),
_as_utc(value.imported_at) or datetime.min.replace(tzinfo=timezone.utc),
),
reverse=True,
)
if item.source_version
),
latest.source_version,
),
refresh_policy=policy.refresh_policy,
review_interval_days=policy.review_interval_days,
next_review_at=next_review_at,
status=status,
historical_series=len(observed_values) > 1 or len(temporal_keys) > 1,
reason=reason,
recommended_action=recommended_action,
integrity=integrity,
)
)
status_rank = {"review_required": 0, "due": 1, "current": 2, "local": 3}
items.sort(key=lambda item: (status_rank[item.status], item.display_name.lower()))
integrity_issue_count = sum(item.integrity.issue_count for item in items)
summary = SourceFreshnessSummary(
source_count=len(items),
dataset_count=len(datasets),
current_count=sum(item.status == "current" for item in items),
due_count=sum(item.status == "due" for item in items),
review_required_count=sum(item.status == "review_required" for item in items),
local_count=sum(item.status == "local" for item in items),
sources_with_integrity_issues=sum(item.integrity.issue_count > 0 for item in items),
integrity_issue_count=integrity_issue_count,
)
return SourceFreshnessReport(
project_id=project_id,
generated_at=generated_at,
summary=summary,
items=items,
limitations=[
"Deze controle leest uitsluitend lokale dataset-, versie- en opslaggegevens.",
"Er worden geen externe catalogi bevraagd, bestanden gedownload of datasets overschreven.",
"Een vaste editie of scenario-publicatie wordt niet verouderd genoemd alleen omdat de publicatiedatum oud is.",
],
)
@staticmethod
def audit_project(db: Session, project_id: UUID, *, now: datetime | None = None) -> SourceFreshnessReport:
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
datasets = db.query(Dataset).filter(Dataset.project_id == project_id).all()
dataset_ids = [dataset.id for dataset in datasets]
versions = (
db.query(DatasetVersion).filter(DatasetVersion.dataset_id.in_(dataset_ids)).all()
if dataset_ids
else []
)
return SourceFreshnessService.build_report(project_id, datasets, versions, now=now)
@@ -0,0 +1,168 @@
from __future__ import annotations
import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
from app.models import Dataset, DatasetVersion
from app.services.source_freshness_service import SourceFreshnessService
NOW = datetime(2026, 7, 16, 12, 0, tzinfo=timezone.utc)
def _dataset(
source_name: str,
*,
imported_at: datetime | None = NOW,
observed_at: datetime | None = None,
source_version: str | None = "edition-1",
storage_path: str | None = None,
checksum: str | None = "abc",
size_bytes: int | None = None,
temporal_series_key: str | None = None,
) -> Dataset:
return Dataset(
id=uuid.uuid4(),
project_id=uuid.uuid4(),
name=f"{source_name} dataset",
dataset_type="vector",
source=source_name,
source_name=source_name,
imported_at=imported_at,
observed_at=observed_at,
source_version=source_version,
storage_path=storage_path,
checksum_sha256=checksum,
size_bytes=size_bytes,
temporal_series_key=temporal_series_key,
status="ready",
)
def _version(dataset: Dataset, *, checksum: str | None = "abc") -> DatasetVersion:
return DatasetVersion(
id=uuid.uuid4(),
dataset_id=dataset.id,
version=1,
source_version=dataset.source_version,
observed_at=dataset.observed_at,
checksum_sha256=checksum,
)
def test_source_freshness_distinguishes_snapshot_annual_edition_and_local_sources(tmp_path: Path) -> None:
existing_file = tmp_path / "snapshot.geojson"
existing_file.write_text("{}", encoding="utf-8")
fresh_grb = _dataset(
"grb",
imported_at=NOW - timedelta(days=20),
storage_path=str(existing_file),
size_bytes=2,
)
old_vrbg = _dataset("vrbg", imported_at=NOW - timedelta(days=120))
current_annual = _dataset(
"statbel",
observed_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
temporal_series_key="population",
source_version="2025",
)
old_annual = _dataset(
"waterinfo",
observed_at=datetime(2023, 1, 1, tzinfo=timezone.utc),
temporal_series_key="water-level",
source_version="2023",
)
fixed_scenario = _dataset("vmm_flood_hazard", observed_at=None, source_version="VMM OGRK")
manual = _dataset("manual", source_version=None, checksum=None)
datasets = [fresh_grb, old_vrbg, current_annual, old_annual, fixed_scenario, manual]
versions = [_version(dataset, checksum=dataset.checksum_sha256) for dataset in datasets]
report = SourceFreshnessService.build_report(fresh_grb.project_id, datasets, versions, now=NOW)
by_source = {item.source_name: item for item in report.items}
assert by_source["grb"].status == "current"
assert by_source["vrbg"].status == "due"
assert by_source["statbel"].status == "current"
assert by_source["waterinfo"].status == "due"
assert by_source["vmm_flood_hazard"].status == "current"
assert by_source["manual"].status == "local"
assert all(item.auto_refresh_supported is False for item in report.items)
assert report.summary.dataset_count == len(datasets)
def test_source_freshness_flags_local_version_and_storage_integrity(tmp_path: Path) -> None:
missing_file = tmp_path / "missing.tif"
dataset = _dataset(
"digitaal_vlaanderen_dhmv",
storage_path=str(missing_file),
checksum="dataset-checksum",
)
version = _version(dataset, checksum="different-version-checksum")
report = SourceFreshnessService.build_report(dataset.project_id, [dataset], [version], now=NOW)
item = report.items[0]
assert item.status == "review_required"
assert item.integrity.checksum_mismatch_count == 1
assert item.integrity.missing_storage_file_count == 1
assert report.summary.sources_with_integrity_issues == 1
assert report.summary.integrity_issue_count == 2
def test_source_freshness_requires_dataset_version_and_marks_temporal_series() -> None:
first = _dataset(
"department_omgeving_land_use",
observed_at=datetime(2022, 1, 1, tzinfo=timezone.utc),
temporal_series_key="land-use",
source_version="2022-v3",
)
second = _dataset(
"department_omgeving_land_use",
observed_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
temporal_series_key="land-use",
source_version="2025-v3",
)
report = SourceFreshnessService.build_report(first.project_id, [first, second], [_version(first)], now=NOW)
item = report.items[0]
assert item.historical_series is True
assert item.status == "review_required"
assert item.integrity.missing_version_count == 1
def test_source_freshness_route_returns_canonical_envelope(monkeypatch) -> None:
from app.api.routes import datasets as dataset_routes
project_id = uuid.uuid4()
expected = SourceFreshnessService.build_report(project_id, [], [], now=NOW)
monkeypatch.setattr(
dataset_routes.SourceFreshnessService,
"audit_project",
lambda db, selected_project_id: expected,
)
response = dataset_routes.audit_dataset_source_freshness(project_id=project_id, db=SimpleNamespace())
assert list(response) == ["data"]
assert response["data"]["project_id"] == project_id
assert response["data"]["summary"]["source_count"] == 0
def test_source_freshness_operator_and_ui_contract_are_read_only() -> None:
root = Path(__file__).resolve().parents[2]
script = (root / "scripts" / "audit_source_freshness.py").read_text(encoding="utf-8")
dockerfile = (root / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
readiness = (root / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
app = (root / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8")
api = (root / "frontend" / "src" / "services" / "api" / "datasets.ts").read_text(encoding="utf-8")
assert "Request(endpoint" in script
assert "method=\"POST\"" not in script
assert "urlopen(request" in script
assert "COPY scripts/audit_source_freshness.py" in dockerfile
assert "py_compile scripts/audit_source_freshness.py" in readiness
assert "<SourceFreshnessPanel" in app
assert "/datasets/source-freshness" in api