Add governed source freshness audit
This commit is contained in:
@@ -1419,3 +1419,27 @@ evidence and existing Dataset. `--force` explicitly refetches the WFS but still
|
||||
fails closed if a different state-2025 checksum is already persisted. The
|
||||
regional output uses the same selection-summary API as Mol; no new endpoint or
|
||||
direct PostGIS write is introduced.
|
||||
|
||||
## Source freshness and version audit
|
||||
|
||||
`GET /api/v1/projects/{project_id}/datasets/source-freshness` derives a
|
||||
read-only source status from persisted Dataset, DatasetVersion and storage
|
||||
evidence. It distinguishes rolling snapshots and annual publications from
|
||||
fixed editions, scenarios, historical archives and local artifacts. Fixed
|
||||
source editions are never called stale solely because they are old.
|
||||
|
||||
The endpoint checks missing versions, checksum disagreement, missing local
|
||||
files and stored-size disagreement. It performs no provider request and no
|
||||
database write. The packaged operator command is suitable for an explicit
|
||||
Unraid cron entry:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/audit_source_freshness.py \
|
||||
--project-id 82a85913-c522-45d7-84a1-02b393d89e55 \
|
||||
--api-url http://127.0.0.1/api/v1 \
|
||||
--fail-on integrity \
|
||||
--output /app/storage/operator-evidence/source-freshness/latest.json
|
||||
```
|
||||
|
||||
Use `--fail-on due` to make a planned review date fail automation, or
|
||||
`--fail-on never` for reporting only. The command never starts a refresh.
|
||||
|
||||
@@ -44,6 +44,7 @@ from app.services.raster_operations_service import RasterOperationsService
|
||||
from app.services.vector_operations_service import VectorOperationsService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.source_freshness_service import SourceFreshnessService
|
||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
||||
from app.services.terrain_analysis_service import TerrainAnalysisService
|
||||
@@ -245,6 +246,15 @@ def list_datasets(
|
||||
return envelope({"items": [item.model_dump() for item in datasets], "total": total, "limit": limit, "offset": offset})
|
||||
|
||||
|
||||
@router.get("/datasets/source-freshness", response_model=dict)
|
||||
def audit_dataset_source_freshness(
|
||||
project_id: UUID,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
report = SourceFreshnessService.audit_project(db, project_id)
|
||||
return envelope(report.model_dump())
|
||||
|
||||
|
||||
@router.get("/datasets/{dataset_id}", response_model=dict)
|
||||
def get_dataset(
|
||||
project_id: UUID,
|
||||
|
||||
@@ -5,6 +5,12 @@ from .project import ProjectCreate, ProjectList, ProjectRead, ProjectUpdate
|
||||
from .area import AreaCreate, AreaList, AreaRead, AreaUpdate
|
||||
from .analysis import ChangeDetectionRequest, ChangeDetectionSummary
|
||||
from .dataset import DatasetCreateResponse, DatasetList
|
||||
from .source_freshness import (
|
||||
SourceFreshnessItem,
|
||||
SourceFreshnessReport,
|
||||
SourceFreshnessSummary,
|
||||
SourceIntegritySummary,
|
||||
)
|
||||
from .detection import (
|
||||
DetectionListResponse,
|
||||
DetectionModelCapability,
|
||||
@@ -131,6 +137,10 @@ __all__ = [
|
||||
"ChangeDetectionSummary",
|
||||
"DatasetCreateResponse",
|
||||
"DatasetList",
|
||||
"SourceFreshnessItem",
|
||||
"SourceFreshnessReport",
|
||||
"SourceFreshnessSummary",
|
||||
"SourceIntegritySummary",
|
||||
"DetectionListResponse",
|
||||
"DetectionModelCapability",
|
||||
"DetectionModelsResponse",
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
SourceFreshnessStatus = Literal["current", "due", "review_required", "local"]
|
||||
SourceRefreshPolicy = Literal["rolling_snapshot", "annual_release", "edition", "scenario", "archive", "local"]
|
||||
|
||||
|
||||
class SourceIntegritySummary(BaseModel):
|
||||
missing_version_count: int = 0
|
||||
checksum_mismatch_count: int = 0
|
||||
missing_storage_file_count: int = 0
|
||||
size_mismatch_count: int = 0
|
||||
|
||||
@property
|
||||
def issue_count(self) -> int:
|
||||
return (
|
||||
self.missing_version_count
|
||||
+ self.checksum_mismatch_count
|
||||
+ self.missing_storage_file_count
|
||||
+ self.size_mismatch_count
|
||||
)
|
||||
|
||||
|
||||
class SourceFreshnessItem(BaseModel):
|
||||
source_name: str
|
||||
display_name: str
|
||||
dataset_count: int
|
||||
ready_count: int
|
||||
version_count: int
|
||||
latest_imported_at: datetime | None = None
|
||||
latest_observed_at: datetime | None = None
|
||||
latest_source_version: str | None = None
|
||||
refresh_policy: SourceRefreshPolicy
|
||||
review_interval_days: int | None = None
|
||||
next_review_at: datetime | None = None
|
||||
status: SourceFreshnessStatus
|
||||
historical_series: bool
|
||||
auto_refresh_supported: bool = False
|
||||
reason: str
|
||||
recommended_action: str
|
||||
integrity: SourceIntegritySummary
|
||||
|
||||
|
||||
class SourceFreshnessSummary(BaseModel):
|
||||
source_count: int
|
||||
dataset_count: int
|
||||
current_count: int
|
||||
due_count: int
|
||||
review_required_count: int
|
||||
local_count: int
|
||||
sources_with_integrity_issues: int
|
||||
integrity_issue_count: int
|
||||
|
||||
|
||||
class SourceFreshnessReport(BaseModel):
|
||||
project_id: UUID
|
||||
generated_at: datetime
|
||||
summary: SourceFreshnessSummary
|
||||
items: list[SourceFreshnessItem]
|
||||
limitations: list[str]
|
||||
@@ -0,0 +1,302 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, DatasetVersion, Project
|
||||
from app.schemas.source_freshness import (
|
||||
SourceFreshnessItem,
|
||||
SourceFreshnessReport,
|
||||
SourceFreshnessSummary,
|
||||
SourceIntegritySummary,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourcePolicy:
|
||||
display_name: str
|
||||
refresh_policy: str
|
||||
review_interval_days: int | None = None
|
||||
|
||||
|
||||
SOURCE_POLICIES: dict[str, SourcePolicy] = {
|
||||
"grb": SourcePolicy("GRB gebouwen en context", "rolling_snapshot", 90),
|
||||
"vrbg": SourcePolicy("VRBG wegenregister", "rolling_snapshot", 90),
|
||||
"digitaal_vlaanderen_buildings_addresses_register": SourcePolicy(
|
||||
"Gebouwen- en adressenregister", "rolling_snapshot", 90
|
||||
),
|
||||
"digitaal_vlaanderen_orthophoto": SourcePolicy("Orthofoto Vlaanderen", "rolling_snapshot", 180),
|
||||
"agentschap_landbouw_zeevisserij_agricultural_parcels": SourcePolicy(
|
||||
"Landbouwgebruikspercelen", "annual_release"
|
||||
),
|
||||
"department_omgeving_land_use": SourcePolicy("Landgebruik Vlaanderen", "annual_release"),
|
||||
"inbo_bwk_natura2000": SourcePolicy("BWK en Natura 2000", "annual_release"),
|
||||
"statbel": SourcePolicy("Statbel bevolking", "annual_release"),
|
||||
"waterinfo": SourcePolicy("Waterinfo meetreeksen", "annual_release"),
|
||||
"digitaal_vlaanderen_dhmv": SourcePolicy("Digitaal Hoogtemodel Vlaanderen", "edition"),
|
||||
"department_omgeving_thematic_raster": SourcePolicy("Omgeving thematische rasters", "edition"),
|
||||
"dov_soil_map": SourcePolicy("DOV bodemkaart", "edition"),
|
||||
"vmm_flood_hazard": SourcePolicy("VMM overstromingskaarten", "scenario"),
|
||||
"historical_landuse": SourcePolicy("Historisch landgebruik", "archive"),
|
||||
"manual": SourcePolicy("Handmatig ingeladen gegevens", "local"),
|
||||
"fixture": SourcePolicy("Test- en demonstratiegegevens", "local"),
|
||||
"map_selection": SourcePolicy("Bewaarde kaartselecties", "local"),
|
||||
}
|
||||
|
||||
DEFAULT_POLICY = SourcePolicy("Niet-geclassificeerde bron", "edition")
|
||||
|
||||
|
||||
def _as_utc(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _source_key(dataset: Dataset) -> str:
|
||||
return (dataset.source_name or dataset.source or "unknown").strip().lower() or "unknown"
|
||||
|
||||
|
||||
def _latest_datetime(values: Iterable[datetime | None]) -> datetime | None:
|
||||
normalized = [_as_utc(value) for value in values if value is not None]
|
||||
return max(normalized) if normalized else None
|
||||
|
||||
|
||||
def _latest_dataset(datasets: list[Dataset]) -> Dataset:
|
||||
return max(
|
||||
datasets,
|
||||
key=lambda item: (
|
||||
_as_utc(item.observed_at) or datetime.min.replace(tzinfo=timezone.utc),
|
||||
_as_utc(item.imported_at) or datetime.min.replace(tzinfo=timezone.utc),
|
||||
str(item.id),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _is_local_storage_path(storage_path: str) -> bool:
|
||||
normalized = storage_path.strip().lower()
|
||||
return bool(normalized) and "://" not in normalized and not normalized.startswith("/vsi")
|
||||
|
||||
|
||||
def _integrity_summary(datasets: list[Dataset], versions_by_dataset: dict[UUID, list[DatasetVersion]]) -> SourceIntegritySummary:
|
||||
summary = SourceIntegritySummary()
|
||||
for dataset in datasets:
|
||||
versions = versions_by_dataset.get(dataset.id, [])
|
||||
latest_version = max(versions, key=lambda item: item.version) if versions else None
|
||||
if dataset.status == "ready" and latest_version is None:
|
||||
summary.missing_version_count += 1
|
||||
if (
|
||||
latest_version is not None
|
||||
and dataset.checksum_sha256
|
||||
and latest_version.checksum_sha256
|
||||
and dataset.checksum_sha256 != latest_version.checksum_sha256
|
||||
):
|
||||
summary.checksum_mismatch_count += 1
|
||||
if dataset.storage_path and _is_local_storage_path(dataset.storage_path):
|
||||
path = Path(dataset.storage_path)
|
||||
try:
|
||||
if not path.is_file():
|
||||
summary.missing_storage_file_count += 1
|
||||
elif dataset.size_bytes is not None and path.stat().st_size != dataset.size_bytes:
|
||||
summary.size_mismatch_count += 1
|
||||
except OSError:
|
||||
summary.missing_storage_file_count += 1
|
||||
return summary
|
||||
|
||||
|
||||
def _classify_source(
|
||||
policy: SourcePolicy,
|
||||
datasets: list[Dataset],
|
||||
integrity: SourceIntegritySummary,
|
||||
now: datetime,
|
||||
) -> tuple[str, datetime | None, str, str]:
|
||||
latest_imported = _latest_datetime(item.imported_at for item in datasets)
|
||||
latest_observed = _latest_datetime(item.observed_at for item in datasets)
|
||||
has_source_version = any(bool((item.source_version or "").strip()) for item in datasets)
|
||||
|
||||
if integrity.issue_count:
|
||||
return (
|
||||
"review_required",
|
||||
None,
|
||||
"De bewaarde dataset- en versie-evidentie bevat een integriteitsafwijking.",
|
||||
"Controleer opslag, checksum en datasetversies voordat deze bron opnieuw wordt gebruikt.",
|
||||
)
|
||||
if policy.refresh_policy == "local":
|
||||
return (
|
||||
"local",
|
||||
None,
|
||||
"Deze bron is lokaal aangemaakt en heeft geen externe publicatiecyclus.",
|
||||
"Geen bronverversing nodig; beheer de lokale dataset via de bestaande werkstroom.",
|
||||
)
|
||||
if policy.refresh_policy == "rolling_snapshot":
|
||||
if latest_imported is None:
|
||||
return (
|
||||
"review_required",
|
||||
None,
|
||||
"De importdatum voor deze rollende bron ontbreekt.",
|
||||
"Controleer de provenance voordat een nieuwe begrensde import wordt gestart.",
|
||||
)
|
||||
next_review = latest_imported + timedelta(days=policy.review_interval_days or 90)
|
||||
if next_review <= now:
|
||||
return (
|
||||
"due",
|
||||
next_review,
|
||||
"De lokale snapshot heeft zijn geplande controledatum bereikt.",
|
||||
"Vergelijk de broncatalogus en voer alleen daarna een begrensde, expliciete verversing uit.",
|
||||
)
|
||||
return (
|
||||
"current",
|
||||
next_review,
|
||||
"De lokale snapshot valt binnen de afgesproken controleperiode.",
|
||||
"Geen actie nodig tot de volgende controledatum.",
|
||||
)
|
||||
if policy.refresh_policy == "annual_release":
|
||||
if latest_observed is None:
|
||||
return (
|
||||
"review_required",
|
||||
None,
|
||||
"De recentste waarnemings- of editieperiode ontbreekt.",
|
||||
"Vul eerst officiële tijds- en versieprovenance aan; download niets automatisch.",
|
||||
)
|
||||
next_review = datetime(latest_observed.year + 2, 1, 1, tzinfo=timezone.utc)
|
||||
if latest_observed.year < now.year - 1:
|
||||
return (
|
||||
"due",
|
||||
next_review,
|
||||
f"De recentste bewaarde jaargang is {latest_observed.year}.",
|
||||
"Controleer of de officiële bron een recentere definitieve jaargang publiceerde.",
|
||||
)
|
||||
return (
|
||||
"current",
|
||||
next_review,
|
||||
f"De recentste bewaarde jaargang is {latest_observed.year}.",
|
||||
"Controleer bij de volgende publicatiecyclus of een nieuwe definitieve jaargang beschikbaar is.",
|
||||
)
|
||||
if not has_source_version:
|
||||
return (
|
||||
"review_required",
|
||||
None,
|
||||
"Deze vaste publicatie heeft geen herkenbare bronversie.",
|
||||
"Leg de officiële editie of scenarioversie vast voordat de bron als gecontroleerd geldt.",
|
||||
)
|
||||
return (
|
||||
"current",
|
||||
None,
|
||||
"Dit is een vaste editie, scenario- of archiefpublicatie met vastgelegde bronversie.",
|
||||
"Vervang deze editie niet automatisch; voeg een nieuwe officiële editie als afzonderlijke versie toe.",
|
||||
)
|
||||
|
||||
|
||||
class SourceFreshnessService:
|
||||
@staticmethod
|
||||
def build_report(
|
||||
project_id: UUID,
|
||||
datasets: list[Dataset],
|
||||
versions: list[DatasetVersion],
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> SourceFreshnessReport:
|
||||
generated_at = _as_utc(now) or datetime.now(timezone.utc)
|
||||
versions_by_dataset: dict[UUID, list[DatasetVersion]] = defaultdict(list)
|
||||
for version in versions:
|
||||
versions_by_dataset[version.dataset_id].append(version)
|
||||
|
||||
datasets_by_source: dict[str, list[Dataset]] = defaultdict(list)
|
||||
for dataset in datasets:
|
||||
datasets_by_source[_source_key(dataset)].append(dataset)
|
||||
|
||||
items: list[SourceFreshnessItem] = []
|
||||
for source_name, source_datasets in datasets_by_source.items():
|
||||
policy = SOURCE_POLICIES.get(source_name, DEFAULT_POLICY)
|
||||
integrity = _integrity_summary(source_datasets, versions_by_dataset)
|
||||
status, next_review_at, reason, recommended_action = _classify_source(
|
||||
policy, source_datasets, integrity, generated_at
|
||||
)
|
||||
if policy is DEFAULT_POLICY and not integrity.issue_count:
|
||||
status = "review_required"
|
||||
next_review_at = None
|
||||
reason = "Voor deze bron is nog geen expliciete publicatie- of controlecyclus vastgelegd."
|
||||
recommended_action = "Classificeer de bron eerst als snapshot, jaargang, vaste editie, scenario, archief of lokaal."
|
||||
latest = _latest_dataset(source_datasets)
|
||||
observed_values = {value for item in source_datasets if (value := _as_utc(item.observed_at)) is not None}
|
||||
temporal_keys = {item.temporal_series_key for item in source_datasets if item.temporal_series_key}
|
||||
items.append(
|
||||
SourceFreshnessItem(
|
||||
source_name=source_name,
|
||||
display_name=policy.display_name if policy is not DEFAULT_POLICY else source_name.replace("_", " ").title(),
|
||||
dataset_count=len(source_datasets),
|
||||
ready_count=sum(item.status == "ready" for item in source_datasets),
|
||||
version_count=sum(len(versions_by_dataset.get(item.id, [])) for item in source_datasets),
|
||||
latest_imported_at=_latest_datetime(item.imported_at for item in source_datasets),
|
||||
latest_observed_at=_latest_datetime(item.observed_at for item in source_datasets),
|
||||
latest_source_version=next(
|
||||
(
|
||||
item.source_version
|
||||
for item in sorted(
|
||||
source_datasets,
|
||||
key=lambda value: (
|
||||
_as_utc(value.observed_at) or datetime.min.replace(tzinfo=timezone.utc),
|
||||
_as_utc(value.imported_at) or datetime.min.replace(tzinfo=timezone.utc),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
if item.source_version
|
||||
),
|
||||
latest.source_version,
|
||||
),
|
||||
refresh_policy=policy.refresh_policy,
|
||||
review_interval_days=policy.review_interval_days,
|
||||
next_review_at=next_review_at,
|
||||
status=status,
|
||||
historical_series=len(observed_values) > 1 or len(temporal_keys) > 1,
|
||||
reason=reason,
|
||||
recommended_action=recommended_action,
|
||||
integrity=integrity,
|
||||
)
|
||||
)
|
||||
|
||||
status_rank = {"review_required": 0, "due": 1, "current": 2, "local": 3}
|
||||
items.sort(key=lambda item: (status_rank[item.status], item.display_name.lower()))
|
||||
integrity_issue_count = sum(item.integrity.issue_count for item in items)
|
||||
summary = SourceFreshnessSummary(
|
||||
source_count=len(items),
|
||||
dataset_count=len(datasets),
|
||||
current_count=sum(item.status == "current" for item in items),
|
||||
due_count=sum(item.status == "due" for item in items),
|
||||
review_required_count=sum(item.status == "review_required" for item in items),
|
||||
local_count=sum(item.status == "local" for item in items),
|
||||
sources_with_integrity_issues=sum(item.integrity.issue_count > 0 for item in items),
|
||||
integrity_issue_count=integrity_issue_count,
|
||||
)
|
||||
return SourceFreshnessReport(
|
||||
project_id=project_id,
|
||||
generated_at=generated_at,
|
||||
summary=summary,
|
||||
items=items,
|
||||
limitations=[
|
||||
"Deze controle leest uitsluitend lokale dataset-, versie- en opslaggegevens.",
|
||||
"Er worden geen externe catalogi bevraagd, bestanden gedownload of datasets overschreven.",
|
||||
"Een vaste editie of scenario-publicatie wordt niet verouderd genoemd alleen omdat de publicatiedatum oud is.",
|
||||
],
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def audit_project(db: Session, project_id: UUID, *, now: datetime | None = None) -> SourceFreshnessReport:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
datasets = db.query(Dataset).filter(Dataset.project_id == project_id).all()
|
||||
dataset_ids = [dataset.id for dataset in datasets]
|
||||
versions = (
|
||||
db.query(DatasetVersion).filter(DatasetVersion.dataset_id.in_(dataset_ids)).all()
|
||||
if dataset_ids
|
||||
else []
|
||||
)
|
||||
return SourceFreshnessService.build_report(project_id, datasets, versions, now=now)
|
||||
@@ -0,0 +1,168 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.models import Dataset, DatasetVersion
|
||||
from app.services.source_freshness_service import SourceFreshnessService
|
||||
|
||||
|
||||
NOW = datetime(2026, 7, 16, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _dataset(
|
||||
source_name: str,
|
||||
*,
|
||||
imported_at: datetime | None = NOW,
|
||||
observed_at: datetime | None = None,
|
||||
source_version: str | None = "edition-1",
|
||||
storage_path: str | None = None,
|
||||
checksum: str | None = "abc",
|
||||
size_bytes: int | None = None,
|
||||
temporal_series_key: str | None = None,
|
||||
) -> Dataset:
|
||||
return Dataset(
|
||||
id=uuid.uuid4(),
|
||||
project_id=uuid.uuid4(),
|
||||
name=f"{source_name} dataset",
|
||||
dataset_type="vector",
|
||||
source=source_name,
|
||||
source_name=source_name,
|
||||
imported_at=imported_at,
|
||||
observed_at=observed_at,
|
||||
source_version=source_version,
|
||||
storage_path=storage_path,
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=size_bytes,
|
||||
temporal_series_key=temporal_series_key,
|
||||
status="ready",
|
||||
)
|
||||
|
||||
|
||||
def _version(dataset: Dataset, *, checksum: str | None = "abc") -> DatasetVersion:
|
||||
return DatasetVersion(
|
||||
id=uuid.uuid4(),
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
source_version=dataset.source_version,
|
||||
observed_at=dataset.observed_at,
|
||||
checksum_sha256=checksum,
|
||||
)
|
||||
|
||||
|
||||
def test_source_freshness_distinguishes_snapshot_annual_edition_and_local_sources(tmp_path: Path) -> None:
|
||||
existing_file = tmp_path / "snapshot.geojson"
|
||||
existing_file.write_text("{}", encoding="utf-8")
|
||||
fresh_grb = _dataset(
|
||||
"grb",
|
||||
imported_at=NOW - timedelta(days=20),
|
||||
storage_path=str(existing_file),
|
||||
size_bytes=2,
|
||||
)
|
||||
old_vrbg = _dataset("vrbg", imported_at=NOW - timedelta(days=120))
|
||||
current_annual = _dataset(
|
||||
"statbel",
|
||||
observed_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||
temporal_series_key="population",
|
||||
source_version="2025",
|
||||
)
|
||||
old_annual = _dataset(
|
||||
"waterinfo",
|
||||
observed_at=datetime(2023, 1, 1, tzinfo=timezone.utc),
|
||||
temporal_series_key="water-level",
|
||||
source_version="2023",
|
||||
)
|
||||
fixed_scenario = _dataset("vmm_flood_hazard", observed_at=None, source_version="VMM OGRK")
|
||||
manual = _dataset("manual", source_version=None, checksum=None)
|
||||
datasets = [fresh_grb, old_vrbg, current_annual, old_annual, fixed_scenario, manual]
|
||||
versions = [_version(dataset, checksum=dataset.checksum_sha256) for dataset in datasets]
|
||||
|
||||
report = SourceFreshnessService.build_report(fresh_grb.project_id, datasets, versions, now=NOW)
|
||||
by_source = {item.source_name: item for item in report.items}
|
||||
|
||||
assert by_source["grb"].status == "current"
|
||||
assert by_source["vrbg"].status == "due"
|
||||
assert by_source["statbel"].status == "current"
|
||||
assert by_source["waterinfo"].status == "due"
|
||||
assert by_source["vmm_flood_hazard"].status == "current"
|
||||
assert by_source["manual"].status == "local"
|
||||
assert all(item.auto_refresh_supported is False for item in report.items)
|
||||
assert report.summary.dataset_count == len(datasets)
|
||||
|
||||
|
||||
def test_source_freshness_flags_local_version_and_storage_integrity(tmp_path: Path) -> None:
|
||||
missing_file = tmp_path / "missing.tif"
|
||||
dataset = _dataset(
|
||||
"digitaal_vlaanderen_dhmv",
|
||||
storage_path=str(missing_file),
|
||||
checksum="dataset-checksum",
|
||||
)
|
||||
version = _version(dataset, checksum="different-version-checksum")
|
||||
|
||||
report = SourceFreshnessService.build_report(dataset.project_id, [dataset], [version], now=NOW)
|
||||
item = report.items[0]
|
||||
|
||||
assert item.status == "review_required"
|
||||
assert item.integrity.checksum_mismatch_count == 1
|
||||
assert item.integrity.missing_storage_file_count == 1
|
||||
assert report.summary.sources_with_integrity_issues == 1
|
||||
assert report.summary.integrity_issue_count == 2
|
||||
|
||||
|
||||
def test_source_freshness_requires_dataset_version_and_marks_temporal_series() -> None:
|
||||
first = _dataset(
|
||||
"department_omgeving_land_use",
|
||||
observed_at=datetime(2022, 1, 1, tzinfo=timezone.utc),
|
||||
temporal_series_key="land-use",
|
||||
source_version="2022-v3",
|
||||
)
|
||||
second = _dataset(
|
||||
"department_omgeving_land_use",
|
||||
observed_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||
temporal_series_key="land-use",
|
||||
source_version="2025-v3",
|
||||
)
|
||||
|
||||
report = SourceFreshnessService.build_report(first.project_id, [first, second], [_version(first)], now=NOW)
|
||||
item = report.items[0]
|
||||
|
||||
assert item.historical_series is True
|
||||
assert item.status == "review_required"
|
||||
assert item.integrity.missing_version_count == 1
|
||||
|
||||
|
||||
def test_source_freshness_route_returns_canonical_envelope(monkeypatch) -> None:
|
||||
from app.api.routes import datasets as dataset_routes
|
||||
|
||||
project_id = uuid.uuid4()
|
||||
expected = SourceFreshnessService.build_report(project_id, [], [], now=NOW)
|
||||
monkeypatch.setattr(
|
||||
dataset_routes.SourceFreshnessService,
|
||||
"audit_project",
|
||||
lambda db, selected_project_id: expected,
|
||||
)
|
||||
|
||||
response = dataset_routes.audit_dataset_source_freshness(project_id=project_id, db=SimpleNamespace())
|
||||
|
||||
assert list(response) == ["data"]
|
||||
assert response["data"]["project_id"] == project_id
|
||||
assert response["data"]["summary"]["source_count"] == 0
|
||||
|
||||
|
||||
def test_source_freshness_operator_and_ui_contract_are_read_only() -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
script = (root / "scripts" / "audit_source_freshness.py").read_text(encoding="utf-8")
|
||||
dockerfile = (root / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (root / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
app = (root / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8")
|
||||
api = (root / "frontend" / "src" / "services" / "api" / "datasets.ts").read_text(encoding="utf-8")
|
||||
|
||||
assert "Request(endpoint" in script
|
||||
assert "method=\"POST\"" not in script
|
||||
assert "urlopen(request" in script
|
||||
assert "COPY scripts/audit_source_freshness.py" in dockerfile
|
||||
assert "py_compile scripts/audit_source_freshness.py" in readiness
|
||||
assert "<SourceFreshnessPanel" in app
|
||||
assert "/datasets/source-freshness" in api
|
||||
Reference in New Issue
Block a user