From 1e0e7b6499b6343b60b22a12e2ebc27c7108b188 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 16 Jul 2026 15:05:42 +0200 Subject: [PATCH] Add governed source freshness audit --- CHANGELOG.md | 13 + backend/README.md | 24 ++ backend/app/api/routes/datasets.py | 10 + backend/app/schemas/__init__.py | 10 + backend/app/schemas/source_freshness.py | 66 ++++ .../app/services/source_freshness_service.py | 302 ++++++++++++++++++ .../test_sprint221_source_freshness_audit.py | 168 ++++++++++ deploy/unraid/Dockerfile.all-in-one | 1 + docs/API_CONTRACTS.md | 18 ++ docs/CODEX_EXECUTION_LOG.md | 30 ++ docs/DATABASE_IMPLEMENTATION_PLAN.md | 11 + docs/TODO.md | 2 + frontend/README.md | 7 + frontend/src/App.tsx | 9 + .../status/SourceFreshnessPanel.tsx | 96 ++++++ frontend/src/hooks/useSourceFreshness.ts | 46 +++ frontend/src/services/api/datasets.ts | 3 + frontend/src/styles/premium.css | 167 ++++++++++ frontend/src/types.ts | 46 +++ scripts/README.md | 20 ++ scripts/audit_source_freshness.py | 98 ++++++ scripts/run_readiness_check.sh | 1 + 22 files changed, 1148 insertions(+) create mode 100644 backend/app/schemas/source_freshness.py create mode 100644 backend/app/services/source_freshness_service.py create mode 100644 backend/tests/test_sprint221_source_freshness_audit.py create mode 100644 frontend/src/components/status/SourceFreshnessPanel.tsx create mode 100644 frontend/src/hooks/useSourceFreshness.ts create mode 100644 scripts/audit_source_freshness.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 68cf9e20..19d8330b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ # Changelog +## Sprint 221 Governed source freshness audit (2026-07-16) + +- Added a project-wide read-only source report derived from Dataset, + DatasetVersion and local storage evidence, with canonical API envelope. +- Classified rolling snapshots, annual releases, fixed editions, scenarios, + archives and local artifacts separately so historical publications are not + mislabeled as stale. +- Added integrity checks for missing versions, checksum disagreement, missing + local files and size mismatches without changing persistence or migrations. +- Added a compact frontend source-status panel and a packaged cron-compatible + operator command. Neither path contacts providers, downloads files or + performs automatic refreshes. + ## Sprint 218 Regional terrain and flood completion (2026-07-16) - Provisioned and audited the complete governed Kempen matrices: 56 DHMV diff --git a/backend/README.md b/backend/README.md index 7d32328c..8ce811bc 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1419,3 +1419,27 @@ evidence and existing Dataset. `--force` explicitly refetches the WFS but still fails closed if a different state-2025 checksum is already persisted. The regional output uses the same selection-summary API as Mol; no new endpoint or direct PostGIS write is introduced. + +## Source freshness and version audit + +`GET /api/v1/projects/{project_id}/datasets/source-freshness` derives a +read-only source status from persisted Dataset, DatasetVersion and storage +evidence. It distinguishes rolling snapshots and annual publications from +fixed editions, scenarios, historical archives and local artifacts. Fixed +source editions are never called stale solely because they are old. + +The endpoint checks missing versions, checksum disagreement, missing local +files and stored-size disagreement. It performs no provider request and no +database write. The packaged operator command is suitable for an explicit +Unraid cron entry: + +```bash +docker exec geointel python /app/scripts/audit_source_freshness.py \ + --project-id 82a85913-c522-45d7-84a1-02b393d89e55 \ + --api-url http://127.0.0.1/api/v1 \ + --fail-on integrity \ + --output /app/storage/operator-evidence/source-freshness/latest.json +``` + +Use `--fail-on due` to make a planned review date fail automation, or +`--fail-on never` for reporting only. The command never starts a refresh. diff --git a/backend/app/api/routes/datasets.py b/backend/app/api/routes/datasets.py index 620a976f..a6c31b40 100644 --- a/backend/app/api/routes/datasets.py +++ b/backend/app/api/routes/datasets.py @@ -44,6 +44,7 @@ from app.services.raster_operations_service import RasterOperationsService from app.services.vector_operations_service import VectorOperationsService from app.services.vector_feature_service import VectorFeatureService from app.services.dataset_service import DatasetService +from app.services.source_freshness_service import SourceFreshnessService from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService from app.services.dhmv_acquisition_service import DhmvAcquisitionService from app.services.terrain_analysis_service import TerrainAnalysisService @@ -245,6 +246,15 @@ def list_datasets( return envelope({"items": [item.model_dump() for item in datasets], "total": total, "limit": limit, "offset": offset}) +@router.get("/datasets/source-freshness", response_model=dict) +def audit_dataset_source_freshness( + project_id: UUID, + db: Session = Depends(get_db), +): + report = SourceFreshnessService.audit_project(db, project_id) + return envelope(report.model_dump()) + + @router.get("/datasets/{dataset_id}", response_model=dict) def get_dataset( project_id: UUID, diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index ef470b55..610db024 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -5,6 +5,12 @@ from .project import ProjectCreate, ProjectList, ProjectRead, ProjectUpdate from .area import AreaCreate, AreaList, AreaRead, AreaUpdate from .analysis import ChangeDetectionRequest, ChangeDetectionSummary from .dataset import DatasetCreateResponse, DatasetList +from .source_freshness import ( + SourceFreshnessItem, + SourceFreshnessReport, + SourceFreshnessSummary, + SourceIntegritySummary, +) from .detection import ( DetectionListResponse, DetectionModelCapability, @@ -131,6 +137,10 @@ __all__ = [ "ChangeDetectionSummary", "DatasetCreateResponse", "DatasetList", + "SourceFreshnessItem", + "SourceFreshnessReport", + "SourceFreshnessSummary", + "SourceIntegritySummary", "DetectionListResponse", "DetectionModelCapability", "DetectionModelsResponse", diff --git a/backend/app/schemas/source_freshness.py b/backend/app/schemas/source_freshness.py new file mode 100644 index 00000000..e36c025e --- /dev/null +++ b/backend/app/schemas/source_freshness.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel + + +SourceFreshnessStatus = Literal["current", "due", "review_required", "local"] +SourceRefreshPolicy = Literal["rolling_snapshot", "annual_release", "edition", "scenario", "archive", "local"] + + +class SourceIntegritySummary(BaseModel): + missing_version_count: int = 0 + checksum_mismatch_count: int = 0 + missing_storage_file_count: int = 0 + size_mismatch_count: int = 0 + + @property + def issue_count(self) -> int: + return ( + self.missing_version_count + + self.checksum_mismatch_count + + self.missing_storage_file_count + + self.size_mismatch_count + ) + + +class SourceFreshnessItem(BaseModel): + source_name: str + display_name: str + dataset_count: int + ready_count: int + version_count: int + latest_imported_at: datetime | None = None + latest_observed_at: datetime | None = None + latest_source_version: str | None = None + refresh_policy: SourceRefreshPolicy + review_interval_days: int | None = None + next_review_at: datetime | None = None + status: SourceFreshnessStatus + historical_series: bool + auto_refresh_supported: bool = False + reason: str + recommended_action: str + integrity: SourceIntegritySummary + + +class SourceFreshnessSummary(BaseModel): + source_count: int + dataset_count: int + current_count: int + due_count: int + review_required_count: int + local_count: int + sources_with_integrity_issues: int + integrity_issue_count: int + + +class SourceFreshnessReport(BaseModel): + project_id: UUID + generated_at: datetime + summary: SourceFreshnessSummary + items: list[SourceFreshnessItem] + limitations: list[str] diff --git a/backend/app/services/source_freshness_service.py b/backend/app/services/source_freshness_service.py new file mode 100644 index 00000000..113e0879 --- /dev/null +++ b/backend/app/services/source_freshness_service.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Iterable +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.core.errors import AppError +from app.models import Dataset, DatasetVersion, Project +from app.schemas.source_freshness import ( + SourceFreshnessItem, + SourceFreshnessReport, + SourceFreshnessSummary, + SourceIntegritySummary, +) + + +@dataclass(frozen=True) +class SourcePolicy: + display_name: str + refresh_policy: str + review_interval_days: int | None = None + + +SOURCE_POLICIES: dict[str, SourcePolicy] = { + "grb": SourcePolicy("GRB gebouwen en context", "rolling_snapshot", 90), + "vrbg": SourcePolicy("VRBG wegenregister", "rolling_snapshot", 90), + "digitaal_vlaanderen_buildings_addresses_register": SourcePolicy( + "Gebouwen- en adressenregister", "rolling_snapshot", 90 + ), + "digitaal_vlaanderen_orthophoto": SourcePolicy("Orthofoto Vlaanderen", "rolling_snapshot", 180), + "agentschap_landbouw_zeevisserij_agricultural_parcels": SourcePolicy( + "Landbouwgebruikspercelen", "annual_release" + ), + "department_omgeving_land_use": SourcePolicy("Landgebruik Vlaanderen", "annual_release"), + "inbo_bwk_natura2000": SourcePolicy("BWK en Natura 2000", "annual_release"), + "statbel": SourcePolicy("Statbel bevolking", "annual_release"), + "waterinfo": SourcePolicy("Waterinfo meetreeksen", "annual_release"), + "digitaal_vlaanderen_dhmv": SourcePolicy("Digitaal Hoogtemodel Vlaanderen", "edition"), + "department_omgeving_thematic_raster": SourcePolicy("Omgeving thematische rasters", "edition"), + "dov_soil_map": SourcePolicy("DOV bodemkaart", "edition"), + "vmm_flood_hazard": SourcePolicy("VMM overstromingskaarten", "scenario"), + "historical_landuse": SourcePolicy("Historisch landgebruik", "archive"), + "manual": SourcePolicy("Handmatig ingeladen gegevens", "local"), + "fixture": SourcePolicy("Test- en demonstratiegegevens", "local"), + "map_selection": SourcePolicy("Bewaarde kaartselecties", "local"), +} + +DEFAULT_POLICY = SourcePolicy("Niet-geclassificeerde bron", "edition") + + +def _as_utc(value: datetime | None) -> datetime | None: + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def _source_key(dataset: Dataset) -> str: + return (dataset.source_name or dataset.source or "unknown").strip().lower() or "unknown" + + +def _latest_datetime(values: Iterable[datetime | None]) -> datetime | None: + normalized = [_as_utc(value) for value in values if value is not None] + return max(normalized) if normalized else None + + +def _latest_dataset(datasets: list[Dataset]) -> Dataset: + return max( + datasets, + key=lambda item: ( + _as_utc(item.observed_at) or datetime.min.replace(tzinfo=timezone.utc), + _as_utc(item.imported_at) or datetime.min.replace(tzinfo=timezone.utc), + str(item.id), + ), + ) + + +def _is_local_storage_path(storage_path: str) -> bool: + normalized = storage_path.strip().lower() + return bool(normalized) and "://" not in normalized and not normalized.startswith("/vsi") + + +def _integrity_summary(datasets: list[Dataset], versions_by_dataset: dict[UUID, list[DatasetVersion]]) -> SourceIntegritySummary: + summary = SourceIntegritySummary() + for dataset in datasets: + versions = versions_by_dataset.get(dataset.id, []) + latest_version = max(versions, key=lambda item: item.version) if versions else None + if dataset.status == "ready" and latest_version is None: + summary.missing_version_count += 1 + if ( + latest_version is not None + and dataset.checksum_sha256 + and latest_version.checksum_sha256 + and dataset.checksum_sha256 != latest_version.checksum_sha256 + ): + summary.checksum_mismatch_count += 1 + if dataset.storage_path and _is_local_storage_path(dataset.storage_path): + path = Path(dataset.storage_path) + try: + if not path.is_file(): + summary.missing_storage_file_count += 1 + elif dataset.size_bytes is not None and path.stat().st_size != dataset.size_bytes: + summary.size_mismatch_count += 1 + except OSError: + summary.missing_storage_file_count += 1 + return summary + + +def _classify_source( + policy: SourcePolicy, + datasets: list[Dataset], + integrity: SourceIntegritySummary, + now: datetime, +) -> tuple[str, datetime | None, str, str]: + latest_imported = _latest_datetime(item.imported_at for item in datasets) + latest_observed = _latest_datetime(item.observed_at for item in datasets) + has_source_version = any(bool((item.source_version or "").strip()) for item in datasets) + + if integrity.issue_count: + return ( + "review_required", + None, + "De bewaarde dataset- en versie-evidentie bevat een integriteitsafwijking.", + "Controleer opslag, checksum en datasetversies voordat deze bron opnieuw wordt gebruikt.", + ) + if policy.refresh_policy == "local": + return ( + "local", + None, + "Deze bron is lokaal aangemaakt en heeft geen externe publicatiecyclus.", + "Geen bronverversing nodig; beheer de lokale dataset via de bestaande werkstroom.", + ) + if policy.refresh_policy == "rolling_snapshot": + if latest_imported is None: + return ( + "review_required", + None, + "De importdatum voor deze rollende bron ontbreekt.", + "Controleer de provenance voordat een nieuwe begrensde import wordt gestart.", + ) + next_review = latest_imported + timedelta(days=policy.review_interval_days or 90) + if next_review <= now: + return ( + "due", + next_review, + "De lokale snapshot heeft zijn geplande controledatum bereikt.", + "Vergelijk de broncatalogus en voer alleen daarna een begrensde, expliciete verversing uit.", + ) + return ( + "current", + next_review, + "De lokale snapshot valt binnen de afgesproken controleperiode.", + "Geen actie nodig tot de volgende controledatum.", + ) + if policy.refresh_policy == "annual_release": + if latest_observed is None: + return ( + "review_required", + None, + "De recentste waarnemings- of editieperiode ontbreekt.", + "Vul eerst officiële tijds- en versieprovenance aan; download niets automatisch.", + ) + next_review = datetime(latest_observed.year + 2, 1, 1, tzinfo=timezone.utc) + if latest_observed.year < now.year - 1: + return ( + "due", + next_review, + f"De recentste bewaarde jaargang is {latest_observed.year}.", + "Controleer of de officiële bron een recentere definitieve jaargang publiceerde.", + ) + return ( + "current", + next_review, + f"De recentste bewaarde jaargang is {latest_observed.year}.", + "Controleer bij de volgende publicatiecyclus of een nieuwe definitieve jaargang beschikbaar is.", + ) + if not has_source_version: + return ( + "review_required", + None, + "Deze vaste publicatie heeft geen herkenbare bronversie.", + "Leg de officiële editie of scenarioversie vast voordat de bron als gecontroleerd geldt.", + ) + return ( + "current", + None, + "Dit is een vaste editie, scenario- of archiefpublicatie met vastgelegde bronversie.", + "Vervang deze editie niet automatisch; voeg een nieuwe officiële editie als afzonderlijke versie toe.", + ) + + +class SourceFreshnessService: + @staticmethod + def build_report( + project_id: UUID, + datasets: list[Dataset], + versions: list[DatasetVersion], + *, + now: datetime | None = None, + ) -> SourceFreshnessReport: + generated_at = _as_utc(now) or datetime.now(timezone.utc) + versions_by_dataset: dict[UUID, list[DatasetVersion]] = defaultdict(list) + for version in versions: + versions_by_dataset[version.dataset_id].append(version) + + datasets_by_source: dict[str, list[Dataset]] = defaultdict(list) + for dataset in datasets: + datasets_by_source[_source_key(dataset)].append(dataset) + + items: list[SourceFreshnessItem] = [] + for source_name, source_datasets in datasets_by_source.items(): + policy = SOURCE_POLICIES.get(source_name, DEFAULT_POLICY) + integrity = _integrity_summary(source_datasets, versions_by_dataset) + status, next_review_at, reason, recommended_action = _classify_source( + policy, source_datasets, integrity, generated_at + ) + if policy is DEFAULT_POLICY and not integrity.issue_count: + status = "review_required" + next_review_at = None + reason = "Voor deze bron is nog geen expliciete publicatie- of controlecyclus vastgelegd." + recommended_action = "Classificeer de bron eerst als snapshot, jaargang, vaste editie, scenario, archief of lokaal." + latest = _latest_dataset(source_datasets) + observed_values = {value for item in source_datasets if (value := _as_utc(item.observed_at)) is not None} + temporal_keys = {item.temporal_series_key for item in source_datasets if item.temporal_series_key} + items.append( + SourceFreshnessItem( + source_name=source_name, + display_name=policy.display_name if policy is not DEFAULT_POLICY else source_name.replace("_", " ").title(), + dataset_count=len(source_datasets), + ready_count=sum(item.status == "ready" for item in source_datasets), + version_count=sum(len(versions_by_dataset.get(item.id, [])) for item in source_datasets), + latest_imported_at=_latest_datetime(item.imported_at for item in source_datasets), + latest_observed_at=_latest_datetime(item.observed_at for item in source_datasets), + latest_source_version=next( + ( + item.source_version + for item in sorted( + source_datasets, + key=lambda value: ( + _as_utc(value.observed_at) or datetime.min.replace(tzinfo=timezone.utc), + _as_utc(value.imported_at) or datetime.min.replace(tzinfo=timezone.utc), + ), + reverse=True, + ) + if item.source_version + ), + latest.source_version, + ), + refresh_policy=policy.refresh_policy, + review_interval_days=policy.review_interval_days, + next_review_at=next_review_at, + status=status, + historical_series=len(observed_values) > 1 or len(temporal_keys) > 1, + reason=reason, + recommended_action=recommended_action, + integrity=integrity, + ) + ) + + status_rank = {"review_required": 0, "due": 1, "current": 2, "local": 3} + items.sort(key=lambda item: (status_rank[item.status], item.display_name.lower())) + integrity_issue_count = sum(item.integrity.issue_count for item in items) + summary = SourceFreshnessSummary( + source_count=len(items), + dataset_count=len(datasets), + current_count=sum(item.status == "current" for item in items), + due_count=sum(item.status == "due" for item in items), + review_required_count=sum(item.status == "review_required" for item in items), + local_count=sum(item.status == "local" for item in items), + sources_with_integrity_issues=sum(item.integrity.issue_count > 0 for item in items), + integrity_issue_count=integrity_issue_count, + ) + return SourceFreshnessReport( + project_id=project_id, + generated_at=generated_at, + summary=summary, + items=items, + limitations=[ + "Deze controle leest uitsluitend lokale dataset-, versie- en opslaggegevens.", + "Er worden geen externe catalogi bevraagd, bestanden gedownload of datasets overschreven.", + "Een vaste editie of scenario-publicatie wordt niet verouderd genoemd alleen omdat de publicatiedatum oud is.", + ], + ) + + @staticmethod + def audit_project(db: Session, project_id: UUID, *, now: datetime | None = None) -> SourceFreshnessReport: + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + datasets = db.query(Dataset).filter(Dataset.project_id == project_id).all() + dataset_ids = [dataset.id for dataset in datasets] + versions = ( + db.query(DatasetVersion).filter(DatasetVersion.dataset_id.in_(dataset_ids)).all() + if dataset_ids + else [] + ) + return SourceFreshnessService.build_report(project_id, datasets, versions, now=now) diff --git a/backend/tests/test_sprint221_source_freshness_audit.py b/backend/tests/test_sprint221_source_freshness_audit.py new file mode 100644 index 00000000..0c7b4196 --- /dev/null +++ b/backend/tests/test_sprint221_source_freshness_audit.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from types import SimpleNamespace + +from app.models import Dataset, DatasetVersion +from app.services.source_freshness_service import SourceFreshnessService + + +NOW = datetime(2026, 7, 16, 12, 0, tzinfo=timezone.utc) + + +def _dataset( + source_name: str, + *, + imported_at: datetime | None = NOW, + observed_at: datetime | None = None, + source_version: str | None = "edition-1", + storage_path: str | None = None, + checksum: str | None = "abc", + size_bytes: int | None = None, + temporal_series_key: str | None = None, +) -> Dataset: + return Dataset( + id=uuid.uuid4(), + project_id=uuid.uuid4(), + name=f"{source_name} dataset", + dataset_type="vector", + source=source_name, + source_name=source_name, + imported_at=imported_at, + observed_at=observed_at, + source_version=source_version, + storage_path=storage_path, + checksum_sha256=checksum, + size_bytes=size_bytes, + temporal_series_key=temporal_series_key, + status="ready", + ) + + +def _version(dataset: Dataset, *, checksum: str | None = "abc") -> DatasetVersion: + return DatasetVersion( + id=uuid.uuid4(), + dataset_id=dataset.id, + version=1, + source_version=dataset.source_version, + observed_at=dataset.observed_at, + checksum_sha256=checksum, + ) + + +def test_source_freshness_distinguishes_snapshot_annual_edition_and_local_sources(tmp_path: Path) -> None: + existing_file = tmp_path / "snapshot.geojson" + existing_file.write_text("{}", encoding="utf-8") + fresh_grb = _dataset( + "grb", + imported_at=NOW - timedelta(days=20), + storage_path=str(existing_file), + size_bytes=2, + ) + old_vrbg = _dataset("vrbg", imported_at=NOW - timedelta(days=120)) + current_annual = _dataset( + "statbel", + observed_at=datetime(2025, 1, 1, tzinfo=timezone.utc), + temporal_series_key="population", + source_version="2025", + ) + old_annual = _dataset( + "waterinfo", + observed_at=datetime(2023, 1, 1, tzinfo=timezone.utc), + temporal_series_key="water-level", + source_version="2023", + ) + fixed_scenario = _dataset("vmm_flood_hazard", observed_at=None, source_version="VMM OGRK") + manual = _dataset("manual", source_version=None, checksum=None) + datasets = [fresh_grb, old_vrbg, current_annual, old_annual, fixed_scenario, manual] + versions = [_version(dataset, checksum=dataset.checksum_sha256) for dataset in datasets] + + report = SourceFreshnessService.build_report(fresh_grb.project_id, datasets, versions, now=NOW) + by_source = {item.source_name: item for item in report.items} + + assert by_source["grb"].status == "current" + assert by_source["vrbg"].status == "due" + assert by_source["statbel"].status == "current" + assert by_source["waterinfo"].status == "due" + assert by_source["vmm_flood_hazard"].status == "current" + assert by_source["manual"].status == "local" + assert all(item.auto_refresh_supported is False for item in report.items) + assert report.summary.dataset_count == len(datasets) + + +def test_source_freshness_flags_local_version_and_storage_integrity(tmp_path: Path) -> None: + missing_file = tmp_path / "missing.tif" + dataset = _dataset( + "digitaal_vlaanderen_dhmv", + storage_path=str(missing_file), + checksum="dataset-checksum", + ) + version = _version(dataset, checksum="different-version-checksum") + + report = SourceFreshnessService.build_report(dataset.project_id, [dataset], [version], now=NOW) + item = report.items[0] + + assert item.status == "review_required" + assert item.integrity.checksum_mismatch_count == 1 + assert item.integrity.missing_storage_file_count == 1 + assert report.summary.sources_with_integrity_issues == 1 + assert report.summary.integrity_issue_count == 2 + + +def test_source_freshness_requires_dataset_version_and_marks_temporal_series() -> None: + first = _dataset( + "department_omgeving_land_use", + observed_at=datetime(2022, 1, 1, tzinfo=timezone.utc), + temporal_series_key="land-use", + source_version="2022-v3", + ) + second = _dataset( + "department_omgeving_land_use", + observed_at=datetime(2025, 1, 1, tzinfo=timezone.utc), + temporal_series_key="land-use", + source_version="2025-v3", + ) + + report = SourceFreshnessService.build_report(first.project_id, [first, second], [_version(first)], now=NOW) + item = report.items[0] + + assert item.historical_series is True + assert item.status == "review_required" + assert item.integrity.missing_version_count == 1 + + +def test_source_freshness_route_returns_canonical_envelope(monkeypatch) -> None: + from app.api.routes import datasets as dataset_routes + + project_id = uuid.uuid4() + expected = SourceFreshnessService.build_report(project_id, [], [], now=NOW) + monkeypatch.setattr( + dataset_routes.SourceFreshnessService, + "audit_project", + lambda db, selected_project_id: expected, + ) + + response = dataset_routes.audit_dataset_source_freshness(project_id=project_id, db=SimpleNamespace()) + + assert list(response) == ["data"] + assert response["data"]["project_id"] == project_id + assert response["data"]["summary"]["source_count"] == 0 + + +def test_source_freshness_operator_and_ui_contract_are_read_only() -> None: + root = Path(__file__).resolve().parents[2] + script = (root / "scripts" / "audit_source_freshness.py").read_text(encoding="utf-8") + dockerfile = (root / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") + readiness = (root / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + app = (root / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8") + api = (root / "frontend" / "src" / "services" / "api" / "datasets.ts").read_text(encoding="utf-8") + + assert "Request(endpoint" in script + assert "method=\"POST\"" not in script + assert "urlopen(request" in script + assert "COPY scripts/audit_source_freshness.py" in dockerfile + assert "py_compile scripts/audit_source_freshness.py" in readiness + assert " { setMapContentMode('dataset') setSelectedProjectId(projectId) @@ -852,6 +855,12 @@ function App(): JSX.Element { activeLayerFeatureCount={mapFeatureCount} selectedAreaHasGeometry={Boolean(areaFeatureCollection)} /> + { void sourceFreshness.refresh() }} + />
Volledige workflowstatus diff --git a/frontend/src/components/status/SourceFreshnessPanel.tsx b/frontend/src/components/status/SourceFreshnessPanel.tsx new file mode 100644 index 00000000..8fec2012 --- /dev/null +++ b/frontend/src/components/status/SourceFreshnessPanel.tsx @@ -0,0 +1,96 @@ +import type { SourceFreshnessItem, SourceFreshnessReport, SourceFreshnessStatus } from '../../types' + +interface SourceFreshnessPanelProps { + report: SourceFreshnessReport | null + loading: boolean + error: string | null + onRefresh: () => void +} + +function statusLabel(status: SourceFreshnessStatus): string { + if (status === 'current') return 'actueel' + if (status === 'due') return 'controle nodig' + if (status === 'review_required') return 'nakijken' + return 'lokaal' +} + +function formatDate(value?: string | null): string { + if (!value) return 'geen datum' + return new Intl.DateTimeFormat('nl-BE', { dateStyle: 'medium' }).format(new Date(value)) +} + +function integrityIssueCount(item: SourceFreshnessItem): number { + return Object.values(item.integrity).reduce((total, value) => total + value, 0) +} + +function SourceRow({ item }: { item: SourceFreshnessItem }): JSX.Element { + const issueCount = integrityIssueCount(item) + return ( +
+
+
+ {item.display_name} + {item.dataset_count} datasets · {item.version_count} versies +
+ {statusLabel(item.status)} +
+

{item.reason}

+
+ Laatste import: {formatDate(item.latest_imported_at)} + {item.latest_source_version ? Editie: {item.latest_source_version} : null} + {item.historical_series ? Historische reeks : null} + {issueCount ? {issueCount} integriteitsafwijking{issueCount === 1 ? '' : 'en'} : null} +
+
+ ) +} + +export function SourceFreshnessPanel({ report, loading, error, onRefresh }: SourceFreshnessPanelProps): JSX.Element { + const attentionItems = report?.items.filter((item) => item.status === 'due' || item.status === 'review_required') ?? [] + const summary = report?.summary + + return ( +
+
+
+

Bronbeheer

+

Actualiteit en versiecontrole

+

Controleert lokale publicaties, versies en bestanden zonder externe bronnen automatisch te wijzigen.

+
+ +
+ + {error ?

{error}

: null} + {!report && !error ?

{loading ? 'Bronstatus wordt gecontroleerd...' : 'Geen projectbronstatus beschikbaar.'}

: null} + {report && summary ? ( + <> +
+ {summary.current_count} actueel + {summary.due_count} controle nodig + {summary.review_required_count} nakijken + {summary.integrity_issue_count} integriteitsfouten +
+ {attentionItems.length ? ( +
+ {attentionItems.map((item) => )} +
+ ) : ( +

Alle externe bronpublicaties hebben geldige lokale versie-evidentie.

+ )} +
+ + Alle {summary.source_count} bronnen bekijken + gecontroleerd {formatDate(report.generated_at)} + +
+ {report.items.map((item) => )} +
+

{report.limitations.join(' ')}

+
+ + ) : null} +
+ ) +} diff --git a/frontend/src/hooks/useSourceFreshness.ts b/frontend/src/hooks/useSourceFreshness.ts new file mode 100644 index 00000000..4c1a2c9c --- /dev/null +++ b/frontend/src/hooks/useSourceFreshness.ts @@ -0,0 +1,46 @@ +import { useCallback, useEffect, useRef, useState } from 'react' + +import { datasetsApi } from '../services/api' +import type { SourceFreshnessReport } from '../types' + +export function useSourceFreshness(selectedProjectId: string | null) { + const [report, setReport] = useState(null) + const [loading, setLoading] = useState(false) + const [error, setError] = useState(null) + const requestSequence = useRef(0) + + const refresh = useCallback(async () => { + const requestId = ++requestSequence.current + if (!selectedProjectId) { + setReport(null) + setError(null) + setLoading(false) + return + } + setLoading(true) + setError(null) + try { + const nextReport = await datasetsApi.sourceFreshness(selectedProjectId) + if (requestSequence.current === requestId) { + setReport(nextReport) + } + } catch (caught) { + if (requestSequence.current === requestId) { + setError(caught instanceof Error ? caught.message : 'De broncontrole kon niet worden geladen.') + } + } finally { + if (requestSequence.current === requestId) { + setLoading(false) + } + } + }, [selectedProjectId]) + + useEffect(() => { + void refresh() + return () => { + requestSequence.current += 1 + } + }, [refresh]) + + return { report, loading, error, refresh } +} diff --git a/frontend/src/services/api/datasets.ts b/frontend/src/services/api/datasets.ts index 4024dd95..3bdb78b5 100644 --- a/frontend/src/services/api/datasets.ts +++ b/frontend/src/services/api/datasets.ts @@ -27,6 +27,7 @@ import type { ThematicRasterAcquireRequest, ThematicRasterProductRead, ThematicRasterSelectionResponse, + SourceFreshnessReport, } from '../../types' const DATASET_PAGE_SIZE = 200 @@ -58,6 +59,8 @@ async function listProjectDatasets(projectId: string): Promise => + apiGet(`/api/v1/projects/${projectId}/datasets/source-freshness`), upload: ( projectId: string, payload: { diff --git a/frontend/src/styles/premium.css b/frontend/src/styles/premium.css index 884b51ee..a6bdd64d 100644 --- a/frontend/src/styles/premium.css +++ b/frontend/src/styles/premium.css @@ -821,6 +821,173 @@ details.ai-lab-model-surface > summary strong { -webkit-line-clamp: 2; } +.source-freshness-panel { + overflow: hidden; + border: 1px solid var(--line); + border-radius: 7px; + background: #ffffff; +} + +.source-freshness-header { + display: flex; + gap: 1rem; + align-items: center; + justify-content: space-between; + padding: 1rem 1.2rem; +} + +.source-freshness-header h2 { + margin: 0; + font-size: 1.05rem; +} + +.source-freshness-header p:last-child { + max-width: 52rem; + margin: 0.3rem 0 0; + color: var(--muted); + font-size: 0.82rem; +} + +.source-freshness-summary { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + border-top: 1px solid var(--line); + border-bottom: 1px solid var(--line); + background: #f8fafb; +} + +.source-freshness-summary span { + padding: 0.7rem 1rem; + border-right: 1px solid var(--line); + color: var(--muted); + font-size: 0.75rem; +} + +.source-freshness-summary span:last-child { + border-right: 0; +} + +.source-freshness-summary strong { + display: block; + margin-bottom: 0.15rem; + color: #24333d; + font-size: 1.05rem; +} + +.source-freshness-attention, +.source-freshness-list { + display: grid; +} + +.source-freshness-row { + padding: 0.75rem 1rem; + border-bottom: 1px solid var(--line); +} + +.source-freshness-row:last-child { + border-bottom: 0; +} + +.source-freshness-row-review_required { + box-shadow: inset 3px 0 0 #b84e4e; +} + +.source-freshness-row-due { + box-shadow: inset 3px 0 0 #ca7a18; +} + +.source-freshness-row-current { + box-shadow: inset 3px 0 0 #2d9272; +} + +.source-freshness-main { + display: flex; + gap: 1rem; + align-items: start; + justify-content: space-between; +} + +.source-freshness-main > div { + display: grid; + gap: 0.15rem; +} + +.source-freshness-main strong { + color: #23313b; + font-size: 0.86rem; +} + +.source-freshness-main span, +.source-freshness-row p, +.source-freshness-meta { + color: var(--muted); + font-size: 0.75rem; +} + +.source-freshness-row p { + margin: 0.4rem 0; +} + +.source-freshness-status { + flex: 0 0 auto; + color: #33424d !important; + font-weight: 750; +} + +.source-freshness-meta { + display: flex; + flex-wrap: wrap; + gap: 0.35rem 1rem; +} + +.source-freshness-ok, +.source-freshness-limitation { + margin: 0; + padding: 0.8rem 1rem; + color: var(--muted); + font-size: 0.78rem; +} + +.source-freshness-details { + border-top: 1px solid var(--line); +} + +.source-freshness-details > summary { + display: flex; + cursor: pointer; + gap: 1rem; + justify-content: space-between; + padding: 0.72rem 1rem; + color: #33424d; + font-size: 0.78rem; + font-weight: 700; +} + +.source-freshness-list { + max-height: 30rem; + overflow: auto; + border-top: 1px solid var(--line); +} + +@media (max-width: 760px) { + .source-freshness-header { + align-items: stretch; + flex-direction: column; + } + + .source-freshness-summary { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .source-freshness-summary span:nth-child(2) { + border-right: 0; + } + + .source-freshness-summary span:nth-child(-n + 2) { + border-bottom: 1px solid var(--line); + } +} + .workflow-guidance-panel { border: 1px solid var(--line); border-left: 1px solid var(--line); diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 42c4a734..8bc93812 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -663,6 +663,52 @@ export interface DatasetListResponse { offset: number } +export type SourceFreshnessStatus = 'current' | 'due' | 'review_required' | 'local' + +export interface SourceIntegritySummary { + missing_version_count: number + checksum_mismatch_count: number + missing_storage_file_count: number + size_mismatch_count: number +} + +export interface SourceFreshnessItem { + source_name: string + display_name: string + dataset_count: number + ready_count: number + version_count: number + latest_imported_at?: string | null + latest_observed_at?: string | null + latest_source_version?: string | null + refresh_policy: 'rolling_snapshot' | 'annual_release' | 'edition' | 'scenario' | 'archive' | 'local' + review_interval_days?: number | null + next_review_at?: string | null + status: SourceFreshnessStatus + historical_series: boolean + auto_refresh_supported: false + reason: string + recommended_action: string + integrity: SourceIntegritySummary +} + +export interface SourceFreshnessReport { + project_id: string + generated_at: string + summary: { + source_count: number + dataset_count: number + current_count: number + due_count: number + review_required_count: number + local_count: number + sources_with_integrity_issues: number + integrity_issue_count: number + } + items: SourceFreshnessItem[] + limitations: string[] +} + export interface GeojsonEnvelopeResponse { data: object } diff --git a/scripts/README.md b/scripts/README.md index 13aec382..e5ce9c4e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1688,6 +1688,26 @@ for an explicit source refetch. operators use canonical APIs and persistent operator-evidence storage. They do not run on application startup. +## Read-only source freshness audit + +Inspect the persisted publication, version and storage evidence for one +project without contacting an external provider: + +```bash +docker exec geointel python /app/scripts/audit_source_freshness.py \ + --project-id 82a85913-c522-45d7-84a1-02b393d89e55 \ + --api-url http://127.0.0.1/api/v1 \ + --fail-on integrity +``` + +Add `--output /app/storage/operator-evidence/source-freshness/latest.json` for +a persistent JSON evidence copy. `--fail-on integrity` exits non-zero only for +missing versions/checksum/file/size evidence; `due` also gates planned source +reviews and `attention` additionally gates unclassified sources. This command +uses one canonical `GET`, changes no application data and performs no source +download. It can therefore be scheduled explicitly through Unraid cron without +turning GeoIntel into a real-time monitoring system. + ## Tower deployment Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime: diff --git a/scripts/audit_source_freshness.py b/scripts/audit_source_freshness.py new file mode 100644 index 00000000..c98bdd8e --- /dev/null +++ b/scripts/audit_source_freshness.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Read-only source freshness and local integrity audit for one GeoIntel project.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Inspect persisted source versions and storage evidence without refreshing source data." + ) + parser.add_argument("--project-id", required=True, help="GeoIntel project UUID") + parser.add_argument("--api-url", default="http://127.0.0.1/api/v1", help="API base URL ending in /api/v1") + parser.add_argument("--timeout", type=float, default=30.0, help="HTTP timeout in seconds") + parser.add_argument("--json", action="store_true", help="Print the canonical report data as JSON") + parser.add_argument("--output", type=Path, help="Optional path for a JSON evidence copy") + parser.add_argument( + "--fail-on", + choices=("never", "integrity", "due", "attention"), + default="integrity", + help="Non-zero exit policy for cron or release automation", + ) + return parser.parse_args() + + +def fetch_report(api_url: str, project_id: str, timeout: float) -> dict: + endpoint = f"{api_url.rstrip('/')}/projects/{project_id}/datasets/source-freshness" + request = Request(endpoint, headers={"Accept": "application/json", "User-Agent": "GeoIntel-source-audit/1.0"}) + try: + with urlopen(request, timeout=timeout) as response: + payload = json.load(response) + except HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body}") from exc + except URLError as exc: + raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc + if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict): + raise RuntimeError("Response is not a canonical GeoIntel data envelope") + return payload["data"] + + +def print_text(report: dict) -> None: + summary = report.get("summary", {}) + print( + "GeoIntel source audit: " + f"{summary.get('source_count', 0)} sources, " + f"{summary.get('dataset_count', 0)} datasets, " + f"{summary.get('due_count', 0)} due, " + f"{summary.get('review_required_count', 0)} review, " + f"{summary.get('integrity_issue_count', 0)} integrity issues" + ) + for item in report.get("items", []): + status = item.get("status", "unknown") + if status not in {"current", "local"} or item.get("integrity", {}).get("missing_version_count", 0): + print(f"- {status:15} {item.get('display_name', item.get('source_name'))}: {item.get('reason', '')}") + print("No external catalog was queried and no dataset was modified.") + + +def should_fail(report: dict, fail_on: str) -> bool: + summary = report.get("summary", {}) + integrity = int(summary.get("integrity_issue_count", 0)) + due = int(summary.get("due_count", 0)) + review = int(summary.get("review_required_count", 0)) + if fail_on == "never": + return False + if fail_on == "integrity": + return integrity > 0 + if fail_on == "due": + return integrity > 0 or due > 0 + return integrity > 0 or due > 0 or review > 0 + + +def main() -> int: + args = parse_args() + try: + report = fetch_report(args.api_url, args.project_id, args.timeout) + except RuntimeError as exc: + print(str(exc), file=sys.stderr) + return 2 + serialized = json.dumps(report, indent=2, sort_keys=True) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(serialized + "\n", encoding="utf-8") + if args.json: + print(serialized) + else: + print_text(report) + return 1 if should_fail(report, args.fail_on) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 90aa99cb..fe049a09 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -65,6 +65,7 @@ ${PYTHON_BIN} -m py_compile scripts/geographic_scopes.py ${PYTHON_BIN} -m py_compile scripts/provision_geographic_scope.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_grb_buildings.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_grb_context.py +${PYTHON_BIN} -m py_compile scripts/audit_source_freshness.py ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py ${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py