Initial public release

scripts/audit_source_freshness.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""Read-only source freshness and local integrity audit for one GeoIntel project."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Inspect persisted source versions and storage evidence without refreshing source data."
+    )
+    parser.add_argument("--project-id", required=True, help="GeoIntel project UUID")
+    parser.add_argument("--api-url", default="http://127.0.0.1/api/v1", help="API base URL ending in /api/v1")
+    parser.add_argument("--timeout", type=float, default=30.0, help="HTTP timeout in seconds")
+    parser.add_argument("--json", action="store_true", help="Print the canonical report data as JSON")
+    parser.add_argument("--output", type=Path, help="Optional path for a JSON evidence copy")
+    parser.add_argument(
+        "--fail-on",
+        choices=("never", "integrity", "due", "attention"),
+        default="integrity",
+        help="Non-zero exit policy for cron or release automation",
+    )
+    parser.add_argument(
+        "--probe-catalogs",
+        action="store_true",
+        help="Explicitly query the allowlisted official GRB and orthophoto metadata catalogs",
+    )
+    parser.add_argument(
+        "--refresh-catalogs",
+        action="store_true",
+        help="Bypass the short server-side catalog cache (implies --probe-catalogs)",
+    )
+    parser.add_argument(
+        "--fail-on-catalog",
+        action="store_true",
+        help="Exit non-zero when an explicitly requested official catalog is degraded or unavailable",
+    )
+    return parser.parse_args()
+
+
+def fetch_report(api_url: str, project_id: str, timeout: float) -> dict:
+    endpoint = f"{api_url.rstrip('/')}/projects/{project_id}/datasets/source-freshness"
+    request = Request(endpoint, headers={"Accept": "application/json", "User-Agent": "GeoIntel-source-audit/1.0"})
+    try:
+        with urlopen(request, timeout=timeout) as response:
+            payload = json.load(response)
+    except HTTPError as exc:
+        body = exc.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body}") from exc
+    except URLError as exc:
+        raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc
+    if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
+        raise RuntimeError("Response is not a canonical GeoIntel data envelope")
+    return payload["data"]
+
+
+def fetch_catalog_report(api_url: str, project_id: str, timeout: float, refresh: bool) -> dict:
+    query = "true" if refresh else "false"
+    endpoint = f"{api_url.rstrip('/')}/projects/{project_id}/datasets/source-catalog-probes?refresh={query}"
+    request = Request(endpoint, headers={"Accept": "application/json", "User-Agent": "GeoIntel-source-audit/1.0"})
+    try:
+        with urlopen(request, timeout=timeout) as response:
+            payload = json.load(response)
+    except HTTPError as exc:
+        body = exc.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"GeoIntel catalog probe returned HTTP {exc.code}: {body}") from exc
+    except URLError as exc:
+        raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc
+    if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
+        raise RuntimeError("Catalog probe response is not a canonical GeoIntel data envelope")
+    return payload["data"]
+
+
+def print_text(report: dict, *, catalog_queried: bool = False) -> None:
+    summary = report.get("summary", {})
+    print(
+        "GeoIntel source audit: "
+        f"{summary.get('source_count', 0)} sources, "
+        f"{summary.get('dataset_count', 0)} datasets, "
+        f"{summary.get('due_count', 0)} due, "
+        f"{summary.get('review_required_count', 0)} review, "
+        f"{summary.get('integrity_issue_count', 0)} integrity issues"
+    )
+    for item in report.get("items", []):
+        status = item.get("status", "unknown")
+        if status not in {"current", "local"} or item.get("integrity", {}).get("missing_version_count", 0):
+            print(f"- {status:15} {item.get('display_name', item.get('source_name'))}: {item.get('reason', '')}")
+    if catalog_queried:
+        print("The official catalog was queried read-only; no feature, raster or dataset was modified.")
+    else:
+        print("No external catalog was queried and no dataset was modified.")
+
+
+def print_catalog_text(report: dict) -> None:
+    summary = report.get("summary", {})
+    print(
+        "Official catalog probe: "
+        f"{summary.get('available_count', 0)}/{summary.get('provider_count', 0)} available, "
+        f"{summary.get('degraded_count', 0)} degraded, "
+        f"{summary.get('unavailable_count', 0)} unavailable, "
+        f"{summary.get('different_version_count', 0)} version differences"
+    )
+    for item in report.get("items", []):
+        remote = item.get("remote_version") or "no official edition"
+        local = item.get("local_source_version") or "not loaded locally"
+        print(
+            f"- {item.get('display_name', item.get('source_name'))}: "
+            f"{item.get('status')} / official={remote} / local={local} / "
+            f"comparison={item.get('comparison_status')}"
+        )
+        print(f"  {item.get('message', '')}")
+
+
+def should_fail(report: dict, fail_on: str) -> bool:
+    summary = report.get("summary", {})
+    integrity = int(summary.get("integrity_issue_count", 0))
+    due = int(summary.get("due_count", 0))
+    review = int(summary.get("review_required_count", 0))
+    if fail_on == "never":
+        return False
+    if fail_on == "integrity":
+        return integrity > 0
+    if fail_on == "due":
+        return integrity > 0 or due > 0
+    return integrity > 0 or due > 0 or review > 0
+
+
+def catalog_should_fail(report: dict) -> bool:
+    summary = report.get("summary", {})
+    return int(summary.get("degraded_count", 0)) > 0 or int(summary.get("unavailable_count", 0)) > 0
+
+
+def main() -> int:
+    args = parse_args()
+    try:
+        report = fetch_report(args.api_url, args.project_id, args.timeout)
+        catalog_report = (
+            fetch_catalog_report(args.api_url, args.project_id, args.timeout, args.refresh_catalogs)
+            if args.probe_catalogs or args.refresh_catalogs
+            else None
+        )
+    except RuntimeError as exc:
+        print(str(exc), file=sys.stderr)
+        return 2
+    output_payload = (
+        {"source_freshness": report, "catalog_probes": catalog_report}
+        if catalog_report is not None
+        else report
+    )
+    serialized = json.dumps(output_payload, indent=2, sort_keys=True)
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(serialized + "\n", encoding="utf-8")
+    if args.json:
+        print(serialized)
+    else:
+        print_text(report, catalog_queried=catalog_report is not None)
+        if catalog_report is not None:
+            print_catalog_text(catalog_report)
+    failed = should_fail(report, args.fail_on)
+    if args.fail_on_catalog and catalog_report is not None:
+        failed = failed or catalog_should_fail(catalog_report)
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())