Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
174 lines
7.3 KiB
Python
174 lines
7.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Read-only source freshness and local integrity audit for one GeoIntel project."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.request import Request, urlopen
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Inspect persisted source versions and storage evidence without refreshing source data."
|
|
)
|
|
parser.add_argument("--project-id", required=True, help="GeoIntel project UUID")
|
|
parser.add_argument("--api-url", default="http://127.0.0.1/api/v1", help="API base URL ending in /api/v1")
|
|
parser.add_argument("--timeout", type=float, default=30.0, help="HTTP timeout in seconds")
|
|
parser.add_argument("--json", action="store_true", help="Print the canonical report data as JSON")
|
|
parser.add_argument("--output", type=Path, help="Optional path for a JSON evidence copy")
|
|
parser.add_argument(
|
|
"--fail-on",
|
|
choices=("never", "integrity", "due", "attention"),
|
|
default="integrity",
|
|
help="Non-zero exit policy for cron or release automation",
|
|
)
|
|
parser.add_argument(
|
|
"--probe-catalogs",
|
|
action="store_true",
|
|
help="Explicitly query the allowlisted official GRB and orthophoto metadata catalogs",
|
|
)
|
|
parser.add_argument(
|
|
"--refresh-catalogs",
|
|
action="store_true",
|
|
help="Bypass the short server-side catalog cache (implies --probe-catalogs)",
|
|
)
|
|
parser.add_argument(
|
|
"--fail-on-catalog",
|
|
action="store_true",
|
|
help="Exit non-zero when an explicitly requested official catalog is degraded or unavailable",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def fetch_report(api_url: str, project_id: str, timeout: float) -> dict:
|
|
endpoint = f"{api_url.rstrip('/')}/projects/{project_id}/datasets/source-freshness"
|
|
request = Request(endpoint, headers={"Accept": "application/json", "User-Agent": "GeoIntel-source-audit/1.0"})
|
|
try:
|
|
with urlopen(request, timeout=timeout) as response:
|
|
payload = json.load(response)
|
|
except HTTPError as exc:
|
|
body = exc.read().decode("utf-8", errors="replace")
|
|
raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body}") from exc
|
|
except URLError as exc:
|
|
raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc
|
|
if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
|
|
raise RuntimeError("Response is not a canonical GeoIntel data envelope")
|
|
return payload["data"]
|
|
|
|
|
|
def fetch_catalog_report(api_url: str, project_id: str, timeout: float, refresh: bool) -> dict:
|
|
query = "true" if refresh else "false"
|
|
endpoint = f"{api_url.rstrip('/')}/projects/{project_id}/datasets/source-catalog-probes?refresh={query}"
|
|
request = Request(endpoint, headers={"Accept": "application/json", "User-Agent": "GeoIntel-source-audit/1.0"})
|
|
try:
|
|
with urlopen(request, timeout=timeout) as response:
|
|
payload = json.load(response)
|
|
except HTTPError as exc:
|
|
body = exc.read().decode("utf-8", errors="replace")
|
|
raise RuntimeError(f"GeoIntel catalog probe returned HTTP {exc.code}: {body}") from exc
|
|
except URLError as exc:
|
|
raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc
|
|
if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
|
|
raise RuntimeError("Catalog probe response is not a canonical GeoIntel data envelope")
|
|
return payload["data"]
|
|
|
|
|
|
def print_text(report: dict, *, catalog_queried: bool = False) -> None:
|
|
summary = report.get("summary", {})
|
|
print(
|
|
"GeoIntel source audit: "
|
|
f"{summary.get('source_count', 0)} sources, "
|
|
f"{summary.get('dataset_count', 0)} datasets, "
|
|
f"{summary.get('due_count', 0)} due, "
|
|
f"{summary.get('review_required_count', 0)} review, "
|
|
f"{summary.get('integrity_issue_count', 0)} integrity issues"
|
|
)
|
|
for item in report.get("items", []):
|
|
status = item.get("status", "unknown")
|
|
if status not in {"current", "local"} or item.get("integrity", {}).get("missing_version_count", 0):
|
|
print(f"- {status:15} {item.get('display_name', item.get('source_name'))}: {item.get('reason', '')}")
|
|
if catalog_queried:
|
|
print("The official catalog was queried read-only; no feature, raster or dataset was modified.")
|
|
else:
|
|
print("No external catalog was queried and no dataset was modified.")
|
|
|
|
|
|
def print_catalog_text(report: dict) -> None:
|
|
summary = report.get("summary", {})
|
|
print(
|
|
"Official catalog probe: "
|
|
f"{summary.get('available_count', 0)}/{summary.get('provider_count', 0)} available, "
|
|
f"{summary.get('degraded_count', 0)} degraded, "
|
|
f"{summary.get('unavailable_count', 0)} unavailable, "
|
|
f"{summary.get('different_version_count', 0)} version differences"
|
|
)
|
|
for item in report.get("items", []):
|
|
remote = item.get("remote_version") or "no official edition"
|
|
local = item.get("local_source_version") or "not loaded locally"
|
|
print(
|
|
f"- {item.get('display_name', item.get('source_name'))}: "
|
|
f"{item.get('status')} / official={remote} / local={local} / "
|
|
f"comparison={item.get('comparison_status')}"
|
|
)
|
|
print(f" {item.get('message', '')}")
|
|
|
|
|
|
def should_fail(report: dict, fail_on: str) -> bool:
|
|
summary = report.get("summary", {})
|
|
integrity = int(summary.get("integrity_issue_count", 0))
|
|
due = int(summary.get("due_count", 0))
|
|
review = int(summary.get("review_required_count", 0))
|
|
if fail_on == "never":
|
|
return False
|
|
if fail_on == "integrity":
|
|
return integrity > 0
|
|
if fail_on == "due":
|
|
return integrity > 0 or due > 0
|
|
return integrity > 0 or due > 0 or review > 0
|
|
|
|
|
|
def catalog_should_fail(report: dict) -> bool:
|
|
summary = report.get("summary", {})
|
|
return int(summary.get("degraded_count", 0)) > 0 or int(summary.get("unavailable_count", 0)) > 0
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
try:
|
|
report = fetch_report(args.api_url, args.project_id, args.timeout)
|
|
catalog_report = (
|
|
fetch_catalog_report(args.api_url, args.project_id, args.timeout, args.refresh_catalogs)
|
|
if args.probe_catalogs or args.refresh_catalogs
|
|
else None
|
|
)
|
|
except RuntimeError as exc:
|
|
print(str(exc), file=sys.stderr)
|
|
return 2
|
|
output_payload = (
|
|
{"source_freshness": report, "catalog_probes": catalog_report}
|
|
if catalog_report is not None
|
|
else report
|
|
)
|
|
serialized = json.dumps(output_payload, indent=2, sort_keys=True)
|
|
if args.output:
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(serialized + "\n", encoding="utf-8")
|
|
if args.json:
|
|
print(serialized)
|
|
else:
|
|
print_text(report, catalog_queried=catalog_report is not None)
|
|
if catalog_report is not None:
|
|
print_catalog_text(catalog_report)
|
|
failed = should_fail(report, args.fail_on)
|
|
if args.fail_on_catalog and catalog_report is not None:
|
|
failed = failed or catalog_should_fail(catalog_report)
|
|
return 1 if failed else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|