From 50952e18f937381471ebc834e6f533e1664d2197 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 20:22:53 +0200 Subject: [PATCH] Add Mol false-negative visual review evidence --- CHANGELOG.md | 9 + backend/README.md | 7 + ..._detection_false_negative_visual_review.py | 266 ++++++++ deploy/unraid/Dockerfile.all-in-one | 1 + docs/AI_PIPELINES.md | 20 + docs/CODEX_EXECUTION_LOG.md | 34 + docs/TODO.md | 3 +- scripts/README.md | 30 + ...on_false_negative_review_contact_sheets.py | 595 ++++++++++++++++++ ...on_false_positive_review_contact_sheets.py | 4 +- scripts/run_readiness_check.sh | 1 + 11 files changed, 967 insertions(+), 3 deletions(-) create mode 100644 backend/tests/test_sprint179_detection_false_negative_visual_review.py create mode 100644 scripts/render_detection_false_negative_review_contact_sheets.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 05b19c42..0d4be047 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ # Changelog +## Sprint 179 Mol detection evidence diagnosis (2026-07-13) + +- Added a read-only, storage-confined false-negative contact-sheet renderer that projects persisted missed GRB geometries onto the exact source tile manifest recorded by the analysis run. +- Added AOI/area-stratified review selection, nearby persisted candidate/reference overlays, explicit manual decision CSVs and separate GeoJSON evidence for references outside inference-tile coverage. +- Added focused rendering, manifest-resolution, decision-default and path-confinement regression coverage plus all-in-one/readiness wiring. +- Completed explicit Donk/Postel visual review: QA alignment dominated 37/48 false-positive and 27/48 false-negative samples; only 7 and 6 respectively were confirmed model errors. +- Found 41 of 638 Donk/Postel false-negative evidence records outside all persisted inference tiles and kept coverage-adjusted recall as a diagnostic only; persisted QA metrics were not changed. +- Recorded a NO-GO for immediate retraining. QA population clipping and box-to-footprint matching diagnostics are the required next pass. + ## Sprint 178 Mol multi-zone operational validation (2026-07-13) - Added documented Mol center, Achterbos, Gompel, Donk and Postel operator zones with municipality and operational-zone provenance. diff --git a/backend/README.md b/backend/README.md index aa780391..5d263c9d 100644 --- a/backend/README.md +++ b/backend/README.md @@ -582,6 +582,13 @@ artifact that separates matched detections, matched references, false positives and false negatives by role. It reads existing persisted `QualityCheck` evidence only and does not rerun inference. +For source-image review of false negatives, run +`scripts/render_detection_false_negative_review_contact_sheets.py` against a +fixed-threshold evidence portfolio. It uses the selected run's persisted tile +manifest, overlays candidate/reference context and explicitly exports reference +features outside tile coverage. The command is read-only and never changes +`QualityCheck`, `Metric`, `Detection` or model state. + ### Run backend ```bash diff --git a/backend/tests/test_sprint179_detection_false_negative_visual_review.py b/backend/tests/test_sprint179_detection_false_negative_visual_review.py new file mode 100644 index 00000000..47a31ff7 --- /dev/null +++ b/backend/tests/test_sprint179_detection_false_negative_visual_review.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +import csv +import json +import subprocess +import sys +from pathlib import Path + +import numpy as np +import rasterio +from PIL import Image +from rasterio.transform import from_bounds +from shapely.geometry import box, mapping + + +ROOT = Path(__file__).resolve().parents[2] + + +def _write_tile(path: Path) -> None: + data = np.full((3, 256, 256), 72, dtype=np.uint8) + data[:, 45:105, 40:110] = np.array([188, 178, 163], dtype=np.uint8)[:, None, None] + data[:, 145:205, 150:225] = np.array([205, 198, 184], dtype=np.uint8)[:, None, None] + path.parent.mkdir(parents=True, exist_ok=True) + with rasterio.open( + path, + "w", + driver="GTiff", + width=256, + height=256, + count=3, + dtype="uint8", + crs="EPSG:4326", + transform=from_bounds(5.0, 51.0, 5.01, 51.01, 256, 256), + ) as dataset: + dataset.write(data) + + +def _feature(role: str, feature_id: str, geometry: dict) -> dict: + return { + "type": "Feature", + "id": f"{role}:{feature_id}", + "properties": { + "qa_evidence_role": role, + "feature_id": feature_id, + "reference_feature_id": feature_id + if role in {"false_negative", "match_reference"} + else None, + "candidate_feature_id": feature_id + if role in {"false_positive", "match_candidate"} + else None, + "analysis_run_id": "run-mol-review", + "quality_check_id": "quality-mol-review", + }, + "geometry": geometry, + } + + +def _write_portfolio(tmp_path: Path, *, escaped_manifest: bool = False) -> tuple[Path, Path]: + storage_root = tmp_path / "storage" + tile_path = storage_root / "tiles" / "mol_donk" / "tile_0000.tif" + _write_tile(tile_path) + manifest_path = storage_root / "tiles" / "mol_donk" / "manifest.json" + if escaped_manifest: + manifest_path = tmp_path / "outside-manifest.json" + manifest_path.write_text( + json.dumps( + { + "tiles": [ + { + "path": str(tile_path), + "bounds": [5.0, 51.0, 5.01, 51.01], + "crs": "EPSG:4326", + } + ] + } + ), + encoding="utf-8", + ) + + portfolio_dir = storage_root / "operator-evidence" / "review" / "portfolio" + summary_path = portfolio_dir / "samples" / "mol_donk" / "quality_matrix_summary.json" + summary_path.parent.mkdir(parents=True, exist_ok=True) + summary_path.write_text( + json.dumps({"items": [{"manifest_path": str(manifest_path)}]}), + encoding="utf-8", + ) + evidence_dir = summary_path.parent / "evidence" + evidence_dir.mkdir(parents=True) + false_negatives = [ + _feature( + "false_negative", + "miss-tiny", + mapping(box(5.001, 51.001, 5.00103, 51.00103)), + ), + _feature( + "false_negative", + "miss-small", + mapping(box(5.002, 51.002, 5.0021, 51.0021)), + ), + _feature( + "false_negative", + "miss-medium", + mapping(box(5.004, 51.004, 5.00418, 51.00418)), + ), + _feature( + "false_negative", + "miss-large", + mapping(box(5.006, 51.006, 5.007, 51.007)), + ), + _feature( + "false_negative", + "outside-source-raster", + mapping(box(5.02, 51.02, 5.021, 51.021)), + ), + ] + evidence_path = evidence_dir / "calibration_evidence.geojson" + evidence_path.write_text( + json.dumps( + { + "type": "FeatureCollection", + "features": false_negatives + + [ + _feature( + "match_candidate", + "candidate-nearby", + mapping(box(5.003, 51.003, 5.004, 51.004)), + ), + _feature( + "match_reference", + "reference-nearby", + mapping(box(5.0031, 51.0031, 5.0041, 51.0041)), + ), + ], + } + ), + encoding="utf-8", + ) + portfolio_path = portfolio_dir / "calibration_evidence_portfolio.json" + portfolio_path.write_text( + json.dumps( + { + "model_asset_id": "model-mol-review", + "samples": [ + { + "sample_slug": "mol_donk", + "copied_summary_path": str(summary_path), + "evidence_geojson_path": str(evidence_path), + "role_counts": {"false_negative": 5}, + } + ], + } + ), + encoding="utf-8", + ) + return portfolio_path, storage_root + + +def test_false_negative_visual_review_uses_persisted_manifest_and_requires_decisions( + tmp_path: Path, +) -> None: + renderer = ROOT / "scripts" / "render_detection_false_negative_review_contact_sheets.py" + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text( + encoding="utf-8" + ) + assert renderer.exists() + assert f"py_compile scripts/{renderer.name}" in readiness + assert f"COPY scripts/{renderer.name}" in dockerfile + + portfolio_path, storage_root = _write_portfolio(tmp_path) + output_dir = tmp_path / "review" + result = subprocess.run( + [ + sys.executable, + str(renderer), + "--portfolio", + str(portfolio_path), + "--storage-root", + str(storage_root), + "--output-dir", + str(output_dir), + "--sample-slugs", + "mol_donk", + "--max-features", + "4", + "--columns", + "2", + "--cards-per-sheet", + "4", + "--thumb-size", + "128", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + report = json.loads( + (output_dir / "detection_false_negative_review_summary.json").read_text( + encoding="utf-8" + ) + ) + assert report["status"] == "review_required" + assert report["population_count"] == 4 + assert report["evidence_population_count"] == 5 + assert report["selected_feature_count"] == 4 + assert report["selected_sample_slugs"] == ["mol_donk"] + assert report["missing_manifest_count"] == 0 + assert report["missing_tile_count"] == 0 + assert report["outside_tile_coverage_count"] == 1 + assert report["context_overlay_feature_count"] > 0 + assert len(report["selected_area_buckets"]) >= 3 + assert {Path(item["source_tile_path"]).name for item in report["selected_features"]} == { + "tile_0000.tif" + } + assert "review required" in result.stdout.lower() + + outside = json.loads( + (output_dir / "false_negatives_outside_tile_coverage.geojson").read_text( + encoding="utf-8" + ) + ) + assert len(outside["features"]) == 1 + assert ( + outside["features"][0]["properties"]["review_exclusion_reason"] + == "outside_tile_coverage" + ) + + with (output_dir / "false_negative_review_decisions.csv").open( + newline="", encoding="utf-8" + ) as handle: + decisions = list(csv.DictReader(handle)) + assert len(decisions) == 4 + assert {row["review_decision"] for row in decisions} == {"unreviewed"} + + sheet = Image.open(output_dir / report["contact_sheets"][0]["path"]).convert("RGB") + assert sheet.width >= 256 + assert sheet.height >= 256 + assert len(sheet.getcolors(maxcolors=1_000_000) or []) > 10 + + +def test_false_negative_visual_review_rejects_manifest_outside_storage( + tmp_path: Path, +) -> None: + renderer = ROOT / "scripts" / "render_detection_false_negative_review_contact_sheets.py" + portfolio_path, storage_root = _write_portfolio(tmp_path, escaped_manifest=True) + result = subprocess.run( + [ + sys.executable, + str(renderer), + "--portfolio", + str(portfolio_path), + "--storage-root", + str(storage_root), + "--output-dir", + str(tmp_path / "review"), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "Tile manifest is outside storage root" in result.stderr diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 8a2545f3..dd0411fc 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -86,6 +86,7 @@ COPY scripts/build_fixed_threshold_evidence_portfolio_inputs.py /app/scripts/bui COPY scripts/audit_detection_false_negative_evidence.py /app/scripts/audit_detection_false_negative_evidence.py COPY scripts/audit_detection_false_positive_evidence.py /app/scripts/audit_detection_false_positive_evidence.py COPY scripts/render_detection_false_positive_review_contact_sheets.py /app/scripts/render_detection_false_positive_review_contact_sheets.py +COPY scripts/render_detection_false_negative_review_contact_sheets.py /app/scripts/render_detection_false_negative_review_contact_sheets.py COPY scripts/validate_detection_false_positive_review_decisions.py /app/scripts/validate_detection_false_positive_review_decisions.py COPY scripts/run_operator_hard_negative_detection_matrix.sh /app/scripts/run_operator_hard_negative_detection_matrix.sh COPY scripts/run_background_corpus_split_matrix.sh /app/scripts/run_background_corpus_split_matrix.sh diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 347571f1..858111af 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -150,6 +150,26 @@ Only records explicitly marked `confirmed_model_false_positive` are emitted by the validator as possible hard-negative review input. The workflow does not train a model, mutate QA persistence, fetch data or infer review decisions. +### Persisted false-negative visual review + +False negatives require the same manual distinction. A missed reference can be +a true model miss, a stale reference, an obscured object, a box-to-footprint +matching failure or an object outside the raster actually presented to the +model. The read-only false-negative renderer resolves the persisted tile +manifest from the fixed-threshold run and overlays: + +- red: the missed GRB/reference footprint; +- blue: nearby persisted candidate detections; +- green: nearby matched reference footprints. + +The decision contract is `confirmed_model_false_negative`, +`reference_gap_or_change`, `qa_alignment_mismatch`, +`imagery_obscured_or_uncertain` or `unreviewed`. References outside every +persisted inference tile are written to a separate exclusion GeoJSON and are +not treated as reviewable model misses. This renderer does not alter persisted +QA metrics; coverage-adjusted values remain audit diagnostics until the QA +service evaluation population is deliberately hardened. + ### Local model asset catalog GeoIntel can list local runtime model files mounted into the backend model diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 05e7b780..e989c72b 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -7301,3 +7301,37 @@ Open: - Ran the interactive PostGIS AOI query against `vector_features`: `126` GRB buildings returned with no truncation. The guided GIS smoke persisted a derived dataset and GeoJSON export, then produced candidate/reference QA F1 `0.9582` with `126` matches, `0` false positives and `11` false negatives; `263` persisted evidence features rendered back on the map. - Remaining evidence caveat: the dataset QA result reports weak CRS-assumption warnings, so those geometry metrics remain explicitly approximate until CRS provenance handling is reviewed. - Host observation outside GeoIntel: Unraid recovered after reboot and serves the app, but still reports one disabled/invalid array device. Storage administration should resolve that independently of application development. + +# Sprint 179 - Mol Donk/Postel detection evidence diagnosis + +## Implementation + +- Added `render_detection_false_negative_review_contact_sheets.py` as a read-only counterpart to the persisted false-positive review workflow. +- Resolved each sample's exact persisted tile manifest from its fixed-threshold run summary; manifests and source tiles are confined to `/app/storage`. +- Projected WGS84 missed-reference geometry onto the real inference tiles and rendered nearby persisted candidate detections plus matched-reference context. +- Added deterministic AOI/area stratification and an explicit five-state manual decision CSV. No decision is inferred and no training input is exported automatically. +- Separated references outside every persisted source tile into `false_negatives_outside_tile_coverage.geojson` instead of hiding them or calling them model misses. +- Added focused rendering, manifest, source-coverage and storage-confinement regression tests; wired the script into readiness compilation and the all-in-one image. + +## Live Mol evidence + +- Re-exported the four-zone fixed-threshold portfolio from existing persisted QualityChecks without rerunning inference or mutating application data. +- The complete portfolio contains `7,283` evidence features. Donk contributes `424` false positives and `553` false negatives; Postel contributes `43` and `85`. +- Rendered and inspected 48 stratified false-positive cases over Donk/Postel. Explicit decisions: `37` QA alignment mismatches, `7` confirmed model false positives, `3` reference gaps/changes and `1` uncertain. The existing validator passed with status `complete`. +- Rendered and inspected 48 stratified false-negative cases. Explicit decisions: `27` QA alignment mismatches, `6` confirmed model false negatives, `3` reference gaps/changes and `12` imagery-obscured/uncertain. +- Found `41/638` false-negative evidence records outside every persisted inference tile: Donk `28`, Postel `13`. Directionally excluding those records raises Donk recall from `0.5519` to `0.5647` and Postel from `0.3796` to `0.4194`; these are audit diagnostics only and no persisted metric was changed. +- The dominant visual mode is rectangle-to-footprint mismatch on large industrial roofs and dense residential blocks, often with a blue persisted candidate already overlapping the red missed GRB footprint. Postel additionally contains many tiny/vegetation-obscured references. +- Persistent evidence and the assessment are stored below `/app/storage/operator-evidence/mol-operational-review/20260713`. + +## Decision and next pass + +- **NO-GO for immediate retraining.** Only `7/48` reviewed false positives and `6/48` reviewed false negatives were confirmed model errors; evaluation alignment and coverage defects dominate the selected evidence. +- Next harden detection QA to restrict candidate/reference populations to persisted raster/tile coverage and expose best-IoU/overlap/unmatched diagnostics. Rerun Donk/Postel QA against the unchanged persisted detections before deciding whether the confirmed model-error subset justifies curated training. + +## Validation + +- Focused false-positive/false-negative audit and visual-review coverage: `8 passed`. +- Ruff passed for the changed Python renderer and regression tests. +- `bash scripts/run_readiness_check.sh`: passed with `487` backend tests, the 81-route API contract audit, one Alembic head, frontend typecheck and production build. +- `python -m alembic upgrade head --sql` rendered the complete migration chain through `202606120900`; no migration changed. +- Local `docker compose config` was unavailable because the Windows workstation has no Docker CLI. The repository-driven Tower deployment remains the required live Docker validation. diff --git a/docs/TODO.md b/docs/TODO.md index 914b8a01..73f68150 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -22,7 +22,8 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add a Mol multi-zone operational pack with independent positive holdouts, background control and persisted map-ready AOIs. - [x] Execute the Mol pack on live Tower/PostGIS with real orthophotos, GRB references, configured YOLO, persisted QA/QC and a zero-building background control. - [x] Persist combined Mol operator evidence under the Unraid storage mount so reports survive all-in-one container replacement. -- [ ] Visually review Mol Postel and Donk false-positive/false-negative evidence, classify the dominant error modes and only then decide whether another model-training pass is justified. +- [x] Visually review Mol Postel and Donk false-positive/false-negative evidence, classify the dominant error modes and only then decide whether another model-training pass is justified. +- [ ] Clip detection QA populations to persisted raster/tile coverage and add box-to-footprint matching diagnostics before reconsidering model training. - [x] Backend FastAPI foundation, health endpoint and service structure. - [x] React/TypeScript frontend foundation and MapLibre workbench. - [x] Map layer visibility, opacity and feature property inspection. diff --git a/scripts/README.md b/scripts/README.md index 391887d9..8a38a620 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -901,6 +901,36 @@ Only explicit `confirmed_model_false_positive` decisions are written to `confirmed_model_false_positives.geojson`; the tool never promotes generic QA false-positives into training labels. +Render persisted false negatives against the exact tile manifest recorded by +the selected analysis run: + +```bash +docker exec geointel /opt/geointel/venv/bin/python \ + /app/scripts/render_detection_false_negative_review_contact_sheets.py \ + --portfolio /app/storage/operator-evidence/model-review/portfolio/calibration_evidence_portfolio.json \ + --storage-root /app/storage \ + --output-dir /app/storage/operator-evidence/model-review/false-negative-visual-review \ + --sample-slugs mol_donk,mol_postel \ + --max-features 48 \ + --columns 4 \ + --cards-per-sheet 12 \ + --thumb-size 256 +``` + +The read-only renderer resolves the one persisted `manifest_path` from each +fixed-threshold sample summary, confines manifests and source tiles to +`--storage-root`, and projects WGS84 missed-reference polygons onto the real +source tiles. Red is the missed reference, blue is persisted candidate +geometry and green is a matched reference. Selection is deterministic and +stratified by AOI and geodetic area bucket. Every CSV decision starts as +`unreviewed`; no positive-training example is inferred. + +Reference features that do not intersect any persisted inference tile are not +silently counted as reviewable model misses. They are reported separately in +`false_negatives_outside_tile_coverage.geojson` with +`review_exclusion_reason=outside_tile_coverage`. Fix the QA evaluation +population before using those records in recall or training decisions. + Docker images install only the GIS runtime by default. To build a local/Tower image with PyTorch/Ultralytics available for the configured-YOLO preflight and runtime path, set: diff --git a/scripts/render_detection_false_negative_review_contact_sheets.py b/scripts/render_detection_false_negative_review_contact_sheets.py new file mode 100644 index 00000000..7e815518 --- /dev/null +++ b/scripts/render_detection_false_negative_review_contact_sheets.py @@ -0,0 +1,595 @@ +#!/usr/bin/env python3 +"""Render persisted detection false negatives for explicit operator review. + +The script is read-only. It resolves the tile manifest recorded by the selected +analysis run, projects missed reference geometries onto those source tiles and +never infers a review decision or mutates application/model state. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from audit_detection_false_negative_evidence import area_bucket +from render_detection_false_positive_review_contact_sheets import ( + build_contact_sheet, + draw_geometry, + load_json, + normalize_raster, + require_dependencies, + resolve_evidence_path, +) + + +JSON_NAME = "detection_false_negative_review_summary.json" +MARKDOWN_NAME = "detection_false_negative_review.md" +DECISIONS_NAME = "false_negative_review_decisions.csv" +DECISION_FIELDS = ( + "reference_feature_id", + "evidence_feature_id", + "sample_slug", + "area_m2", + "area_bucket", + "analysis_run_id", + "quality_check_id", + "source_tile_path", + "review_decision", + "review_notes", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Render persisted detection false-negative evidence for manual review." + ) + parser.add_argument("--portfolio", required=True, type=Path) + parser.add_argument("--storage-root", default="/app/storage", type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument( + "--sample-slugs", + default="", + help="Optional comma-separated AOI slugs. All portfolio samples are used by default.", + ) + parser.add_argument("--max-features", type=int, default=48) + parser.add_argument("--columns", type=int, default=4) + parser.add_argument("--cards-per-sheet", type=int, default=16) + parser.add_argument("--thumb-size", type=int, default=256) + return parser.parse_args() + + +def resolve_confined_file(raw: str, storage_root: Path, description: str) -> Path: + root = storage_root.expanduser().resolve() + candidate = Path(raw).expanduser() + if not candidate.is_absolute(): + candidate = root / candidate + candidate = candidate.resolve() + try: + candidate.relative_to(root) + except ValueError as exc: + raise SystemExit(f"{description} is outside storage root: {candidate}") from exc + if not candidate.is_file(): + raise SystemExit(f"{description} is not readable: {candidate}") + return candidate + + +def resolve_manifest_path( + sample: dict[str, Any], storage_root: Path +) -> Path: + raw_summary = str( + sample.get("copied_summary_path") or sample.get("source_summary_path") or "" + ).strip() + if not raw_summary: + raise SystemExit( + f"Portfolio sample {sample.get('sample_slug')} has no persisted run summary path" + ) + summary_path = resolve_confined_file(raw_summary, storage_root, "Run summary") + summary = load_json(summary_path) + items = [item for item in summary.get("items") or [] if isinstance(item, dict)] + manifest_paths = { + str(item.get("manifest_path") or "").strip() + for item in items + if str(item.get("manifest_path") or "").strip() + } + if len(manifest_paths) != 1: + raise SystemExit( + f"Expected one persisted tile manifest for {sample.get('sample_slug')}; " + f"found {len(manifest_paths)}" + ) + return resolve_confined_file( + next(iter(manifest_paths)), storage_root, "Tile manifest" + ) + + +def load_tiles( + manifest_path: Path, + storage_root: Path, + dependencies: dict[str, Any], +) -> list[dict[str, Any]]: + manifest = load_json(manifest_path) + rasterio = dependencies["rasterio"] + box = dependencies["box"] + raw_paths = [ + str(tile.get("path") or "").strip() + for tile in manifest.get("tiles") or [] + if isinstance(tile, dict) + ] or [str(path).strip() for path in manifest.get("tile_paths") or []] + if not raw_paths: + raise SystemExit(f"Tile manifest contains no source tiles: {manifest_path}") + + tiles = [] + for raw_path in raw_paths: + tile_path = resolve_confined_file(raw_path, storage_root, "Source tile") + with rasterio.open(tile_path) as source: + if not source.crs: + raise SystemExit(f"Source tile has no CRS: {tile_path}") + tiles.append( + { + "path": str(tile_path), + "crs": source.crs, + "geometry": box(*source.bounds), + } + ) + return tiles + + +def choose_source_tile( + geometry: Any, + tiles: list[dict[str, Any]], + dependencies: dict[str, Any], +) -> str | None: + Transformer = dependencies["Transformer"] + transform_geometry = dependencies["transform"] + candidates: list[tuple[float, float, str]] = [] + for tile in tiles: + transformer = Transformer.from_crs("EPSG:4326", tile["crs"], always_xy=True) + projected = transform_geometry(transformer.transform, geometry) + intersection = projected.intersection(tile["geometry"]) + if intersection.is_empty: + continue + candidates.append( + ( + float(intersection.area), + -float(projected.centroid.distance(tile["geometry"].centroid)), + tile["path"], + ) + ) + return max(candidates)[2] if candidates else None + + +def stable_sort_key(record: dict[str, Any]) -> str: + identity = f"{record['sample_slug']}:{record['reference_feature_id']}" + return hashlib.sha256(identity.encode("utf-8")).hexdigest() + + +def stratified_selection( + records: list[dict[str, Any]], limit: int +) -> list[dict[str, Any]]: + grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for record in records: + grouped[(record["sample_slug"], record["area_bucket"])].append(record) + for values in grouped.values(): + values.sort(key=stable_sort_key) + + selected: list[dict[str, Any]] = [] + keys = sorted(grouped) + depth = 0 + while len(selected) < limit: + added = False + for key in keys: + values = grouped[key] + if depth < len(values): + selected.append(values[depth]) + added = True + if len(selected) == limit: + break + if not added: + break + depth += 1 + return selected + + +def read_population( + portfolio_path: Path, + selected_slugs: set[str], + storage_root: Path, + dependencies: dict[str, Any], +) -> tuple[ + list[dict[str, Any]], + dict[str, list[dict[str, Any]]], + list[dict[str, Any]], +]: + portfolio = load_json(portfolio_path) + samples = portfolio.get("samples") or [] + if not isinstance(samples, list): + raise SystemExit("Portfolio samples must be a list") + geod = dependencies["Geod"](ellps="WGS84") + shape = dependencies["shape"] + population: list[dict[str, Any]] = [] + context: dict[str, list[dict[str, Any]]] = defaultdict(list) + outside_tile_coverage: list[dict[str, Any]] = [] + + for sample in samples: + if not isinstance(sample, dict): + raise SystemExit("Portfolio contains a non-object sample") + slug = str(sample.get("sample_slug") or "").strip().lower() + if not slug or (selected_slugs and slug not in selected_slugs): + continue + tiles = load_tiles( + resolve_manifest_path(sample, storage_root), storage_root, dependencies + ) + evidence_path = resolve_evidence_path(portfolio_path, sample) + evidence = load_json(evidence_path) + if evidence.get("type") != "FeatureCollection": + raise SystemExit(f"Evidence must be a FeatureCollection: {evidence_path}") + false_negative_count = 0 + for feature in evidence.get("features") or []: + if not isinstance(feature, dict): + raise SystemExit(f"Evidence contains a non-object feature: {evidence_path}") + properties = feature.get("properties") or {} + role = str(properties.get("qa_evidence_role") or "") + if role not in { + "false_negative", + "false_positive", + "match_candidate", + "match_reference", + }: + continue + geometry = shape(feature.get("geometry")) + if geometry.is_empty or not geometry.is_valid or geometry.geom_type not in { + "Polygon", + "MultiPolygon", + }: + raise SystemExit( + f"Detection QA evidence has invalid polygon geometry: {feature.get('id')}" + ) + if role != "false_negative": + context[slug].append( + {"role": role, "geometry": feature.get("geometry")} + ) + continue + + false_negative_count += 1 + reference_id = str( + properties.get("reference_feature_id") + or properties.get("source_feature_id") + or properties.get("feature_id") + or feature.get("id") + ) + area_m2 = abs(float(geod.geometry_area_perimeter(geometry)[0])) + source_tile_path = choose_source_tile(geometry, tiles, dependencies) + if source_tile_path is None: + outside_tile_coverage.append( + { + "reference_feature_id": reference_id, + "evidence_feature_id": str(feature.get("id") or reference_id), + "sample_slug": slug, + "area_m2": area_m2, + "area_bucket": area_bucket(area_m2), + "geometry": feature.get("geometry"), + "properties": properties, + } + ) + continue + population.append( + { + "reference_feature_id": reference_id, + "evidence_feature_id": str(feature.get("id") or reference_id), + "sample_slug": slug, + "area_m2": area_m2, + "area_bucket": area_bucket(area_m2), + "analysis_run_id": properties.get("analysis_run_id"), + "quality_check_id": properties.get("quality_check_id"), + "source_tile_path": source_tile_path, + "geometry": feature.get("geometry"), + "properties": properties, + } + ) + declared = (sample.get("role_counts") or {}).get("false_negative") + if declared is not None and int(declared) != false_negative_count: + raise SystemExit( + f"Portfolio role count drift for {slug}: declared {declared}, " + f"found {false_negative_count}" + ) + if not population: + raise SystemExit( + "No reviewable false-negative evidence intersects the selected tile manifests" + ) + return population, context, outside_tile_coverage + + +def render_card( + record: dict[str, Any], + context: list[dict[str, Any]], + thumb_size: int, + dependencies: dict[str, Any], + projection_cache: dict[tuple[str, str], list[dict[str, Any]]], +) -> tuple[Any, int]: + rasterio = dependencies["rasterio"] + Image = dependencies["Image"] + ImageDraw = dependencies["ImageDraw"] + ImageFont = dependencies["ImageFont"] + numpy = dependencies["numpy"] + Transformer = dependencies["Transformer"] + shape = dependencies["shape"] + transform_geometry = dependencies["transform"] + Polygon = dependencies["Polygon"] + + header_height = 88 + with rasterio.open(record["source_tile_path"]) as source: + pixels = normalize_raster(source.read(), numpy) + transformer = Transformer.from_crs("EPSG:4326", source.crs, always_xy=True) + target = transform_geometry(transformer.transform, shape(record["geometry"])) + cache_key = (record["sample_slug"], source.crs.to_string()) + projected_context = projection_cache.get(cache_key) + if projected_context is None: + projected_context = [ + { + "role": item["role"], + "geometry": transform_geometry( + transformer.transform, shape(item["geometry"]) + ), + } + for item in context + ] + projection_cache[cache_key] = projected_context + inverse = ~source.transform + pixel_points = [inverse * (x, y) for x, y in target.envelope.exterior.coords] + columns = [point[0] for point in pixel_points] + rows = [point[1] for point in pixel_points] + x_min, x_max = min(columns), max(columns) + y_min, y_max = min(rows), max(rows) + bbox_width = max(1.0, x_max - x_min) + bbox_height = max(1.0, y_max - y_min) + crop_width = min(source.width, max(128, math.ceil(bbox_width * 4))) + crop_height = min(source.height, max(128, math.ceil(bbox_height * 4))) + center_x = (x_min + x_max) / 2 + center_y = (y_min + y_max) / 2 + crop_left = max(0, min(source.width - crop_width, round(center_x - crop_width / 2))) + crop_top = max(0, min(source.height - crop_height, round(center_y - crop_height / 2))) + crop_right = crop_left + crop_width + crop_bottom = crop_top + crop_height + image = Image.fromarray(pixels, mode="RGB").crop( + (crop_left, crop_top, crop_right, crop_bottom) + ) + scale = min(thumb_size / image.width, thumb_size / image.height) + render_width = max(1, round(image.width * scale)) + render_height = max(1, round(image.height * scale)) + image = image.resize((render_width, render_height), Image.Resampling.BILINEAR) + offset_x = (thumb_size - render_width) // 2 + offset_y = (thumb_size - render_height) // 2 + card = Image.new( + "RGB", (thumb_size, thumb_size + header_height), color=(242, 245, 247) + ) + card.paste(image, (offset_x, header_height + offset_y)) + draw = ImageDraw.Draw(card) + font = ImageFont.load_default() + draw.rectangle((0, 0, thumb_size, header_height), fill=(22, 29, 38)) + draw.text((6, 6), record["sample_slug"], fill=(255, 255, 255), font=font) + draw.text( + (6, 23), + f"{record['area_bucket'].split('_', 1)[0]} | {record['area_m2']:.1f} m2", + fill=(197, 215, 231), + font=font, + ) + draw.text( + (6, 40), + Path(record["source_tile_path"]).name[:24], + fill=(197, 215, 231), + font=font, + ) + draw.text((6, 56), "red: missed GRB", fill=(235, 238, 241), font=font) + draw.text( + (6, 72), "blue: candidate | green: matched ref", fill=(235, 238, 241), font=font + ) + + crop_bounds = Polygon( + [ + source.transform * (crop_left, crop_top), + source.transform * (crop_right, crop_top), + source.transform * (crop_right, crop_bottom), + source.transform * (crop_left, crop_bottom), + ] + ) + overlay = Image.new("RGBA", (thumb_size, thumb_size), (0, 0, 0, 0)) + overlay_draw = ImageDraw.Draw(overlay) + overlay_count = 0 + for item in projected_context: + geometry = item["geometry"] + if geometry.is_empty or not geometry.intersects(crop_bounds): + continue + color = ( + (39, 174, 96) + if item["role"] == "match_reference" + else (52, 152, 219) + ) + draw_geometry( + overlay_draw, + geometry, + inverse, + scale, + offset_x, + offset_y, + crop_left, + crop_top, + color, + ) + overlay_count += 1 + draw_geometry( + overlay_draw, + target, + inverse, + scale, + offset_x, + offset_y, + crop_left, + crop_top, + (231, 76, 60), + ) + card.paste(overlay, (0, header_height), overlay) + return card, overlay_count + + +def write_decisions(selected: list[dict[str, Any]], output_path: Path) -> None: + with output_path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=DECISION_FIELDS) + writer.writeheader() + for record in selected: + writer.writerow( + { + key: record.get(key, "") + for key in DECISION_FIELDS + if key not in {"review_decision", "review_notes"} + } + | {"review_decision": "unreviewed", "review_notes": ""} + ) + + +def write_markdown(report: dict[str, Any], output_dir: Path) -> None: + lines = [ + "# Detection false-negative visual review", + "", + "> A QA false-negative is not automatically a model miss. Review imagery, the persisted GRB footprint and nearby candidate geometry.", + "", + f"- Status: `{report['status']}`", + f"- Population: {report['population_count']}", + f"- Selected for manual review: {report['selected_feature_count']}", + f"- Outside tile coverage (excluded): {report['outside_tile_coverage_count']}", + f"- AOIs: {', '.join(report['selected_sample_slugs'])}", + "", + "## Allowed decisions", + "", + "- `confirmed_model_false_negative`: imagery confirms the referenced building and no suitable detection covers it.", + "- `reference_gap_or_change`: the persisted reference is absent or stale in the imagery.", + "- `qa_alignment_mismatch`: a nearby detection exists but geometry/alignment or matching tolerance prevented a match.", + "- `imagery_obscured_or_uncertain`: the image does not support a confident decision.", + "- `unreviewed`: no operator decision has been made.", + "", + "Only explicitly confirmed model misses may inform a future positive-training review set.", + "", + "## Contact sheets", + "", + ] + for sheet in report["contact_sheets"]: + lines.extend([f"![{sheet['path']}]({sheet['path']})", ""]) + (output_dir / MARKDOWN_NAME).write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def run(args: argparse.Namespace) -> dict[str, Any]: + if min(args.max_features, args.columns, args.cards_per_sheet, args.thumb_size) <= 0: + raise SystemExit("Review limits, columns and thumbnail size must be positive") + dependencies = require_dependencies("False-negative visual review") + portfolio_path = args.portfolio.expanduser().resolve() + storage_root = args.storage_root.expanduser().resolve() + requested_slugs = { + value.strip().lower() for value in args.sample_slugs.split(",") if value.strip() + } + population, context, outside_tile_coverage = read_population( + portfolio_path, requested_slugs, storage_root, dependencies + ) + available_slugs = {record["sample_slug"] for record in population} + missing_slugs = requested_slugs - available_slugs + if missing_slugs: + raise SystemExit( + "Selected sample slug is absent from the portfolio: " + + ", ".join(sorted(missing_slugs)) + ) + selected = stratified_selection(population, min(args.max_features, len(population))) + + output_dir = args.output_dir.expanduser().resolve() + output_dir.mkdir(parents=True, exist_ok=True) + cards = [] + overlay_count = 0 + projection_cache: dict[tuple[str, str], list[dict[str, Any]]] = {} + for record in selected: + card, count = render_card( + record, + context.get(record["sample_slug"], []), + args.thumb_size, + dependencies, + projection_cache, + ) + cards.append(card) + overlay_count += count + contact_sheets = [] + for index in range(0, len(cards), args.cards_per_sheet): + batch = cards[index : index + args.cards_per_sheet] + path = output_dir / f"false_negative_review_{index // args.cards_per_sheet + 1:03d}.png" + build_contact_sheet(batch, args.columns, path, dependencies["Image"]) + contact_sheets.append({"path": path.name, "feature_count": len(batch)}) + + portfolio = load_json(portfolio_path) + report = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "schema_version": 1, + "status": "review_required", + "portfolio_path": str(portfolio_path), + "model_asset_id": portfolio.get("model_asset_id"), + "model_sha256": portfolio.get("model_sha256"), + "population_count": len(population), + "evidence_population_count": len(population) + len(outside_tile_coverage), + "selected_feature_count": len(selected), + "selected_sample_slugs": sorted({record["sample_slug"] for record in selected}), + "selected_area_buckets": sorted({record["area_bucket"] for record in selected}), + "missing_manifest_count": 0, + "missing_tile_count": 0, + "outside_tile_coverage_count": len(outside_tile_coverage), + "outside_tile_coverage_geojson_path": ( + "false_negatives_outside_tile_coverage.geojson" + ), + "context_overlay_feature_count": overlay_count, + "contact_sheets": contact_sheets, + "selected_features": selected, + } + (output_dir / JSON_NAME).write_text( + json.dumps(report, indent=2, sort_keys=True), encoding="utf-8" + ) + (output_dir / "false_negatives_outside_tile_coverage.geojson").write_text( + json.dumps( + { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": record["evidence_feature_id"], + "geometry": record["geometry"], + "properties": { + **record["properties"], + "review_exclusion_reason": "outside_tile_coverage", + "sample_slug": record["sample_slug"], + "area_m2": record["area_m2"], + "area_bucket": record["area_bucket"], + }, + } + for record in outside_tile_coverage + ], + }, + indent=2, + sort_keys=True, + ), + encoding="utf-8", + ) + write_decisions(selected, output_dir / DECISIONS_NAME) + write_markdown(report, output_dir) + return report + + +def main() -> int: + args = parse_args() + report = run(args) + print("Detection false-negative review required") + print(f"Selected features: {report['selected_feature_count']}") + print(f"Summary: {args.output_dir.expanduser().resolve() / JSON_NAME}") + print(f"Decisions: {args.output_dir.expanduser().resolve() / DECISIONS_NAME}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/render_detection_false_positive_review_contact_sheets.py b/scripts/render_detection_false_positive_review_contact_sheets.py index c3b06067..b6ec8b69 100644 --- a/scripts/render_detection_false_positive_review_contact_sheets.py +++ b/scripts/render_detection_false_positive_review_contact_sheets.py @@ -135,7 +135,7 @@ def stratified_selection( return selected -def require_dependencies() -> dict[str, Any]: +def require_dependencies(purpose: str = "False-positive visual review") -> dict[str, Any]: try: import numpy import rasterio @@ -145,7 +145,7 @@ def require_dependencies() -> dict[str, Any]: from shapely.ops import transform except ImportError as exc: raise SystemExit( - "False-positive visual review requires the backend GIS/raster extras" + f"{purpose} requires the backend GIS/raster extras" ) from exc return { "numpy": numpy, diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index fec8060c..15f787c1 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -51,6 +51,7 @@ ${PYTHON_BIN} -m py_compile scripts/build_fixed_threshold_evidence_portfolio_inp ${PYTHON_BIN} -m py_compile scripts/audit_detection_false_negative_evidence.py ${PYTHON_BIN} -m py_compile scripts/audit_detection_false_positive_evidence.py ${PYTHON_BIN} -m py_compile scripts/render_detection_false_positive_review_contact_sheets.py +${PYTHON_BIN} -m py_compile scripts/render_detection_false_negative_review_contact_sheets.py ${PYTHON_BIN} -m py_compile scripts/validate_detection_false_positive_review_decisions.py ${PYTHON_BIN} -m py_compile scripts/activate_promoted_yolo_candidate.py ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py