From 64dac0d9b740c34b6316f6212cdd343550d41aea Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Jul 2026 02:06:13 +0200 Subject: [PATCH] Classify operator background corpus --- CHANGELOG.md | 7 ++ ...int156_background_corpus_classification.py | 99 +++++++++++++++++++ docs/AI_PIPELINES.md | 10 +- docs/CODEX_EXECUTION_LOG.md | 38 +++++++ docs/TODO.md | 3 +- scripts/README.md | 15 ++- scripts/export_operator_yolo_tile_dataset.py | 2 + scripts/prepare_operator_real_data_samples.py | 16 ++- ...operator_hard_negative_detection_matrix.sh | 44 +++++++-- 9 files changed, 218 insertions(+), 16 deletions(-) create mode 100644 backend/tests/test_sprint156_background_corpus_classification.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d60d890..eafffa56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 156 Background corpus classification (2026-07-10) + +- Added explicit operator background categories to prepared sample manifests: `pure_empty_negative` when GRB returns zero reference buildings and `sparse_building_context` when contextual GRB buildings are present. +- Added `OPERATOR_BACKGROUND_CATEGORIES` to `scripts/run_operator_hard_negative_detection_matrix.sh` so strict default-promotion false-positive gates can run on pure-empty negatives separately from sparse-context review samples. +- Preserved `background_category` in exported YOLO tile metadata for training auditability. +- No model default, backend API, database migration, provider fetching, fake detection output or model download behavior changed. + ## Sprint 155 Detection operator profiles (2026-07-09) - Added explicit Detection Lab operator profiles for the inactive `geointel-building-yolov8s-aoi1024bg512r3e50-pt` local model asset. diff --git a/backend/tests/test_sprint156_background_corpus_classification.py b/backend/tests/test_sprint156_background_corpus_classification.py new file mode 100644 index 00000000..b0afcf7c --- /dev/null +++ b/backend/tests/test_sprint156_background_corpus_classification.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import sys + + +ROOT = Path(__file__).resolve().parents[2] + + +def load_sample_preparer(): + script_path = ROOT / "scripts" / "prepare_operator_real_data_samples.py" + spec = importlib.util.spec_from_file_location("operator_sample_preparer_s156", script_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_background_samples_are_classified_by_actual_reference_density() -> None: + module = load_sample_preparer() + + background = module.OperatorSample( + slug="background", + display_name="Background", + center_lon=5.0, + center_lat=51.0, + sample_role="background_candidate", + allow_empty_reference=True, + ) + reference = module.OperatorSample( + slug="reference", + display_name="Reference", + center_lon=5.0, + center_lat=51.0, + ) + + assert module.background_category_for_sample(background, 0) == "pure_empty_negative" + assert module.background_category_for_sample(background, 3) == "sparse_building_context" + assert module.background_category_for_sample(reference, 30) == "reference_aoi" + + +def test_prepare_sample_manifest_records_background_category_from_cached_reference( + tmp_path: Path, + monkeypatch, +) -> None: + module = load_sample_preparer() + sample = module.OperatorSample( + slug="background", + display_name="Background", + center_lon=5.0, + center_lat=51.0, + sample_role="background_candidate", + allow_empty_reference=True, + ) + raster_path, reference_path = module.sample_artifact_paths(sample, tmp_path) + raster_path.write_bytes(b"placeholder raster") + reference_path.write_text( + json.dumps( + { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, + "properties": {}, + } + ], + } + ), + encoding="utf-8", + ) + monkeypatch.setattr(module, "raster_summary", lambda path: {"path": str(path)}) + monkeypatch.setattr(module, "sample_bounds", lambda current: ((0.0, 0.0, 1.0, 1.0), [4.9, 50.9, 5.1, 51.1])) + + prepared = module.prepare_sample(sample, tmp_path, force=False) + + assert prepared["background_category"] == "sparse_building_context" + assert prepared["reference_feature_count"] == 1 + + +def test_hard_negative_matrix_can_filter_background_categories() -> None: + script = (ROOT / "scripts" / "run_operator_hard_negative_detection_matrix.sh").read_text(encoding="utf-8") + + assert "OPERATOR_BACKGROUND_CATEGORIES" in script + assert "background_category" in script + assert "pure_empty_negative" in script + assert "sparse_building_context" in script + assert "background_category_counts" in script + + +def test_yolo_tile_export_preserves_background_category_provenance() -> None: + script = (ROOT / "scripts" / "export_operator_yolo_tile_dataset.py").read_text(encoding="utf-8") + + assert "background_category = str(sample.get(\"background_category\") or \"reference_aoi\")" in script + assert "\"background_category\": background_category" in script diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 35f1d321..8c506c6d 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -247,6 +247,7 @@ considered as a default: ```bash OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \ OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos" \ QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8n-expanded160e50-pt geointel-building-yolov8n-tile30-pt yolov8s-building-segmentation-pt" \ QUALITY_TILE_SIZES="640" \ @@ -257,8 +258,13 @@ bash scripts/run_operator_hard_negative_detection_matrix.sh http://192.168.10.15 The hard-negative matrix uploads only background rasters and counts detections as false-positive pressure. It does not run QA/QC or invent reference metrics -for empty/sparse background AOIs. The first expanded local model improved dense -AOI F1, but Kasterlee-bos false positives block default promotion. +for empty/sparse background AOIs. Operator manifests classify background +samples as `pure_empty_negative` when GRB returns zero reference buildings and +`sparse_building_context` when contextual buildings are present. Use +`OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative"` for default-promotion +hard-negative gates, then run `sparse_building_context` as a separate review +matrix. The first expanded local model improved dense AOI F1, but Kasterlee-bos +false positives block default promotion. The current inactive AOI1024 background-aware local model asset, `geointel-building-yolov8s-aoi1024bg512r3e50-pt`, is exposed in Detection Lab diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 715a21b3..a0456cad 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -6197,3 +6197,41 @@ Open: - The profiles are review/demo aids only. The background corpus still needs to be split into pure-empty negatives and sparse-building contextual AOIs before retraining or recalibrating for a default detector decision. - No backend API contract, migration, provider fetching, fake detection output, model download behavior or active runtime default changed. + +# Sprint 156 - Background corpus classification + +## What changed + +- Added explicit background category classification to operator sample preparation: + - `pure_empty_negative` when a background candidate has zero GRB reference buildings. + - `sparse_building_context` when a background candidate has one or more GRB reference buildings. + - `reference_aoi` for normal positive reference samples. +- Persisted `background_category` into generated operator sample manifests and reference GeoJSON metadata. +- Added `OPERATOR_BACKGROUND_CATEGORIES` to `scripts/run_operator_hard_negative_detection_matrix.sh` so the strict default-promotion hard-negative gate can run only on `pure_empty_negative` samples, while `sparse_building_context` samples can be reviewed separately. +- Preserved `background_category` in YOLO tile export metadata so negative-tile provenance survives training dataset audits. +- Updated operator pipeline docs, TODO and changelog. + +## What was tested + +- Added regression coverage in `backend/tests/test_sprint156_background_corpus_classification.py`. +- Ran `python -m pytest tests/test_sprint156_background_corpus_classification.py -q`. +- Ran `python -m pytest tests/test_sprint156_background_corpus_classification.py tests/test_sprint131_operator_sample_expansion.py tests/test_sprint132_operator_hard_negative_matrix.py tests/test_sprint130_operator_yolo_tile_dataset.py -q`: 17 passed. +- Ran `python -m compileall backend/app`. +- Ran `python -m pytest` in `backend`: 439 passed. +- Ran `cd frontend && npm run typecheck`. +- Ran `cd frontend && npm run build`. +- Ran `bash scripts/run_readiness_check.sh`. +- Ran `cd backend && python -m alembic heads` and `cd backend && python -m alembic upgrade head --sql`. +- Ran `bash -n scripts/live_migration_smoke.sh` and `bash -n scripts/run_operator_hard_negative_detection_matrix.sh`. + +## Known limitations + +- This pass adds the cleaner corpus/gate contract only. It does not regenerate Tower manifests, retrain YOLO, rerun the live hard-negative matrices or change any model default. +- No backend API contract, database migration, provider fetching, fake detection output or model download behavior changed. + +## Next recommended pass + +- Redeploy/rebuild the runtime scripts, regenerate the operator sample manifest, then run: + - `OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative"` for the strict default-promotion false-positive gate. + - `OPERATOR_BACKGROUND_CATEGORIES="sparse_building_context"` for contextual review evidence. +- Retrain or recalibrate the inactive AOI1024 local model candidate only after those two matrices are available. diff --git a/docs/TODO.md b/docs/TODO.md index d6aa2524..640cb64a 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -121,7 +121,8 @@ This file now starts with the current implementation status. Older preparation/b - [x] Train and gate `geointel-building-yolov8s-aoi1024clean512e50-pt` through seven positive AOIs and nine hard-negative/background AOIs. - [x] Train and gate background-aware `geointel-building-yolov8s-aoi1024bg512r3e50-pt`; it is the strongest positive-AOI candidate so far but remains inactive because full background-candidate false-positive pressure still blocks default promotion. - [x] Add explicit operator detection profiles for local model assets: balanced review around threshold `0.15` and conservative high-precision review around threshold `0.35`, both clearly marked as non-default-approved until promotion gates pass. -- [ ] Split the background corpus into pure-empty negatives and sparse-building contextual AOIs, then retrain or recalibrate against the cleaner gate. +- [x] Add pure-empty versus sparse-building contextual background corpus classification to operator manifests, hard-negative matrix filters and YOLO tile provenance. +- [ ] Retrain or recalibrate against the cleaner pure-empty gate plus separate sparse-context inspection matrix. - [ ] Promote a V1 default building detector only after it passes seven positive AOIs, clean hard-negative/background gates and persisted QA/QC evidence without fake detections or model downloads. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index 50b8973d..5d2a883f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -187,8 +187,11 @@ Kasterlee-bos, Dessel-heide, Ravels-bos, Meerhout-bos, Geel-Bel, Arendonk-heide and Herenthout-bos. Normal reference AOIs still fail when GRB returns no buildings; background candidates are explicitly marked with `sample_role` and may write an empty reference FeatureCollection for -negative-tile training. The helper fetches only the explicit documented AOIs, -records Digitaal Vlaanderen attribution and reuses existing files by default. +negative-tile training. Generated manifests also classify background samples as +`pure_empty_negative` when GRB returns zero reference buildings or +`sparse_building_context` when GRB returns one or more contextual buildings. +The helper fetches only the explicit documented AOIs, records Digitaal +Vlaanderen attribution and reuses existing files by default. Use `--force` only when the local runtime artifacts should be regenerated. GRB building references are fetched through the provider's OGC API `rel=next` pagination links, so dense AOIs are not silently limited to the @@ -472,6 +475,7 @@ before changing model defaults: ```bash OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \ OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos dessel_heide ravels_bos meerhout_bos geel_bel arendonk_heide herenthout_bos" \ QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8n-expanded160e50-pt geointel-building-yolov8n-tile30-pt yolov8s-building-segmentation-pt" \ QUALITY_TILE_SIZES="640" \ @@ -485,7 +489,12 @@ The hard-negative matrix uploads only the background raster, generates tiles, runs configured-YOLO detection and counts persisted detections as `false_positive_pressure`. It does not upload a reference vector and does not run QA/QC, because empty or sparse background AOIs do not have a meaningful -precision/recall target. In the first live run, `geointel-building-yolov8n-expanded160e50-pt` +precision/recall target. Use `OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative"` +for the strict default-promotion false-positive gate. Run +`OPERATOR_BACKGROUND_CATEGORIES="sparse_building_context"` separately for +contextual review; sparse-context detections should be inspected, not counted +as fake precision/recall metrics. In the first live run, +`geointel-building-yolov8n-expanded160e50-pt` was clean on Postel-bos and Lommel-heide at thresholds `0.25` and `0.15`, but produced 38 detections on Kasterlee-bos even at `0.25`. That blocks it from becoming a V1 default until a hard-negative-balanced candidate improves. diff --git a/scripts/export_operator_yolo_tile_dataset.py b/scripts/export_operator_yolo_tile_dataset.py index fa93fd53..2c1c4c08 100644 --- a/scripts/export_operator_yolo_tile_dataset.py +++ b/scripts/export_operator_yolo_tile_dataset.py @@ -315,6 +315,7 @@ def export_sample_tiles( ) -> list[dict[str, Any]]: sample_slug = str(sample["sample_slug"]) sample_role = str(sample.get("sample_role") or "reference") + background_category = str(sample.get("background_category") or "reference_aoi") split = "val" if sample_slug.lower() in val_slugs else "train" raster_path = resolve_manifest_path(str(sample["raster_path"]), manifest_path) reference_path = resolve_manifest_path(str(sample["reference_path"]), manifest_path) @@ -366,6 +367,7 @@ def export_sample_tiles( { "sample_slug": sample_slug, "sample_role": sample_role, + "background_category": background_category, "split": split, "tile_index": tile_index, "repeat_index": repeat_index, diff --git a/scripts/prepare_operator_real_data_samples.py b/scripts/prepare_operator_real_data_samples.py index bf97d774..3b7de9c1 100644 --- a/scripts/prepare_operator_real_data_samples.py +++ b/scripts/prepare_operator_real_data_samples.py @@ -22,6 +22,9 @@ GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data") DEFAULT_GRB_PAGE_LIMIT = 1000 DEFAULT_GRB_MAX_FEATURES = 100000 +REFERENCE_AOI_CATEGORY = "reference_aoi" +PURE_EMPTY_BACKGROUND_CATEGORY = "pure_empty_negative" +SPARSE_BACKGROUND_CATEGORY = "sparse_building_context" requests: Any = None rasterio: Any = None Transformer: Any = None @@ -261,6 +264,12 @@ def selected_samples(raw: str) -> list[OperatorSample]: return [SAMPLES[slug] for slug in slugs] +def background_category_for_sample(sample: OperatorSample, reference_feature_count: int) -> str: + if sample.sample_role != "background_candidate": + return REFERENCE_AOI_CATEGORY + return PURE_EMPTY_BACKGROUND_CATEGORY if reference_feature_count <= 0 else SPARSE_BACKGROUND_CATEGORY + + def apply_sample_overrides( sample: OperatorSample, *, @@ -471,6 +480,7 @@ def fetch_reference( reference["sample_slug"] = sample.slug reference["sample_role"] = sample.sample_role reference["allow_empty_reference"] = sample.allow_empty_reference + reference["background_category"] = background_category_for_sample(sample, len(features)) reference["reference_page_limit"] = page_limit reference["reference_max_features"] = max_features reference["reference_pages_fetched"] = len(pages) @@ -484,6 +494,7 @@ def fetch_reference( props.setdefault("reference_layer_name", "buildings") props.setdefault("sample_slug", sample.slug) props.setdefault("sample_role", sample.sample_role) + props.setdefault("background_category", background_category_for_sample(sample, len(features))) reference_path.write_text(json.dumps(reference, ensure_ascii=False), encoding="utf-8") return prepared_url(GRB_GBG_URL, ogc_params), len(features) @@ -513,6 +524,7 @@ def prepare_sample( ) else: reference_feature_count = geojson_feature_count(reference_path) + background_category = background_category_for_sample(sample, reference_feature_count) return { "sample_slug": sample.slug, @@ -524,6 +536,7 @@ def prepare_sample( "height": sample.height, "sample_role": sample.sample_role, "allow_empty_reference": sample.allow_empty_reference, + "background_category": background_category, "raster_path": str(ortho_path), "reference_path": str(reference_path), "reference_feature_count": reference_feature_count, @@ -558,7 +571,8 @@ def write_readme(output_dir: Path, samples: list[dict[str, Any]]) -> None: lines.append( f"- `{sample['sample_slug']}`: `{Path(sample['raster_path']).name}` and " f"`{Path(sample['reference_path']).name}`, " - f"{sample['reference_feature_count']} reference features, role `{sample['sample_role']}`." + f"{sample['reference_feature_count']} reference features, role `{sample['sample_role']}`, " + f"background category `{sample['background_category']}`." ) lines.append("") lines.append("Purpose: configured-YOLO detection + persisted QA/QC validation with operator-provided files.") diff --git a/scripts/run_operator_hard_negative_detection_matrix.sh b/scripts/run_operator_hard_negative_detection_matrix.sh index f130c1fe..a376fb56 100644 --- a/scripts/run_operator_hard_negative_detection_matrix.sh +++ b/scripts/run_operator_hard_negative_detection_matrix.sh @@ -13,6 +13,7 @@ Usage: Optional environment: OPERATOR_SAMPLE_MANIFEST_PATH Manifest from prepare_operator_real_data_samples.py. OPERATOR_BACKGROUND_SAMPLE_SLUGS Optional comma/space separated filter. Defaults to samples marked background_candidate or allow_empty_reference. + OPERATOR_BACKGROUND_CATEGORIES Optional comma/space separated filter, e.g. pure_empty_negative or sparse_building_context. HARD_NEGATIVE_OUTPUT_DIR Output directory, default: artifacts/detection-hard-negatives/. QUALITY_MODEL_ASSET_IDS Space/comma separated local model asset IDs. Default: active configured model asset. QUALITY_TILE_SIZES Space/comma separated raster tile sizes, default: 640. @@ -33,6 +34,7 @@ cd "$ROOT" BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" OPERATOR_SAMPLE_MANIFEST_PATH="${OPERATOR_SAMPLE_MANIFEST_PATH:-storage/operator-data/operator_samples_manifest.json}" OPERATOR_BACKGROUND_SAMPLE_SLUGS="${OPERATOR_BACKGROUND_SAMPLE_SLUGS:-}" +OPERATOR_BACKGROUND_CATEGORIES="${OPERATOR_BACKGROUND_CATEGORIES:-}" HARD_NEGATIVE_OUTPUT_DIR="${HARD_NEGATIVE_OUTPUT_DIR:-artifacts/detection-hard-negatives/$(date -u +%Y%m%dT%H%M%SZ)}" QUALITY_MODEL_ASSET_IDS="${QUALITY_MODEL_ASSET_IDS:-${REAL_MODEL_ASSET_ID:-__active__}}" QUALITY_TILE_SIZES="${QUALITY_TILE_SIZES:-640}" @@ -79,6 +81,7 @@ matrix_manifest="${HARD_NEGATIVE_OUTPUT_DIR}/hard_negative_requests.tsv" "${ROOT}" \ "${OPERATOR_SAMPLE_MANIFEST_PATH}" \ "${OPERATOR_BACKGROUND_SAMPLE_SLUGS}" \ + "${OPERATOR_BACKGROUND_CATEGORIES}" \ "${sample_manifest_tsv}" <<'PY' import json import sys @@ -87,7 +90,8 @@ from pathlib import Path root = Path(sys.argv[1]).resolve() manifest_path = Path(sys.argv[2]) slug_filter_raw = sys.argv[3] -output_path = Path(sys.argv[4]) +category_filter_raw = sys.argv[4] +output_path = Path(sys.argv[5]) payload = json.loads(manifest_path.read_text(encoding="utf-8-sig")) samples = payload.get("samples") or [] @@ -99,6 +103,11 @@ requested_slugs = { for value in slug_filter_raw.replace(",", " ").split() if value.strip() } +requested_categories = { + value.strip().lower() + for value in category_filter_raw.replace(",", " ").split() + if value.strip() +} def resolve_path(raw: str) -> str: @@ -129,11 +138,22 @@ with output_path.open("w", encoding="utf-8") as handle: continue raster_path = resolve_path(str(sample.get("raster_path") or "")) reference_count = int(sample.get("reference_feature_count") or 0) - handle.write(f"{sample_slug}\t{raster_path}\t{sample_role}\t{allow_empty_reference}\t{reference_count}\n") + background_category = str(sample.get("background_category") or "").lower() + if not background_category: + if sample_role == "background_candidate" or allow_empty_reference: + background_category = "pure_empty_negative" if reference_count == 0 else "sparse_building_context" + else: + background_category = "reference_aoi" + if requested_categories and background_category not in requested_categories: + continue + handle.write( + f"{sample_slug}\t{raster_path}\t{sample_role}\t{allow_empty_reference}\t" + f"{reference_count}\t{background_category}\n" + ) selected += 1 if selected == 0: - raise SystemExit("No background_candidate samples matched OPERATOR_BACKGROUND_SAMPLE_SLUGS") + raise SystemExit("No background_candidate samples matched OPERATOR_BACKGROUND_SAMPLE_SLUGS/OPERATOR_BACKGROUND_CATEGORIES") PY model_requests_normalized="$(printf '%s' "${QUALITY_MODEL_ASSET_IDS}" | tr ',' ' ')" @@ -236,14 +256,15 @@ echo "== GeoIntel operator hard-negative detection matrix ==" echo "Base URL: ${BASE_URL}" echo "Manifest: ${OPERATOR_SAMPLE_MANIFEST_PATH}" echo "Background filter: ${OPERATOR_BACKGROUND_SAMPLE_SLUGS:-background_candidate samples}" +echo "Background categories: ${OPERATOR_BACKGROUND_CATEGORIES:-all}" echo "Models: ${model_requests_normalized}" echo "Tile sizes: ${tile_sizes_normalized}" echo "Tile overlaps: ${tile_overlaps_normalized}" echo "Thresholds: ${thresholds_normalized}" echo "Output: ${HARD_NEGATIVE_OUTPUT_DIR}" -while IFS=$'\t' read -r sample_slug raster_path sample_role allow_empty_reference reference_feature_count; do - echo "-- Background sample ${sample_slug}: role=${sample_role} allow_empty_reference=${allow_empty_reference} reference_features=${reference_feature_count} --" +while IFS=$'\t' read -r sample_slug raster_path sample_role allow_empty_reference reference_feature_count background_category; do + echo "-- Background sample ${sample_slug}: role=${sample_role} category=${background_category} allow_empty_reference=${allow_empty_reference} reference_features=${reference_feature_count} --" while IFS=$'\t' read -r model_request tile_size tile_overlap threshold run_label; do sample_output_dir="${HARD_NEGATIVE_OUTPUT_DIR}/${sample_slug}" mkdir -p "${sample_output_dir}" @@ -379,6 +400,7 @@ PY "${sample_role}" \ "${allow_empty_reference}" \ "${reference_feature_count}" \ + "${background_category}" \ "${model_request}" \ "${model_asset_id}" \ "${tile_size}" \ @@ -401,6 +423,7 @@ import sys sample_role, allow_empty_reference, reference_feature_count, + background_category, model_request, model_asset_id, tile_size, @@ -414,7 +437,7 @@ import sys detections_list_count, tile_count, run_log, -) = sys.argv[1:19] +) = sys.argv[1:20] detections = int(detection_count) listed = int(detections_list_count) @@ -426,6 +449,7 @@ summary = { "sample_role": sample_role, "allow_empty_reference": allow_empty_reference == "True", "reference_feature_count": int(reference_feature_count), + "background_category": background_category, "model_request": model_request, "model_asset_id": model_asset_id, "tile_size": int(tile_size), @@ -443,7 +467,7 @@ summary = { with open(output_path, "w", encoding="utf-8") as handle: json.dump(summary, handle, indent=2, sort_keys=True) print( - "sample={sample_slug} model={model_asset_id} tile={tile_size} overlap={tile_overlap} " + "sample={sample_slug} category={background_category} model={model_asset_id} tile={tile_size} overlap={tile_overlap} " "threshold={threshold} detections={detection_count} false_positive_pressure={false_positive_pressure}".format( **summary ) @@ -460,6 +484,7 @@ done < "${sample_manifest_tsv}" import glob import json import sys +from collections import Counter from datetime import datetime, timezone from pathlib import Path @@ -486,6 +511,7 @@ summary = { "base_url": base_url, "operator_sample_manifest_path": manifest_path, "sample_count": len({item["sample_slug"] for item in items}), + "background_category_counts": dict(Counter(str(item.get("background_category") or "unknown") for item in items)), "run_count": len(items), "best_by_lowest_pressure": best_by_lowest_pressure, "items": items, @@ -495,10 +521,10 @@ summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding= print("") print("Operator hard-negative detection summary") -print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\tfalse_positive_pressure") +print("sample\tcategory\tmodel\ttile\toverlap\tthreshold\tdetections\tfalse_positive_pressure") for item in items: print( - "{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{false_positive_pressure}".format( + "{sample_slug}\t{background_category}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{false_positive_pressure}".format( **item ) )