From 1689dce9287470994e9974800a60baa4761244b7 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 12:15:27 +0200 Subject: [PATCH] Promote focused small-building detector --- CHANGELOG.md | 10 +++ backend/README.md | 31 ++++++-- backend/app/schemas/detection.py | 20 ++++-- backend/app/schemas/segmentation.py | 14 ++-- backend/tests/test_docker_runtime_config.py | 35 +++++++--- .../tests/test_schema_model_field_warnings.py | 17 +++++ ...st_sprint130_operator_yolo_tile_dataset.py | 41 ++++++++++- ...est_sprint131_operator_sample_expansion.py | 47 ++++++++++++- ...t_sprint155_detection_operator_profiles.py | 6 +- ...print170_detection_false_negative_audit.py | 22 ++++++ deploy/unraid/Dockerfile.all-in-one | 24 ++++++- docs/AI_PIPELINES.md | 36 ++++++---- docs/CODEX_EXECUTION_LOG.md | 65 +++++++++++++++++ docs/TODO.md | 9 ++- frontend/README.md | 2 +- .../components/detection/detectionProfiles.ts | 23 +++++- scripts/README.md | 28 +++++++- ...audit_detection_false_negative_evidence.py | 70 +++++++++++++++++-- scripts/export_operator_yolo_tile_dataset.py | 57 +++++++++++++-- scripts/prepare_operator_real_data_samples.py | 50 ++++++++++++- 20 files changed, 546 insertions(+), 61 deletions(-) create mode 100644 backend/tests/test_schema_model_field_warnings.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 06c46a33..dd4d27e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ # Changelog +## Sprint 174 Focused small-building model promotion (2026-07-13) + +- Expanded the real operator corpus with four focused training AOIs and two independent validation AOIs, while keeping Turnhout, Retie and Westerlo outside the tile-training corpus as operation-level holdouts. +- Exported and visually audited 198 tiles with 58,820 real GRB-derived labels; the accepted corpus contained no invalid labels, missing files or low-variance review selections. +- Trained `geointel-building-yolov8s-smallbld-minpx3-img640-ft30-pt` from the previous active local model without downloading weights; the trained asset SHA256 is `a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1`. +- At tile `512`, overlap `64`, threshold `0.15` and QA match IoU `0.25`, seven persisted AOIs reached mean precision `0.5898`, recall `0.5770`, F1 `0.5825` and minimum F1 `0.5528`; all three pure-empty controls remained at zero detections. +- Fixed-reference evidence reduced false negatives from 7,753 to 6,182, including 745 fewer 25-100 m2 misses and 181 fewer sub-25 m2 misses. Precision is lower, so the UI states the increased false-positive review load explicitly. +- Added persistent false-negative area summaries/GeoJSON, explicit tile-corpus sample selection provenance, missing runtime evaluation scripts, Docker COPY-source regression checks and warning-free Pydantic model-field schemas. +- Guarded activation updated only the Tower AI/model environment values; no fake outputs, provider fetching, API contract or database migration changed. + ## Sprint 173 Expanded building model promotion (2026-07-13) - Trained and fully gated the inactive `geointel-building-yolov8s-aoi1024expandedminpx4vis035e50-pt` candidate from the 20-source expanded real-data corpus. diff --git a/backend/README.md b/backend/README.md index bbba759f..aa09e65f 100644 --- a/backend/README.md +++ b/backend/README.md @@ -345,6 +345,12 @@ Then point `OPERATOR_YOLO_DATASET_DIR` at training wrapper. Tile-level output remains operator tooling outside the V1 browser product. +Use `--samples` (or `OPERATOR_YOLO_SAMPLES`) when an experiment needs a +deliberate manifest subset. The generated summary records the source manifest +count plus selected and excluded sample slugs. Unknown samples and any selected +manifest holdout that is omitted from `--val-samples` fail before files are +written. + The backend also exposes a read-only model asset catalog for the mounted model directory: @@ -462,13 +468,24 @@ docker exec -it geointel python3 /app/scripts/prepare_operator_real_data_samples ``` The helper writes GeoTIFF orthophotos, GRB GBG building GeoJSON files and -`operator_samples_manifest.json` under `/app/storage/operator-data`. The corpus -contains dense reference AOIs for Geel, Mol, Turnhout, Herentals, Balen, Retie -and Westerlo plus explicitly marked background candidates for Postel-bos, -Lommel-heide and Kasterlee-bos. Background candidates can persist empty GRB -FeatureCollections for negative-tile training; normal reference AOIs still fail -when GRB returns no buildings. These are runtime artifacts only and are not -committed to Git. +`operator_samples_manifest.json` under `/app/storage/operator-data`. In +addition to the established positive and background AOIs, the registry contains +Beerse, Rijkevorsel, Hoogstraten and Vorselaar as focused small-building +training AOIs. Vosselaar and Grobbendonk are independent validation AOIs and +must not be exported into the training split. Background candidates can persist +empty GRB FeatureCollections for negative-tile training; normal reference AOIs +still fail when GRB returns no buildings. These are runtime artifacts only and +are not committed to Git. + +The current recommended local building model is +`geointel-building-yolov8s-smallbld-minpx3-img640-ft30-pt` with tile size +`512`, overlap `64` and confidence threshold `0.15`. Its SHA256 is +`a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1`. +The promotion evidence covers seven positive AOIs at QA match IoU `0.25` and +three pure-empty background AOIs. The model improves recall and persisted +false-negative counts, but has lower precision than the previous balanced +model; operators must review and persist QA/QC rather than treating detections +as ground truth. For model-quality calibration, run the confidence sweep wrapper: diff --git a/backend/app/schemas/detection.py b/backend/app/schemas/detection.py index 3c278ac4..fd8546ea 100644 --- a/backend/app/schemas/detection.py +++ b/backend/app/schemas/detection.py @@ -3,10 +3,12 @@ from __future__ import annotations from datetime import datetime from uuid import UUID -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class DetectionModelCapability(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + model_id: str display_name: str framework: str @@ -23,6 +25,8 @@ class DetectionModelsResponse(BaseModel): class ModelAssetRead(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + model_asset_id: str filename: str display_name: str @@ -39,12 +43,16 @@ class ModelAssetRead(BaseModel): class ModelAssetListResponse(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + items: list[ModelAssetRead] total: int model_directory: str class DetectionRunRequest(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + project_id: UUID dataset_id: UUID model_id: str @@ -63,6 +71,8 @@ class DetectionQaRequest(BaseModel): class DetectionRunResponse(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + analysis_run_id: UUID job_id: UUID project_id: UUID @@ -75,6 +85,8 @@ class DetectionRunResponse(BaseModel): class DetectionRunRead(BaseModel): + model_config = ConfigDict(from_attributes=True, protected_namespaces=()) + id: UUID project_id: UUID dataset_id: UUID | None = None @@ -90,8 +102,6 @@ class DetectionRunRead(BaseModel): started_at: datetime | None = None finished_at: datetime | None = None - model_config = {"from_attributes": True} - class DetectionRunListResponse(BaseModel): items: list[DetectionRunRead] @@ -99,6 +109,8 @@ class DetectionRunListResponse(BaseModel): class DetectionRead(BaseModel): + model_config = ConfigDict(from_attributes=True, protected_namespaces=()) + id: UUID project_id: UUID dataset_id: UUID | None = None @@ -113,8 +125,6 @@ class DetectionRead(BaseModel): properties_json: dict | None = None created_at: datetime | None = None - model_config = {"from_attributes": True} - class DetectionListResponse(BaseModel): items: list[DetectionRead] diff --git a/backend/app/schemas/segmentation.py b/backend/app/schemas/segmentation.py index 8f66a6e2..5098c251 100644 --- a/backend/app/schemas/segmentation.py +++ b/backend/app/schemas/segmentation.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime from uuid import UUID -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from app.schemas.detection import DetectionModelCapability @@ -16,6 +16,8 @@ class SegmentationModelsResponse(BaseModel): class SegmentationRunRequest(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + project_id: UUID dataset_id: UUID model_id: str @@ -33,6 +35,8 @@ class SegmentationQaRequest(BaseModel): class SegmentationRunResponse(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + analysis_run_id: UUID job_id: UUID project_id: UUID @@ -45,6 +49,8 @@ class SegmentationRunResponse(BaseModel): class SegmentationRunRead(BaseModel): + model_config = ConfigDict(from_attributes=True, protected_namespaces=()) + id: UUID project_id: UUID dataset_id: UUID | None = None @@ -60,8 +66,6 @@ class SegmentationRunRead(BaseModel): started_at: datetime | None = None finished_at: datetime | None = None - model_config = {"from_attributes": True} - class SegmentationRunListResponse(BaseModel): items: list[SegmentationRunRead] @@ -69,6 +73,8 @@ class SegmentationRunListResponse(BaseModel): class SegmentationRead(BaseModel): + model_config = ConfigDict(from_attributes=True, protected_namespaces=()) + id: UUID project_id: UUID dataset_id: UUID | None = None @@ -87,8 +93,6 @@ class SegmentationRead(BaseModel): provenance_json: dict | None = None created_at: datetime | None = None - model_config = {"from_attributes": True} - class SegmentationListResponse(BaseModel): items: list[SegmentationRead] diff --git a/backend/tests/test_docker_runtime_config.py b/backend/tests/test_docker_runtime_config.py index 878def58..7206ecac 100644 --- a/backend/tests/test_docker_runtime_config.py +++ b/backend/tests/test_docker_runtime_config.py @@ -58,14 +58,33 @@ def test_all_in_one_dockerfile_can_opt_into_ai_dependencies_without_base_install def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None: dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") - assert "COPY scripts/prepare_operator_real_data_samples.py /app/scripts/prepare_operator_real_data_samples.py" in dockerfile - assert "COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py" in dockerfile - assert "COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py" in dockerfile - assert ( - "COPY scripts/render_operator_yolo_label_qa_contact_sheets.py " - "/app/scripts/render_operator_yolo_label_qa_contact_sheets.py" - ) in dockerfile - assert "COPY scripts/train_operator_yolo_detector.sh /app/scripts/train_operator_yolo_detector.sh" in dockerfile + for line in dockerfile.splitlines(): + if line.startswith("COPY scripts/"): + source_path = line.split()[1] + assert (ROOT / source_path).is_file() + + required_runtime_scripts = { + "prepare_operator_real_data_samples.py", + "export_operator_yolo_tile_dataset.py", + "audit_operator_yolo_dataset_quality.py", + "render_operator_yolo_label_qa_contact_sheets.py", + "train_operator_yolo_detector.sh", + "verify_real_data_detection_qa_workflow.sh", + "run_detection_quality_matrix.sh", + "run_multi_sample_detection_quality_matrix.sh", + "export_detection_calibration_evidence.sh", + "assemble_detection_calibration_evidence_portfolio.sh", + "build_fixed_threshold_evidence_portfolio_inputs.py", + "audit_detection_false_negative_evidence.py", + "run_operator_hard_negative_detection_matrix.sh", + "run_background_corpus_split_matrix.sh", + "build_background_corpus_split_report.py", + "build_detection_model_promotion_report.py", + "run_split_background_promotion_workflow.sh", + "activate_promoted_yolo_candidate.py", + } + for script_name in required_runtime_scripts: + assert f"COPY scripts/{script_name} /app/scripts/{script_name}" in dockerfile def test_all_in_one_dockerfile_copies_operator_scripts_after_dependency_install() -> None: diff --git a/backend/tests/test_schema_model_field_warnings.py b/backend/tests/test_schema_model_field_warnings.py new file mode 100644 index 00000000..f3dae7a5 --- /dev/null +++ b/backend/tests/test_schema_model_field_warnings.py @@ -0,0 +1,17 @@ +from pydantic import BaseModel + +from app.schemas import detection, segmentation + + +def test_model_prefixed_api_fields_are_explicitly_supported() -> None: + schemas = [ + value + for module in (detection, segmentation) + for value in vars(module).values() + if isinstance(value, type) + and issubclass(value, BaseModel) + and any(field_name.startswith("model_") for field_name in value.model_fields) + ] + + assert schemas + assert all(schema.model_config.get("protected_namespaces") == () for schema in schemas) diff --git a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py index 4651b293..eeb8c572 100644 --- a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py +++ b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py @@ -69,6 +69,7 @@ def test_operator_yolo_tile_dataset_export_help_does_not_require_gis_dependencie assert "Export operator real-data samples to a tile-level YOLO detection dataset" in result.stdout assert "--tile-size" in result.stdout assert "--stride" in result.stdout + assert "--samples" in result.stdout assert "--negative-keep-ratio" in result.stdout assert "--min-label-visible-ratio" in result.stdout assert "--background-negative-repeat" in result.stdout @@ -84,10 +85,19 @@ def test_default_validation_split_is_explicit_and_rejects_holdout_leakage() -> N {"sample_slug": "retie", "recommended_split": "val"}, {"sample_slug": "westerlo", "recommended_split": "val"}, {"sample_slug": "arendonk_heide", "recommended_split": "val"}, + {"sample_slug": "vosselaar_center", "recommended_split": "val"}, + {"sample_slug": "grobbendonk_center", "recommended_split": "val"}, ] assert module.DEFAULT_VALIDATION_SAMPLE_SLUGS == frozenset( - {"turnhout", "retie", "westerlo", "arendonk_heide"} + { + "turnhout", + "retie", + "westerlo", + "arendonk_heide", + "vosselaar_center", + "grobbendonk_center", + } ) assert module.validate_validation_split( samples, @@ -100,6 +110,35 @@ def test_default_validation_split_is_explicit_and_rejects_holdout_leakage() -> N module.validate_validation_split(samples, {"turnhout", "missing"}) +def test_manifest_sample_selection_keeps_external_holdouts_out_of_targeted_dataset() -> None: + module = load_tile_exporter() + samples = [ + {"sample_slug": "geel", "recommended_split": "train"}, + {"sample_slug": "beerse_center", "recommended_split": "train"}, + {"sample_slug": "vosselaar_center", "recommended_split": "val"}, + {"sample_slug": "turnhout", "recommended_split": "val"}, + {"sample_slug": "retie", "recommended_split": "val"}, + {"sample_slug": "westerlo", "recommended_split": "val"}, + ] + + selected, excluded = module.select_manifest_samples( + samples, + {"geel", "beerse_center", "vosselaar_center"}, + ) + + assert [sample["sample_slug"] for sample in selected] == [ + "geel", + "beerse_center", + "vosselaar_center", + ] + assert excluded == ["retie", "turnhout", "westerlo"] + assert module.validate_validation_split(selected, {"vosselaar_center"}) == { + "vosselaar_center" + } + with pytest.raises(SystemExit, match="unknown samples"): + module.select_manifest_samples(samples, {"geel", "missing"}) + + def test_validation_coverage_reports_holdouts_without_retained_tiles() -> None: module = load_tile_exporter() coverage = module.validation_sample_coverage( diff --git a/backend/tests/test_sprint131_operator_sample_expansion.py b/backend/tests/test_sprint131_operator_sample_expansion.py index b245b750..fbb80d0f 100644 --- a/backend/tests/test_sprint131_operator_sample_expansion.py +++ b/backend/tests/test_sprint131_operator_sample_expansion.py @@ -51,7 +51,7 @@ def test_operator_training_expansion_preserves_geographically_separate_holdouts( expected_holdouts = {"turnhout", "retie", "westerlo", "arendonk_heide"} assert module.TRAINING_EXPANSION_SAMPLE_SLUGS == frozenset(expected_expansion) - assert module.DEFAULT_VALIDATION_SAMPLE_SLUGS == frozenset(expected_holdouts) + assert expected_holdouts.issubset(module.DEFAULT_VALIDATION_SAMPLE_SLUGS) assert all(module.SAMPLES[slug].sample_role == "reference" for slug in expected_expansion) assert all(not module.SAMPLES[slug].allow_empty_reference for slug in expected_expansion) assert all(module.recommended_split_for_sample(module.SAMPLES[slug]) == "train" for slug in expected_expansion) @@ -78,6 +78,51 @@ def test_operator_training_expansion_preserves_geographically_separate_holdouts( ) >= 2_000 +def test_small_building_expansion_has_separate_training_and_validation_centers() -> None: + module = load_sample_preparer() + + expected_training = { + "beerse_center", + "rijkevorsel_center", + "hoogstraten_center", + "vorselaar_center", + } + expected_validation = {"vosselaar_center", "grobbendonk_center"} + + assert module.SMALL_BUILDING_TRAINING_SAMPLE_SLUGS == frozenset(expected_training) + assert module.SMALL_BUILDING_VALIDATION_SAMPLE_SLUGS == frozenset(expected_validation) + assert expected_validation.issubset(module.DEFAULT_VALIDATION_SAMPLE_SLUGS) + assert all(module.SAMPLES[slug].sample_role == "reference" for slug in expected_training | expected_validation) + assert all( + module.recommended_split_for_sample(module.SAMPLES[slug]) == "train" + for slug in expected_training + ) + assert all( + module.recommended_split_for_sample(module.SAMPLES[slug]) == "val" + for slug in expected_validation + ) + + def distance_m(left, right) -> float: + radius_m = 6_371_008.8 + left_lat = math.radians(left.center_lat) + right_lat = math.radians(right.center_lat) + delta_lat = right_lat - left_lat + delta_lon = math.radians(right.center_lon - left.center_lon) + haversine = ( + math.sin(delta_lat / 2) ** 2 + + math.cos(left_lat) * math.cos(right_lat) * math.sin(delta_lon / 2) ** 2 + ) + return 2 * radius_m * math.asin(math.sqrt(haversine)) + + protected_holdouts = expected_validation | {"turnhout", "retie", "westerlo"} + for training_slug in expected_training: + training_sample = module.SAMPLES[training_slug] + assert min( + distance_m(training_sample, module.SAMPLES[holdout_slug]) + for holdout_slug in protected_holdouts + ) >= 2_000 + + def test_operator_background_candidates_are_unique_enough_for_hard_negative_training() -> None: module = load_sample_preparer() diff --git a/backend/tests/test_sprint155_detection_operator_profiles.py b/backend/tests/test_sprint155_detection_operator_profiles.py index 1ce30630..ede00af2 100644 --- a/backend/tests/test_sprint155_detection_operator_profiles.py +++ b/backend/tests/test_sprint155_detection_operator_profiles.py @@ -9,8 +9,10 @@ def test_detection_operator_profiles_define_explicit_yolo_candidates_and_promote source = profiles.read_text(encoding="utf-8") assert "DETECTION_OPERATOR_PROFILES" in source + assert "geointel-building-yolov8s-smallbld-minpx3-img640-ft30-pt" in source assert "geointel-building-yolov8s-aoi1024expandedminpx4vis035e50-pt" in source assert "geointel-building-yolov8s-aoi1024bg512r3e50-pt" in source + assert "small-building-balanced-review" in source assert "expanded-balanced-review" in source assert "conservative-review" in source assert "confidenceThreshold: 0.15" in source @@ -18,10 +20,12 @@ def test_detection_operator_profiles_define_explicit_yolo_candidates_and_promote assert "defaultApproved: true" in source assert "promotionRecommendation: 'promote_candidate'" in source assert "positiveSampleCount: 7" in source + assert "f1: 0.5824578631584316" in source assert "f1: 0.5432865390636915" in source assert "maxBackgroundDetections: 0" in source assert "pure-empty gate passed" in source - assert "persistent small-building misses" in source + assert "1,571 fewer false negatives" in source + assert "higher false-positive review load" in source def test_detection_lab_surfaces_profiles_as_deliberate_operator_actions() -> None: diff --git a/backend/tests/test_sprint170_detection_false_negative_audit.py b/backend/tests/test_sprint170_detection_false_negative_audit.py index 6817c9ef..283e1592 100644 --- a/backend/tests/test_sprint170_detection_false_negative_audit.py +++ b/backend/tests/test_sprint170_detection_false_negative_audit.py @@ -199,6 +199,28 @@ def test_false_negative_audit_finds_persistent_reference_misses(tmp_path: Path) assert candidate["false_negative_count"] == 1 assert candidate["false_negative_rate"] == 1 / 3 assert active["false_negative_area_m2"]["median"] > 0 + assert sample["persistent_false_negative_area_m2"]["count"] == 1 + assert sample["persistent_false_negative_area_m2"]["median"] > 0 + assert sum( + bucket["count"] for bucket in sample["persistent_area_buckets"].values() + ) == 1 + assert sum( + bucket["share"] for bucket in sample["persistent_area_buckets"].values() + ) == 1.0 + persistent_evidence = json.loads( + (output_dir / "persistent_false_negatives.geojson").read_text(encoding="utf-8") + ) + assert persistent_evidence["type"] == "FeatureCollection" + assert len(persistent_evidence["features"]) == 1 + persistent_feature = persistent_evidence["features"][0] + assert persistent_feature["properties"]["qa_evidence_role"] == "persistent_false_negative" + assert persistent_feature["properties"]["sample_slug"] == "geel" + assert persistent_feature["properties"]["persistent_reference_id"] == "source:persistent-small" + assert persistent_feature["properties"]["area_m2"] > 0 + assert persistent_feature["properties"]["area_bucket"] in sample["persistent_area_buckets"] + assert report["persistent_evidence_geojson_path"] == str( + output_dir / "persistent_false_negatives.geojson" + ) assert report["recommendations"] assert (output_dir / "detection_false_negative_audit.md").is_file() diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 2ff176e7..2573facd 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -75,12 +75,34 @@ COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_y COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py COPY scripts/train_operator_yolo_detector.sh /app/scripts/train_operator_yolo_detector.sh +COPY scripts/verify_real_data_detection_qa_workflow.sh /app/scripts/verify_real_data_detection_qa_workflow.sh +COPY scripts/run_detection_quality_matrix.sh /app/scripts/run_detection_quality_matrix.sh +COPY scripts/run_multi_sample_detection_quality_matrix.sh /app/scripts/run_multi_sample_detection_quality_matrix.sh +COPY scripts/export_detection_calibration_evidence.sh /app/scripts/export_detection_calibration_evidence.sh +COPY scripts/assemble_detection_calibration_evidence_portfolio.sh /app/scripts/assemble_detection_calibration_evidence_portfolio.sh +COPY scripts/build_fixed_threshold_evidence_portfolio_inputs.py /app/scripts/build_fixed_threshold_evidence_portfolio_inputs.py +COPY scripts/audit_detection_false_negative_evidence.py /app/scripts/audit_detection_false_negative_evidence.py +COPY scripts/run_operator_hard_negative_detection_matrix.sh /app/scripts/run_operator_hard_negative_detection_matrix.sh +COPY scripts/run_background_corpus_split_matrix.sh /app/scripts/run_background_corpus_split_matrix.sh +COPY scripts/build_background_corpus_split_report.py /app/scripts/build_background_corpus_split_report.py +COPY scripts/build_detection_model_promotion_report.py /app/scripts/build_detection_model_promotion_report.py +COPY scripts/run_split_background_promotion_workflow.sh /app/scripts/run_split_background_promotion_workflow.sh +COPY scripts/activate_promoted_yolo_candidate.py /app/scripts/activate_promoted_yolo_candidate.py COPY deploy/unraid/nginx-all-in-one.conf /etc/nginx/conf.d/default.conf COPY deploy/unraid/all-in-one-start.sh /usr/local/bin/geointel-all-in-one-start COPY --from=frontend-build /frontend/dist/ /usr/share/nginx/html/ RUN chmod +x /usr/local/bin/geointel-all-in-one-start \ - && chmod +x /app/scripts/train_operator_yolo_detector.sh + && chmod +x \ + /app/scripts/train_operator_yolo_detector.sh \ + /app/scripts/verify_real_data_detection_qa_workflow.sh \ + /app/scripts/run_detection_quality_matrix.sh \ + /app/scripts/run_multi_sample_detection_quality_matrix.sh \ + /app/scripts/export_detection_calibration_evidence.sh \ + /app/scripts/assemble_detection_calibration_evidence_portfolio.sh \ + /app/scripts/run_operator_hard_negative_detection_matrix.sh \ + /app/scripts/run_background_corpus_split_matrix.sh \ + /app/scripts/run_split_background_promotion_workflow.sh VOLUME ["/var/lib/postgresql/data", "/app/storage"] diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index b13304b3..c860b026 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -335,17 +335,22 @@ hard-negative gates, then run `sparse_building_context` as a separate review matrix. The first expanded local model improved dense AOI F1, but Kasterlee-bos false positives block default promotion. -The expanded-AOI local model asset, -`geointel-building-yolov8s-aoi1024expandedminpx4vis035e50-pt`, is the current +The focused small-building local model asset, +`geointel-building-yolov8s-smallbld-minpx3-img640-ft30-pt`, is the current recommended Detection Lab operator profile. Use tile size `512`, overlap `64` -and confidence threshold `0.15`. Persisted QA/QC across seven positive AOIs -measured mean precision `0.6471`, recall `0.4700` and F1 `0.5433`; the strict -three-sample pure-empty background gate produced zero detections. The previous -`geointel-building-yolov8s-aoi1024bg512r3e50-pt` model remains available as a -legacy conservative `0.35` review profile. Sparse-context detections remain -review-only evidence, not a default-promotion blocker. Persistent misses are -concentrated in small buildings, so every production-like run still requires -persisted QA/QC against suitable reference data. +and confidence threshold `0.15`. Persisted QA/QC at match IoU `0.25` across +seven positive AOIs measured mean precision `0.5898`, recall `0.5770` and F1 +`0.5825`; minimum per-AOI F1 was `0.5528`. The strict three-sample pure-empty +background gate produced zero detections. Compared with the previous balanced +profile, the same persisted reference populations contain 1,571 fewer false +negatives, including 745 fewer misses in the 25-100 m2 bucket and 181 fewer +below 25 m2. This recall gain increases the false-positive review load, so the +previous `geointel-building-yolov8s-aoi1024expandedminpx4vis035e50-pt` profile +remains available as a higher-precision legacy `0.15` choice. The older +`geointel-building-yolov8s-aoi1024bg512r3e50-pt` remains the conservative +`0.35` profile. Sparse-context detections remain review-only evidence, not a +default-promotion blocker. Every production-like run still requires persisted +QA/QC against suitable reference data. To update a Tower/Unraid `.env` from a promoted report, use the guarded activation helper. It validates the exact report candidate key, verifies that @@ -355,8 +360,8 @@ when `--apply` is supplied: ```bash python scripts/activate_promoted_yolo_candidate.py \ - --promotion-report artifacts/detection-model-promotion/split-aware/aoi1024expandedminpx4vis035e50-split/detection_model_promotion_report.json \ - --candidate-key 'geointel-building-yolov8s-aoi1024expandedminpx4vis035e50-pt|512|64|0.15' \ + --promotion-report storage/operator-data/model-review/small-building-candidate/promotion/detection_model_promotion_report.json \ + --candidate-key 'geointel-building-yolov8s-smallbld-minpx3-img640-ft30-pt|512|64|0.15' \ --models-dir /mnt/user/appdata/geointel/models \ --env-file /mnt/user/appdata/geointel/.env \ --json @@ -440,6 +445,13 @@ and records the positive/negative tile counts. This gives the training smoke more image samples while preserving the same explicit operator-data and QA/QC validation boundary. +Focused small-building experiments use Beerse, Rijkevorsel, Hoogstraten and +Vorselaar as training AOIs, with Vosselaar and Grobbendonk retained as +independent validation AOIs. The exporter accepts an explicit `--samples` +subset and records `source_manifest_sample_count`, `selected_sample_slugs` and +`excluded_sample_slugs` in its summary. Manifest-backed validation samples +cannot silently enter training. + For visual error inspection, export the persisted QA evidence from a calibration summary: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 51ff9d31..eeca2304 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -7071,3 +7071,68 @@ Open: - Embedded PostGIS live migration smoke passed with PostGIS `3.6`, required tables/indexes, database collation and the single Alembic head `202606120900`. - Browser validation confirmed that the recommended profile selects the exact active asset and threshold `0.15`, while live preflight displays CPU dependency/model readiness and the expected missing-manifest guard before dataset handoff. - Browser console warnings/errors: `0`. + +# Sprint 174 - Focused small-building recovery and promotion + +## Data and training evidence + +- Converted the Sprint 173 persistent false-negative audit into one focused real-data experiment instead of extending the same corpus blindly. +- Added Beerse, Rijkevorsel, Hoogstraten and Vorselaar as training AOIs and Vosselaar/Grobbendonk as independent tile-level validation AOIs. +- Kept Turnhout, Retie and Westerlo outside the tile corpus as operation-level holdouts. +- Exported `/app/storage/operator-data/yolo-building-aoi1024-smallbld-minpx3vis035` from an explicit 23-sample manifest subset: + - 198 retained tiles; + - 180 positive and 18 negative tiles; + - 58,820 real GRB-derived labels; + - 48 visually reviewed tiles; + - zero invalid labels, missing images, missing label files or low-variance review selections. +- The accepted `min-label-px=3` corpus retained 1,228 more genuine small-building labels than the comparable `min-label-px=4` export. +- Trained one inactive 30-epoch CPU candidate from the previous active local model: + - model: `geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt`; + - model SHA256: `a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1`; + - dataset-summary SHA256: `49b2a07d2105d08356431757b83eafc1498eaf1fb76965b1efe05b776824942a`; + - dataset-YAML SHA256: `3a2ea97c35a18072a1ab6738cd673c0ecec5344b19461c91d72a15e138d46e8d`; + - no model download and no fake training or QA data. + +## Persisted promotion evidence + +- Evaluated the exact fixed profile `tile=512`, `overlap=64`, `confidence=0.15` with QA match IoU explicitly fixed at `0.25`. +- Seven positive AOIs produced: + - mean precision `0.5898197518`; + - mean recall `0.5769921004`; + - mean F1 `0.5824578632`; + - minimum per-AOI F1 `0.5527837436`. +- Every AOI improved F1 relative to the previous balanced model. Turnhout improved from `0.4897494305` to `0.5527837436`. +- The strict pure-empty gate covered Postel, Lommel and Arendonk and produced zero detections for every sample. +- The formal promotion report recommended the exact key `geointel-building-yolov8s-smallbld-minpx3-img640-ft30-pt|512|64|0.15`. +- Fixed-reference object evidence used identical GRB feature identities and reduced false negatives from `7,753` to `6,182`: + - 1,571 fewer total false negatives; + - 745 fewer misses in the 25-100 m2 bucket; + - 181 fewer misses below 25 m2; + - all seven AOIs improved. +- Remaining persistent misses total `5,838`, concentrated in Turnhout, Herentals and Geel and still dominated by small buildings. +- Mean precision decreased from `0.6470590036` to `0.5898197518`. The new profile is therefore a recall-balanced operator default with a higher false-positive review load, not ground truth. + +## Repository hardening + +- Added explicit `--samples` / `OPERATOR_YOLO_SAMPLES` corpus selection with selected/excluded sample provenance and unknown-sample rejection. +- Added persistent false-negative area statistics, size buckets and combined GeoJSON review evidence. +- Copied the complete operator evaluation/promotion toolchain into the all-in-one image and added a regression that rejects every Docker `COPY scripts/...` source that does not exist. +- Removed Pydantic protected-namespace warnings for legitimate `model_*` API fields while preserving all schema field names and response contracts. +- Updated Detection Lab profiles: the new small-building profile is recommended, the previous expanded profile remains the higher-precision legacy choice, and the background-aware `0.35` profile remains conservative. +- Guarded activation first returned `ready_to_apply`; the reviewed `--apply` pass updated only `GEOINTEL_INSTALL_AI`, `YOLO_ENABLED`, `YOLO_MODELS_DIR` and `YOLO_MODEL_PATH` in the Tower environment. + +## Local validation + +- `python -m compileall backend/app`: passed. +- `python -m pytest`: 472 passed. +- `python -m ruff check` for all changed Python modules/tests: passed. +- `npm run typecheck`: passed. +- `npm run build`: passed; app bundle `215.64 kB`, MapLibre bundle `801.82 kB` before gzip. +- `bash scripts/run_readiness_check.sh`: passed with 472 tests. +- `python -m alembic heads`: one head, `202606120900`. +- `python -m alembic upgrade head --sql`: complete migration chain rendered successfully. +- Shell syntax checks passed for live migration and the full operator evaluation/promotion chain. + +## Next recommended pass + +- After redeploy, verify the active model SHA, local model-load preflight, live PostGIS migration smoke and browser profile selection. Then review false-positive evidence and the remaining 5,838 persistent misses before any further training. diff --git a/docs/TODO.md b/docs/TODO.md index 101259a3..181c7d60 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -137,7 +137,8 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add deterministic dataset/base/trained-model SHA256 provenance to future operator training summaries. - [x] Review per-AOI false-negative evidence, expand positive sample/label coverage and verify the resulting candidate improves false-negative rate in every validated AOI. - [x] Complete rebuild/restart and browser/runtime smoke for the guarded promoted V1 building detector activation. -- [ ] Expand focused small-building training evidence only after reviewing persistent false negatives from the promoted model; do not start another blind training run. +- [x] Expand focused small-building training evidence after reviewing persistent false negatives, train one inactive candidate and pass it through positive, pure-empty and fixed-reference promotion evidence before guarded activation. +- [ ] Review the remaining 5,838 persistent false negatives and the increased false-positive load before any further model training; do not start another blind run. ## Sprint 8 status @@ -494,7 +495,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Regenerate Tower AOI1024 operator samples with paged GRB references, then re-export and audit labels before any new training attempt. - [x] Improve AOI1024 label quality before retraining: `yolo-building-aoi1024-cleanpx12vis035` now audits `ok` with 14,632 labels, `min_label_px=12`, `min_label_visible_ratio=0.35`, median normalized box area `0.001373291015625` and small-box share `0.0`. - [x] Train and reject `geointel-building-yolov8s-aoi1024cleanpx12vis035e50-pt` through the positive/background promotion gate. -- [ ] Keep every local YOLO candidate inactive until positive-AOI and hard-negative promotion reports recommend default activation. +- [x] Keep every local YOLO candidate inactive until positive-AOI and hard-negative promotion reports recommend default activation. # Sprint 171 - Positive AOI expansion and small-building recovery @@ -505,4 +506,6 @@ This file now starts with the current implementation status. Older preparation/b - [x] Refresh the full AOI1024 operator manifest on Tower and fetch only missing AOIs. - [x] Export and audit a low-minimum-label dataset without changing the active model. - [x] Render and inspect a sample-balanced label contact sheet before training. -- [ ] Finish the inactive expanded-minpx4 candidate and run the full promotion gate. +- [x] Finish the inactive expanded-minpx4 candidate and run the full promotion gate. +- [x] Export the focused 23-sample minpx3 corpus with independent Vosselaar/Grobbendonk validation and external Turnhout/Retie/Westerlo holdouts. +- [x] Train, audit and guarded-activate the focused small-building candidate only after all persisted promotion gates passed. diff --git a/frontend/README.md b/frontend/README.md index 42b62a09..c7ff0dae 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -124,7 +124,7 @@ AI Lab run controls explicitly explain when no raster dataset is available, inst - Detection Lab now exposes the `yolo-configured` capability reported by the backend. - When `yolo-configured` is selected, users can provide an existing raster tile manifest path. - Detection Lab lists local model assets from `GET /api/v1/detection/model-assets` so operators can choose an existing mounted model file instead of editing only one hidden `YOLO_MODEL_PATH` slot. -- Detection Lab exposes explicit operator profiles for the local AOI1024 building detector: balanced review at threshold `0.15` remains candidate-only, while conservative review at threshold `0.35` is marked as the promoted profile after the split-background pure-empty gate passed. +- Detection Lab exposes explicit operator profiles for mounted local building detectors. The focused small-building model is the recommended recall-balanced `0.15` profile; the previous expanded-AOI `0.15` model remains available for higher precision, and the background-aware `0.35` model remains the conservative review choice. Applying a profile never downloads weights, changes runtime environment or starts inference automatically. - Applying a profile deliberately selects the local model asset and threshold for the browser-run request; runtime default activation remains a separate guarded `.env` operation through `scripts/activate_promoted_yolo_candidate.py`. - Detection Lab includes a read-only YOLO runtime preflight panel with backend status, dependency visibility, local model configuration, `torch`/`ultralytics` versions, CUDA state and `YOLO_CONFIG_DIR`. - The UI still does not download models or create fake detections; backend status and error codes remain the source of truth. diff --git a/frontend/src/components/detection/detectionProfiles.ts b/frontend/src/components/detection/detectionProfiles.ts index 93fd7ad6..5e77725b 100644 --- a/frontend/src/components/detection/detectionProfiles.ts +++ b/frontend/src/components/detection/detectionProfiles.ts @@ -15,9 +15,26 @@ export interface DetectionOperatorProfile { } export const DETECTION_OPERATOR_PROFILES: DetectionOperatorProfile[] = [ + { + id: 'small-building-balanced-review', + displayName: 'Recommended small-building review', + modelAssetId: 'geointel-building-yolov8s-smallbld-minpx3-img640-ft30-pt', + confidenceThreshold: 0.15, + defaultApproved: true, + promotionRecommendation: 'promote_candidate', + precision: 0.5898197517793451, + recall: 0.576992100419565, + f1: 0.5824578631584316, + positiveSampleCount: 7, + maxBackgroundDetections: 0, + description: + 'Recommended recall-balanced profile for building review, with improved small-building coverage across seven Kempen AOIs.', + limitationMessage: + 'The pure-empty gate passed and persisted QA found 1,571 fewer false negatives than the previous balanced profile; expect a higher false-positive review load.', + }, { id: 'expanded-balanced-review', - displayName: 'Recommended balanced review', + displayName: 'Legacy expanded balanced review', modelAssetId: 'geointel-building-yolov8s-aoi1024expandedminpx4vis035e50-pt', confidenceThreshold: 0.15, defaultApproved: true, @@ -27,9 +44,9 @@ export const DETECTION_OPERATOR_PROFILES: DetectionOperatorProfile[] = [ f1: 0.5432865390636915, positiveSampleCount: 7, maxBackgroundDetections: 0, - description: 'Recommended expanded-AOI profile for balanced building review across the validated Kempen samples.', + description: 'Legacy expanded-AOI profile for review sessions where precision matters more than the newest recall gain.', limitationMessage: - 'Default-approved after the pure-empty gate passed; persistent small-building misses still require operator QA.', + 'The pure-empty gate passed; this profile has fewer false positives but more persistent small-building misses than the recommended profile.', }, { id: 'conservative-review', diff --git a/scripts/README.md b/scripts/README.md index c9457056..86588764 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -531,8 +531,27 @@ Current Tower audit status: `0.000694274766`, small-box share `0.3832694151486098`, no invalid labels and no missing label files. The balanced visual pass rendered 40 tiles across all 19 source samples that retained at least one tile, with no invalid labels, - missing images or low-variance selections. A new candidate may be trained, - but remains inactive until positive and split-background promotion gates pass. + missing images or low-variance selections. Its promoted model remains the + higher-precision legacy `0.15` operator profile. +- `yolo-building-aoi1024-smallbld-minpx3vis035`: focused small-building corpus + exported from an explicit 23-sample subset. Beerse, Rijkevorsel, Hoogstraten + and Vorselaar extend training; Vosselaar and Grobbendonk are validation-only; + Turnhout, Retie and Westerlo remain external operation-level holdouts. The + Tower export retained 198 tiles and 58,820 labels. Its small-object-aware + audit passed with no invalid/missing labels, and the 48-tile balanced visual + review contained no missing, invalid or low-variance selections. The trained + `geointel-building-yolov8s-smallbld-minpx3-img640-ft30-pt` candidate passed + seven positive-AOI and three pure-empty background gates at tile `512`, + overlap `64`, threshold `0.15` and QA match IoU `0.25`. Mean F1 is `0.5825` + and all pure-empty samples remain at zero detections. Persisted comparison + found 1,571 fewer false negatives than the previous balanced model, with a + lower mean precision and therefore a higher operator review load. + +Use `--samples` or `OPERATOR_YOLO_SAMPLES` to make an experimental corpus +membership explicit. Dataset summaries preserve the complete manifest count, +selected sample slugs and excluded sample slugs. Split validation still applies +after filtering, so a manifest-backed holdout cannot be selected as training by +omitting it from `--val-samples`. After rebuilding the all-in-one image, the operator scripts are available inside the container at `/app/scripts/...`. Before rebuilding, use the host checkout or @@ -765,7 +784,10 @@ python scripts/audit_detection_false_negative_evidence.py \ ``` The audit reports false-negative rates and area buckets per AOI/model, plus -reference buildings missed by every compared portfolio. Stable +reference buildings missed by every compared portfolio. It writes the combined +`persistent_false_negatives.geojson`, records geodetic persistent-miss area and +adds persistent area buckets so operators can inspect the shared misses on a +map instead of relying only on counts. Stable `source_feature_id` values are preferred; a normalized geometry fingerprint is used only when source IDs are absent. Invalid or missing geometry fails the audit instead of being silently skipped. The tools do not run inference, diff --git a/scripts/audit_detection_false_negative_evidence.py b/scripts/audit_detection_false_negative_evidence.py index a5e656c5..973f2427 100644 --- a/scripts/audit_detection_false_negative_evidence.py +++ b/scripts/audit_detection_false_negative_evidence.py @@ -119,6 +119,7 @@ def audit_feature_collection(payload: dict[str, Any], geod: Any, shape: Any) -> reference_ids: set[str] = set() false_negative_areas: list[float] = [] matched_reference_areas: list[float] = [] + reference_records: dict[str, dict[str, Any]] = {} bucket_counts = { label: {"false_negative": 0, "matched_reference": 0, "total_reference": 0, "false_negative_rate": None} for label, _, _ in AREA_BUCKETS @@ -144,6 +145,11 @@ def audit_feature_collection(payload: dict[str, Any], geod: Any, shape: Any) -> bucket = area_bucket(area_m2) reference_id = stable_reference_id(feature, geometry) reference_ids.add(reference_id) + reference_records[reference_id] = { + "area_m2": area_m2, + "area_bucket": bucket, + "feature": feature, + } bucket_role = "false_negative" if role == "false_negative" else "matched_reference" bucket_counts[bucket][bucket_role] += 1 bucket_counts[bucket]["total_reference"] += 1 @@ -161,6 +167,7 @@ def audit_feature_collection(payload: dict[str, Any], geod: Any, shape: Any) -> return { "false_negative_ids": false_negative_ids, "reference_ids": reference_ids, + "reference_records": reference_records, "false_negative_count": len(false_negative_areas), "matched_reference_count": len(matched_reference_areas), "total_reference_count": total_reference, @@ -285,6 +292,7 @@ def run_audit(portfolio_args: list[str], output_dir: Path) -> tuple[Path, Path]: ) sample_reports: list[dict[str, Any]] = [] + persistent_evidence_features: list[dict[str, Any]] = [] for sample_slug in sorted(expected_slugs): portfolio_rows = [] false_negative_sets = [] @@ -297,7 +305,7 @@ def run_audit(portfolio_args: list[str], output_dir: Path) -> tuple[Path, Path]: { key: value for key, value in raw.items() - if key not in {"false_negative_ids", "reference_ids"} + if key not in {"false_negative_ids", "reference_ids", "reference_records"} } ) if any(reference_ids != reference_sets[0] for reference_ids in reference_sets[1:]): @@ -309,16 +317,61 @@ def run_audit(portfolio_args: list[str], output_dir: Path) -> tuple[Path, Path]: f"Sample {sample_slug} has different reference populations across portfolios ({counts})" ) persistent_ids = sorted(set.intersection(*false_negative_sets)) + reference_records = portfolio_samples[parsed_portfolios[0][0]][sample_slug]["reference_records"] + persistent_areas = [float(reference_records[reference_id]["area_m2"]) for reference_id in persistent_ids] + persistent_bucket_counts = { + label: {"count": 0, "share": 0.0} + for label, _, _ in AREA_BUCKETS + } + for reference_id in persistent_ids: + record = reference_records[reference_id] + persistent_bucket_counts[record["area_bucket"]]["count"] += 1 + source_feature = record["feature"] + properties = dict(source_feature.get("properties") or {}) + properties.update( + { + "qa_evidence_role": "persistent_false_negative", + "sample_slug": sample_slug, + "persistent_reference_id": reference_id, + "area_m2": record["area_m2"], + "area_bucket": record["area_bucket"], + "compared_portfolios": labels, + } + ) + persistent_evidence_features.append( + { + "type": "Feature", + "id": f"{sample_slug}:{reference_id}", + "properties": properties, + "geometry": source_feature["geometry"], + } + ) + if persistent_ids: + for values in persistent_bucket_counts.values(): + values["share"] = values["count"] / len(persistent_ids) sample_reports.append( { "sample_slug": sample_slug, "reference_population_count": len(reference_sets[0]), "persistent_false_negative_count": len(persistent_ids), "persistent_reference_ids": persistent_ids, + "persistent_false_negative_area_m2": area_stats(persistent_areas), + "persistent_area_buckets": persistent_bucket_counts, "portfolios": portfolio_rows, } ) + output_dir = output_dir.expanduser().resolve() + output_dir.mkdir(parents=True, exist_ok=True) + persistent_geojson_path = output_dir / "persistent_false_negatives.geojson" + persistent_geojson_path.write_text( + json.dumps( + {"type": "FeatureCollection", "features": persistent_evidence_features}, + indent=2, + sort_keys=True, + ), + encoding="utf-8", + ) report = { "generated_at": datetime.now(timezone.utc).isoformat(), "schema_version": 1, @@ -328,10 +381,9 @@ def run_audit(portfolio_args: list[str], output_dir: Path) -> tuple[Path, Path]: "portfolios": portfolio_meta, "sample_count": len(sample_reports), "samples": sample_reports, + "persistent_evidence_geojson_path": str(persistent_geojson_path), "recommendations": build_recommendations(sample_reports), } - output_dir = output_dir.expanduser().resolve() - output_dir.mkdir(parents=True, exist_ok=True) json_path = output_dir / "detection_false_negative_audit.json" json_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") @@ -345,18 +397,26 @@ def run_audit(portfolio_args: list[str], output_dir: Path) -> tuple[Path, Path]: "", "## AOI comparison", "", - "| AOI | Persistent misses | " + "| AOI | Persistent misses | Persistent tiny/small | Persistent median m2 | " + " | ".join(f"{label} FN rate" for label, _ in parsed_portfolios) + " |", - "|---|---:|" + "---:|" * len(parsed_portfolios), + "|---|---:|---:|---:|" + "---:|" * len(parsed_portfolios), ] for sample in sample_reports: rates = [ f"{(row['false_negative_rate'] or 0.0):.3f}" for row in sample["portfolios"] ] + persistent_buckets = sample["persistent_area_buckets"] + persistent_small_count = sum( + persistent_buckets[key]["count"] + for key in ("tiny_lt_25_m2", "small_25_100_m2") + ) + persistent_median = sample["persistent_false_negative_area_m2"]["median"] + persistent_median_text = f"{persistent_median:.1f}" if persistent_median is not None else "n/a" lines.append( f"| {sample['sample_slug']} | {sample['persistent_false_negative_count']} | " + f"{persistent_small_count} | {persistent_median_text} | " + " | ".join(rates) + " |" ) diff --git a/scripts/export_operator_yolo_tile_dataset.py b/scripts/export_operator_yolo_tile_dataset.py index 19ad6362..29da78d3 100644 --- a/scripts/export_operator_yolo_tile_dataset.py +++ b/scripts/export_operator_yolo_tile_dataset.py @@ -26,7 +26,14 @@ PURE_EMPTY_BACKGROUND_CATEGORY = "pure_empty_negative" SPARSE_BACKGROUND_CATEGORY = "sparse_building_context" LOW_VARIANCE_NEGATIVE_SKIP_REASON = "low_visual_variance_negative" DEFAULT_VALIDATION_SAMPLE_SLUGS = frozenset( - {"turnhout", "retie", "westerlo", "arendonk_heide"} + { + "turnhout", + "retie", + "westerlo", + "arendonk_heide", + "vosselaar_center", + "grobbendonk_center", + } ) DEFAULT_VALIDATION_SAMPLES = ",".join(sorted(DEFAULT_VALIDATION_SAMPLE_SLUGS)) rasterio: Any = None @@ -69,12 +76,20 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--tile-size", type=int, default=int(os.environ.get("OPERATOR_YOLO_TILE_SIZE", "256"))) parser.add_argument("--stride", type=int, default=int(os.environ.get("OPERATOR_YOLO_TILE_STRIDE", "128"))) + parser.add_argument( + "--samples", + default=os.environ.get("OPERATOR_YOLO_SAMPLES", ""), + help=( + "Optional comma/space separated manifest sample slugs to export. " + "An empty value keeps every manifest sample." + ), + ) parser.add_argument( "--val-samples", default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", DEFAULT_VALIDATION_SAMPLES), help=( "Comma/space separated sample slugs assigned to validation. " - "Defaults to the documented Turnhout, Retie, Westerlo and Arendonk-heide holdouts." + "Defaults to all documented validation samples." ), ) parser.add_argument( @@ -152,6 +167,31 @@ def split_slugs(raw: str) -> set[str]: return {value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()} +def select_manifest_samples( + samples: list[dict[str, Any]], + requested_slugs: set[str], +) -> tuple[list[dict[str, Any]], list[str]]: + manifest_slugs = { + str(sample.get("sample_slug") or "").strip().lower() + for sample in samples + if str(sample.get("sample_slug") or "").strip() + } + if not requested_slugs: + return samples, [] + unknown = requested_slugs - manifest_slugs + if unknown: + raise SystemExit( + "YOLO sample selection references unknown samples: " + ", ".join(sorted(unknown)) + ) + selected = [ + sample + for sample in samples + if str(sample.get("sample_slug") or "").strip().lower() in requested_slugs + ] + excluded = sorted(manifest_slugs - requested_slugs) + return selected, excluded + + def validate_validation_split(samples: list[dict[str, Any]], val_slugs: set[str]) -> set[str]: sample_slugs = { str(sample.get("sample_slug") or "").strip().lower() @@ -533,9 +573,13 @@ def main() -> int: ensure_yolo_directories(args.output_dir) manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig")) - samples = manifest.get("samples") or [] - if not samples: + manifest_samples = manifest.get("samples") or [] + if not manifest_samples: raise SystemExit("Operator sample manifest contains no samples") + samples, excluded_sample_slugs = select_manifest_samples( + manifest_samples, + split_slugs(args.samples), + ) val_slugs = validate_validation_split(samples, split_slugs(args.val_samples)) exported_tiles: list[dict[str, Any]] = [] for sample in samples: @@ -584,7 +628,12 @@ def main() -> int: "min_label_visible_ratio": args.min_label_visible_ratio, "drop_low_variance_negatives": args.drop_low_variance_negatives, "blank_range_threshold": args.blank_range_threshold, + "source_manifest_sample_count": len(manifest_samples), "source_sample_count": len(samples), + "selected_sample_slugs": sorted( + str(sample.get("sample_slug") or "").strip().lower() for sample in samples + ), + "excluded_sample_slugs": excluded_sample_slugs, "validation_sample_slugs": sorted(val_slugs), **validation_coverage, "tile_count": len(kept_tiles), diff --git a/scripts/prepare_operator_real_data_samples.py b/scripts/prepare_operator_real_data_samples.py index 0301af68..41cb951f 100644 --- a/scripts/prepare_operator_real_data_samples.py +++ b/scripts/prepare_operator_real_data_samples.py @@ -28,8 +28,20 @@ SPARSE_BACKGROUND_CATEGORY = "sparse_building_context" TRAINING_EXPANSION_SAMPLE_SLUGS = frozenset( {"olen_center", "lille_center", "oud_turnhout_center", "kasterlee_center"} ) +SMALL_BUILDING_TRAINING_SAMPLE_SLUGS = frozenset( + {"beerse_center", "rijkevorsel_center", "hoogstraten_center", "vorselaar_center"} +) +SMALL_BUILDING_VALIDATION_SAMPLE_SLUGS = frozenset( + {"vosselaar_center", "grobbendonk_center"} +) DEFAULT_VALIDATION_SAMPLE_SLUGS = frozenset( - {"turnhout", "retie", "westerlo", "arendonk_heide"} + { + "turnhout", + "retie", + "westerlo", + "arendonk_heide", + *SMALL_BUILDING_VALIDATION_SAMPLE_SLUGS, + } ) requests: Any = None rasterio: Any = None @@ -122,6 +134,42 @@ SAMPLES: dict[str, OperatorSample] = { center_lon=4.9678120, center_lat=51.2407915, ), + "beerse_center": OperatorSample( + slug="beerse_center", + display_name="Beerse center small-building training expansion", + center_lon=4.8534, + center_lat=51.3192, + ), + "rijkevorsel_center": OperatorSample( + slug="rijkevorsel_center", + display_name="Rijkevorsel center small-building training expansion", + center_lon=4.7604, + center_lat=51.3487, + ), + "hoogstraten_center": OperatorSample( + slug="hoogstraten_center", + display_name="Hoogstraten center small-building training expansion", + center_lon=4.7609, + center_lat=51.4002, + ), + "vorselaar_center": OperatorSample( + slug="vorselaar_center", + display_name="Vorselaar center small-building training expansion", + center_lon=4.7731, + center_lat=51.2020, + ), + "vosselaar_center": OperatorSample( + slug="vosselaar_center", + display_name="Vosselaar center small-building validation", + center_lon=4.8899, + center_lat=51.3095, + ), + "grobbendonk_center": OperatorSample( + slug="grobbendonk_center", + display_name="Grobbendonk center small-building validation", + center_lon=4.7358, + center_lat=51.1907, + ), "postel_bos": OperatorSample( slug="postel_bos", display_name="Postel forest background candidate",