audit: establish accuracy phase 1 baseline

This commit is contained in:
Jens
2026-08-01 18:57:38 +02:00
parent 0c019bb22f
commit 3d442ef43f
48 changed files with 15390 additions and 1 deletions
+3
View File
@@ -11,3 +11,6 @@ Dockerfile text eol=lf
*.ts text eol=lf *.ts text eol=lf
*.css text eol=lf *.css text eol=lf
*.json text eol=lf *.json text eol=lf
# Retained audit exports preserve source bytes, including tool-emitted whitespace.
artifacts/evidence/accuracy/** -whitespace
+7 -1
View File
@@ -18,7 +18,13 @@ build/
/frontend-src.tar.gz /frontend-src.tar.gz
# Large local data # Large local data
/artifacts/ /artifacts/*
!/artifacts/evidence/
/artifacts/evidence/*
!/artifacts/evidence/accuracy/
/artifacts/evidence/accuracy/*
!/artifacts/evidence/accuracy/P1/
!/artifacts/evidence/accuracy/P1/**
/.cache/ /.cache/
/datasets/raw/* /datasets/raw/*
/datasets/processed/* /datasets/processed/*
@@ -0,0 +1,5 @@
command=python -m alembic heads
exit_code=0
captured_at=2026-08-01T18:37:24.0020021+02:00
202607260001 (head)
@@ -0,0 +1,500 @@
-- command=python -m alembic upgrade head --sql
-- exit_code=0
-- captured_at=2026-08-01T18:37:24.0299035+02:00
BEGIN;
INFO [alembic.runtime.migration] Context impl PostgresqlImpl.
INFO [alembic.runtime.migration] Generating static SQL
INFO [alembic.runtime.migration] Will assume transactional DDL.
CREATE TABLE alembic_version (
version_num VARCHAR(32) NOT NULL,
CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)
);
INFO [alembic.runtime.migration] Running upgrade -> 202601110001, Initial PostGIS schema for Sprint 1 foundation.
-- Running upgrade -> 202601110001
CREATE EXTENSION IF NOT EXISTS postgis;
CREATE EXTENSION IF NOT EXISTS postgis_topology;
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE TABLE projects (
id UUID NOT NULL,
name TEXT NOT NULL,
description TEXT,
region TEXT DEFAULT 'Kempen' NOT NULL,
status TEXT DEFAULT 'active' NOT NULL,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
PRIMARY KEY (id)
);
CREATE TABLE areas (
id UUID NOT NULL,
project_id UUID NOT NULL,
name TEXT NOT NULL,
geometry geometry(MULTIPOLYGON,4326) NOT NULL,
original_crs TEXT,
area_m2 FLOAT,
bbox geometry(POLYGON,4326),
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
PRIMARY KEY (id),
FOREIGN KEY(project_id) REFERENCES projects (id) ON DELETE CASCADE
);
CREATE INDEX idx_areas_geometry ON areas USING gist (geometry);
CREATE INDEX idx_areas_bbox ON areas USING gist (bbox);
CREATE TABLE datasets (
id UUID NOT NULL,
project_id UUID NOT NULL,
area_id UUID,
name TEXT NOT NULL,
dataset_type TEXT NOT NULL,
source TEXT NOT NULL,
storage_path TEXT,
derived_from_dataset_id UUID,
crs TEXT,
bounds_json JSON,
resolution_json JSON,
bands_json JSON,
metadata_json JSON,
status TEXT DEFAULT 'created' NOT NULL,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
PRIMARY KEY (id),
FOREIGN KEY(project_id) REFERENCES projects (id) ON DELETE CASCADE,
FOREIGN KEY(area_id) REFERENCES areas (id) ON DELETE SET NULL,
FOREIGN KEY(derived_from_dataset_id) REFERENCES datasets (id) ON DELETE SET NULL
);
CREATE TABLE dataset_versions (
id UUID NOT NULL,
dataset_id UUID NOT NULL,
version INTEGER DEFAULT '1' NOT NULL,
storage_path TEXT,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
PRIMARY KEY (id),
FOREIGN KEY(dataset_id) REFERENCES datasets (id) ON DELETE CASCADE
);
CREATE TABLE analysis_runs (
id UUID NOT NULL,
project_id UUID NOT NULL,
area_id UUID,
analysis_type TEXT NOT NULL,
status TEXT NOT NULL,
parameters_json JSON NOT NULL,
started_at TIMESTAMP WITH TIME ZONE,
finished_at TIMESTAMP WITH TIME ZONE,
error_message TEXT,
PRIMARY KEY (id),
FOREIGN KEY(project_id) REFERENCES projects (id) ON DELETE CASCADE,
FOREIGN KEY(area_id) REFERENCES areas (id) ON DELETE SET NULL
);
CREATE TABLE exports (
id UUID NOT NULL,
project_id UUID NOT NULL,
analysis_run_id UUID,
export_type TEXT NOT NULL,
storage_path TEXT NOT NULL,
metadata_json JSON,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
PRIMARY KEY (id),
FOREIGN KEY(project_id) REFERENCES projects (id) ON DELETE CASCADE,
FOREIGN KEY(analysis_run_id) REFERENCES analysis_runs (id) ON DELETE SET NULL
);
CREATE INDEX ix_areas_geometry ON areas USING gist (geometry);
CREATE INDEX ix_areas_project_id ON areas (project_id);
CREATE INDEX ix_datasets_project_id ON datasets (project_id);
INSERT INTO alembic_version (version_num) VALUES ('202601110001') RETURNING alembic_version.version_num;
INFO [alembic.runtime.migration] Running upgrade 202601110001 -> 202601120001, Add dataset storage metadata columns.
-- Running upgrade 202601110001 -> 202601120001
ALTER TABLE datasets ADD COLUMN original_filename TEXT;
ALTER TABLE datasets ADD COLUMN stored_filename TEXT;
ALTER TABLE datasets ADD COLUMN content_type TEXT;
ALTER TABLE datasets ADD COLUMN size_bytes INTEGER;
ALTER TABLE datasets ADD COLUMN checksum_sha256 TEXT;
ALTER TABLE datasets ALTER COLUMN status SET DEFAULT 'uploaded';
UPDATE alembic_version SET version_num='202601120001' WHERE alembic_version.version_num = '202601110001';
INFO [alembic.runtime.migration] Running upgrade 202601120001 -> 20260611212435, Add lightweight job table for sprint-3 async architecture foundation.
-- Running upgrade 202601120001 -> 20260611212435
CREATE TABLE jobs (
id UUID NOT NULL,
job_type TEXT NOT NULL,
status TEXT DEFAULT 'queued' NOT NULL,
project_id UUID NOT NULL,
dataset_id UUID,
input_dataset_id UUID,
output_dataset_id UUID,
parameters_json JSON NOT NULL,
result_json JSON,
error_message TEXT,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
started_at TIMESTAMP WITH TIME ZONE,
finished_at TIMESTAMP WITH TIME ZONE,
PRIMARY KEY (id),
FOREIGN KEY(project_id) REFERENCES projects (id) ON DELETE CASCADE,
FOREIGN KEY(dataset_id) REFERENCES datasets (id) ON DELETE SET NULL,
FOREIGN KEY(input_dataset_id) REFERENCES datasets (id) ON DELETE SET NULL,
FOREIGN KEY(output_dataset_id) REFERENCES datasets (id) ON DELETE SET NULL
);
CREATE INDEX ix_jobs_project_id ON jobs (project_id);
CREATE INDEX ix_jobs_status ON jobs (status);
UPDATE alembic_version SET version_num='20260611212435' WHERE alembic_version.version_num = '202601120001';
INFO [alembic.runtime.migration] Running upgrade 20260611212435 -> 202606120001, Add dataset reference and provenance metadata columns.
-- Running upgrade 20260611212435 -> 202606120001
ALTER TABLE datasets ADD COLUMN dataset_role TEXT DEFAULT 'source' NOT NULL;
ALTER TABLE datasets ADD COLUMN source_name TEXT;
ALTER TABLE datasets ADD COLUMN reference_layer_name TEXT;
ALTER TABLE datasets ADD COLUMN source_metadata JSON;
ALTER TABLE datasets ADD COLUMN provenance_metadata JSON;
ALTER TABLE datasets ADD COLUMN imported_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL;
UPDATE alembic_version SET version_num='202606120001' WHERE alembic_version.version_num = '20260611212435';
INFO [alembic.runtime.migration] Running upgrade 202606120001 -> 202606120700, Add Sprint 7A vector feature and QA persistence foundation.
-- Running upgrade 202606120001 -> 202606120700
CREATE TABLE vector_features (
id UUID NOT NULL,
dataset_id UUID NOT NULL,
feature_class TEXT,
source_feature_id TEXT,
properties_json JSON,
geometry geometry(GEOMETRY,4326) NOT NULL,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
PRIMARY KEY (id),
FOREIGN KEY(dataset_id) REFERENCES datasets (id) ON DELETE CASCADE
);
CREATE INDEX ix_vector_features_dataset_id ON vector_features (dataset_id);
CREATE INDEX ix_vector_features_geometry ON vector_features USING gist (geometry);
CREATE TABLE quality_checks (
id UUID NOT NULL,
project_id UUID NOT NULL,
job_id UUID,
analysis_run_id UUID,
candidate_dataset_id UUID,
reference_dataset_id UUID NOT NULL,
check_type TEXT NOT NULL,
status TEXT NOT NULL,
score FLOAT,
parameters_json JSON,
findings_json JSON,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
completed_at TIMESTAMP WITH TIME ZONE,
PRIMARY KEY (id),
FOREIGN KEY(project_id) REFERENCES projects (id) ON DELETE CASCADE,
FOREIGN KEY(job_id) REFERENCES jobs (id) ON DELETE SET NULL,
FOREIGN KEY(analysis_run_id) REFERENCES analysis_runs (id) ON DELETE SET NULL,
FOREIGN KEY(candidate_dataset_id) REFERENCES datasets (id) ON DELETE SET NULL,
FOREIGN KEY(reference_dataset_id) REFERENCES datasets (id) ON DELETE CASCADE
);
CREATE INDEX ix_quality_checks_project_id ON quality_checks (project_id);
CREATE INDEX ix_quality_checks_reference_dataset_id ON quality_checks (reference_dataset_id);
CREATE INDEX ix_quality_checks_candidate_dataset_id ON quality_checks (candidate_dataset_id);
CREATE INDEX ix_quality_checks_analysis_run_id ON quality_checks (analysis_run_id);
CREATE TABLE metrics (
id UUID NOT NULL,
quality_check_id UUID,
analysis_run_id UUID,
metric_key TEXT NOT NULL,
metric_value FLOAT,
metric_unit TEXT,
label TEXT,
metadata_json JSON,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
PRIMARY KEY (id),
FOREIGN KEY(quality_check_id) REFERENCES quality_checks (id) ON DELETE CASCADE,
FOREIGN KEY(analysis_run_id) REFERENCES analysis_runs (id) ON DELETE SET NULL
);
CREATE INDEX ix_metrics_quality_check_id ON metrics (quality_check_id);
CREATE INDEX ix_metrics_analysis_run_id ON metrics (analysis_run_id);
UPDATE alembic_version SET version_num='202606120700' WHERE alembic_version.version_num = '202606120001';
INFO [alembic.runtime.migration] Running upgrade 202606120700 -> 202606120800, Add Sprint 8 detection foundation.
-- Running upgrade 202606120700 -> 202606120800
ALTER TABLE analysis_runs ADD COLUMN dataset_id UUID;
ALTER TABLE analysis_runs ADD FOREIGN KEY(dataset_id) REFERENCES datasets (id) ON DELETE SET NULL;
ALTER TABLE analysis_runs ADD COLUMN job_id UUID;
ALTER TABLE analysis_runs ADD FOREIGN KEY(job_id) REFERENCES jobs (id) ON DELETE SET NULL;
ALTER TABLE analysis_runs ADD COLUMN model_name VARCHAR(255);
ALTER TABLE analysis_runs ADD COLUMN model_version VARCHAR(120);
ALTER TABLE analysis_runs ADD COLUMN result_json JSON;
ALTER TABLE analysis_runs ADD COLUMN created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL;
CREATE TABLE detections (
id UUID NOT NULL,
project_id UUID NOT NULL,
dataset_id UUID,
analysis_run_id UUID,
job_id UUID,
model_name VARCHAR(255) NOT NULL,
model_version VARCHAR(120),
class_name VARCHAR(120) NOT NULL,
confidence FLOAT NOT NULL,
geometry geometry(GEOMETRY,4326) NOT NULL,
bbox_json JSON,
source_tile_path VARCHAR(500),
properties_json JSON,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
PRIMARY KEY (id),
FOREIGN KEY(project_id) REFERENCES projects (id) ON DELETE CASCADE,
FOREIGN KEY(dataset_id) REFERENCES datasets (id) ON DELETE SET NULL,
FOREIGN KEY(analysis_run_id) REFERENCES analysis_runs (id) ON DELETE SET NULL,
FOREIGN KEY(job_id) REFERENCES jobs (id) ON DELETE SET NULL
);
CREATE INDEX ix_detections_project_id ON detections (project_id);
CREATE INDEX ix_detections_dataset_id ON detections (dataset_id);
CREATE INDEX ix_detections_analysis_run_id ON detections (analysis_run_id);
CREATE INDEX ix_detections_class_name ON detections (class_name);
CREATE INDEX ix_detections_geometry ON detections USING gist (geometry);
UPDATE alembic_version SET version_num='202606120800' WHERE alembic_version.version_num = '202606120700';
INFO [alembic.runtime.migration] Running upgrade 202606120800 -> 202606120900, Add Sprint 9 segmentation foundation.
-- Running upgrade 202606120800 -> 202606120900
CREATE TABLE segmentations (
id UUID NOT NULL,
project_id UUID NOT NULL,
dataset_id UUID,
job_id UUID,
analysis_run_id UUID,
model_name VARCHAR(255) NOT NULL,
model_version VARCHAR(120),
class_name VARCHAR(120) NOT NULL,
confidence FLOAT,
geometry geometry(MULTIPOLYGON,4326) NOT NULL,
bbox_json JSON,
area_m2 FLOAT,
mask_path TEXT,
source_tile_path VARCHAR(500),
tile_index INTEGER,
properties_json JSON,
provenance_json JSON,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
PRIMARY KEY (id),
FOREIGN KEY(project_id) REFERENCES projects (id) ON DELETE CASCADE,
FOREIGN KEY(dataset_id) REFERENCES datasets (id) ON DELETE SET NULL,
FOREIGN KEY(job_id) REFERENCES jobs (id) ON DELETE SET NULL,
FOREIGN KEY(analysis_run_id) REFERENCES analysis_runs (id) ON DELETE SET NULL
);
CREATE INDEX ix_segmentations_project_id ON segmentations (project_id);
CREATE INDEX ix_segmentations_dataset_id ON segmentations (dataset_id);
CREATE INDEX ix_segmentations_analysis_run_id ON segmentations (analysis_run_id);
CREATE INDEX ix_segmentations_job_id ON segmentations (job_id);
CREATE INDEX ix_segmentations_class_name ON segmentations (class_name);
CREATE INDEX ix_segmentations_geometry ON segmentations USING gist (geometry);
UPDATE alembic_version SET version_num='202606120900' WHERE alembic_version.version_num = '202606120800';
INFO [alembic.runtime.migration] Running upgrade 202606120900 -> 202607140001, Add temporal dataset metadata and durable dataset-version provenance.
-- Running upgrade 202606120900 -> 202607140001
ALTER TABLE datasets ADD COLUMN temporal_series_key VARCHAR(255);
ALTER TABLE datasets ADD COLUMN observed_at TIMESTAMP WITH TIME ZONE;
ALTER TABLE datasets ADD COLUMN valid_from TIMESTAMP WITH TIME ZONE;
ALTER TABLE datasets ADD COLUMN valid_to TIMESTAMP WITH TIME ZONE;
ALTER TABLE datasets ADD COLUMN temporal_granularity VARCHAR(32);
ALTER TABLE datasets ADD COLUMN source_version VARCHAR(120);
ALTER TABLE dataset_versions ADD COLUMN source_version VARCHAR(120);
ALTER TABLE dataset_versions ADD COLUMN observed_at TIMESTAMP WITH TIME ZONE;
ALTER TABLE dataset_versions ADD COLUMN valid_from TIMESTAMP WITH TIME ZONE;
ALTER TABLE dataset_versions ADD COLUMN valid_to TIMESTAMP WITH TIME ZONE;
ALTER TABLE dataset_versions ADD COLUMN checksum_sha256 VARCHAR(64);
ALTER TABLE dataset_versions ADD COLUMN source_metadata JSON;
ALTER TABLE dataset_versions ADD COLUMN provenance_metadata JSON;
CREATE INDEX ix_datasets_project_temporal_series_observed ON datasets (project_id, temporal_series_key, observed_at);
CREATE UNIQUE INDEX ix_dataset_versions_dataset_version ON dataset_versions (dataset_id, version);
CREATE INDEX ix_vector_features_dataset_source_feature ON vector_features (dataset_id, source_feature_id);
ALTER TABLE datasets ADD CONSTRAINT ck_datasets_temporal_valid_range CHECK (valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from);
ALTER TABLE dataset_versions ADD CONSTRAINT ck_dataset_versions_temporal_valid_range CHECK (valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from);
UPDATE alembic_version SET version_num='202607140001' WHERE alembic_version.version_num = '202606120900';
INFO [alembic.runtime.migration] Running upgrade 202607140001 -> 202607150001, Add durable operator review decisions for detection QA evidence.
-- Running upgrade 202607140001 -> 202607150001
CREATE TABLE detection_reviews (
id UUID NOT NULL,
project_id UUID NOT NULL,
quality_check_id UUID NOT NULL,
analysis_run_id UUID,
evidence_role VARCHAR(32) NOT NULL,
evidence_feature_id VARCHAR(255) NOT NULL,
detection_id UUID,
reference_feature_id UUID,
decision VARCHAR(64) DEFAULT 'unreviewed' NOT NULL,
notes TEXT,
reviewed_by VARCHAR(120) DEFAULT 'operator' NOT NULL,
created_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
PRIMARY KEY (id),
CONSTRAINT ck_detection_reviews_evidence_role CHECK (evidence_role IN ('false_positive', 'false_negative')),
CONSTRAINT ck_detection_reviews_decision CHECK (decision IN ('confirmed_model_false_positive', 'confirmed_model_false_negative', 'reference_gap_or_change', 'qa_alignment_mismatch', 'imagery_obscured_or_uncertain', 'uncertain', 'unreviewed')),
FOREIGN KEY(analysis_run_id) REFERENCES analysis_runs (id) ON DELETE SET NULL,
FOREIGN KEY(detection_id) REFERENCES detections (id) ON DELETE SET NULL,
FOREIGN KEY(project_id) REFERENCES projects (id) ON DELETE CASCADE,
FOREIGN KEY(quality_check_id) REFERENCES quality_checks (id) ON DELETE CASCADE,
FOREIGN KEY(reference_feature_id) REFERENCES vector_features (id) ON DELETE SET NULL,
CONSTRAINT uq_detection_reviews_evidence UNIQUE (quality_check_id, evidence_role, evidence_feature_id)
);
CREATE INDEX ix_detection_reviews_project_id ON detection_reviews (project_id);
CREATE INDEX ix_detection_reviews_quality_check_id ON detection_reviews (quality_check_id);
CREATE INDEX ix_detection_reviews_analysis_run_id ON detection_reviews (analysis_run_id);
CREATE INDEX ix_detection_reviews_decision ON detection_reviews (decision);
UPDATE alembic_version SET version_num='202607150001' WHERE alembic_version.version_num = '202607140001';
INFO [alembic.runtime.migration] Running upgrade 202607150001 -> 202607160001, Index partitioned vector features by dataset and municipality.
-- Running upgrade 202607150001 -> 202607160001
CREATE INDEX ix_vector_features_dataset_municipality ON vector_features (dataset_id, (properties_json ->> 'municipality'));
UPDATE alembic_version SET version_num='202607160001' WHERE alembic_version.version_num = '202607150001';
INFO [alembic.runtime.migration] Running upgrade 202607160001 -> 202607260001, Add resumable AOI parent and partition operations.
-- Running upgrade 202607160001 -> 202607260001
CREATE TABLE aoi_operations (
id UUID NOT NULL,
project_id UUID NOT NULL,
area_id UUID,
parent_job_id UUID,
operation_type VARCHAR(128) NOT NULL,
status VARCHAR(32) NOT NULL,
geometry geometry(MULTIPOLYGON,4326) NOT NULL,
request_json JSON NOT NULL,
plan_json JSON NOT NULL,
result_json JSON,
error_message TEXT,
created_at TIMESTAMP WITH TIME ZONE DEFAULT now(),
started_at TIMESTAMP WITH TIME ZONE,
finished_at TIMESTAMP WITH TIME ZONE,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT now(),
PRIMARY KEY (id),
CONSTRAINT ck_aoi_operations_status CHECK (status IN ('queued', 'running', 'partial', 'success', 'failed', 'cancelled')),
FOREIGN KEY(project_id) REFERENCES projects (id) ON DELETE CASCADE,
FOREIGN KEY(area_id) REFERENCES areas (id) ON DELETE SET NULL,
FOREIGN KEY(parent_job_id) REFERENCES jobs (id) ON DELETE SET NULL
);
CREATE INDEX ix_aoi_operations_project_status ON aoi_operations (project_id, status);
CREATE INDEX ix_aoi_operations_geometry ON aoi_operations USING gist (geometry);
CREATE TABLE aoi_operation_partitions (
id UUID NOT NULL,
operation_id UUID NOT NULL,
child_job_id UUID,
partition_key VARCHAR(255) NOT NULL,
provider_key VARCHAR(120) NOT NULL,
product_key VARCHAR(120) NOT NULL,
ordinal INTEGER NOT NULL,
status VARCHAR(32) NOT NULL,
geometry geometry(MULTIPOLYGON,4326) NOT NULL,
attempt_count INTEGER DEFAULT '0' NOT NULL,
max_attempts INTEGER DEFAULT '3' NOT NULL,
checkpoint_json JSON,
result_json JSON,
error_message TEXT,
created_at TIMESTAMP WITH TIME ZONE DEFAULT now(),
started_at TIMESTAMP WITH TIME ZONE,
finished_at TIMESTAMP WITH TIME ZONE,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT now(),
PRIMARY KEY (id),
CONSTRAINT ck_aoi_operation_partitions_status CHECK (status IN ('queued', 'running', 'success', 'failed', 'skipped')),
CONSTRAINT uq_aoi_operation_partition_key UNIQUE (operation_id, partition_key),
FOREIGN KEY(operation_id) REFERENCES aoi_operations (id) ON DELETE CASCADE,
FOREIGN KEY(child_job_id) REFERENCES jobs (id) ON DELETE SET NULL
);
CREATE INDEX ix_aoi_operation_partitions_operation_status ON aoi_operation_partitions (operation_id, status);
CREATE INDEX ix_aoi_operation_partitions_geometry ON aoi_operation_partitions USING gist (geometry);
UPDATE alembic_version SET version_num='202607260001' WHERE alembic_version.version_num = '202607160001';
COMMIT;
@@ -0,0 +1,28 @@
workdir=backend
command=python -m pytest -W error::DeprecationWarning
exit_code=2
captured_at=2026-08-01T18:33:34.1591431+02:00
============================= test session starts =============================
platform win32 -- Python 3.13.2, pytest-8.4.0, pluggy-1.6.0
rootdir: C:\Projects\geointel\backend
configfile: pyproject.toml
plugins: asyncio-1.0.0, cov-6.3.0, anyio-4.12.1
asyncio: mode=Mode.STRICT, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function
collected 1194 items / 1 error
=================================== ERRORS ====================================
__________ ERROR collecting tests/test_operator_polygon_label_qa.py ___________
ImportError while importing test module 'C:\Projects\geointel\backend\tests\test_operator_polygon_label_qa.py'.
Hint: make sure your test modules/packages have valid Python names.
Traceback:
C:\Program Files\Python313\Lib\importlib\__init__.py:88: in import_module
return _bootstrap._gcd_import(name[level:], package, level)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
tests\test_operator_polygon_label_qa.py:1: in <module>
from scripts.render_operator_polygon_label_qa import geometry_rings
E ModuleNotFoundError: No module named 'scripts.render_operator_polygon_label_qa'
=========================== short test summary info ===========================
ERROR tests/test_operator_polygon_label_qa.py
!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!
============================== 1 error in 5.69s ===============================
File diff suppressed because one or more lines are too long
@@ -0,0 +1,325 @@
command=python -m pytest backend/tests -q -p no:cacheprovider -W error::DeprecationWarning --junitxml=artifacts/evidence/accuracy/P1/backend-full-suite.junit.xml
exit_code=1
captured_at=2026-08-01T18:32:02.4460806+02:00
........................................................................ [ 6%]
........................................................................ [ 12%]
.........................................F..........F..................F [ 18%]
..................F..................................................... [ 24%]
.........F.............................................................. [ 30%]
........................................................................ [ 36%]
.....F.F.......................................FF....................... [ 42%]
........................................................................ [ 48%]
..F..................................................................... [ 54%]
........................................................................ [ 60%]
.......F................................................................ [ 66%]
........................................................................ [ 72%]
........................................FFFF...........F................ [ 78%]
...............................................F........................ [ 84%]
........................................................................ [ 90%]
........................................................................ [ 96%]
............................................. [100%]
================================== FAILURES ===================================
_ test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbox _
def test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbox() -> None:
root = Path(__file__).parents[2]
focus = (root / "frontend" / "src" / "config" / "primaryFocus.ts").read_text(encoding="utf-8")
workspace_hook = (root / "frontend" / "src" / "hooks" / "useProjectWorkspace.ts").read_text(encoding="utf-8")
coverage_hook = (root / "frontend" / "src" / "hooks" / "useCoverageResolver.ts").read_text(encoding="utf-8")
map_workspace = (root / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8")
assert "Belgium and North Sea Workbench" in focus
assert "nationalProject" in workspace_hook
assert "return nationalProject.id" in workspace_hook
assert "NATIONAL_WORKSPACE_REGION" in workspace_hook
assert "externalApi.resolveCoverage" in coverage_hook
assert "coverage.outside_supported_scope" in map_workspace
assert "coverageStatusLabel" in map_workspace
> assert "coverageSelectionAvailable" in map_workspace
E assert 'coverageSelectionAvailable' in "import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react'\nimport { BoxSelect, Ch...ijken.</p>\n )}\n </div>\n </div>\n </details>\n </div>\n </section>\n )\n}\n"
backend\tests\test_rc4_national_coverage.py:432: AssertionError
____ test_runtime_configuration_is_validated_before_container_replacement _____
def test_runtime_configuration_is_validated_before_container_replacement() -> None:
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
validation_index = run_script.index("validate_runtime_config")
> replacement_index = run_script.index("docker compose down")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E ValueError: substring not found
backend\tests\test_rc5_release_deployment.py:65: ValueError
___________ test_rc9_loading_and_accessibility_states_are_explicit ____________
def test_rc9_loading_and_accessibility_states_are_explicit() -> None:
app = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8")
map_workspace = (
ROOT / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx"
).read_text(encoding="utf-8")
geo_map = (ROOT / "frontend" / "src" / "components" / "GeoMap.tsx").read_text(
encoding="utf-8"
)
assert "workspaceDataLoading" in app
assert 'role="status" aria-live="polite"' in app
assert "Databronnen worden gecontroleerd" in map_workspace
> assert "Beschikbaarheid controleren" in map_workspace
E assert 'Beschikbaarheid controleren' in "import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react'\nimport { BoxSelect, Ch...ijken.</p>\n )}\n </div>\n </div>\n </details>\n </div>\n </section>\n )\n}\n"
backend\tests\test_rc9_ux_release_contract.py:33: AssertionError
___________ test_all_in_one_deploy_embeds_immutable_build_identity ____________
def test_all_in_one_deploy_embeds_immutable_build_identity() -> None:
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
deploy_powershell = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8")
deploy_shell = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8")
assert "ARG GEOINTEL_BUILD_SHA=unknown" in dockerfile
assert 'GEOINTEL_BUILD_SHA="${GEOINTEL_BUILD_SHA}"' in dockerfile
assert 'GEOINTEL_BUILD_TIME="${GEOINTEL_BUILD_TIME}"' in dockerfile
assert 'org.opencontainers.image.revision="${GEOINTEL_BUILD_SHA}"' in dockerfile
> assert 'GEOINTEL_BUILD_SHA="$(git rev-parse HEAD)"' in release_script
E assert 'GEOINTEL_BUILD_SHA="$(git rev-parse HEAD)"' in '#!/usr/bin/env bash\nset -euo pipefail\n\nROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"\ncd "$ROOT"\n\nG...ge.revision"}} created={{index .Config.Labels "org.opencontainers.image.created"}}\' \\\n "$GEOINTEL_RELEASE_IMAGE"\n'
backend\tests\test_rc_runtime_observability.py:47: AssertionError
_____________ test_frontend_exposes_map_bbox_selection_contracts ______________
def test_frontend_exposes_map_bbox_selection_contracts() -> None:
api_client = (ROOT / "frontend" / "src" / "services" / "api" / "datasets.ts").read_text(encoding="utf-8")
map_workspace = (ROOT / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8")
geomap = (ROOT / "frontend" / "src" / "components" / "GeoMap.tsx").read_text(encoding="utf-8")
app = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8")
extract_hook = (ROOT / "frontend" / "src" / "hooks" / "useMapSelectionExtract.ts").read_text(encoding="utf-8")
theme_hook = (ROOT / "frontend" / "src" / "hooks" / "useMapThemeSelectionInsights.ts").read_text(encoding="utf-8")
assert "selectVectorFeatures" in api_client
assert "Area selection" in map_workspace
assert "Teken rechthoek" in map_workspace
assert "Objecten in gebied ophalen" in map_workspace
assert "Gebiedsdownload bewaren" in map_workspace
assert "bboxSelectionMode" in geomap
assert "selection-bbox" in geomap
assert "selection-result" in geomap
assert "useMapSelectionExtract" in app
assert "area_id: areaId" in extract_hook
assert "area_id: areaId" in theme_hook
> assert "analyzeSelection(selectedAreaBbox, selectedMapArea?.id)" in map_workspace
E assert 'analyzeSelection(selectedAreaBbox, selectedMapArea?.id)' in "import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react'\nimport { BoxSelect, Ch...ijken.</p>\n )}\n </div>\n </div>\n </details>\n </div>\n </section>\n )\n}\n"
backend\tests\test_sprint106_map_bbox_extract.py:356: AssertionError
______ test_frontend_declares_national_scope_as_primary_operating_focus _______
def test_frontend_declares_national_scope_as_primary_operating_focus() -> None:
focus = (ROOT / "frontend" / "src" / "config" / "primaryFocus.ts").read_text(
encoding="utf-8"
)
project_hook = (ROOT / "frontend" / "src" / "hooks" / "useProjectWorkspace.ts").read_text(
encoding="utf-8"
)
map_source = (ROOT / "frontend" / "src" / "components" / "GeoMap.tsx").read_text(
encoding="utf-8"
)
navigation = (
ROOT
/ "frontend"
/ "src"
/ "components"
/ "shell"
/ "WorkbenchNavigation.tsx"
).read_text(encoding="utf-8")
assert "NATIONAL_WORKSPACE_PROJECT_NAME = 'Belgium and North Sea Workbench'" in focus
assert "NATIONAL_WORKSPACE_REGION = 'Belgie en Belgische Noordzee'" in focus
assert "NATIONAL_MAP_CENTER" in focus
assert "return nationalProject.id" in project_hook
assert "hasMappedAnalysisContext(data)" in project_hook
assert "dataset.dataset_type === 'raster'" in project_hook
assert "dataset.dataset_type === 'vector' || dataset.dataset_type === 'geojson'" in project_hook
assert "PRIMARY_FOCUS_AREA_NAME" not in project_hook
assert "PRIMARY_FOCUS_AREA_GEOJSON" not in project_hook
assert "center: NATIONAL_MAP_CENTER" in map_source
assert "zoom: NATIONAL_MAP_ZOOM" in map_source
assert "GeoIntel" in navigation
> assert "Atlas Workbench" in navigation
E AssertionError: assert 'Atlas Workbench' in 'import {\n Activity,\n Bot,\n Database,\n Download,\n Map,\n ScanSearch,\n Settings,\n ShieldCheck,\n type L... )\n })}\n </div>\n ))}\n </nav>\n <ItWorxSignature />\n </aside>\n )\n}\n'
backend\tests\test_sprint177_mol_primary_focus.py:38: AssertionError
______ test_product_docs_record_national_scope_and_mol_regression_focus _______
def test_product_docs_record_national_scope_and_mol_regression_focus() -> None:
readme = (ROOT / "README.md").read_text(encoding="utf-8")
vision = (ROOT / "docs" / "PRODUCT_VISION.md").read_text(encoding="utf-8")
> assert "Belgium and the Belgian North Sea" in readme
E assert 'Belgium and the Belgian North Sea' in '# GeoIntel\n\n<p align="center">\n <img src="frontend/public/geointel-icon.png" width="92" alt="GeoIntel logo">\n</p...\n\nActieve mijlpaal: **v1.0.0 Belgium and Belgian North Sea**.\n\nGeoIntel is een project van Jens / ITWorx.tech.\n'
backend\tests\test_sprint177_mol_primary_focus.py:62: AssertionError
_____________ test_map_first_explorer_is_the_default_product_flow _____________
def test_map_first_explorer_is_the_default_product_flow() -> None:
app = read("frontend/src/App.tsx")
workspace = read("frontend/src/components/map/MapWorkspace.tsx")
assert "useState<WorkspaceKey>('map')" in app
assert "Gebied analyseren" in workspace
> assert "<h3>Focus op de kaart <small>optioneel</small></h3>" in workspace
E assert '<h3>Focus op de kaart <small>optioneel</small></h3>' in "import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react'\nimport { BoxSelect, Ch...ijken.</p>\n )}\n </div>\n </div>\n </details>\n </div>\n </section>\n )\n}\n"
backend\tests\test_sprint186_map_first_geographic_explorer.py:19: AssertionError
___________ test_map_rectangle_drag_is_wired_to_automatic_analysis ____________
def test_map_rectangle_drag_is_wired_to_automatic_analysis() -> None:
workspace = read("frontend/src/components/map/MapWorkspace.tsx")
geomap = read("frontend/src/components/GeoMap.tsx")
styles = read("frontend/src/styles/app.css")
assert "onMapBboxPreview={handleMapBboxPreview}" in workspace
assert "onMapBboxSelect={handleMapBboxSelect}" in workspace
assert "void analyzeSelection(bbox, areaIdForSelection(bbox))" in workspace
assert "const areaIdForSelection" in workspace
assert "bbox && selectedMapArea ? selectedMapArea.id : undefined" in workspace
> assert "void analyzeSelection(selectedAreaBbox, selectedMapArea?.id)" in workspace
E assert 'void analyzeSelection(selectedAreaBbox, selectedMapArea?.id)' in "import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react'\nimport { BoxSelect, Ch...ijken.</p>\n )}\n </div>\n </div>\n </details>\n </div>\n </section>\n )\n}\n"
backend\tests\test_sprint186_map_first_geographic_explorer.py:44: AssertionError
__ test_evolution_theme_catalog_distinguishes_history_from_current_only_data __
def test_evolution_theme_catalog_distinguishes_history_from_current_only_data() -> None:
workspace = read("frontend/src/components/map/MapWorkspace.tsx")
assert "analysisMode === 'current'" in workspace
> assert "Boolean(dataset || onDemandProduct)" in workspace
E assert 'Boolean(dataset || onDemandProduct)' in "import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react'\nimport { BoxSelect, Ch...ijken.</p>\n )}\n </div>\n </div>\n </details>\n </div>\n </section>\n )\n}\n"
backend\tests\test_sprint200_temporal_explorer_handoff.py:25: AssertionError
_____ test_regional_map_uses_logical_partition_groups_and_exact_analysis ______
def test_regional_map_uses_logical_partition_groups_and_exact_analysis() -> None:
app = (ROOT / "frontend/src/App.tsx").read_text(encoding="utf-8")
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
hook = (ROOT / "frontend/src/hooks/useMapThemeSelectionInsights.ts").read_text(encoding="utf-8")
api = (ROOT / "frontend/src/services/api/datasets.ts").read_text(encoding="utf-8")
assert "regionalScopeSelected" in workspace
assert "rasterPartitionsForDataset" in workspace
assert "imageOverlays={activeImageOverlays}" in workspace
> assert "de juiste gemeentelijke rasters worden automatisch gecombineerd" in workspace
E assert 'de juiste gemeentelijke rasters worden automatisch gecombineerd' in "import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react'\nimport { BoxSelect, Ch...ijken.</p>\n )}\n </div>\n </div>\n </details>\n </div>\n </section>\n )\n}\n"
backend\tests\test_sprint219_regional_raster_explorer.py:30: AssertionError
___ test_frontend_uses_partitioned_bathymetry_selection_for_regional_scope ____
def test_frontend_uses_partitioned_bathymetry_selection_for_regional_scope() -> None:
app = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8")
index = (ROOT / "frontend" / "index.html").read_text(encoding="utf-8")
focus = (
ROOT / "frontend" / "src" / "config" / "primaryFocus.ts"
).read_text(encoding="utf-8")
map_workspace = (
ROOT / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx"
).read_text(encoding="utf-8")
theme_hook = (
ROOT / "frontend" / "src" / "hooks" / "useMapThemeSelectionInsights.ts"
).read_text(encoding="utf-8")
dataset_api = (
ROOT / "frontend" / "src" / "services" / "api" / "datasets.ts"
).read_text(encoding="utf-8")
assert "isPartitionedBathymetry" in map_workspace
assert "regionalPartitionedThemeActive" in map_workspace
> assert "datasetAvailabilityLabel(dataset, partitions)" in map_workspace
E assert 'datasetAvailabilityLabel(dataset, partitions)' in "import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react'\nimport { BoxSelect, Ch...ijken.</p>\n )}\n </div>\n </div>\n </details>\n </div>\n </section>\n )\n}\n"
backend\tests\test_sprint236_bathymetry_expansion.py:478: AssertionError
____ test_flanders_workspace_exposes_governed_thematic_products_on_demand _____
def test_flanders_workspace_exposes_governed_thematic_products_on_demand() -> None:
workspace = read("frontend/src/components/map/MapWorkspace.tsx")
product_hook = read("frontend/src/hooks/useOfficialMapProducts.ts")
selection_hook = read("frontend/src/hooks/useMapThemeSelectionInsights.ts")
api = read("frontend/src/services/api/datasets.ts")
assert "activeScopeProject?.name === FLANDERS_WORKSPACE_PROJECT_NAME" in workspace
assert "new Map<DataThemeId, OnDemandMapProduct>" in workspace
> assert "'Automatisch'" in workspace
E assert "'Automatisch'" in "import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react'\nimport { BoxSelect, Ch...ijken.</p>\n )}\n </div>\n </div>\n </details>\n </div>\n </section>\n )\n}\n"
backend\tests\test_sprint237_flanders_thematic_on_demand.py:19: AssertionError
________ test_selection_reads_and_bounded_acquires_all_relevant_themes ________
def test_selection_reads_and_bounded_acquires_all_relevant_themes() -> None:
workspace = read("frontend/src/components/map/MapWorkspace.tsx")
app = read("frontend/src/App.tsx")
selection_hook = read("frontend/src/hooks/useMapThemeSelectionInsights.ts")
> assert "for (const theme of [activeTheme])" in workspace
E assert 'for (const theme of [activeTheme])' in "import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react'\nimport { BoxSelect, Ch...ijken.</p>\n )}\n </div>\n </div>\n </details>\n </div>\n </section>\n )\n}\n"
backend\tests\test_sprint237_flanders_thematic_on_demand.py:37: AssertionError
______ test_regional_on_demand_sources_require_a_bounded_drawn_selection ______
def test_regional_on_demand_sources_require_a_bounded_drawn_selection() -> None:
workspace = read("frontend/src/components/map/MapWorkspace.tsx")
assert "regionalOnDemandThemeActive" in workspace
> assert "regionalRasterThemeActive || regionalOnDemandThemeActive" in workspace
E assert 'regionalRasterThemeActive || regionalOnDemandThemeActive' in "import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react'\nimport { BoxSelect, Ch...ijken.</p>\n )}\n </div>\n </div>\n </details>\n </div>\n </section>\n )\n}\n"
backend\tests\test_sprint237_flanders_thematic_on_demand.py:56: AssertionError
_____ test_grb_frontend_and_contracts_use_only_the_governed_backend_path ______
def test_grb_frontend_and_contracts_use_only_the_governed_backend_path() -> None:
selection_hook = (ROOT / "frontend/src/hooks/useMapThemeSelectionInsights.ts").read_text(encoding="utf-8")
catalog_hook = (ROOT / "frontend/src/hooks/useOfficialMapProducts.ts").read_text(encoding="utf-8")
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
contracts = (ROOT / "docs/API_CONTRACTS.md").read_text(encoding="utf-8")
assert "datasetsApi.acquireGrb" in selection_hook
assert "datasetsApi.listGrbProducts" in catalog_hook
assert "officialMapProducts.grb" in workspace
assert "result[product.key] = null" in workspace
assert ": onDemandThemeActive\n ? null\n : mapFeatureCollection" in workspace
assert "onSetContextLayerLabel" in workspace
> assert "'Automatisch'" in workspace
E assert "'Automatisch'" in "import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react'\nimport { BoxSelect, Ch...ijken.</p>\n )}\n </div>\n </div>\n </details>\n </div>\n </section>\n )\n}\n"
backend\tests\test_sprint239_bounded_grb_acquisition.py:394: AssertionError
__________ test_app_entrypoint_has_clean_encoding_and_react_imports ___________
def test_app_entrypoint_has_clean_encoding_and_react_imports() -> None:
app_path = ROOT / "frontend" / "src" / "App.tsx"
app_bytes = app_path.read_bytes()
app = app_path.read_text(encoding="utf-8")
assert not app_bytes.startswith(b"\xef\xbb\xbf")
assert "import { useEffect, useMemo, useRef, useState } from 'react'" in app
assert "FormEvent" not in app
> assert app.count("useEffect(") == 1
E assert 3 == 1
E + where 3 = <built-in method count of str object at 0x000001FED98D8C30>('useEffect(')
E + where <built-in method count of str object at 0x000001FED98D8C30> = "import { useEffect, useMemo, useRef, useState } from 'react'\nimport { CircleAlert, LogOut, ShieldCheck, UserRound } ...ator' : 'open')}\n loggingOut={loggingOut}\n onLogout={handleLogout}\n />\n )\n}\n\nexport default App\n".count
backend\tests\test_sprint39_frontend_orchestration_hooks.py:40: AssertionError
- generated xml file: C:\Projects\geointel\artifacts\evidence\accuracy\P1\backend-full-suite.junit.xml -
=========================== short test summary info ===========================
FAILED backend\tests\test_rc4_national_coverage.py::test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbox
FAILED backend\tests\test_rc5_release_deployment.py::test_runtime_configuration_is_validated_before_container_replacement
FAILED backend\tests\test_rc9_ux_release_contract.py::test_rc9_loading_and_accessibility_states_are_explicit
FAILED backend\tests\test_rc_runtime_observability.py::test_all_in_one_deploy_embeds_immutable_build_identity
FAILED backend\tests\test_sprint106_map_bbox_extract.py::test_frontend_exposes_map_bbox_selection_contracts
FAILED backend\tests\test_sprint177_mol_primary_focus.py::test_frontend_declares_national_scope_as_primary_operating_focus
FAILED backend\tests\test_sprint177_mol_primary_focus.py::test_product_docs_record_national_scope_and_mol_regression_focus
FAILED backend\tests\test_sprint186_map_first_geographic_explorer.py::test_map_first_explorer_is_the_default_product_flow
FAILED backend\tests\test_sprint186_map_first_geographic_explorer.py::test_map_rectangle_drag_is_wired_to_automatic_analysis
FAILED backend\tests\test_sprint200_temporal_explorer_handoff.py::test_evolution_theme_catalog_distinguishes_history_from_current_only_data
FAILED backend\tests\test_sprint219_regional_raster_explorer.py::test_regional_map_uses_logical_partition_groups_and_exact_analysis
FAILED backend\tests\test_sprint236_bathymetry_expansion.py::test_frontend_uses_partitioned_bathymetry_selection_for_regional_scope
FAILED backend\tests\test_sprint237_flanders_thematic_on_demand.py::test_flanders_workspace_exposes_governed_thematic_products_on_demand
FAILED backend\tests\test_sprint237_flanders_thematic_on_demand.py::test_selection_reads_and_bounded_acquires_all_relevant_themes
FAILED backend\tests\test_sprint237_flanders_thematic_on_demand.py::test_regional_on_demand_sources_require_a_bounded_drawn_selection
FAILED backend\tests\test_sprint239_bounded_grb_acquisition.py::test_grb_frontend_and_contracts_use_only_the_governed_backend_path
FAILED backend\tests\test_sprint39_frontend_orchestration_hooks.py::test_app_entrypoint_has_clean_encoding_and_react_imports
17 failed, 1180 passed in 70.63s (0:01:10)
@@ -0,0 +1,266 @@
# Phase 1 command and result ledger
Captured: 2026-08-01
Audited repository baseline: `0c019bb22f816db1e4b7a68379bcad08924d9a21`
Branch: `codex/geointel-accuracy-program`
This ledger records the commands and claim boundaries behind the retained
evidence. JSON, JUnit, SQL and full text logs in this directory are the source
of truth when this summary and a raw artifact differ.
## Repository and static inventory
```powershell
python scripts/run_accuracy_phase1_baseline.py --output-dir artifacts/evidence/accuracy/P1
```
Result: completed. The baseline found 2,495 tracked files, including a tracked
`geointel/` mirror with 1,153 files; 68 paired files differ from root. No model
checkpoint is tracked in the local checkout. The mirror is excluded by the root
`.dockerignore`; the residual risk is source/import/maintenance ambiguity, not
official all-in-one build-context inclusion.
Evidence:
- `phase1-baseline-summary.json`
- `repository-inventory.json`
- `local-artifact-inventory.json`
- `static-risk-signals.json`
## Deterministic contract reproductions
```powershell
python scripts/reproduce_accuracy_phase1_findings.py
```
Result: 7 of 7 expected violations reproduced. They cover cross-theme coverage
contamination, metres-as-degrees buffering, Lambert-as-4326 persistence,
caller-spoofed source authority, mutable-name model scope, mutable-name legal
scope and silently ignored Area PATCH geometry.
Evidence: `forensic-reproductions.json`.
## Tower runtime, database and storage references
The collector was streamed into the running `geointel` container and executed
with read-only SQL and a 30-second statement timeout:
```text
docker exec -i geointel env PYTHONPATH=/app/backend python -
```
Result:
- Python 3.11.2, PyTorch 2.11.0+cu128, CUDA 12.8, Ultralytics 8.4.99;
- NVIDIA GeForce RTX 4080 SUPER visible on `cuda:0`;
- Alembic head `202607260001`;
- 5,816 direct database storage references checked, 0 missing;
- four Geel detections contain Lambert-domain coordinates while the geometry
column reports SRID 4326;
- no database or storage row was modified.
A broader recursive `audit_data_operations.py` storage scan was stopped by its
244-second execution timeout and produced no retained result. It is not counted
as a pass. The bounded, direct-reference collector above completed and is the
only storage-completeness claim made here.
Evidence:
- `tower-runtime-database-snapshot.json`
- `tower-runtime-database-snapshot-detailed.json`
## Real GPU inference smoke
The read-only collector used the production `YoloDetectionAdapter`, the active
model and one existing Geel tile:
```text
python - --model-path /app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt \
--tile-path /app/storage/tiles/cb80638d-dbef-48ac-b19c-cec7c3efc96e/ae0ff76d-70c0-404f-b777-54d14517179a/b191c968-7d56-4e0d-afbb-8b5baaa62470/tile_0000.tif \
--manifest-path /app/storage/tiles/cb80638d-dbef-48ac-b19c-cec7c3efc96e/ae0ff76d-70c0-404f-b777-54d14517179a/b191c968-7d56-4e0d-afbb-8b5baaa62470/manifest.json \
--confidence 0.5 --image-size 640 --max-detections 1000 --device cuda:0 --seed 20260801
```
Result: passed; 17 raw building detections; 0.883694486 s synchronized
inference; model SHA-256
`a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1`.
Claim boundary: this proves one runtime execution. It does not prove that any
box is correct, that confidence is calibrated, or that the model generalizes.
Evidence: `tower-gpu-inference-smoke.json`.
## ML/data-lineage inventory
The bounded collector read V56/V58/V62/V66 plus model, checkpoint, report and
manifest metadata from the mounted Tower volume:
```text
docker exec -i geointel python - --app-root /app
```
Result:
- 26 model assets, 229 training checkpoints, 424 training JSON reports and
36 operator manifests inventoried;
- V56 has 180 AOIs and 0 completed human reviews;
- three pure-empty background-test AOIs: Flanders 2, Wallonia 1, Brussels 0;
- zero exact cross-split raster-hash duplicates;
- bounded 64-bit dHash screen: 180 rasters, minimum Hamming distance 17,
zero cross-split pairs at or below 4;
- minimum cross-split AOI bbox distance 95.720336 m and 24 pairs below 2 km;
- V58/V62 evidence is calibration-only; at threshold 0.15 aggregate F1 is
0.512905 while Flanders recall and F1 are 0;
- no protected-test, background-test promotion or national-release evidence.
Claim boundary: exact/dHash and bbox-distance screens do not establish
municipality, flight-strip, instance, semantic or imagery-edition independence.
Evidence: `tower-ml-data-lineage-snapshot.json` and
`tower-key-artifact-hashes.json` (V62 best/last checkpoints and V56 tile audit).
## Backend tests
### Full root-context suite
```powershell
$env:PYTHONPATH='C:\Projects\geointel\backend;C:\Projects\geointel'
python -m pytest backend/tests -q -p no:cacheprovider \
-W error::DeprecationWarning \
--junitxml=artifacts/evidence/accuracy/P1/backend-full-suite.junit.xml
```
Result: exit 1; 1,180 passed and 17 failed in 70.63 seconds. The 17 failures are
stale source/contract assertions, including expectations that conflict with the
new explicit user-selected model flow. They remain release-blocking until
replaced with reviewed behavior tests.
Evidence:
- `backend-full-suite.txt`
- `backend-full-suite.junit.xml`
### Actual backend CI working directory
```powershell
cd backend
python -m pytest -W error::DeprecationWarning
```
Result: exit 2 during collection; 1,194 items collected plus one import error:
```text
ModuleNotFoundError: No module named 'scripts.render_operator_polygon_label_qa'
```
Evidence: `backend-ci-entrypoint.txt`.
### Phase 1 tooling
```powershell
python -m pytest tests/test_accuracy_phase1_baseline.py -q -p no:cacheprovider
```
Result: 4 passed.
Evidence: `phase1-tooling-tests.txt`.
## Lint
```powershell
python -m ruff check scripts/collect_accuracy_phase1_inference_smoke.py \
scripts/collect_accuracy_phase1_ml_lineage.py \
scripts/collect_accuracy_phase1_runtime.py \
scripts/reproduce_accuracy_phase1_findings.py \
scripts/run_accuracy_phase1_baseline.py tests/test_accuracy_phase1_baseline.py
```
Result: all new Phase 1 code passed.
```powershell
python -m ruff check backend scripts tests
```
Result: exit 1; 112 findings: E402 13, E701 2, E702 69, F401 23, F403 1,
F811 2 and F841 2.
Evidence:
- `repository-ruff-baseline.txt`
- `repository-ruff-baseline.json`
## Frontend
```powershell
cd frontend
npm run test:unit
npm run typecheck
npm run build
```
Results:
- Vitest: 16 files, 51 tests passed;
- TypeScript typecheck: passed;
- production build: passed, 1,896 modules transformed.
The attempted generic `npm test -- --run` failed because no `test` script
exists; the configured command is `test:unit`. The test strategy also calls
for lint, but `npm run lint` fails with `Missing script: "lint"`.
Evidence:
- `frontend-vitest.txt` (failed non-existent generic command, retained)
- `frontend-vitest-unit.txt` (correct configured command, passed)
- `frontend-typecheck.txt`
- `frontend-build.txt`
- `frontend-lint.txt`
## API, migrations and golden QA
```powershell
python scripts/audit_api_contracts.py
```
Result: 147 implemented routes match documentation; 10 explicitly tracked
non-envelope endpoints.
Evidence: `openapi-contract-audit.txt`.
```powershell
cd backend
python -m alembic heads
python -m alembic upgrade head --sql
```
Result: one head, `202607260001`; complete offline upgrade rendered 496 SQL/log
lines. This is not a local live-PostGIS migration test.
Evidence:
- `alembic-heads.txt`
- `alembic-offline-upgrade.sql`
```powershell
python scripts/run_golden_qa_benchmark.py --json
```
Result: two runs have semantically identical metric results but different file
hashes because run identities use UUID4.
Evidence:
- `golden-qa-run-1.json`
- `golden-qa-run-2.json`
- `golden-qa-reproducibility.json`
## Explicitly not executed as a success gate
- no protected test or background-test portfolio was opened;
- no training, calibration fitting, checkpoint promotion or active-model change;
- no label was accepted on behalf of a human reviewer;
- no production database migration or data repair;
- no dataset, checkpoint, cache, output or user-owned untracked file deleted,
rewritten or moved;
- no segmentation, SAM or solar asset was inferred to be production-ready from
file presence.
@@ -0,0 +1,116 @@
{
"findings": [
{
"expected": "A small buildings partition remains partial; a roads bbox cannot complete buildings coverage.",
"id": "P1-COV-001",
"observed": {
"fully_covered": true,
"matched_dataset_ids": [
"00000000-0000-4000-8000-000000000001"
]
},
"reproduced": true,
"severity": "critical",
"source": "backend/app/services/coverage_registry_service.py:481-505"
},
{
"expected": "A 100 metre buffer is projected to a metric CRS and spans roughly hundreds of metres.",
"id": "P1-CRS-001",
"observed": {
"bounds_epsg4326": [
-95.0,
-49.0,
105.0,
151.0
],
"longitude_span_degrees": 200.0
},
"reproduced": true,
"severity": "critical",
"source": "backend/app/services/vector_operations_service.py:179-203"
},
{
"expected": "Non-WGS84 input is transformed to EPSG:4326 or rejected before persistence.",
"id": "P1-CRS-002",
"observed": {
"stored_coordinates": [
150000.0,
210000.0
],
"stored_srid": 4326
},
"reproduced": true,
"severity": "critical",
"source": "backend/app/services/vector_feature_service.py:271-299"
},
{
"expected": "Only server-attested source identities can produce authoritative operational coverage.",
"id": "P1-AUTH-001",
"observed": {
"caller_controlled_source_name": "grb",
"coverage_status": "operational",
"reported_authority": "authoritative"
},
"reproduced": true,
"severity": "critical",
"source": "backend/app/api/routes/datasets.py:142-163; backend/app/services/coverage_registry_service.py:463-559"
},
{
"expected": "Validation scope is bound to immutable geometry/source/checksum evidence.",
"id": "P1-AI-001",
"observed": {
"accepted": true,
"area_name": "Mol validation bypass",
"geometry_bounds": [
-75.0,
35.0,
-74.9,
35.1
]
},
"reproduced": true,
"severity": "critical",
"source": "backend/app/services/detection_service.py:223-232"
},
{
"expected": "Renaming an Area cannot change its legal coverage-zone identity.",
"id": "P1-COV-002",
"observed": {
"canonical": {
"outside": false,
"zones": [
"flanders"
]
},
"renamed_same_geometry": {
"outside": true,
"zones": []
}
},
"reproduced": true,
"severity": "high",
"source": "backend/app/services/coverage_registry_service.py:56-65,425-447"
},
{
"expected": "PATCH /areas/{area_id} either validates and applies geometry or rejects the field.",
"id": "P1-API-001",
"observed": {
"geometry_silently_ignored": true,
"parsed_payload": {
"crs": null,
"name": "Renamed"
}
},
"reproduced": true,
"severity": "high",
"source": "backend/app/schemas/area.py:15-17; backend/app/services/area_service.py:154-171"
}
],
"purpose": "Read-only deterministic reproductions of Phase-1 contract violations.",
"schema_version": 1,
"summary": {
"all_reproduced": true,
"reproduced": 7,
"total": 7
}
}
@@ -0,0 +1,31 @@
command=npm run build
exit_code=0
captured_at=2026-08-01T18:34:50.3212385+02:00
> geointel-frontend@1.0.0 build
> tsc -b && vite build
vite v7.3.6 building client environment for production...
transforming...
✓ 1896 modules transformed.
rendering chunks...
computing gzip size...
dist/index.html  0.98 kB │ gzip: 0.49 kB
dist/assets/manrope-latin-500-normal-BYYD-dBL.woff2  14.04 kB
dist/assets/manrope-latin-600-normal-4f0koTD-.woff2  14.17 kB
dist/assets/manrope-latin-700-normal-BZp_XxE4.woff2  14.21 kB
dist/assets/public-sans-latin-600-normal-Fru-LXNs.woff2  14.59 kB
dist/assets/public-sans-latin-400-normal-8Rpg0ruU.woff2  14.63 kB
dist/assets/public-sans-latin-500-normal-NlrCPXnF.woff2  14.66 kB
dist/assets/manrope-latin-500-normal-DMZssgOp.woff  18.24 kB
dist/assets/manrope-latin-600-normal-BqgrALkZ.woff  18.38 kB
dist/assets/manrope-latin-700-normal-DGRFkw-m.woff  18.41 kB
dist/assets/public-sans-latin-400-normal-SBbinRkI.woff  18.49 kB
dist/assets/public-sans-latin-500-normal-vCxiVFAq.woff  18.52 kB
dist/assets/public-sans-latin-600-normal-BR59oU-I.woff  18.56 kB
dist/assets/index-BNS8Tp4f.css 426.59 kB │ gzip: 69.40 kB
dist/assets/vendor-CrqJdhs-.js 140.80 kB │ gzip: 45.27 kB
dist/assets/index-DR8RUeAl.js 457.16 kB │ gzip: 119.71 kB
dist/assets/maplibre-DEBTaTl_.js 802.71 kB │ gzip: 217.71 kB
✓ built in 9.80s
@@ -0,0 +1,12 @@
command=npm run lint
exit_code=1
captured_at=2026-08-01T18:36:26.6162306+02:00
npm error Missing script: "lint"
npm error
npm error Did you mean this?
npm error npm link # Symlink a package folder
npm error
npm error To see a list of scripts, run:
npm error npm run
npm error A complete log of this run can be found in: C:\Users\Jens\AppData\Local\npm-cache\_logs\2026-08-01T16_36_25_353Z-debug-0.log
@@ -0,0 +1,8 @@
command=npm run typecheck
exit_code=0
captured_at=2026-08-01T18:34:27.3623155+02:00
> geointel-frontend@1.0.0 typecheck
> tsc -p tsconfig.json --noEmit
@@ -0,0 +1,35 @@
command=npm run test:unit
exit_code=0
captured_at=2026-08-01T18:35:24.4298867+02:00
> geointel-frontend@1.0.0 test:unit
> vitest run
 RUN  v3.2.6 C:/Projects/geointel/frontend
✓ src/lib/datasetCapabilities.test.ts (2 tests) 5ms
✓ src/lib/performanceBudget.test.ts (3 tests) 41ms
✓ src/components/map/mapWorkspaceUtils.test.ts (14 tests) 36ms
✓ src/hooks/useMapThemeSelectionInsights.test.ts (2 tests) 43ms
✓ src/hooks/useCoverageResolver.test.tsx (4 tests) 47ms
✓ src/hooks/useTemporalComparison.test.tsx (3 tests) 36ms
✓ src/components/WorkbenchStatusStrip.test.tsx (2 tests) 151ms
✓ src/hooks/useWorkbenchBootstrap.test.tsx (3 tests) 60ms
✓ src/components/map/LiveAnalysisJourney.test.tsx (3 tests) 59ms
✓ src/components/overview/ProjectAtlasIllustration.test.tsx (2 tests) 267ms
✓ src/components/models/ModelSelector.test.tsx (2 tests) 342ms
✓ src/components/detection/AiPipelineIllustration.test.tsx (1 test) 296ms
✓ src/components/auth/LandingProjectStory.test.tsx (2 tests) 332ms
✓ src/components/map/MunicipalitySearch.test.tsx (2 tests) 483ms
✓ MunicipalitySearch > searches the official municipality catalog and activates a result  468ms
✓ src/components/auth/LandingPage.test.tsx (4 tests) 825ms
✓ LandingPage > presents the professionalised landing page and submits the operator login  471ms
✓ src/lib/bathymetryRaster.test.ts (2 tests) 3ms
 Test Files  16 passed (16)
 Tests  51 passed (51)
 Start at  18:35:16
 Duration  7.37s (transform 2.45s, setup 0ms, collect 14.66s, tests 3.03s, environment 45.46s, prepare 8.07s)
@@ -0,0 +1,9 @@
command=npm test -- --run
exit_code=1
captured_at=2026-08-01T18:34:11.9904224+02:00
npm error Missing script: "test"
npm error
npm error To see a list of scripts, run:
npm error npm run
npm error A complete log of this run can be found in: C:\Users\Jens\AppData\Local\npm-cache\_logs\2026-08-01T16_34_11_785Z-debug-0.log
@@ -0,0 +1,11 @@
{
"schema_version": 1,
"command": "python scripts/run_golden_qa_benchmark.py --json",
"run_1_exit_code": 0,
"run_2_exit_code": 0,
"run_1_sha256": "ec96862b024987b59e56579920361582e718126b07885732335ac1b4d51b61bc",
"run_2_sha256": "4f900711a706dd56dd16b368e261761051f6d52a38bfd421e7d1c5f66ee51e49",
"byte_identical": false,
"semantic_results_equal": true,
"interpretation": "Metric values are stable, but UUID4-backed run identity makes retained JSON byte-nondeterministic."
}
@@ -0,0 +1,177 @@
{
"benchmark_id": "golden-buildings-partial-match-v1",
"fixtures": {
"candidate": "fixtures/golden/predicted_buildings.geojson",
"reference": "fixtures/golden/reference_buildings.geojson"
},
"metrics": {
"f1": 0.5,
"false_negative_count": 1,
"false_positive_count": 1,
"mean_iou": 0.8339768339652592,
"precision": 0.5,
"recall": 0.5
},
"persistence": {
"commit_count": 4,
"metric_count": 24,
"metric_keys": [
"f1",
"false_negative_count",
"false_positive_count",
"mean_iou",
"precision",
"recall"
],
"quality_check_count": 4
},
"scenario_count": 4,
"scenarios": [
{
"benchmark_id": "golden-buildings-partial-match-v1",
"description": "Two reference building polygons and two candidate building polygons: one candidate matches ref-1, one candidate is a false positive, and ref-2 is a false negative.",
"fixtures": {
"candidate": "fixtures/golden/predicted_buildings.geojson",
"reference": "fixtures/golden/reference_buildings.geojson"
},
"metrics": {
"f1": 0.5,
"false_negative_count": 1,
"false_positive_count": 1,
"mean_iou": 0.8339768339652592,
"precision": 0.5,
"recall": 0.5
},
"persistence": {
"commit_count": 1,
"metric_count": 6,
"metric_keys": [
"precision",
"recall",
"f1",
"mean_iou",
"false_positive_count",
"false_negative_count"
],
"quality_check_count": 1
},
"quality_check_id": "5fe92da9-2d8d-46a4-9992-2de89e3cf511",
"result_counts": {
"candidate_feature_count": 2,
"matches": 1,
"reference_feature_count": 2
},
"status": "passed"
},
{
"benchmark_id": "golden-buildings-perfect-match-v1",
"description": "One candidate building polygon exactly matches one reference polygon.",
"fixtures": {
"candidate": "fixtures/golden/predicted_buildings_perfect.geojson",
"reference": "fixtures/golden/reference_buildings_perfect.geojson"
},
"metrics": {
"f1": 1.0,
"false_negative_count": 0,
"false_positive_count": 0,
"mean_iou": 1.0,
"precision": 1.0,
"recall": 1.0
},
"persistence": {
"commit_count": 1,
"metric_count": 6,
"metric_keys": [
"precision",
"recall",
"f1",
"mean_iou",
"false_positive_count",
"false_negative_count"
],
"quality_check_count": 1
},
"quality_check_id": "7b8c1444-31e6-4780-92cb-f2060f4a4048",
"result_counts": {
"candidate_feature_count": 1,
"matches": 1,
"reference_feature_count": 1
},
"status": "passed"
},
{
"benchmark_id": "golden-buildings-no-overlap-v1",
"description": "One candidate polygon and one reference polygon do not overlap.",
"fixtures": {
"candidate": "fixtures/golden/predicted_buildings_no_overlap.geojson",
"reference": "fixtures/golden/reference_buildings_no_overlap.geojson"
},
"metrics": {
"f1": null,
"false_negative_count": 1,
"false_positive_count": 1,
"mean_iou": null,
"precision": 0.0,
"recall": 0.0
},
"persistence": {
"commit_count": 1,
"metric_count": 6,
"metric_keys": [
"precision",
"recall",
"f1",
"mean_iou",
"false_positive_count",
"false_negative_count"
],
"quality_check_count": 1
},
"quality_check_id": "1ffb1817-b237-4ca8-aad7-ff3312ac7e03",
"result_counts": {
"candidate_feature_count": 1,
"matches": 0,
"reference_feature_count": 1
},
"status": "passed"
},
{
"benchmark_id": "golden-buildings-multipolygon-match-v1",
"description": "One candidate MultiPolygon exactly matches one reference MultiPolygon.",
"fixtures": {
"candidate": "fixtures/golden/predicted_buildings_multipolygon.geojson",
"reference": "fixtures/golden/reference_buildings_multipolygon.geojson"
},
"metrics": {
"f1": 1.0,
"false_negative_count": 0,
"false_positive_count": 0,
"mean_iou": 1.0,
"precision": 1.0,
"recall": 1.0
},
"persistence": {
"commit_count": 1,
"metric_count": 6,
"metric_keys": [
"precision",
"recall",
"f1",
"mean_iou",
"false_positive_count",
"false_negative_count"
],
"quality_check_count": 1
},
"quality_check_id": "60e6a321-2aaf-467b-9281-f30dd4179333",
"result_counts": {
"candidate_feature_count": 1,
"matches": 1,
"reference_feature_count": 1
},
"status": "passed"
}
],
"status": "passed",
"version": 1
}
@@ -0,0 +1,177 @@
{
"benchmark_id": "golden-buildings-partial-match-v1",
"fixtures": {
"candidate": "fixtures/golden/predicted_buildings.geojson",
"reference": "fixtures/golden/reference_buildings.geojson"
},
"metrics": {
"f1": 0.5,
"false_negative_count": 1,
"false_positive_count": 1,
"mean_iou": 0.8339768339652592,
"precision": 0.5,
"recall": 0.5
},
"persistence": {
"commit_count": 4,
"metric_count": 24,
"metric_keys": [
"f1",
"false_negative_count",
"false_positive_count",
"mean_iou",
"precision",
"recall"
],
"quality_check_count": 4
},
"scenario_count": 4,
"scenarios": [
{
"benchmark_id": "golden-buildings-partial-match-v1",
"description": "Two reference building polygons and two candidate building polygons: one candidate matches ref-1, one candidate is a false positive, and ref-2 is a false negative.",
"fixtures": {
"candidate": "fixtures/golden/predicted_buildings.geojson",
"reference": "fixtures/golden/reference_buildings.geojson"
},
"metrics": {
"f1": 0.5,
"false_negative_count": 1,
"false_positive_count": 1,
"mean_iou": 0.8339768339652592,
"precision": 0.5,
"recall": 0.5
},
"persistence": {
"commit_count": 1,
"metric_count": 6,
"metric_keys": [
"precision",
"recall",
"f1",
"mean_iou",
"false_positive_count",
"false_negative_count"
],
"quality_check_count": 1
},
"quality_check_id": "8d030591-a701-46ee-ae59-7f529db053d3",
"result_counts": {
"candidate_feature_count": 2,
"matches": 1,
"reference_feature_count": 2
},
"status": "passed"
},
{
"benchmark_id": "golden-buildings-perfect-match-v1",
"description": "One candidate building polygon exactly matches one reference polygon.",
"fixtures": {
"candidate": "fixtures/golden/predicted_buildings_perfect.geojson",
"reference": "fixtures/golden/reference_buildings_perfect.geojson"
},
"metrics": {
"f1": 1.0,
"false_negative_count": 0,
"false_positive_count": 0,
"mean_iou": 1.0,
"precision": 1.0,
"recall": 1.0
},
"persistence": {
"commit_count": 1,
"metric_count": 6,
"metric_keys": [
"precision",
"recall",
"f1",
"mean_iou",
"false_positive_count",
"false_negative_count"
],
"quality_check_count": 1
},
"quality_check_id": "bc99f243-ee42-40bf-ad71-22da29e5278f",
"result_counts": {
"candidate_feature_count": 1,
"matches": 1,
"reference_feature_count": 1
},
"status": "passed"
},
{
"benchmark_id": "golden-buildings-no-overlap-v1",
"description": "One candidate polygon and one reference polygon do not overlap.",
"fixtures": {
"candidate": "fixtures/golden/predicted_buildings_no_overlap.geojson",
"reference": "fixtures/golden/reference_buildings_no_overlap.geojson"
},
"metrics": {
"f1": null,
"false_negative_count": 1,
"false_positive_count": 1,
"mean_iou": null,
"precision": 0.0,
"recall": 0.0
},
"persistence": {
"commit_count": 1,
"metric_count": 6,
"metric_keys": [
"precision",
"recall",
"f1",
"mean_iou",
"false_positive_count",
"false_negative_count"
],
"quality_check_count": 1
},
"quality_check_id": "b12249f0-24e3-4bbf-837a-e4a962d1bca9",
"result_counts": {
"candidate_feature_count": 1,
"matches": 0,
"reference_feature_count": 1
},
"status": "passed"
},
{
"benchmark_id": "golden-buildings-multipolygon-match-v1",
"description": "One candidate MultiPolygon exactly matches one reference MultiPolygon.",
"fixtures": {
"candidate": "fixtures/golden/predicted_buildings_multipolygon.geojson",
"reference": "fixtures/golden/reference_buildings_multipolygon.geojson"
},
"metrics": {
"f1": 1.0,
"false_negative_count": 0,
"false_positive_count": 0,
"mean_iou": 1.0,
"precision": 1.0,
"recall": 1.0
},
"persistence": {
"commit_count": 1,
"metric_count": 6,
"metric_keys": [
"precision",
"recall",
"f1",
"mean_iou",
"false_positive_count",
"false_negative_count"
],
"quality_check_count": 1
},
"quality_check_id": "4dabce04-4c65-4dac-807d-9f9cbaa8c72f",
"result_counts": {
"candidate_feature_count": 1,
"matches": 1,
"reference_feature_count": 1
},
"status": "passed"
}
],
"status": "passed",
"version": 1
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,5 @@
command=python scripts/audit_api_contracts.py
exit_code=0
captured_at=2026-08-01T18:37:24.4984309+02:00
API contract audit OK: 147 implemented routes match docs; 10 explicit non-envelope endpoints tracked.
@@ -0,0 +1,36 @@
{
"baseline_scope": [
"tracked repository state",
"static code/test/migration inventory",
"tracked mirror comparison",
"local artifact inventory",
"mock/fixture/placeholder/fallback triage signals"
],
"finding_count": 2,
"findings": [
{
"evidence": {
"paired_different_file_count": 68,
"tracked_mirror_file_count": 1153
},
"id": "P1-REPO-001",
"severity": "high",
"title": "Tracked nested repository mirror creates ambiguous source state"
},
{
"id": "P1-ML-LOCAL-001",
"interpretation": "Production model truth must be verified on the mounted Tower volume.",
"severity": "info",
"title": "No local checkpoint is available in the repository checkout"
}
],
"generated_at": "2026-08-01T15:53:53.482303+00:00",
"schema_version": 1,
"separate_required_evidence": [
"Tower database/storage audit",
"Tower CUDA/model preflight and representative inference",
"corpus leakage/duplicate/label/time review",
"independent human visual review"
],
"status": "findings_present"
}
@@ -0,0 +1,6 @@
command=python -m pytest tests/test_accuracy_phase1_baseline.py -q -p no:cacheprovider
exit_code=0
captured_at=2026-08-01T18:33:32.0526074+02:00
.... [100%]
4 passed in 1.24s
@@ -0,0 +1,651 @@
{
"code": {
"api_route_decorator_count": 148,
"frontend_test_file_count": 16,
"groups": {
"backend": {
"file_count": 129,
"line_count": 32583
},
"backend_tests": {
"file_count": 235,
"line_count": 36067
},
"frontend": {
"file_count": 112,
"line_count": 23534
},
"frontend_e2e": {
"file_count": 2,
"line_count": 734
},
"migrations": {
"file_count": 11,
"line_count": 610
},
"root_tests": {
"file_count": 7,
"line_count": 261
},
"scripts": {
"file_count": 158,
"line_count": 43180
}
},
"pytest_test_function_count": 1182
},
"generated_at": "2026-08-01T15:53:27.980241+00:00",
"git": {
"branch": "codex/geointel-accuracy-program",
"head": "0c019bb22f816db1e4b7a68379bcad08924d9a21",
"status_porcelain": [
" M .gitignore",
"?? .codex-artifacts/",
"?? scripts/run_accuracy_phase1_baseline.py"
],
"top_level_tracked_counts": {
".codex-input": 109,
".dockerignore": 1,
".env.example": 1,
".gitattributes": 1,
".gitea": 1,
".github": 4,
".gitignore": 1,
".gitkeep": 1,
".playwright-mcp": 1,
"AGENTS.md": 1,
"CHANGELOG.md": 1,
"CODEX_START.md": 1,
"M10_UPDATE_MANIFEST.txt": 1,
"M11_UPDATE_MANIFEST.txt": 1,
"M12_UPDATE_MANIFEST.txt": 1,
"M13_UPDATE_MANIFEST.txt": 1,
"M14_UPDATE_MANIFEST.txt": 1,
"M5_UPDATE_MANIFEST.txt": 1,
"M9_UPDATE_MANIFEST.txt": 1,
"Makefile": 1,
"README.md": 1,
"RELEASE_NOTES": 8,
"VERSION": 1,
"adr": 7,
"backend": 406,
"checklists": 2,
"contracts": 7,
"data": 27,
"datasets": 4,
"demo": 7,
"deploy": 13,
"docker-compose.unraid.yml": 1,
"docker-compose.yml": 1,
"docs": 263,
"exports": 1,
"fixtures": 12,
"frontend": 154,
"geointel": 1153,
"knowledge": 6,
"models": 1,
"output": 1,
"prompts": 57,
"release": 1,
"rfc": 5,
"scripts": 167,
"security": 1,
"skills": 7,
"storage": 11,
"test-results": 4,
"tests": 21,
"tickets": 14
},
"tracked_file_count": 2495
},
"migrations": {
"count": 11,
"heads_from_static_chain": [
"202607260001"
],
"missing_revision_identifiers": 0,
"records": [
{
"down_revision": null,
"path": "backend/alembic/versions/202601110001_initial.py",
"revision": "202601110001"
},
{
"down_revision": "202601110001",
"path": "backend/alembic/versions/202601120001_dataset_storage_metadata.py",
"revision": "202601120001"
},
{
"down_revision": "202601120001",
"path": "backend/alembic/versions/20260611212435_add_jobs_table.py",
"revision": "20260611212435"
},
{
"down_revision": "20260611212435",
"path": "backend/alembic/versions/202606120001_add_dataset_reference_metadata.py",
"revision": "202606120001"
},
{
"down_revision": "202606120001",
"path": "backend/alembic/versions/202606120700_sprint7a_persistence_foundation.py",
"revision": "202606120700"
},
{
"down_revision": "202606120700",
"path": "backend/alembic/versions/202606120800_sprint8_detection_foundation.py",
"revision": "202606120800"
},
{
"down_revision": "202606120800",
"path": "backend/alembic/versions/202606120900_sprint9_segmentation_foundation.py",
"revision": "202606120900"
},
{
"down_revision": "202606120900",
"path": "backend/alembic/versions/202607140001_temporal_dataset_foundation.py",
"revision": "202607140001"
},
{
"down_revision": "202607140001",
"path": "backend/alembic/versions/202607150001_detection_reviews.py",
"revision": "202607150001"
},
{
"down_revision": "202607150001",
"path": "backend/alembic/versions/202607160001_vector_feature_municipality_index.py",
"revision": "202607160001"
},
{
"down_revision": "202607160001",
"path": "backend/alembic/versions/202607260001_aoi_operations.py",
"revision": "202607260001"
}
]
},
"repo_root": "C:\\Projects\\geointel",
"schema_version": 1,
"tracked_mirror": {
"different_files": [
{
"mirror_sha256": "74a27de50351950c6fa64cea9761f20f88f94500afaa5b0a83d750e260d07139",
"mirror_size_bytes": 261,
"path": ".dockerignore",
"root_sha256": "d6ef949ac81b034249bf8907a47d7fba1265e3cde0abd679a0884f321a57143a",
"root_size_bytes": 1059
},
{
"mirror_sha256": "68b3c2a8c7cbf56c16ab809af1a40f88aa4bf97f7071f004fe9651ae69952cff",
"mirror_size_bytes": 6558,
"path": ".env.example",
"root_sha256": "53c822e80b04bcc752eb41c5020a2f57345df4c7b7af53054cb6df6320b631da",
"root_size_bytes": 6657
},
{
"mirror_sha256": "6c6952d6e5fd0f763216617e3bd1ea9e5dbc91174a001a9293dcd54e49b552e7",
"mirror_size_bytes": 893,
"path": ".gitignore",
"root_sha256": "cc7a9c3ee59df1f72789b7e24d195bfce53854fb5be89ba1fd90afb7b9ef41df",
"root_size_bytes": 1166
},
{
"mirror_sha256": "077058b5c46d1f4069be44cd21deaddce669cf31f04f8ec8f0d2d6d57302be28",
"mirror_size_bytes": 253593,
"path": "CHANGELOG.md",
"root_sha256": "a46d9ca5f69ce5d806d76b30c156d5801fea0158c696004ceb83e0a2fa0548d7",
"root_size_bytes": 253841
},
{
"mirror_sha256": "b7ad2801709886035ae9b57130d4fb1808d4ee0520fee346bb8553fea7444340",
"mirror_size_bytes": 9662,
"path": "README.md",
"root_sha256": "9834f00b7439b49e6b9b976c1c12a6f94d30231ea10ecb69e7fe48029471bce1",
"root_size_bytes": 9504
},
{
"mirror_sha256": "fac098f49a784c2c7faff0dec7ab50f7dc9ffe3decdc48d05596d3fb72e7ecff",
"mirror_size_bytes": 21710,
"path": "backend/app/core/config.py",
"root_sha256": "6ac8dce2fcc32a9b47ca7d6bd9355ec5257476a90be7ea10052ff7b6e545cbaa",
"root_size_bytes": 21547
},
{
"mirror_sha256": "3f8282014fbc66059a7147aee9672a285227c1b5fc596cdc45edff4493ab4246",
"mirror_size_bytes": 15395,
"path": "backend/app/main.py",
"root_sha256": "27947e7b7762ca2226d9710a8438ff0b484cd142d3dbc51d76114d0447362634",
"root_size_bytes": 18001
},
{
"mirror_sha256": "f6a21cbc3d644d6cee922c8ddfe9ab41e35e9749cfcef2fc77095047fcb53a5e",
"mirror_size_bytes": 7546,
"path": "backend/tests/test_auth.py",
"root_sha256": "df31b406424b96884be1c7427e46a6caad78b6535a5c19924f4a12674ff0dcbf",
"root_size_bytes": 9638
},
{
"mirror_sha256": "860adf4f218de693856d52e2a31683754e218391643230623f3f02c61a663e40",
"mirror_size_bytes": 4313,
"path": "backend/tests/test_belgium_training_loop.py",
"root_sha256": "f77121b687ecd3fef50782a47a39fb361c3829fbf610fe3937b15c7d06d1fbc2",
"root_size_bytes": 10919
},
{
"mirror_sha256": "c89fd7a641b2969c72b96fedd1aff23f6c88eec56bfdd3581184365b86c6f0c1",
"mirror_size_bytes": 1344,
"path": "backend/tests/test_belgium_training_portfolio.py",
"root_sha256": "b93152cdf4c5894dbc614529fe9823bb6472fc45cebe24be8ad9cf900d5e98c9",
"root_size_bytes": 2021
},
{
"mirror_sha256": "8a40fd3f5daa010b203639c5114ff4c080294eb82316a2c3a06aac500cdd2c2e",
"mirror_size_bytes": 7556,
"path": "backend/tests/test_building_label_normalization.py",
"root_sha256": "6236040626f8ab770a1495052fa5c6f25739cef3a2d1d04d7cc3e232e3839421",
"root_size_bytes": 10317
},
{
"mirror_sha256": "9912c38ef5e24efc7048945dc9e7035c904f504972b56d3b6081561e42b07943",
"mirror_size_bytes": 20311,
"path": "backend/tests/test_docker_runtime_config.py",
"root_sha256": "1e6aea61938d9bb90707d724a186312a4a7475b15b891d69555b52187590b057",
"root_size_bytes": 20315
},
{
"mirror_sha256": "a91d378320b20ebe1ae5223d2049445802de81df26e86378e2ad33a032a9c643",
"mirror_size_bytes": 4827,
"path": "backend/tests/test_failure_driven_yolo_sampling.py",
"root_sha256": "c4343f42509189ea867b7cf285473b2e1687e55c5238fa7022b44edae06eec37",
"root_size_bytes": 16948
},
{
"mirror_sha256": "3cf7c2979234bfcb6095630d586731e8b9f2053fbced328c5820997db1bd2fdf",
"mirror_size_bytes": 4973,
"path": "backend/tests/test_rc6_supply_chain.py",
"root_sha256": "d4f0e29bc3a5820b378ffa904052bc0d56c3ccd2133b8406cebadad9ad702d59",
"root_size_bytes": 4973
},
{
"mirror_sha256": "a175e97aaa2adfc28a37ab175040fd3c4e1c37bf0e28d733564ffd30bc689600",
"mirror_size_bytes": 1016,
"path": "backend/tests/test_sam_roof_label_refinement.py",
"root_sha256": "c4fbab6d9ab2e94ee3014a46979e8489471b7eb375aaccbd7f2ae204c6b6f647",
"root_size_bytes": 1473
},
{
"mirror_sha256": "fc301f89b6c149d7b012bac3d2942902e74794d84fa18090e5485f1f7102f823",
"mirror_size_bytes": 12548,
"path": "backend/tests/test_sprint130_operator_yolo_tile_dataset.py",
"root_sha256": "ae3caa4d74a5e7ba4db3628ce5dd403b20fc08e37c522a7b85e657d75e11a1b6",
"root_size_bytes": 12941
},
{
"mirror_sha256": "c6b3f5a1f283c7940140734d35bbc44ec31c6eaadb6363abcb6fcdad4a97f176",
"mirror_size_bytes": 10437,
"path": "backend/tests/test_sprint31_unraid_template.py",
"root_sha256": "cf5f0eaf93510a7ebf70386691513275d0cca69b59d6a86b35d6ba486aa35f21",
"root_size_bytes": 10611
},
{
"mirror_sha256": "a8fff182a2968b2dae2c561bd23ebc6732b5d3a45fe886641ca9538550a126ce",
"mirror_size_bytes": 12232,
"path": "deploy/unraid/Dockerfile.all-in-one",
"root_sha256": "b30d16b8ac28acef25961674e1cacdb067bd43e3925adcb68a605f0a505eed80",
"root_size_bytes": 12326
},
{
"mirror_sha256": "cb4f487125c750137420f71b1b31309f596693186dc7e8b0d3f6a49afb69a6b9",
"mirror_size_bytes": 13687,
"path": "deploy/unraid/README.md",
"root_sha256": "8c72e0aa8448f83db73fbf3862280f414d4d31056ee5e8b2176c4356d996c38c",
"root_size_bytes": 13866
},
{
"mirror_sha256": "9851cb85ae5904b1def720055037ee1dbe3a405b767e76b0cf5c592a84afed7f",
"mirror_size_bytes": 5541,
"path": "deploy/unraid/deploy-release.sh",
"root_sha256": "21c358def5773975e0bfb37a1b2be441b616291a3f62944862abed7d799d3611",
"root_size_bytes": 7885
},
{
"mirror_sha256": "041d1cfc5d184a93650d650487db9b86d52a80d2ef4ee5fd7528d0ef4ba76459",
"mirror_size_bytes": 40243,
"path": "deploy/unraid/geointel-unraid-template.xml",
"root_sha256": "4d3da0654a308ed938ec745087e6484b710444a08ca5a54304f17934bba8a21f",
"root_size_bytes": 40252
},
{
"mirror_sha256": "d6ac993c9c08972a521b58136cb2029b89df34c67eb76818d1eeb631d42d3149",
"mirror_size_bytes": 7839,
"path": "deploy/unraid/geointel.env.example",
"root_sha256": "85d7f899827fd17debc1d8cffbe672337b470e5f18c390e587b5341d12c485d5",
"root_size_bytes": 8329
},
{
"mirror_sha256": "4839b5fd2ddd4d0c2e35b3dee6ae04d1f12ecf17cfe3a07a32d5accca64915b1",
"mirror_size_bytes": 23318,
"path": "deploy/unraid/run-dockerman-container.sh",
"root_sha256": "b5a348f0cd96eb09e81239d1fcbd5ad1395451c1371f0ef3af40446f027a0d2b",
"root_size_bytes": 23483
},
{
"mirror_sha256": "e5def10ebc15cf796fdbc3de3583fe6b8a761fb90e91ac61300ea363b5503d75",
"mirror_size_bytes": 10203,
"path": "docker-compose.unraid.yml",
"root_sha256": "30d4a6d809306e6ce8cdf5d9cbc5e0f510612db32a9398100e45e09d439f3d25",
"root_size_bytes": 10202
},
{
"mirror_sha256": "a9071fd2a61c8cfe9a43aed4a378770939d2ae3b35888c433356e017cb8aec48",
"mirror_size_bytes": 11887,
"path": "docker-compose.yml",
"root_sha256": "7e4f93a2ec69548a94556e06d71623e652e23acc311ca24cd56fefd7cda8fcd3",
"root_size_bytes": 11886
},
{
"mirror_sha256": "427a484990ade76e85d9d7cbba2835095ea4ca08a1b3b9836193ce9835f77231",
"mirror_size_bytes": 101810,
"path": "docs/API_CONTRACTS.md",
"root_sha256": "305db12f3f44fe6287f7b48543df5b4e461e3c22b54fa40f2e900470b9258ae7",
"root_size_bytes": 102260
},
{
"mirror_sha256": "694f87467374352486c9f134d08a432d94ec15fb3c85d9dbbc69f47a57fc8407",
"mirror_size_bytes": 6732,
"path": "docs/BELGIUM_BUILDING_TRAINING_LOOP.md",
"root_sha256": "39b5032a6d7983fd816440ddcf6fb79d3c4be822145e954aeb3c7f46c2f82701",
"root_size_bytes": 11239
},
{
"mirror_sha256": "665e8f21c3988909d4428ccc736479111f6118d6c470827f9399d53857f4a204",
"mirror_size_bytes": 818967,
"path": "docs/CODEX_EXECUTION_LOG.md",
"root_sha256": "a91e4c95d30e26eb075f7977ed25f435685f45d54fb889d085274273ef782451",
"root_size_bytes": 858488
},
{
"mirror_sha256": "e7b135e9550a79472be7ae4974bc699811b4ca4a257f6fcc1ae7ced1781c9e68",
"mirror_size_bytes": 2785,
"path": "docs/DEPENDENCY_POLICY.md",
"root_sha256": "3c2fd4680091c642c4429b5ec68a0d403aaf3cce44c7712c368fc0fbb61ad5c5",
"root_size_bytes": 2987
},
{
"mirror_sha256": "1cc29d86c7acdf76ebe62782e9aa66ae1dc179f6e6e017d80b6df21217feb05a",
"mirror_size_bytes": 9480,
"path": "docs/PROJECT_PROFESSIONALIZATION_AUDIT_2026-07-27.md",
"root_sha256": "836de43d4c81bdc5748b44a50848226d434dffb05a26fc2813e4abe318d1e537",
"root_size_bytes": 9597
},
{
"mirror_sha256": "c052edad0eff3e6042642d3e33e75bb5ea718baa49cc562cc4566aa3a3308441",
"mirror_size_bytes": 66244,
"path": "docs/TODO.md",
"root_sha256": "76834351437c69ef63838c6f802e73b844eb0e8c774b3709016b4b8de43ac239",
"root_size_bytes": 75459
},
{
"mirror_sha256": "70d5fccd4be3f5568b8d022a76aafcd30806366090d3c841e72fd3e80fa7b2cb",
"mirror_size_bytes": 59353,
"path": "frontend/src/App.tsx",
"root_sha256": "05c367d0f321d7f5787825943cd61c07e85742a017567436dd80eaf1d9d60746",
"root_size_bytes": 59689
},
{
"mirror_sha256": "e6a15ffc9a95c57a2c9710822385e0d7fdf48115b2109cca2ba1533d494a9f7f",
"mirror_size_bytes": 21860,
"path": "frontend/src/components/GeoMap.tsx",
"root_sha256": "64a2fe78846f51c9227ed5927dff9f7b9df4ee3ce04a17cf0d18b4d1595a149f",
"root_size_bytes": 22935
},
{
"mirror_sha256": "c5202c44be3154ce2cb395681c65d3bde3866603cdbf6f7da8c30a77969e6471",
"mirror_size_bytes": 6371,
"path": "frontend/src/components/assistant/GeoAssistantPanel.tsx",
"root_sha256": "250d8781246a3e7d218699de38c80a9816cf1718fbea8d45b46b1a58bd91768a",
"root_size_bytes": 7963
},
{
"mirror_sha256": "ab91189c74d1af534334855957f308e48846f1184c4862017c2472c29b687724",
"mirror_size_bytes": 16246,
"path": "frontend/src/components/auth/LandingPage.tsx",
"root_sha256": "dbd1f3df060c62c7340d711db24d0f17444406923a5c05dd32fc56fdeba9fc85",
"root_size_bytes": 16489
},
{
"mirror_sha256": "a41b9221ba39f1e1f6e07a12e800eaa94ee877a38d550cb48bdb5617dccb85c8",
"mirror_size_bytes": 50857,
"path": "frontend/src/components/detection/DetectionLab.tsx",
"root_sha256": "366e65b6d0aabb50da8b18a9e7c9769ddc03de5f841c367eb8db8b1b54efa171",
"root_size_bytes": 51643
},
{
"mirror_sha256": "c5a1595e2799d848852f8891856323faa4863f8c1bfcbfbc7f5e6b64cee4d2ed",
"mirror_size_bytes": 175335,
"path": "frontend/src/components/map/MapWorkspace.tsx",
"root_sha256": "6f57d1ea8201cd97e228a9b562173457a98caf0231b7439b71abdd1a1bef43aa",
"root_size_bytes": 175688
},
{
"mirror_sha256": "012608ad5ea891bcaeb920c279720aa8f6091a0c6d8735dcf9e33210301a5c84",
"mirror_size_bytes": 7614,
"path": "frontend/src/components/overview/OverviewWorkspace.tsx",
"root_sha256": "4b091b6c3fd4d3a7065604a92267521d549abe8217de2d899fe37405d941d1ce",
"root_size_bytes": 10080
},
{
"mirror_sha256": "80898f659302de06b50d66d2a54b99c4262f13f574eaaad95a2c6d75c0081d36",
"mirror_size_bytes": 19502,
"path": "frontend/src/components/segmentation/SegmentationLab.tsx",
"root_sha256": "5e4ee646bc6e998d0dc83b0dfc76e055ef39981024b7d00efdafe7cdef9a2dae",
"root_size_bytes": 19571
},
{
"mirror_sha256": "00a189596cd8752d24392707288d9b557c411738bb860f19c0983db39dd9cd1c",
"mirror_size_bytes": 2657,
"path": "frontend/src/components/shell/WorkbenchNavigation.tsx",
"root_sha256": "2216e8a6158b314ee0ab1661e7ff34dadcb4a5f2917f0f88ce4cc895cfda76e7",
"root_size_bytes": 2616
},
{
"mirror_sha256": "ef09733b1ba174c9f02dd84eb9e5e1cee87ce3bcfbaf7d5f09c9a16f967ec784",
"mirror_size_bytes": 4153,
"path": "frontend/src/hooks/useDemoWorkflow.ts",
"root_sha256": "15d1b3db25b941285b2ab3460ca4b886e382062550e8a9d2377a62c9fe09a06c",
"root_size_bytes": 4076
},
{
"mirror_sha256": "074f3e0dc7f94c39899b3c456bbde3c78f9fdd26cbf5664b7af3ad7a048eee58",
"mirror_size_bytes": 23013,
"path": "frontend/src/hooks/useDetectionWorkflow.ts",
"root_sha256": "6a75f632a4b66fdb43c78e4a77b71c21566813b94af07ec49bb9fc9658c438ff",
"root_size_bytes": 23097
},
{
"mirror_sha256": "a3f4b018086b86364db6adfac28633d6a4d8feaca4f873fabc82bfb45c650a20",
"mirror_size_bytes": 7570,
"path": "frontend/src/hooks/useExportWorkflow.ts",
"root_sha256": "a04b02ea9604ba294f94e2df8a289fdf45f4e014bef2721d1270b28239bff36c",
"root_size_bytes": 7818
},
{
"mirror_sha256": "05fd1dea79a335a6607a5994bbd39bd83b5c215a352abb2747a02ec1dda0d2fb",
"mirror_size_bytes": 3695,
"path": "frontend/src/hooks/useGeoAssistant.ts",
"root_sha256": "af8d707c8639dca1fe2df735e965c36d74b2696057f82f318ac7dff78564107e",
"root_size_bytes": 4771
},
{
"mirror_sha256": "b38fda4b32212a3bf12b89183d23c3dadeceb104ae40952bf539785f565273ea",
"mirror_size_bytes": 3463,
"path": "frontend/src/hooks/useMapSelectionQa.ts",
"root_sha256": "0785c198d3860ed330538bbb56ba4745262c3b32fab099164f1ab2d8421526fc",
"root_size_bytes": 3482
},
{
"mirror_sha256": "75efa3a406a5cfe400a209eadc23317086b4742068599fa492b6f20c075a0a9e",
"mirror_size_bytes": 4923,
"path": "frontend/src/hooks/useQualityWorkflow.ts",
"root_sha256": "662377348c1cfaa462955792166d5752a25056c38d3ce8375799ae46a5c4df11",
"root_size_bytes": 4942
},
{
"mirror_sha256": "52cf7dd8fae31ec2bcc0f1dbec124319c389bd2c45dea5c6f6aad5b9960a43d5",
"mirror_size_bytes": 9426,
"path": "frontend/src/hooks/useSegmentationWorkflow.ts",
"root_sha256": "1864807b63ecaf778df7f53218fa06a6f30a3fa40e39830e32aad81e9cbedbb4",
"root_size_bytes": 9491
},
{
"mirror_sha256": "153296d88e9549a9227adc552c614ccbe931658e0174061313ad819e7ff9136b",
"mirror_size_bytes": 3528,
"path": "frontend/src/hooks/useWorkbenchBootstrap.test.tsx",
"root_sha256": "150d755f5f6dbb70404a410eee4ab447a36ce86c90c65ed3528afdd9a2cb1fc2",
"root_size_bytes": 3565
},
{
"mirror_sha256": "9c19508fd628f004f095e9f29cfaec608c91282dc0b42bb6a96f2fa36da64142",
"mirror_size_bytes": 3084,
"path": "frontend/src/hooks/useWorkbenchBootstrap.ts",
"root_sha256": "86646f79ac20672fe1ba04b8d97c899b8dbad37b53b94a03b9778035fda3a953",
"root_size_bytes": 2946
},
{
"mirror_sha256": "7441e17e4c9bee54847317fcde7d4520c2c96bc5c78a0c9ceaac7a63cf81186c",
"mirror_size_bytes": 2536,
"path": "frontend/src/services/api/detection.ts",
"root_sha256": "a56da21fb640431b49b1a77530d4b993d1852b9a4adf2811c0f800a9b0a9288b",
"root_size_bytes": 2682
},
{
"mirror_sha256": "287165ff54e427b30ec574ed995213ad02e7e94220874c8064a57cf6facc417a",
"mirror_size_bytes": 1804,
"path": "frontend/src/services/api/exports.ts",
"root_sha256": "7f9f1da4694143d1c3a768950fb13b894d89f0c37b9cf6160362d1db55c939a3",
"root_size_bytes": 2253
},
{
"mirror_sha256": "11637de6de25f2830f600d01c4a1a45dedfcdc75ffa9121a218d7e5c7f857d64",
"mirror_size_bytes": 1725,
"path": "frontend/src/services/api/qa.ts",
"root_sha256": "fa3fcfd0a9e0e123bf2b821ae2b2a0d98062eaaae15a0a49dd13f3caed9bfb3c",
"root_size_bytes": 1788
},
{
"mirror_sha256": "24be3bf6499337864ae2d99fac6409c908b473926e7b31bb61624e2f2563e0ae",
"mirror_size_bytes": 2178,
"path": "frontend/src/services/api/segmentation.ts",
"root_sha256": "f4666f07aea007513a915e30f2c487b54e7eb5f3da171f8aac634c09a60558b2",
"root_size_bytes": 2324
},
{
"mirror_sha256": "a2aba5359ff65b328a9c01ed966ed29e235bf7020083a086c6815978229a4c6f",
"mirror_size_bytes": 137520,
"path": "frontend/src/styles/app.css",
"root_sha256": "bacced4a7ef1a8253ed4cf6999140230d434f8efcf8654691bbafb1595a7bb44",
"root_size_bytes": 137816
},
{
"mirror_sha256": "f60090c6ea5046cb8db4bb3e72105baf4b5e4e1aceeae7bee08619d237859c60",
"mirror_size_bytes": 34362,
"path": "frontend/src/styles/landing.css",
"root_sha256": "a21b6568d9a9394cd513107b1b3e2ea616b5e8ea117107264607dcefdcbe74a8",
"root_size_bytes": 34383
},
{
"mirror_sha256": "ef6c5e988a0558cf5f25a158bcbf968b5a23fa0f2256ea802ffc29776f773dbf",
"mirror_size_bytes": 4935,
"path": "scripts/assess_belgium_building_training_iteration.py",
"root_sha256": "3eb022330efd838bac23f4af1d6b70f3f589f8133797a22a25c20b8a0f9a27ef",
"root_size_bytes": 4954
},
{
"mirror_sha256": "b50620a8b8d179d2d91a129db50609939f8f28ee272bd5380bca9a11bc6518d4",
"mirror_size_bytes": 7110,
"path": "scripts/build_failure_driven_yolo_sampling.py",
"root_sha256": "35d662a838b78873b6298145f722bc41b38aa96af47be5e4e9f01f96e98422e6",
"root_size_bytes": 15234
},
{
"mirror_sha256": "22e8c202b49fc3c31fbc3c74fae79855474e3c9fbb9c61710dbbbda7626ba787",
"mirror_size_bytes": 2986,
"path": "scripts/build_regional_yolo_dataset.py",
"root_sha256": "c28f677ff81f94b4a7d0337cec26995b646c2d576da9a0079fb9007e78b5edf8",
"root_size_bytes": 6874
},
{
"mirror_sha256": "d0f859e810839fc34585f0f65877808bef952452b87c78b236450a1b3107ba4a",
"mirror_size_bytes": 2652,
"path": "scripts/deploy_tower.ps1",
"root_sha256": "5470e5a7ef9ca513e29050d9cc3bc5cff32395420c4c78ae4685302a75cb5196",
"root_size_bytes": 2967
},
{
"mirror_sha256": "ccef16aaf01ec16a297eec423f993afa705f45dd052919b005a231f087ff7469",
"mirror_size_bytes": 1471,
"path": "scripts/deploy_tower.sh",
"root_sha256": "77e04343e60bd1ef3846a459f8c4b70c0cd539f1900fb8c24fb3964b7062d785",
"root_size_bytes": 1925
},
{
"mirror_sha256": "c96d6c3bb70a59256870bbbf464e5b5ba1c599e24c38a5c0b8ff19d57fff5930",
"mirror_size_bytes": 15603,
"path": "scripts/evaluate_belgium_building_candidate.py",
"root_sha256": "640312d1de91362b7b906f7e468e9ebf09c80958d11c8cda5e3b1663b85fafde",
"root_size_bytes": 17832
},
{
"mirror_sha256": "96fe2345e35eeceb5b11c15760b61a33070a2ab2602d623af5cb99b20f37270e",
"mirror_size_bytes": 28472,
"path": "scripts/export_operator_yolo_tile_dataset.py",
"root_sha256": "5e84eae418c5f855bc8bff10f9967a331df40c7736d93ea2ed0905f59fab04a5",
"root_size_bytes": 29051
},
{
"mirror_sha256": "91275894803fb0913c0786bedd8ff1e3ac46d0e52c28edb2ecd7e4c4f1698cf1",
"mirror_size_bytes": 13510,
"path": "scripts/normalize_belgium_building_labels.py",
"root_sha256": "e231f6e34585531c162807b67fa24721455a56af66fab5b1d72835a8f46db774",
"root_size_bytes": 14792
},
{
"mirror_sha256": "ba0b5fa26c01c739e74d8791bf17db7159159aceaac79d485cc4e8d61d2c6bba",
"mirror_size_bytes": 17734,
"path": "scripts/provision_belgium_building_training_portfolio.py",
"root_sha256": "17da30d3ea480b4d0d86d5c2a454264202cb02443e4dbef9c082342ccfca08ca",
"root_size_bytes": 26891
},
{
"mirror_sha256": "5deb04f2563cb30028a34dcd7d5b275855822076495e5d36560ab6674a2a86a2",
"mirror_size_bytes": 8777,
"path": "scripts/refine_yolo_labels_with_sam.py",
"root_sha256": "472985d75d8800e5e15634477bf9f7fcd01ecbf5c67985e25d3ca0872a8dc603",
"root_size_bytes": 11442
},
{
"mirror_sha256": "5820d017710dae0b4ae518e2db22d8cbab6b72e708be0e7f9c68a9f631f5e6ef",
"mirror_size_bytes": 16456,
"path": "scripts/render_operator_yolo_label_qa_contact_sheets.py",
"root_sha256": "e1a16e5ab2d0b0d50144ac49d7805eb9a1814cc31471c19d109cfa83a295d243",
"root_size_bytes": 17322
},
{
"mirror_sha256": "06a7f7ba3c067de259c8fce62d3949979fb9296d1665f14a4e6f41039dadc9ae",
"mirror_size_bytes": 9516,
"path": "scripts/rotate_belgium_building_holdouts.py",
"root_sha256": "e0d35afe7c719715b19d7a6b7be75ae64d2c2c65e006cfc798eea3bbf18bedf9",
"root_size_bytes": 9920
},
{
"mirror_sha256": "2d3677e57c3089fe8a9b9f1d999589d4d938b94b193f24adcfc15f343287df35",
"mirror_size_bytes": 12958,
"path": "scripts/run_belgium_building_training_loop.py",
"root_sha256": "2ec1d535fc7bf41b6278acf7132eff7b91a081067590f2a6949b4a92781d8f3a",
"root_size_bytes": 20572
}
],
"paired_different_file_count": 68,
"paired_identical_file_count": 1085,
"risk": "The tracked geointel/ repository mirror can create ambiguous imports, stale tests and local/deployment drift; Docker excludes it but local tools may not.",
"tracked_mirror_file_count": 1153
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,663 @@
{
"counts": {
"fallback": 27,
"fixture": 94,
"heuristic": 1,
"mock": 31,
"not_configured": 24,
"placeholder": 39,
"todo": 2
},
"examples": {
"fallback": [
{
"line": 1046,
"path": "backend/app/services/export_service.py",
"text": "def _filename(name: str | None, fallback: str, suffix: str) -> str:"
},
{
"line": 1047,
"path": "backend/app/services/export_service.py",
"text": "raw_name = name or fallback"
},
{
"line": 1050,
"path": "backend/app/services/export_service.py",
"text": "cleaned = fallback"
},
{
"line": 29,
"path": "backend/app/services/mdk_bathymetry_acquisition_service.py",
"text": "No depth values are ever synthesized, no insecure TLS fallback exists and the"
},
{
"line": 173,
"path": "backend/app/services/mdk_bathymetry_probe_service.py",
"text": "message=\"MDK WCS TLS certificate validation failed; insecure fallback is prohibited.\","
},
{
"line": 186,
"path": "backend/app/services/mdk_bathymetry_probe_service.py",
"text": "\"MDK WCS TLS certificate validation failed; insecure fallback is prohibited.\""
},
{
"line": 440,
"path": "backend/app/services/raster_operations_service.py",
"text": "# best effort fallback; keep computed dimensions."
},
{
"line": 19,
"path": "backend/app/services/raster_service.py",
"text": "except Exception as exc: # pragma: no cover - exercised via API-level fallback tests"
},
{
"line": 26,
"path": "backend/app/services/storage_service.py",
"text": "fallback = \"upload\""
},
{
"line": 28,
"path": "backend/app/services/storage_service.py",
"text": "return fallback"
},
{
"line": 37,
"path": "backend/app/services/storage_service.py",
"text": "return cleaned or fallback"
},
{
"line": 76,
"path": "frontend/src/hooks/useGeoAssistant.ts",
"text": "const fallback = result.default_model && result.items.some((model) => model.name === result.default_model)"
},
{
"line": 79,
"path": "frontend/src/hooks/useGeoAssistant.ts",
"text": "setDefaultModel(fallback)"
},
{
"line": 9,
"path": "frontend/src/lib/authError.ts",
"text": "export function formatAuthError(error: unknown, fallback: string): string {"
},
{
"line": 10,
"path": "frontend/src/lib/authError.ts",
"text": "if (!(error instanceof Error)) return fallback"
},
{
"line": 21,
"path": "frontend/src/lib/authError.ts",
"text": "return fallback"
},
{
"line": 24,
"path": "frontend/src/lib/authError.ts",
"text": "return error.message || fallback"
},
{
"line": 1,
"path": "frontend/src/lib/formatError.ts",
"text": "export function formatError(error: unknown, fallback: string): string {"
},
{
"line": 6,
"path": "frontend/src/lib/formatError.ts",
"text": "return fallback"
},
{
"line": 501,
"path": "scripts/provision_belgium_north_sea_scope.py",
"text": "def _max_admin_modification(payloads: dict[str, dict[str, Any]], fallback: str) -> str:"
},
{
"line": 507,
"path": "scripts/provision_belgium_north_sea_scope.py",
"text": "value = max(values) if values else fallback"
},
{
"line": 111,
"path": "scripts/refine_yolo_labels_with_sam.py",
"text": "parser.add_argument(\"--fallback-policy\", choices=(\"retain\", \"drop\"), default=\"retain\")"
},
{
"line": 42,
"path": "scripts/run_accuracy_phase1_baseline.py",
"text": "\"fallback\": re.compile(r\"\\b(?:fallback|fall back)\\b\", re.IGNORECASE),"
},
{
"line": 347,
"path": "scripts/run_accuracy_phase1_baseline.py",
"text": "\"mock/fixture/placeholder/fallback triage signals\","
},
{
"line": 52,
"path": "scripts/run_golden_qa_benchmark.py",
"text": "# Backwards-compatible fallback for older checkouts and local smoke scripts."
}
],
"fixture": [
{
"line": 3,
"path": "backend/app/providers/__init__.py",
"text": "from app.providers import base, fixture, grb, manual, osm, registry"
},
{
"line": 5,
"path": "backend/app/providers/__init__.py",
"text": "__all__ = [\"base\", \"fixture\", \"grb\", \"manual\", \"osm\", \"registry\"]"
},
{
"line": 9,
"path": "backend/app/providers/fixture.py",
"text": "provider_name=\"fixture\","
},
{
"line": 10,
"path": "backend/app/providers/fixture.py",
"text": "display_name=\"Fixture data\","
},
{
"line": 11,
"path": "backend/app/providers/fixture.py",
"text": "authority_level=\"fixture\","
},
{
"line": 14,
"path": "backend/app/providers/fixture.py",
"text": "supported_query_modes=[\"fixture\"],"
},
{
"line": 15,
"path": "backend/app/providers/fixture.py",
"text": "fetch_signature=\"tests/fixtures and demo fixture upload flow\","
},
{
"line": 16,
"path": "backend/app/providers/fixture.py",
"text": "limitation_message=\"Fixture provider represents local test/demo fixtures only.\","
},
{
"line": 7,
"path": "backend/app/providers/registry.py",
"text": "from app.providers.fixture import FixtureProvider"
},
{
"line": 37,
"path": "backend/app/providers/registry.py",
"text": "\"fixture\": FixtureProvider(),"
},
{
"line": 112,
"path": "backend/app/providers/registry.py",
"text": "provider_name=\"fixture\","
},
{
"line": 114,
"path": "backend/app/providers/registry.py",
"text": "message=\"Fixture provider data must use checked-in demo/test fixture flows.\","
},
{
"line": 122,
"path": "backend/app/services/demo_workflow_service.py",
"text": ".filter(Dataset.source_name == \"fixture\")"
},
{
"line": 129,
"path": "backend/app/services/demo_workflow_service.py",
"text": ".filter(Dataset.source_name == \"fixture\")"
},
{
"line": 148,
"path": "backend/app/services/demo_workflow_service.py",
"text": ".filter(Dataset.source_name == \"fixture\")"
},
{
"line": 213,
"path": "backend/app/services/demo_workflow_service.py",
"text": "source=\"fixture\","
},
{
"line": 218,
"path": "backend/app/services/demo_workflow_service.py",
"text": "\"fixture\": True,"
},
{
"line": 298,
"path": "backend/app/services/demo_workflow_service.py",
"text": "source=\"fixture\","
},
{
"line": 300,
"path": "backend/app/services/demo_workflow_service.py",
"text": "source_name=\"fixture\","
},
{
"line": 303,
"path": "backend/app/services/demo_workflow_service.py",
"text": "\"fixture\": True,"
},
{
"line": 423,
"path": "backend/app/services/demo_workflow_service.py",
"text": ".filter(Dataset.source_name == \"fixture\")"
},
{
"line": 430,
"path": "backend/app/services/demo_workflow_service.py",
"text": ".filter(Dataset.source_name == \"fixture\")"
},
{
"line": 472,
"path": "backend/app/services/demo_workflow_service.py",
"text": "description=\"Offline fixture workflow: reference buildings, predicted buildings and persisted QA metrics.\","
},
{
"line": 497,
"path": "backend/app/services/demo_workflow_service.py",
"text": "source_name=\"fixture\","
},
{
"line": 509,
"path": "backend/app/services/demo_workflow_service.py",
"text": "source_name=\"fixture\","
}
],
"heuristic": [
{
"line": 39,
"path": "scripts/run_accuracy_phase1_baseline.py",
"text": "\"heuristic\": re.compile(r\"\\bheuristic(?:s)?\\b\", re.IGNORECASE),"
}
],
"mock": [
{
"line": 7,
"path": "frontend/src/components/auth/LandingPage.test.tsx",
"text": "vi.mock('../../services/api/auth', () => ({"
},
{
"line": 34,
"path": "frontend/src/components/auth/LandingPage.test.tsx",
"text": "vi.mocked(login).mockReset()"
},
{
"line": 35,
"path": "frontend/src/components/auth/LandingPage.test.tsx",
"text": "vi.mocked(loginAsGuest).mockReset()"
},
{
"line": 42,
"path": "frontend/src/components/auth/LandingPage.test.tsx",
"text": "vi.mocked(login).mockResolvedValue(operatorSession)"
},
{
"line": 57,
"path": "frontend/src/components/auth/LandingPage.test.tsx",
"text": "vi.mocked(loginAsGuest).mockResolvedValue(guestSession)"
},
{
"line": 75,
"path": "frontend/src/components/auth/LandingPage.test.tsx",
"text": "vi.mocked(login).mockRejectedValue(new Error('Gebruikersnaam of wachtwoord is onjuist.'))"
},
{
"line": 8,
"path": "frontend/src/components/map/MunicipalitySearch.test.tsx",
"text": "vi.mock('../../services/api/areas', () => ({"
},
{
"line": 14,
"path": "frontend/src/components/map/MunicipalitySearch.test.tsx",
"text": "vi.mocked(areasApi.searchMunicipalities).mockResolvedValue({"
},
{
"line": 5,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "const mocks = vi.hoisted(() => ({"
},
{
"line": 9,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "vi.mock('../services/api', () => ({"
},
{
"line": 11,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "resolveCoverage: mocks.resolveCoverage,"
},
{
"line": 35,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "mocks.resolveCoverage.mockResolvedValue(coverageResult)"
},
{
"line": 51,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "expect(mocks.resolveCoverage).not.toHaveBeenCalled()"
},
{
"line": 58,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "expect(mocks.resolveCoverage).not.toHaveBeenCalled()"
},
{
"line": 64,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "expect(mocks.resolveCoverage).toHaveBeenCalledWith({"
},
{
"line": 89,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "mocks.resolveCoverage.mockRejectedValueOnce(new Error('provider unavailable'))"
},
{
"line": 102,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "mocks.resolveCoverage.mockImplementationOnce("
},
{
"line": 5,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "const mocks = vi.hoisted(() => ({"
},
{
"line": 9,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "vi.mock('../services/api/temporal', () => ({"
},
{
"line": 11,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "compare: mocks.compare,"
},
{
"line": 34,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "expect(mocks.compare).not.toHaveBeenCalled()"
},
{
"line": 43,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "expect(mocks.compare).not.toHaveBeenCalled()"
},
{
"line": 53,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "mocks.compare.mockResolvedValueOnce(comparison)"
},
{
"line": 64,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "expect(mocks.compare).toHaveBeenCalledWith('project-1', {"
},
{
"line": 37,
"path": "scripts/run_accuracy_phase1_baseline.py",
"text": "\"mock\": re.compile(r\"\\bmock(?:ed|ing|s)?\\b\", re.IGNORECASE),"
}
],
"not_configured": [
{
"line": 166,
"path": "backend/app/api/routes/external.py",
"text": "status=response.get(\"status\", \"not_configured\"),"
},
{
"line": 149,
"path": "backend/app/api/routes/health.py",
"text": "yolo_status = configured_yolo.status if configured_yolo else \"not_configured\""
},
{
"line": 79,
"path": "backend/app/providers/base.py",
"text": "status=\"configured\" if self.is_configured else \"not_configured\","
},
{
"line": 94,
"path": "backend/app/providers/base.py",
"text": "\"status\": \"not_configured\","
},
{
"line": 96,
"path": "backend/app/providers/registry.py",
"text": "status=\"not_configured\","
},
{
"line": 9,
"path": "backend/app/schemas/coverage.py",
"text": "CoverageStatus = Literal[\"operational\", \"partial\", \"not_configured\", \"unsupported\"]"
},
{
"line": 53,
"path": "backend/app/services/coverage_registry_service.py",
"text": "STATUS_ORDER = (\"unsupported\", \"not_configured\", \"partial\", \"operational\")"
},
{
"line": 340,
"path": "backend/app/services/coverage_registry_service.py",
"text": "integration_status=\"not_configured\","
},
{
"line": 572,
"path": "backend/app/services/coverage_registry_service.py",
"text": "\"not_configured\""
},
{
"line": 266,
"path": "backend/app/services/geo_assistant_service.py",
"text": "status=\"not_configured\","
},
{
"line": 42,
"path": "backend/app/services/model_registry_service.py",
"text": "status=\"not_configured\","
},
{
"line": 98,
"path": "backend/app/services/model_registry_service.py",
"text": "status=\"not_configured\","
},
{
"line": 127,
"path": "backend/app/services/model_registry_service.py",
"text": "status = \"not_configured\""
},
{
"line": 169,
"path": "backend/app/services/model_registry_service.py",
"text": "status = \"not_configured\""
},
{
"line": 211,
"path": "backend/app/services/model_registry_service.py",
"text": "status = \"not_configured\""
},
{
"line": 36,
"path": "backend/app/services/yolo_preflight_service.py",
"text": "\"status\": \"not_configured\","
},
{
"line": 40,
"path": "frontend/src/components/detection/DetectionModelManagement.tsx",
"text": "if (value === 'not_configured') return 'niet geconfigureerd'"
},
{
"line": 295,
"path": "frontend/src/components/map/MapWorkspace.tsx",
"text": "not_configured: 'Niet gekoppeld',"
},
{
"line": 1050,
"path": "frontend/src/components/map/MapWorkspace.tsx",
"text": "{ operational: 0, partial: 0, not_configured: 0, unsupported: 0 },"
},
{
"line": 1051,
"path": "frontend/src/components/map/MapWorkspace.tsx",
"text": ") ?? { operational: 0, partial: 0, not_configured: 0, unsupported: 0 },"
},
{
"line": 3365,
"path": "frontend/src/components/map/MapWorkspace.tsx",
"text": "<span>{coverageCounts.not_configured} niet gekoppeld</span>"
},
{
"line": 6,
"path": "frontend/src/components/models/modelOptions.ts",
"text": "const configured = model.configured && model.status !== 'not_configured'"
},
{
"line": 1072,
"path": "frontend/src/types.ts",
"text": "export type CoverageStatus = 'operational' | 'partial' | 'not_configured' | 'unsupported'"
},
{
"line": 40,
"path": "scripts/run_accuracy_phase1_baseline.py",
"text": "\"not_configured\": re.compile(r\"\\bnot_configured\\b\", re.IGNORECASE),"
}
],
"placeholder": [
{
"line": 36,
"path": "backend/app/services/model_registry_service.py",
"text": "model_id=\"yolo-placeholder\","
},
{
"line": 37,
"path": "backend/app/services/model_registry_service.py",
"text": "display_name=\"YOLO detector placeholder\","
},
{
"line": 92,
"path": "backend/app/services/model_registry_service.py",
"text": "model_id=\"segmentation-placeholder\","
},
{
"line": 93,
"path": "backend/app/services/model_registry_service.py",
"text": "display_name=\"Segmentation placeholder\","
},
{
"line": 94,
"path": "backend/app/services/model_registry_service.py",
"text": "framework=\"placeholder\","
},
{
"line": 99,
"path": "backend/app/services/model_registry_service.py",
"text": "limitation_message=\"Segmentation inference is not configured for this placeholder; no model is downloaded or executed.\","
},
{
"line": 187,
"path": "frontend/src/components/assistant/GeoAssistantPanel.tsx",
"text": "placeholder=\"Bijvoorbeeld: hoeveel bos verdween er sinds 2013?\""
},
{
"line": 246,
"path": "frontend/src/components/datasets/DatasetPanel.tsx",
"text": "placeholder=\"bijvoorbeeld eigen luchtbeeld\""
},
{
"line": 306,
"path": "frontend/src/components/datasets/DatasetPanel.tsx",
"text": "placeholder=\"Naam, bron, gemeente of type\""
},
{
"line": 276,
"path": "frontend/src/components/datasets/RasterControls.tsx",
"text": "placeholder=\"EPSG:31370\""
},
{
"line": 345,
"path": "frontend/src/components/datasets/RasterControls.tsx",
"text": "placeholder=\"optioneel\""
},
{
"line": 44,
"path": "frontend/src/components/detection/DetectionLab.tsx",
"text": "if (modelName === 'yolo-placeholder') return 'Niet-geconfigureerd gebouwmodel'"
},
{
"line": 516,
"path": "frontend/src/components/detection/DetectionLab.tsx",
"text": "placeholder=\"Pad naar de aangemaakte beeldtegels\""
},
{
"line": 583,
"path": "frontend/src/components/detection/DetectionLab.tsx",
"text": "placeholder=\"0.50 0.25 0.15\""
},
{
"line": 715,
"path": "frontend/src/components/detection/DetectionLab.tsx",
"text": "placeholder=\"bijvoorbeeld gebouw\""
},
{
"line": 28,
"path": "frontend/src/components/detection/DetectionModelManagement.tsx",
"text": "if (model.model_id === 'yolo-placeholder') return 'Gebouwmodel nog niet geconfigureerd'"
},
{
"line": 402,
"path": "frontend/src/components/exports/ExportCenter.tsx",
"text": "placeholder=\"Type, id of pad\""
},
{
"line": 2312,
"path": "frontend/src/components/map/MapWorkspace.tsx",
"text": "placeholder=\"Zoek themas\""
},
{
"line": 82,
"path": "frontend/src/components/map/MunicipalitySearch.tsx",
"text": "placeholder=\"Gemeentenaam of NIS-code\""
},
{
"line": 98,
"path": "frontend/src/components/project/AreaPanel.tsx",
"text": "placeholder=\"Naam van het gebied\""
},
{
"line": 106,
"path": "frontend/src/components/project/AreaPanel.tsx",
"text": "placeholder=\"EPSG:4326\""
},
{
"line": 142,
"path": "frontend/src/components/project/AreaPanel.tsx",
"text": "placeholder=\"Bijvoorbeeld Brussel, Namen of Noordzee\""
},
{
"line": 119,
"path": "frontend/src/components/project/ProjectPanel.tsx",
"text": "placeholder=\"Naam van de werkruimte\""
},
{
"line": 127,
"path": "frontend/src/components/project/ProjectPanel.tsx",
"text": "placeholder=\"Korte beschrijving\""
},
{
"line": 135,
"path": "frontend/src/components/project/ProjectPanel.tsx",
"text": "placeholder=\"Regio\""
}
],
"todo": [
{
"line": 23,
"path": "scripts/codex_pass_end_check.sh",
"text": "\"TODO: implement later\\|placeholder only\\|fake completed\" \\"
},
{
"line": 6,
"path": "scripts/contract_drift_grep.sh",
"text": "if grep -R \"TODO: decide\\|TBD\\|placeholder only\\|fake real\" -n docs contracts backend frontend prompts 2>/dev/null; then"
}
]
},
"interpretation": "Triage signals only; production impact requires a traced contract/runtime path."
}
@@ -0,0 +1,215 @@
{
"captured_at": "2026-08-01T18:13:54.6553+02:00",
"claim_boundary": "One production-adapter inference completed on one existing tile. No accuracy, calibration, geographic-generalization, or release claim follows from this smoke.",
"configuration": {
"confidence": 0.5,
"deterministic_algorithms": true,
"device": "cuda:0",
"image_size": 640,
"max_detections": 1000,
"seed": 20260801
},
"input": {
"manifest_path": "/app/storage/tiles/cb80638d-dbef-48ac-b19c-cec7c3efc96e/ae0ff76d-70c0-404f-b777-54d14517179a/b191c968-7d56-4e0d-afbb-8b5baaa62470/manifest.json",
"manifest_sha256": "6ab8a96bf2a1405e224932afb90255311a09bbeaed4e9a2fdcdf8b1bc2230abd",
"raster": {
"available": true,
"bounds": [
193277.53584918313,
205708.27443503588,
193777.53584918313,
206208.27443503588
],
"count": 3,
"crs": "EPSG:31370",
"dtypes": [
"uint8",
"uint8",
"uint8"
],
"height": 512,
"nodata": null,
"transform": [
0.9765625,
0.0,
193277.53584918313,
0.0,
-0.9765625,
206208.27443503588,
0.0,
0.0,
1.0
],
"width": 512
},
"tile_path": "/app/storage/tiles/cb80638d-dbef-48ac-b19c-cec7c3efc96e/ae0ff76d-70c0-404f-b777-54d14517179a/b191c968-7d56-4e0d-afbb-8b5baaa62470/tile_0000.tif",
"tile_sha256": "134a9e86850c92c577c73bc6ee57a9df7d4c1c513ae6450263e800b6dd47b6ee",
"tile_size_bytes": 669227
},
"model": {
"path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt",
"sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1",
"size_bytes": 22516074
},
"output": {
"class_counts": {
"building": 17
},
"confidence": {
"maximum": 0.8144345879554749,
"mean": 0.5594814244438621,
"minimum": 0.5008771419525146
},
"count": 17,
"sample": [
{
"bbox": [
125.5418930053711,
157.9995880126953,
187.2771453857422,
232.72903442382812
],
"class_name": "building",
"confidence": 0.8144345879554749,
"properties": {
"class_id": 0
}
},
{
"bbox": [
38.20725631713867,
487.8446350097656,
56.458614349365234,
510.8298034667969
],
"class_name": "building",
"confidence": 0.6531882286071777,
"properties": {
"class_id": 0
}
},
{
"bbox": [
177.36990356445312,
229.9929656982422,
248.5861053466797,
267.96173095703125
],
"class_name": "building",
"confidence": 0.6213690638542175,
"properties": {
"class_id": 0
}
},
{
"bbox": [
367.4935607910156,
474.3263244628906,
407.673828125,
498.2358093261719
],
"class_name": "building",
"confidence": 0.6148342490196228,
"properties": {
"class_id": 0
}
},
{
"bbox": [
435.2783203125,
131.2145233154297,
453.61016845703125,
166.76919555664062
],
"class_name": "building",
"confidence": 0.5637134313583374,
"properties": {
"class_id": 0
}
},
{
"bbox": [
407.32525634765625,
317.380126953125,
416.885009765625,
326.8919982910156
],
"class_name": "building",
"confidence": 0.547556459903717,
"properties": {
"class_id": 0
}
},
{
"bbox": [
395.66558837890625,
356.8952331542969,
413.51416015625,
377.1672668457031
],
"class_name": "building",
"confidence": 0.5378748774528503,
"properties": {
"class_id": 0
}
},
{
"bbox": [
394.0309143066406,
143.1604461669922,
408.4023132324219,
167.8805389404297
],
"class_name": "building",
"confidence": 0.5360639095306396,
"properties": {
"class_id": 0
}
},
{
"bbox": [
125.3250503540039,
56.1500358581543,
157.87998962402344,
68.70291900634766
],
"class_name": "building",
"confidence": 0.5342658162117004,
"properties": {
"class_id": 0
}
},
{
"bbox": [
213.7998504638672,
77.86518859863281,
230.8061065673828,
90.1888198852539
],
"class_name": "building",
"confidence": 0.5319290161132812,
"properties": {
"class_id": 0
}
}
]
},
"read_only": true,
"runtime": {
"cuda_available": true,
"cuda_device_index": 0,
"cuda_device_name": "NVIDIA GeForce RTX 4080 SUPER",
"cuda_peak_memory_bytes": 82307072,
"cuda_runtime": "12.8",
"python": "3.11.2 (main, May 12 2026, 05:17:27) [GCC 12.2.0]",
"torch": "2.11.0+cu128",
"ultralytics": "8.4.99"
},
"schema_version": 1,
"status": "passed",
"timing_seconds": {
"inference": 0.8836944859940559,
"model_load": 0.48524460900807753,
"total": 1.3689390950021334
}
}
@@ -0,0 +1,25 @@
{
"schema_version": 1,
"captured_at": "2026-08-01T18:46:19.288288+02:00",
"read_only": true,
"artifacts": [
{
"path": "/app/storage/training/building-be-v62-v58-flanders-balanced-transfer-r1/runs/flanders/weights/best.pt",
"exists": true,
"size_bytes": 456136473,
"sha256": "889ee5b3bfe722803542d15a53a51a716872cf0e8516da5e8291e0a2605adbdb"
},
{
"path": "/app/storage/training/building-be-v62-v58-flanders-balanced-transfer-r1/runs/flanders/weights/last.pt",
"exists": true,
"size_bytes": 456136985,
"sha256": "08d12061f017c45c123c824ffa043bc07794beef251c903299eb1e0954636c8b"
},
{
"path": "/app/storage/training/building-be-v56-rotated-quality-audit-r1/operator_yolo_dataset_quality_audit.json",
"exists": true,
"size_bytes": 153659,
"sha256": "da0d4648ee2365ab949f1b00518ddba0a17cb722a41cb228525b4d2e7657ce36"
}
]
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,503 @@
{
"active_model": {
"classes": "building",
"configured_path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt",
"device": "cuda:0",
"exists": true,
"model_id": "yolo-configured",
"model_version": "",
"require_cuda": "true",
"sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1",
"size_bytes": 22516074,
"validated_area_names": "Mol,Kempen",
"validation_scope_enforced": "true"
},
"database": {
"analysis_statuses": [
{
"count": 2,
"status": "failed"
},
{
"count": 1144,
"status": "success"
}
],
"confidence_integrity": [
{
"detections_outside_unit_interval": 0,
"segmentations_outside_unit_interval": 0
}
],
"dataset_lineage_gaps": [
{
"missing_checksum": 0,
"missing_crs": 0,
"missing_imported_at": 0,
"missing_observed_at": 2377,
"missing_provenance_metadata": 0,
"missing_source_metadata": 0,
"missing_source_version": 1761,
"total": 3377
}
],
"dataset_statuses": [
{
"count": 3377,
"status": "ready"
}
],
"dataset_version_gaps": [
{
"missing_checksum": 0,
"missing_provenance_metadata": 0,
"missing_source_metadata": 0,
"missing_storage_path": 0,
"total": 1671
}
],
"geometry_integrity": {
"areas": {
"rows": [
{
"empty": 0,
"invalid": 0,
"outside_epsg4326_domain": 0,
"populated": 387,
"wrong_srid": 0
}
],
"status": "ok"
},
"detections": {
"rows": [
{
"empty": 0,
"invalid": 0,
"outside_epsg4326_domain": 4,
"populated": 299233,
"wrong_srid": 0
}
],
"status": "ok"
},
"segmentations": {
"rows": [
{
"empty": 0,
"invalid": 0,
"outside_epsg4326_domain": 0,
"populated": 0,
"wrong_srid": 0
}
],
"status": "ok"
},
"vector_features": {
"rows": [
{
"empty": 0,
"invalid": 0,
"outside_epsg4326_domain": 0,
"populated": 6689447,
"wrong_srid": 0
}
],
"status": "ok"
}
},
"job_statuses": [
{
"count": 135,
"status": "failed"
},
{
"count": 4325,
"status": "success"
}
],
"metric_nulls": [
{
"metric_key": "f1",
"null_values": 58,
"total": 697
},
{
"metric_key": "false_negative_count",
"null_values": 0,
"total": 697
},
{
"metric_key": "false_positive_count",
"null_values": 0,
"total": 697
},
{
"metric_key": "mean_iou",
"null_values": 90,
"total": 697
},
{
"metric_key": "precision",
"null_values": 58,
"total": 697
},
{
"metric_key": "recall",
"null_values": 1,
"total": 697
}
],
"migration_heads": [
"202607260001"
],
"model_run_summary": [
{
"analysis_type": "detection",
"count": 1144,
"model_name": "yolo-configured",
"model_version": "",
"status": "success"
},
{
"analysis_type": "detection",
"count": 2,
"model_name": "yolo-configured",
"model_version": "",
"status": "failed"
}
],
"outside_domain_detection_records": [
{
"analysis_run_id": "7ba34274-411d-45e3-8f54-c37baec598b1",
"analysis_status": "success",
"analysis_type": "detection",
"bbox_json": {
"x_max": 376.3067321777344,
"x_min": 256.26025390625,
"y_max": 273.9410705566406,
"y_min": 157.967041015625
},
"class_name": "building",
"confidence": 0.6873681545257568,
"created_at": "2026-07-06 23:58:50.968501+00:00",
"dataset_id": "ae0ff76d-70c0-404f-b777-54d14517179a",
"dataset_name": "geel_orthophoto_wms_512.tif",
"dataset_source_name": "manual",
"detection_id": "18f27c89-3793-429c-8fc4-41f8801a9c31",
"max_x": 193645.02289232545,
"max_y": 206054.00974654406,
"min_x": 193527.79000338845,
"min_y": 205940.75385832042,
"model_name": "yolo-configured",
"model_version": "",
"project_name": "GeoIntel Real Data Validation 20260706T235847Z",
"source_tile_path": "/app/storage/tiles/cb80638d-dbef-48ac-b19c-cec7c3efc96e/ae0ff76d-70c0-404f-b777-54d14517179a/b191c968-7d56-4e0d-afbb-8b5baaa62470/tile_0000.tif"
},
{
"analysis_run_id": "7ba34274-411d-45e3-8f54-c37baec598b1",
"analysis_status": "success",
"analysis_type": "detection",
"bbox_json": {
"x_max": 353.90289306640625,
"x_min": 252.73110961914062,
"y_max": 352.81695556640625,
"y_min": 319.88580322265625
},
"class_name": "building",
"confidence": 0.5060421824455261,
"created_at": "2026-07-06 23:58:50.968501+00:00",
"dataset_id": "ae0ff76d-70c0-404f-b777-54d14517179a",
"dataset_name": "geel_orthophoto_wms_512.tif",
"dataset_source_name": "manual",
"detection_id": "2d6aed95-d8b0-488b-87dd-59b242bf2426",
"max_x": 193623.1441431933,
"max_y": 205895.88595532626,
"min_x": 193524.34357342057,
"min_y": 205863.72662686557,
"model_name": "yolo-configured",
"model_version": "",
"project_name": "GeoIntel Real Data Validation 20260706T235847Z",
"source_tile_path": "/app/storage/tiles/cb80638d-dbef-48ac-b19c-cec7c3efc96e/ae0ff76d-70c0-404f-b777-54d14517179a/b191c968-7d56-4e0d-afbb-8b5baaa62470/tile_0000.tif"
},
{
"analysis_run_id": "7ba34274-411d-45e3-8f54-c37baec598b1",
"analysis_status": "success",
"analysis_type": "detection",
"bbox_json": {
"x_max": 508.56524658203125,
"x_min": 330.76678466796875,
"y_max": 511.4638671875,
"y_min": 380.63653564453125
},
"class_name": "building",
"confidence": 0.581618070602417,
"created_at": "2026-07-06 23:58:50.968501+00:00",
"dataset_id": "ae0ff76d-70c0-404f-b777-54d14517179a",
"dataset_name": "geel_orthophoto_wms_512.tif",
"dataset_source_name": "manual",
"detection_id": "6e9eadb8-a3c5-4f12-b5ef-d00c00571428",
"max_x": 193774.1815977984,
"max_y": 205836.55906819552,
"min_x": 193600.55028733544,
"min_y": 205708.7980022356,
"model_name": "yolo-configured",
"model_version": "",
"project_name": "GeoIntel Real Data Validation 20260706T235847Z",
"source_tile_path": "/app/storage/tiles/cb80638d-dbef-48ac-b19c-cec7c3efc96e/ae0ff76d-70c0-404f-b777-54d14517179a/b191c968-7d56-4e0d-afbb-8b5baaa62470/tile_0000.tif"
},
{
"analysis_run_id": "7ba34274-411d-45e3-8f54-c37baec598b1",
"analysis_status": "success",
"analysis_type": "detection",
"bbox_json": {
"x_max": 54.1800422668457,
"x_min": 0.481353759765625,
"y_max": 511.86688232421875,
"y_min": 456.6719665527344
},
"class_name": "building",
"confidence": 0.5783078670501709,
"created_at": "2026-07-06 23:58:50.968501+00:00",
"dataset_id": "ae0ff76d-70c0-404f-b777-54d14517179a",
"dataset_name": "geel_orthophoto_wms_512.tif",
"dataset_source_name": "manual",
"detection_id": "bec4f3af-5ca2-4a38-a547-21aeb19a70fd",
"max_x": 193330.44604670934,
"max_y": 205762.30571769923,
"min_x": 193278.00592121415,
"min_y": 205708.40443276614,
"model_name": "yolo-configured",
"model_version": "",
"project_name": "GeoIntel Real Data Validation 20260706T235847Z",
"source_tile_path": "/app/storage/tiles/cb80638d-dbef-48ac-b19c-cec7c3efc96e/ae0ff76d-70c0-404f-b777-54d14517179a/b191c968-7d56-4e0d-afbb-8b5baaa62470/tile_0000.tif"
}
],
"postgis_version": "POSTGIS=\"3.6.4 94d984b\" [EXTENSION] PGSQL=\"160\" GEOS=\"3.11.1-CAPI-1.17.1\" PROJ=\"9.1.1 NETWORK_ENABLED=OFF URL_ENDPOINT=https://cdn.proj.org USER_WRITABLE_DIRECTORY=/tmp/proj DATABASE_PATH=/usr/share/proj/proj.db\" LIBXML=\"2.9.14\" LIBJSON=\"0.16\" LIBPROTOBUF=\"1.4.1\" WAGYU=\"0.5.0 (Internal)\" (core procs from \"3.4.3 e365945\" need upgrade) TOPOLOGY (topology procs from \"3.4.3 e365945\" need upgrade)",
"source_summary": [
{
"count": 1751,
"reference_count": 660,
"source_name": "manual",
"status": "ready"
},
{
"count": 339,
"reference_count": 0,
"source_name": "vmm_flood_hazard",
"status": "ready"
},
{
"count": 273,
"reference_count": 273,
"source_name": "vmm_vha_bathymetry_profiles",
"status": "ready"
},
{
"count": 210,
"reference_count": 0,
"source_name": "department_omgeving_thematic_raster",
"status": "ready"
},
{
"count": 179,
"reference_count": 179,
"source_name": "grb",
"status": "ready"
},
{
"count": 168,
"reference_count": 0,
"source_name": "digitaal_vlaanderen_orthophoto",
"status": "ready"
},
{
"count": 107,
"reference_count": 0,
"source_name": "spw_orthophoto",
"status": "ready"
},
{
"count": 68,
"reference_count": 68,
"source_name": "spw_picc",
"status": "ready"
},
{
"count": 60,
"reference_count": 0,
"source_name": "digitaal_vlaanderen_dhmv",
"status": "ready"
},
{
"count": 36,
"reference_count": 36,
"source_name": "urbis",
"status": "ready"
},
{
"count": 26,
"reference_count": 0,
"source_name": "urbis_orthophoto",
"status": "ready"
},
{
"count": 26,
"reference_count": 26,
"source_name": "waterinfo",
"status": "ready"
},
{
"count": 25,
"reference_count": 25,
"source_name": "department_omgeving_land_use",
"status": "ready"
},
{
"count": 21,
"reference_count": 21,
"source_name": "historical_landuse",
"status": "ready"
},
{
"count": 18,
"reference_count": 18,
"source_name": "agentschap_landbouw_zeevisserij_agricultural_parcels",
"status": "ready"
},
{
"count": 15,
"reference_count": 15,
"source_name": "statbel",
"status": "ready"
},
{
"count": 13,
"reference_count": 13,
"source_name": "dov_soil_map",
"status": "ready"
},
{
"count": 12,
"reference_count": 12,
"source_name": "inbo_bwk_natura2000",
"status": "ready"
},
{
"count": 7,
"reference_count": 0,
"source_name": "spw_walous_land_cover",
"status": "ready"
},
{
"count": 6,
"reference_count": 0,
"source_name": "vrbg",
"status": "ready"
},
{
"count": 5,
"reference_count": 1,
"source_name": "fixture",
"status": "ready"
},
{
"count": 4,
"reference_count": 4,
"source_name": "ngi_adminvector",
"status": "ready"
},
{
"count": 3,
"reference_count": 0,
"source_name": "map_selection",
"status": "ready"
},
{
"count": 1,
"reference_count": 1,
"source_name": "digitaal_vlaanderen_buildings_addresses_register",
"status": "ready"
},
{
"count": 1,
"reference_count": 1,
"source_name": "rbins_marine_reporting_units",
"status": "ready"
},
{
"count": 1,
"reference_count": 1,
"source_name": "rbins_msp_2026",
"status": "ready"
},
{
"count": 1,
"reference_count": 0,
"source_name": "spw_bathymetry",
"status": "ready"
},
{
"count": 1,
"reference_count": 0,
"source_name": "spw_terrain",
"status": "ready"
}
],
"table_counts": {
"analysis_runs": 1146,
"aoi_operation_partitions": 61,
"aoi_operations": 4,
"areas": 387,
"dataset_versions": 1671,
"datasets": 3377,
"detection_reviews": 0,
"detections": 299233,
"exports": 768,
"jobs": 4460,
"metrics": 4182,
"projects": 1097,
"quality_checks": 697,
"segmentations": 0,
"vector_features": 6689447
},
"version": "PostgreSQL 16.14 (Debian 16.14-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit"
},
"generated_at": "2026-08-01T16:06:01.768127+00:00",
"mode": "read_only",
"runtime": {
"cuda": {
"available": true,
"device_count": 1,
"device_names": [
"NVIDIA GeForce RTX 4080 SUPER"
],
"runtime_version": "12.8"
},
"packages": {
"fastapi": "0.139.2",
"geoalchemy2": "0.20.0",
"geointel-backend": "0.1.0",
"geopandas": "1.1.4",
"pyproj": "3.7.2",
"rasterio": "1.4.4",
"shapely": "2.1.2",
"sqlalchemy": "2.0.51",
"torch": "2.11.0+cu128",
"torchvision": "0.26.0+cu128",
"ultralytics": "8.4.99"
},
"platform": "Linux-6.12.54-Unraid-x86_64-with-glibc2.36",
"python": "3.11.2"
},
"schema_version": 1,
"statement_timeout_ms": 30000,
"storage_references": {
"checked_count": 5816,
"missing_count": 0,
"missing_records": [],
"records_truncated": false,
"scope": "direct datasets, dataset_versions and exports storage_path columns"
},
"storage_root": "/app/storage"
}
@@ -0,0 +1,397 @@
{
"active_model": {
"classes": "building",
"configured_path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt",
"device": "cuda:0",
"exists": true,
"model_id": "yolo-configured",
"model_version": "",
"require_cuda": "true",
"sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1",
"size_bytes": 22516074,
"validated_area_names": "Mol,Kempen",
"validation_scope_enforced": "true"
},
"database": {
"analysis_statuses": [
{
"count": 2,
"status": "failed"
},
{
"count": 1144,
"status": "success"
}
],
"confidence_integrity": [
{
"detections_outside_unit_interval": 0,
"segmentations_outside_unit_interval": 0
}
],
"dataset_lineage_gaps": [
{
"missing_checksum": 0,
"missing_crs": 0,
"missing_imported_at": 0,
"missing_observed_at": 2377,
"missing_provenance_metadata": 0,
"missing_source_metadata": 0,
"missing_source_version": 1761,
"total": 3377
}
],
"dataset_statuses": [
{
"count": 3377,
"status": "ready"
}
],
"dataset_version_gaps": [
{
"missing_checksum": 0,
"missing_provenance_metadata": 0,
"missing_source_metadata": 0,
"missing_storage_path": 0,
"total": 1671
}
],
"geometry_integrity": {
"areas": {
"rows": [
{
"empty": 0,
"invalid": 0,
"outside_epsg4326_domain": 0,
"populated": 387,
"wrong_srid": 0
}
],
"status": "ok"
},
"detections": {
"rows": [
{
"empty": 0,
"invalid": 0,
"outside_epsg4326_domain": 4,
"populated": 299233,
"wrong_srid": 0
}
],
"status": "ok"
},
"segmentations": {
"rows": [
{
"empty": 0,
"invalid": 0,
"outside_epsg4326_domain": 0,
"populated": 0,
"wrong_srid": 0
}
],
"status": "ok"
},
"vector_features": {
"rows": [
{
"empty": 0,
"invalid": 0,
"outside_epsg4326_domain": 0,
"populated": 6689447,
"wrong_srid": 0
}
],
"status": "ok"
}
},
"job_statuses": [
{
"count": 135,
"status": "failed"
},
{
"count": 4325,
"status": "success"
}
],
"metric_nulls": [
{
"metric_key": "f1",
"null_values": 58,
"total": 697
},
{
"metric_key": "false_negative_count",
"null_values": 0,
"total": 697
},
{
"metric_key": "false_positive_count",
"null_values": 0,
"total": 697
},
{
"metric_key": "mean_iou",
"null_values": 90,
"total": 697
},
{
"metric_key": "precision",
"null_values": 58,
"total": 697
},
{
"metric_key": "recall",
"null_values": 1,
"total": 697
}
],
"migration_heads": [
"202607260001"
],
"model_run_summary": [
{
"analysis_type": "detection",
"count": 1144,
"model_name": "yolo-configured",
"model_version": "",
"status": "success"
},
{
"analysis_type": "detection",
"count": 2,
"model_name": "yolo-configured",
"model_version": "",
"status": "failed"
}
],
"postgis_version": "POSTGIS=\"3.6.4 94d984b\" [EXTENSION] PGSQL=\"160\" GEOS=\"3.11.1-CAPI-1.17.1\" PROJ=\"9.1.1 NETWORK_ENABLED=OFF URL_ENDPOINT=https://cdn.proj.org USER_WRITABLE_DIRECTORY=/tmp/proj DATABASE_PATH=/usr/share/proj/proj.db\" LIBXML=\"2.9.14\" LIBJSON=\"0.16\" LIBPROTOBUF=\"1.4.1\" WAGYU=\"0.5.0 (Internal)\" (core procs from \"3.4.3 e365945\" need upgrade) TOPOLOGY (topology procs from \"3.4.3 e365945\" need upgrade)",
"source_summary": [
{
"count": 1751,
"reference_count": 660,
"source_name": "manual",
"status": "ready"
},
{
"count": 339,
"reference_count": 0,
"source_name": "vmm_flood_hazard",
"status": "ready"
},
{
"count": 273,
"reference_count": 273,
"source_name": "vmm_vha_bathymetry_profiles",
"status": "ready"
},
{
"count": 210,
"reference_count": 0,
"source_name": "department_omgeving_thematic_raster",
"status": "ready"
},
{
"count": 179,
"reference_count": 179,
"source_name": "grb",
"status": "ready"
},
{
"count": 168,
"reference_count": 0,
"source_name": "digitaal_vlaanderen_orthophoto",
"status": "ready"
},
{
"count": 107,
"reference_count": 0,
"source_name": "spw_orthophoto",
"status": "ready"
},
{
"count": 68,
"reference_count": 68,
"source_name": "spw_picc",
"status": "ready"
},
{
"count": 60,
"reference_count": 0,
"source_name": "digitaal_vlaanderen_dhmv",
"status": "ready"
},
{
"count": 36,
"reference_count": 36,
"source_name": "urbis",
"status": "ready"
},
{
"count": 26,
"reference_count": 0,
"source_name": "urbis_orthophoto",
"status": "ready"
},
{
"count": 26,
"reference_count": 26,
"source_name": "waterinfo",
"status": "ready"
},
{
"count": 25,
"reference_count": 25,
"source_name": "department_omgeving_land_use",
"status": "ready"
},
{
"count": 21,
"reference_count": 21,
"source_name": "historical_landuse",
"status": "ready"
},
{
"count": 18,
"reference_count": 18,
"source_name": "agentschap_landbouw_zeevisserij_agricultural_parcels",
"status": "ready"
},
{
"count": 15,
"reference_count": 15,
"source_name": "statbel",
"status": "ready"
},
{
"count": 13,
"reference_count": 13,
"source_name": "dov_soil_map",
"status": "ready"
},
{
"count": 12,
"reference_count": 12,
"source_name": "inbo_bwk_natura2000",
"status": "ready"
},
{
"count": 7,
"reference_count": 0,
"source_name": "spw_walous_land_cover",
"status": "ready"
},
{
"count": 6,
"reference_count": 0,
"source_name": "vrbg",
"status": "ready"
},
{
"count": 5,
"reference_count": 1,
"source_name": "fixture",
"status": "ready"
},
{
"count": 4,
"reference_count": 4,
"source_name": "ngi_adminvector",
"status": "ready"
},
{
"count": 3,
"reference_count": 0,
"source_name": "map_selection",
"status": "ready"
},
{
"count": 1,
"reference_count": 1,
"source_name": "digitaal_vlaanderen_buildings_addresses_register",
"status": "ready"
},
{
"count": 1,
"reference_count": 1,
"source_name": "rbins_marine_reporting_units",
"status": "ready"
},
{
"count": 1,
"reference_count": 1,
"source_name": "rbins_msp_2026",
"status": "ready"
},
{
"count": 1,
"reference_count": 0,
"source_name": "spw_bathymetry",
"status": "ready"
},
{
"count": 1,
"reference_count": 0,
"source_name": "spw_terrain",
"status": "ready"
}
],
"table_counts": {
"analysis_runs": 1146,
"aoi_operation_partitions": 61,
"aoi_operations": 4,
"areas": 387,
"dataset_versions": 1671,
"datasets": 3377,
"detection_reviews": 0,
"detections": 299233,
"exports": 768,
"jobs": 4460,
"metrics": 4182,
"projects": 1097,
"quality_checks": 697,
"segmentations": 0,
"vector_features": 6689447
},
"version": "PostgreSQL 16.14 (Debian 16.14-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit"
},
"generated_at": "2026-08-01T16:03:11.003029+00:00",
"mode": "read_only",
"runtime": {
"cuda": {
"available": true,
"device_count": 1,
"device_names": [
"NVIDIA GeForce RTX 4080 SUPER"
],
"runtime_version": "12.8"
},
"packages": {
"fastapi": "0.139.2",
"geoalchemy2": "0.20.0",
"geointel-backend": "0.1.0",
"geopandas": "1.1.4",
"pyproj": "3.7.2",
"rasterio": "1.4.4",
"shapely": "2.1.2",
"sqlalchemy": "2.0.51",
"torch": "2.11.0+cu128",
"torchvision": "0.26.0+cu128",
"ultralytics": "8.4.99"
},
"platform": "Linux-6.12.54-Unraid-x86_64-with-glibc2.36",
"python": "3.11.2"
},
"schema_version": 1,
"statement_timeout_ms": 30000,
"storage_references": {
"checked_count": 5816,
"missing_count": 0,
"missing_records": [],
"records_truncated": false,
"scope": "direct datasets, dataset_versions and exports storage_path columns"
},
"storage_root": "/app/storage"
}
+68
View File
@@ -12240,3 +12240,71 @@ Open:
- De productiedemo toont zeven niet-administratieve werkruimtes, drie detectiemodellen en vier segmentatiemodellen; het geconfigureerde YOLO-model wordt als beschikbaar getoond. - De productiedemo toont zeven niet-administratieve werkruimtes, drie detectiemodellen en vier segmentatiemodellen; het geconfigureerde YOLO-model wordt als beschikbaar getoond.
- Modelkalibratie, technische modelinstellingen, uploads en systeembeheer zijn niet zichtbaar voor gasten. Een directe gast-POST naar projectbeheer retourneert HTTP 403 `GUEST_READ_ONLY`; het modelregister retourneert HTTP 200 binnen dezelfde sessie. - Modelkalibratie, technische modelinstellingen, uploads en systeembeheer zijn niet zichtbaar voor gasten. Een directe gast-POST naar projectbeheer retourneert HTTP 403 `GUEST_READ_ONLY`; het modelregister retourneert HTTP 200 binnen dezelfde sessie.
- Visuele browseracceptatie: horizontale overflow `0`; browserconsole `0` waarschuwingen en `0` fouten. - Visuele browseracceptatie: horizontale overflow `0`; browserconsole `0` waarschuwingen en `0` fouten.
## 2026-08-01 - Accuracy Improvement Program Phase 1 forensic baseline
### Executed scope
- Audited repository root and tracked nested mirror, API/services/schemas,
migrations, frontend, CI/readiness, container/deployment configuration,
Tower Postgres/PostGIS, direct storage references, active CUDA runtime,
mounted models/checkpoints/manifests/evaluations and Belgian building
corpus lineage.
- Created the seven required documents and machine-readable status under
`docs/accuracy-program/`.
- Added read-only collectors for repository state, Tower runtime/database,
Tower ML/data lineage and one production-adapter GPU inference, plus a
deterministic reproducer for seven critical/high contract violations.
- Retained JSON, JUnit, SQL and command logs under
`artifacts/evidence/accuracy/P1/`; no dataset, checkpoint, cache, output,
user-owned untracked file or production database row was deleted or rewritten.
### Proven findings
- Reproduced cross-theme coverage contamination, metres-as-degrees buffering,
Lambert coordinates persisted under SRID 4326, caller-spoofable official
authority, mutable-name YOLO scope, mutable-name legal coverage and silently
ignored Area PATCH geometry.
- Tower database contains four successful Geel detections with Lambert-domain
coordinates while stored as SRID 4326. Direct storage-reference audit checked
5,816 references with zero missing; the broader recursive scan timed out and
is not counted as a pass.
- The building training loop contains a protected-test feedback path. V56 has
180 AOIs but 0 human review decisions, only three pure-empty background-test
AOIs and 24 cross-split AOI pairs below 2 km. V58/V62 are calibration-only,
fail Flanders at the reported operating point and have no protected-test or
promotion evidence.
- Runtime lineage is incomplete: every persisted detection run has an empty
model version and no tile-manifest hash; three runs lack a model hash.
- Repository source of truth is ambiguous through 1,153 tracked nested mirror
files, including 68 root/mirror differences. Root `.dockerignore` correctly
excludes the mirror from the official all-in-one context.
### Runtime and verification evidence
- Real read-only inference passed through the production adapter on the Tower
RTX 4080 SUPER, PyTorch 2.11.0+cu128/CUDA 12.8, with active model SHA
`a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1`.
One existing EPSG:31370 tile produced 17 raw detections in 0.8837 s; this is
runtime evidence only and not an accuracy result.
- Full backend suite: 1,180 passed, 17 stale source/contract assertions failed.
The actual backend CI working directory fails during collection on
`scripts.render_operator_polygon_label_qa`.
- Phase-1 tooling: 4 tests passed; all new audit files pass Ruff.
- Repository Ruff baseline: 112 findings. Frontend `test:unit`: 51/51 passed;
typecheck and build passed; the required `npm run lint` script is absent.
- OpenAPI audit passed for 147 routes plus 10 declared non-envelope endpoints.
Alembic has one head, `202607260001`, and the complete offline upgrade
rendered successfully.
- Two golden-QA runs produced equal semantic metrics but different retained
bytes because run identity uses UUID4.
### Decision
- Phase 1 is complete as a forensic and executable baseline.
- Release promotion, national validation, scope widening and immediate training
remain blocked.
- Phase 2 may start only as the dependency-ordered, test-first remediation in
`docs/accuracy-program/06-implementation-roadmap.md`: fix CRS/authority/
coverage/lineage and protected-test isolation, complete human corpus review,
rebuild independent data, freeze metrics, then train on `cuda:0`.
+34
View File
@@ -1075,3 +1075,37 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Sta projectgebonden analyse-, QA-, assistent- en exportacties toe. - [x] Sta projectgebonden analyse-, QA-, assistent- en exportacties toe.
- [x] Behoud server-side blokkades op instellingen, beheer, uploads, reviews en cross-projectverzoeken. - [x] Behoud server-side blokkades op instellingen, beheer, uploads, reviews en cross-projectverzoeken.
- [x] Verifieer en redeploy de exacte commit naar Tower `/mnt/user/appdata/geointel`. - [x] Verifieer en redeploy de exacte commit naar Tower `/mnt/user/appdata/geointel`.
# Accuracy Improvement Program Phase 2 (2026-08-01)
- [x] Complete the Phase-1 forensic inventory, data lineage, reproducible
baseline, risk register, metric framework, roadmap, status and retained
evidence manifest.
- [ ] P2-00: freeze evidence and enforce a promotion/scope-widening lock.
- [ ] P2-01: establish one canonical CI command, repair import shadowing,
replace 17 stale source assertions with reviewed behavior tests, close the
112 Ruff findings and add real frontend lint.
- [ ] P2-02: fix CRS ingest, metre buffering and Area geometry/CRS updates;
auditably quarantine or repair the four legacy Geel detections.
- [ ] P2-03: isolate coverage by source/theme/layer/zone, make official source
identity server-attested and replace mutable-name legal/model scope checks.
- [ ] P2-04: make derived persistence transactional, require complete
RunManifest hashes and expose every fallback/persistence failure.
- [ ] P2-05: remove every protected-test feedback path, introduce a test vault
and make the sampler reject protected IDs, paths and assessment fields.
- [ ] P2-06: complete representative human V56 review; add independent AOIs,
Brussels pure backgrounds, difficult negatives and corrected temporal/dense
PICC/UrbIS labels; freeze a new immutable corpus rather than changing V56.
- [ ] P2-07: implement the metric/null/error-taxonomy contract, stratified
incumbent baseline, calibration and reproducibility rerun; freeze numeric
gates before protected-test access.
- [ ] P2-08: train the pre-registered candidate matrix on the Tower RTX 4080
SUPER with pinned PyTorch/CUDA, full hashes, seeds and zero CPU fallback.
- [ ] P2-09: run a calibration-only improvement loop; use only independently
provisioned, reviewed train data for another corpus version.
- [ ] P2-10: open protected test/background-test exactly once for one immutable
pre-registered candidate; never retrain that candidate family from its result.
- [ ] P2-11: promote and redeploy only after every gate passes, with model card,
image/model/config hashes, shadow evidence and tested rollback.
- [ ] P2-12: monitor drift into a human reviewqueue without automatic
self-training or silent promotion.
@@ -0,0 +1,189 @@
# Phase 1 execution contract
- Status: **executed**
- Audit date: 2026-08-01
- Audit branch: `codex/geointel-accuracy-program`
- Audited baseline commit: `0c019bb22f816db1e4b7a68379bcad08924d9a21`
- Product scope: Belgium and the Belgian North Sea
- Release conclusion: **blocked**
- Phase 2 conclusion: **safe to start as controlled remediation, not as training or promotion**
## 1. Purpose
This contract turns the accuracy program into an evidence-bearing engineering
program. Phase 1 establishes what exists, what actually runs, where data and
model results originate, which claims are supportable, and which defects must
be corrected before any new training or release promotion.
Phase 1 is complete when the required inventory, lineage, baseline, risk,
metric, roadmap, machine-readable status and retained evidence exist and can be
reproduced. Completion of Phase 1 does **not** mean that GeoIntel is accurate,
nationally validated, release-ready, or fully trained.
## 2. Non-negotiable rules
1. No quality, coverage, authority, calibration or national-readiness claim may
be stronger than the retained evidence.
2. A green unit test that uses a fixture, mock, placeholder or source-text
assertion is not evidence of live geospatial or model quality.
3. Protected test data is never used for training, calibration, failure-driven
sampling, threshold selection or stopping decisions.
4. Dataset, label, tile, checkpoint, runtime and evaluation identities are
immutable hashes, not filenames or mutable display names.
5. Geometry entering persistence is validated, assigned an explicit source CRS,
transformed to the canonical CRS and checked for valid domain and units.
6. Official or authoritative status is server-attested; caller-controlled source
names can never establish authority.
7. Failed, unavailable and `not_configured` states remain explicit. A fallback
cannot silently be reported as a successful primary result.
8. Human review is only complete when reviewer identity, decision, timestamp,
sampled object/AOI identity and reviewed artifact hash are persisted.
9. Phase 1 does not delete, rewrite or promote datasets, checkpoints, caches,
outputs, migrations or server state. The GPU smoke is read-only and persists
no detections.
10. User-owned untracked files, including `.codex-artifacts/`, server
`.env.bak.*`, `.forgeflow/`, `storage/evaluation/` and
`storage/source-cache/`, remain untouched.
## 3. Evidence vocabulary
Every statement in this program uses one of these classes:
| Class | Meaning |
|---|---|
| observed | Directly read from the repository, database, storage, runtime or retained artifact |
| reproduced | Deterministic read-only reproducer demonstrated the behavior |
| tested | An executable test or build gate ran and its result is retained or recorded |
| inferred | A conclusion drawn from observed evidence; it must be labelled as an inference |
| planned | Not implemented or not yet evidenced |
| unavailable | The required system or artifact could not be inspected; reason is stated |
| blocked | A required gate cannot pass without remediation |
The words “implemented”, “configured”, “available”, “validated” and “promoted”
are not synonyms:
- **implemented** means code exists;
- **configured** means a runtime selects it;
- **available** means dependencies/assets can be loaded;
- **validated** requires a registered evaluation and acceptance decision;
- **promoted** requires every release gate and an auditable activation record.
## 4. Authorized Phase 1 mutations
The only authorized repository mutations are:
- audit collectors and deterministic reproducer scripts;
- focused tests for those collectors;
- `docs/accuracy-program/`;
- retained evidence under `artifacts/evidence/accuracy/P1/`;
- required execution-log and TODO updates;
- a selective `.gitignore` exception that tracks only the Phase 1 evidence
tree while keeping other generated `artifacts/` ignored.
No product behavior, model checkpoint, dataset, label or production database row
is changed in Phase 1.
## 5. Executed investigation surface
The audit covered:
- root and nested tracked source trees, Git state and ignored/untracked state;
- backend API routes, schemas, services, ORM entities and Alembic chain;
- frontend TypeScript, tests, typecheck and production build;
- CI/readiness scripts, container definitions and Tower deployment identity;
- Postgres/PostGIS table counts, statuses, lineage gaps and geometry integrity;
- mounted storage paths referenced by the database;
- active NVIDIA/PyTorch/Ultralytics runtime and active model hash;
- one existing Geel raster tile through the production YOLO adapter on
`cuda:0`;
- available Belgian building corpus, split, label, tiling, calibration,
checkpoint and human-review evidence;
- mocks, fixtures, fallbacks, placeholders and documentation drift.
Detailed outcomes live in documents 01 through 06 and in the P1 evidence tree.
## 6. Reproducibility entry points
Run from the repository root:
```powershell
python scripts/run_accuracy_phase1_baseline.py --output-dir artifacts/evidence/accuracy/P1
python scripts/reproduce_accuracy_phase1_findings.py
python -m pytest tests/test_accuracy_phase1_baseline.py -q -p no:cacheprovider
```
The runtime/database collector is intentionally executed inside the deployed
application container and uses read-only SQL with a statement timeout:
```text
scripts/collect_accuracy_phase1_runtime.py
```
The GPU collector is streamed into the deployed container and uses the
production adapter with an existing checksummed model, manifest and tile:
```text
scripts/collect_accuracy_phase1_inference_smoke.py
```
Its retained JSON includes the model, manifest and tile SHA-256 values, package
versions, CUDA device, seed, configuration, raster CRS and raw output summary.
It explicitly forbids deriving an accuracy or release claim from one smoke.
## 7. Proven release blockers
Phase 1 reproduced or observed all of the following:
- cross-theme coverage contamination can incorrectly return
`operational`;
- a nominal metre buffer is applied as degrees;
- Lambert coordinates can be persisted unchanged under SRID 4326;
- caller-controlled source identity can be elevated to authoritative coverage;
- YOLO validation scope is bypassable through a mutable Area-name substring;
- legal coverage identity changes when an Area display name changes;
- Area PATCH silently ignores a documented geometry field;
- four persisted Geel detections contain Lambert-domain coordinates while their
geometry column is labelled SRID 4326;
- the training loop contains a future protected-test feedback path;
- V56 has zero accepted human AOI reviews out of 180;
- split independence, pure-background coverage, temporal label alignment and
unique-object evaluation are insufficient;
- no candidate has protected-test, background-test and promotion evidence;
- the real CI pytest work directory fails during collection, the full root suite
has 17 stale assertion failures, Ruff has 112 findings and frontend lint is
not configured;
- the production AI image is not built or exercised by CI;
- a tracked nested source mirror contains 1,153 files, 68 of which differ from
root, creating source-of-truth ambiguity.
## 8. Decision gates
| Gate | Phase 1 decision |
|---|---|
| Runtime CUDA and active model load | pass |
| One production-adapter GPU inference | pass, runtime-only evidence |
| Database/storage reachability | pass for inspected references |
| Geometry/CRS integrity | fail |
| Authoritative-source integrity | fail |
| Coverage correctness | fail |
| Strict protected-test isolation | fail in code design |
| Human label acceptance | fail |
| Split independence and negatives | fail |
| Reproducible national metric baseline | absent |
| Backend release gate | fail |
| Lint and frontend lint | fail / absent |
| National model promotion | blocked |
| Start test-first Phase 2 remediation | allowed |
## 9. Stop and escalation conditions
Training and model promotion remain prohibited until Phase 2 has closed the
CRS, authority, coverage and protected-test isolation blockers and a frozen,
human-approved corpus exists. If a required source snapshot, reviewer decision,
immutable hash or independent holdout cannot be produced, the corresponding
claim remains blocked; it is never replaced by synthetic success.
The protected test may be opened exactly once for a pre-registered candidate
after all calibration gates pass. A failure after that opening creates a new
model-development cycle and requires a newly governed protected set; its errors
must not feed back into the same training lineage.
@@ -0,0 +1,158 @@
# GeoIntel Accuracy Improvement Program — 01 Systeeminventaris
## 1. Doel, peildatum en claimgrens
Dit document beschrijft de aantoonbaar aanwezige GeoIntel-componenten op 1 augustus 2026. Het is een forensische inventaris, geen kwaliteitscertificaat. `implemented` betekent dat code en een contractpad bestaan; `configured` betekent dat de betreffende runtimeconfiguratie daadwerkelijk is waargenomen; `fixture/mock` betekent dat het pad alleen test- of demobewijs levert; `planned/unproven` betekent dat geen uitvoerbaar productiebewijs is gevonden.
De inventaris is opgebouwd uit:
- checkout `C:\Projects\geointel` op branch `codex/geointel-accuracy-program`, basis-HEAD `0c019bb22f816db1e4b7a68379bcad08924d9a21`;
- Tower-checkout `/mnt/user/appdata/geointel` op branch `main`, dezelfde HEAD;
- draaiende container `/geointel`, image `geointel-all-in-one:0c019bb22f81-wipfdc62947dfb2-ai`, status `running`, health `healthy`;
- read-only runtime-/databasecollectie in `C:\Projects\geointel\artifacts\evidence\accuracy\P1\tower-runtime-database-snapshot-detailed.json`;
- read-only GPU-smoke in `C:\Projects\geointel\artifacts\evidence\accuracy\P1\tower-gpu-inference-smoke.json`;
- retained model-, checkpoint-, corpus- en splitinventaris in `C:\Projects\geointel\artifacts\evidence\accuracy\P1\tower-ml-data-lineage-snapshot.json`;
- lokale repository- en artifactinventaris in dezelfde evidence-map.
Geen bevinding in dit document bewijst nationale modelkwaliteit, menselijk gevalideerde labels, calibratie, geografische generalisatie of releasegeschiktheid.
## 2. Uitvoerbare productarchitectuur
| Onderdeel | Concrete code-/configpaden | Status | Bewezen runtime-/teststatus | Accuracy-grens |
|---|---|---|---|---|
| React/TypeScript workbench | `frontend/src/App.tsx`, `frontend/src/components`, `frontend/src/hooks`, `frontend/src/services` | implemented | 16 frontend-testbestanden; Vitest 51/51, typecheck en productiebuild slaagden in de Phase-1-baseline | UI-werking bewijst geen inhoudelijke GIS- of modeljuistheid |
| Mapweergave en gebiedsselectie | `frontend/src/components/map/MapWorkspace.tsx`, MapLibre dependency in `frontend/package.json` | implemented | compileert en is unit-getest; geen volledige browser-E2E-uitvoering in de releasegate | selectie-UX is geen bewijs dat server-side scope/CRS correct is |
| FastAPI HTTP-laag | `backend/app/main.py`, `backend/app/api/routes/*.py` | implemented | 148 route-decorators statisch gevonden; OpenAPI-audit zag 147 routes, waarvan 10 contractueel toegestane niet-envelope routes | route-aanwezigheid bewijst geen correcte gegevensinhoud |
| Projecten en AOI's | `project_service.py`, `area_service.py`, routes `projects.py` en `areas.py` | implemented, persisted | 1.097 projecten en 387 gebieden in Tower; alle 387 gevulde gebiedsgeometrieën geldig, SRID 4326 en binnen het 4326-domein | Area PATCH en naamgebaseerde wettelijke scope hebben bewezen contract-/integriteitsgaten |
| Datasetinname en versies | `dataset_service.py`, `storage_service.py`, `datasets.py` | implemented, persisted | 3.377 datasets, alle status `ready`; 1.671 datasetversies; 5.816 directe storage-referenties gecontroleerd, 0 ontbrekend | `ready` is een verwerkingsstatus, geen bronautoriteit of inhoudelijke kwaliteitsclaim |
| Rasterverwerking | `raster_service.py`, `raster_operations_service.py`, acquisitieservices voor orthofoto/DHMV/WALOUS/SPW | implemented | raster- en GIS-gerichte tests slaagden; bestaande Tower-tiles konden read-only worden geopend | geen volledige corpusbrede rasteralignment-/nodata-/resolutieaudit afgerond |
| Vectorverwerking | `vector_feature_service.py`, `vector_operations_service.py`, officiële-vectoracquisitie | implemented, persisted | 6.689.447 `vector_features`; alle gevulde geometrieën geldig, SRID 4326 en binnen domein in de gebonden DB-query | buffer, import-CRS en afgeleide-featurepersistentie hebben kritieke gereproduceerde fouten |
| Bron- en dekkingsresolutie | `providers/*`, `coverage_registry_service.py`, `source_catalog_probe_service.py` | implemented | 28 waargenomen `source_name`-waarden in de DB; officiële en fixtureproviders zijn afzonderlijk gemodelleerd | bronautoriteit kan via uploadmetadata worden gespooft; dekking kan door een andere themalaag worden overschat |
| Achtergrondjobs/AOI-operations | `job_service.py`, `aoi_operation_service.py`, `aoi_operation_worker.py`, `backend/app/workers` | implemented, configured | 4.460 jobs: 4.325 success, 135 failed; 4 AOI-operations en 61 partitions; geen lopende job/run tijdens snapshot | status `success` is geen accuracy-gate; foutpercentages zijn niet per workflow genormaliseerd |
| Objectdetectieadapter | `detection_service.py`, `detection_georeferencing.py`, `yolo_adapter.py`, `yolo_preflight_service.py` | implemented, configured op Tower | 1.146 detectieruns: 1.144 success, 2 failed; 299.233 detecties; één read-only productieadapter-smoke op GPU slaagde | actieve modelscope wordt op een muteerbare gebiedsnaam afgedwongen; modelversie is leeg; smoke bewijst geen accuracy |
| Modelregister | `model_registry_service.py`, runtimeinstellingen in `backend/app/core/config.py` | mixed | actief YOLO-pad bestaat; register toont daarnaast expliciet `yolo-placeholder` en `manual-fixture-detector` | placeholder/fixture mogen niet als productie-inference of kwaliteitsbewijs worden geteld |
| Segmentatie | `segmentation_service.py`, `segmentation_adapter.py`, routes `segmentation.py` | abstraction implemented; production unproven | modelregister bevat `segmentation-placeholder` en `fixture-segmenter`; Tower bevat 0 segmentaties | geen bewijs van geconfigureerde productie-segmentatie of modelkwaliteit |
| QA/QC en metrics | `qa_service.py`, `quality_service.py`, `detection_qa_service.py`, routes `qa.py` en `quality_checks.py` | implemented, persisted | 697 quality checks en 4.182 metricrijen; 4-scenario golden fixturebenchmark slaagde | metricnulls en ongedefinieerde no-overlap-F1 vereisen een expliciet contract; fixturebenchmark is geen veldbenchmark |
| Exports | `export_service.py`, route `exports.py` | implemented, persisted | 768 exportrecords; directe storage-referenties zaten in de 5.816 gecontroleerde records | een export kan een foutief upstreamresultaat correct serialiseren |
| Demo/fixtures | `demo_workflow_service.py`, `providers/fixture.py`, `fixtures/*` | fixture/demo only | expliciet gemarkeerde fixtureprovider en handmatige fixturemodellen | nooit meenemen in productie- of nationale accuracyclaims |
| Training- en audittooling | `scripts/run_belgium_building_training_loop.py`, `scripts/evaluate_belgium_building_candidate.py`, corpus-/label-/qualityscripts | implemented tooling; release linkage incomplete | omvangrijke Tower-runs en checkpoints aanwezig; v56 automated audits en v62 previewcalibratie aantoonbaar | protected-testfeedback lekt terug naar volgende sampling; 0/180 v56-corpussamples menselijk gereviewd; geen geldige nationale promotieclaim |
| Deployment | `deploy/unraid/Dockerfile.all-in-one`, `docker-compose.unraid.yml`, `deploy/unraid/*` | implemented, configured op Tower | all-in-one-container gezond; NVIDIA GPU doorgegeven | compose-default `cu130` en feitelijke image/runtime `cu128` zijn supply-chain/builddrift; CI bouwt standaard zonder AI |
| Documentatie/contracts | `docs/API_CONTRACTS.md`, `docs/DATABASE_IMPLEMENTATION_PLAN.md`, scope freezes en DoD | implemented but drifted | contracttests bestaan | meerdere documenten beschrijven historische of geplande toestand die niet met de runtime overeenstemt |
## 3. Repositoryomvang en bronstaat
De machineleesbare inventaris staat in `artifacts/evidence/accuracy/P1/repository-inventory.json`.
| Groep | Bestanden | Regels |
|---|---:|---:|
| backend applicatie | 129 | 32.583 |
| backend tests | 235 | 36.067 |
| frontend broncode | 112 | 23.534 |
| frontend E2E-scripts | 2 | 734 |
| migraties | 11 | 610 |
| root tests | 7 | 261 |
| scripts | 158 | 43.180 |
Aanvullend zijn 1.182 `test_*`-functies, 16 frontend-testbestanden en 148 API-route-decorators statisch geteld. Dit zijn omvangmetingen, geen bewijs dat iedere test of route correct is.
De checkout bevat 2.495 tracked files. Daarvan zijn 1.153 bestanden een tweede, tracked repositorykopie onder `geointel/`: 1.085 gepaarde bestanden zijn byte-identiek en 68 verschillen van hun roottegenhanger. Docker sluit `/geointel` expliciet uit, maar lokale imports, zoekresultaten en scripts kunnen toch de verkeerde kopie raken. Verder zijn onder meer 109 tracked bestanden onder `.codex-input`, 27 onder `data`, één onder `output` en vier onder `test-results` aangetroffen. Phase 1 verwijdert of overschrijft deze user-/historieartefacten niet.
## 4. Runtime en persistence op Tower
### 4.1 Platform
| Eigenschap | Waargenomen waarde |
|---|---|
| Host/containerplatform | Linux 6.12.54 Unraid, x86_64 |
| Python | 3.11.2 |
| PostgreSQL | 16.14 |
| PostGIS | extension 3.6.4; core procedures melden nog 3.4.3 en `need upgrade` |
| Alembic-head | `202607260001` (één lineaire statische en runtime-head) |
| PyTorch | `2.11.0+cu128` |
| torchvision | `0.26.0+cu128` |
| Ultralytics | `8.4.99` |
| CUDA-runtime | 12.8 |
| GPU | één NVIDIA GeForce RTX 4080 SUPER; CUDA beschikbaar |
| GIS-libraries | GeoPandas 1.1.4, Rasterio 1.4.4, Shapely 2.1.2, pyproj 3.7.2, GeoAlchemy2 0.20.0 |
### 4.2 Persistente aantallen
| Tabel | Rijen | Tabel | Rijen |
|---|---:|---|---:|
| `projects` | 1.097 | `areas` | 387 |
| `datasets` | 3.377 | `dataset_versions` | 1.671 |
| `vector_features` | 6.689.447 | `analysis_runs` | 1.146 |
| `detections` | 299.233 | `segmentations` | 0 |
| `detection_reviews` | 0 | `quality_checks` | 697 |
| `metrics` | 4.182 | `jobs` | 4.460 |
| `exports` | 768 | `aoi_operations` | 4 |
| `aoi_operation_partitions` | 61 | | |
De snapshot gebruikte `statement_timeout=30000` ms en read-only queries. De volledige recursieve storageaudit eindigde in een time-out en is dus geen pass; de aparte controle van alle 5.816 directe `storage_path`-referenties voltooide wel en vond 0 ontbrekende bestanden. Niet-gerefereerde, geneste of semantisch verkeerde artefacten vallen buiten dat resultaat.
## 5. Modellen en trainingsartefacten
### 5.1 Actief productiepad
| Veld | Waarde |
|---|---|
| Containerpad | `/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt` |
| Hostpad | `/mnt/user/appdata/geointel/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt` |
| SHA-256 | `a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1` |
| Grootte | 22.516.074 bytes |
| Geconfigureerde klasse | `building` |
| Device | `cuda:0`, CUDA verplicht |
| Geconfigureerde validatienamen | `Mol,Kempen` |
| Modelversie | leeg |
De hash identificeert het actieve bestand. Er is geen aangetoonde release-/dataset-/commitketen die deze hash koppelt aan een menselijk gereviewd, beschermd testresultaat. De scopecheck gebruikt bovendien een substring in `Area.name`, niet een onveranderlijke geografische scope.
De GPU-smoke gebruikte één bestaande 512×512 RGB GeoTIFF in EPSG:31370 en de productieadapter met seed `20260801`, deterministische algoritmen, `imgsz=640`, confidence `0.5` en `max_det=1000`. Het model produceerde 17 `building`-boxes in 0,884 s inference; dat bewijst alleen dat één adapter-call op de NVIDIA-GPU uitvoer gaf.
### 5.2 Niet-actieve recente checkpoints
De retained Tower-inventaris telt 26 modelassets in `/app/models`, 229 trainingscheckpoints met samen 28.512.052.142 bytes, 424 JSON-trainingsrapporten en 36 operator-manifests. Dit zijn aanwezigheidsaantallen; de collector claimt uitdrukkelijk geen human label acceptance, strikte splitonafhankelijkheid, protected-testprestatie, nationale geldigheid of release readiness.
Onder de niet-actieve checkpoints bestaan onder meer:
- `/mnt/user/appdata/geointel/storage/training/building-be-v62-v58-flanders-balanced-transfer-r1/runs/flanders/weights/best.pt`, 456.136.473 bytes, SHA-256 `889ee5b3bfe722803542d15a53a51a716872cf0e8516da5e8291e0a2605adbdb`;
- `/mnt/user/appdata/geointel/storage/training/building-be-v62-v58-flanders-balanced-transfer-r1/runs/flanders/weights/last.pt`, 456.136.985 bytes, SHA-256 `08d12061f017c45c123c824ffa043bc07794beef251c903299eb1e0954636c8b`.
Aanwezigheid is geen activatie of promotie. Het v62-pad heeft in de geïnspecteerde run alleen preview-calibratiebewijs op 144 tiles; geen protected-test-, achtergrondtest- of releasebundel werd daar aangetroffen. De meest recente geïnspecteerde v66-manifestversie bevat slechts drie Vlaamse `train`-samples, nul background-test-samples en geen andere regio/split; ze is een gerichte proef, geen opvolgende nationale corpusrelease.
### 5.3 Lokale checkout
De lokale artifactscanner inventariseerde 193 bestanden (84.874.287 bytes) onder `models`, `datasets`, `data`, `storage`, `artifacts` en `output`: 114 rasters, 32 overige bestanden, 21 runtime-databasebestanden, 13 visuele-reviewbestanden, 6 evaluatie/auditbestanden, 5 manifests en 2 vectors. Er staat lokaal geen modelcheckpoint. Grote bestanden zijn bewust niet allemaal gehasht; aanwezigheid van contact sheets telt niet als menselijke review.
## 6. Build-, CI- en configuratiedrift
- De lokale audit draaide met Python 3.13.2, Node 24.18.1 en npm 11.16.0; CI is ingericht op Python 3.11 en Node 20. Een lokaal groen resultaat is daarom niet automatisch een CI-runtimebewijs.
- `docker-compose.unraid.yml` bouwt standaard met `GEOINTEL_INSTALL_AI=false` en noemt als Torch-index `cu130`; de actieve Tower-runtime is een AI-image met Torch/CUDA `cu128`. `deploy/unraid/Dockerfile.all-in-one` en deployscripts vormen dus een tweede, afwijkend buildpad.
- GitHub- en Gitea-workflows voeren `scripts/run_readiness_check.sh` uit. Die gate compileert code, draait backendtests, Alembic heads, frontend unit/typecheck/build, maar controleert de E2E- en veel live-workflows alleen met `node --check` of `bash -n`.
- `frontend/package.json` heeft geen `lint`-script en geen ESLint-configuratie, hoewel linting in de teststrategie wordt verwacht. `python -m ruff check backend scripts tests` meldde 112 bevindingen (E402 13, E701 2, E702 69, F401 23, F403 1, F811 2, F841 2) en zit niet in de huidige readinessgate.
- De CI bouwt niet aantoonbaar de AI/CUDA-image en controleert geen hash-lock voor de PyTorch/Ultralytics-artifactketen.
## 7. Reproduceerbare inventariscommando's
Vanaf `C:\Projects\geointel`:
```powershell
python scripts\run_accuracy_phase1_baseline.py --output-dir artifacts\evidence\accuracy\P1
python scripts\collect_accuracy_phase1_runtime.py --help
python scripts\collect_accuracy_phase1_inference_smoke.py --help
git rev-parse HEAD
git status --short
```
De laatste twee collectors zijn ontworpen om in de Tower-container tegen `/app/storage` en `/app/models` te draaien; de bewaarde outputs zijn de JSON-bestanden in `artifacts/evidence/accuracy/P1`. Voor serveridentiteit is read-only geverifieerd:
```bash
cd /mnt/user/appdata/geointel
git rev-parse HEAD
git branch --show-current
docker inspect geointel --format '{{.Config.Image}} {{.State.Status}} {{.State.Health.Status}}'
```
## 8. Inventarisconclusie
GeoIntel is geen mockplatform: de workbench, API, PostGIS-persistentie, provider- en analysekaders, jobs, exports, QA/QC en een echte CUDA-YOLO-adapter zijn aantoonbaar geïmplementeerd en draaien. Even aantoonbaar is dat meerdere kritieke correctheidsgrenzen ontbreken of omzeild kunnen worden. Segmentatie is niet als productiemodel bewezen, fixturepaden blijven aanwezig, recente training is niet naar een geldige releaseketen gepromoveerd en geen enkel Phase-1-resultaat ondersteunt “100% getraind” of nationale nauwkeurigheid.
+250
View File
@@ -0,0 +1,250 @@
# GeoIntel Accuracy Improvement Program — 02 Data-lineage
## 1. Scope en bewijsregel
Deze lineage-audit volgt een resultaat terug naar project/AOI, bronbestand, datasetversie, tile of vectorfeature, model of algoritme, runparameters, QA/QC en export. Een veld dat in het schema bestaat maar leeg is, geldt niet als lineagebewijs. Een door de gebruiker aangeleverde string geldt evenmin als bewijs van officiële bronautoriteit.
De runtimecijfers komen uit de gebonden read-only snapshot van 1 augustus 2026 in:
- `C:\Projects\geointel\artifacts\evidence\accuracy\P1\tower-runtime-database-snapshot-detailed.json`;
- `C:\Projects\geointel\artifacts\evidence\accuracy\P1\tower-gpu-inference-smoke.json`;
- `C:\Projects\geointel\artifacts\evidence\accuracy\P1\tower-ml-data-lineage-snapshot.json`.
Containerpaden beginnen met `/app`; dezelfde persistente volumes staan op de host onder `/mnt/user/appdata/geointel`.
## 2. Canonieke resultaatketen
```mermaid
flowchart LR
S["Externe bron of upload"] --> F["Bestand / API-respons"]
F --> D["Dataset + DatasetVersion"]
D --> V["VectorFeature(s)"]
D --> T["Rastertiles + manifest"]
V --> A["AnalysisRun / geoprocessing"]
T --> M["Modelbestand + adapter"]
M --> R["Detection / Segmentation"]
A --> Q["QualityCheck + Metric"]
R --> Q
A --> E["Export"]
R --> E
Q --> E
```
De feitelijke persistencevelden staan in `backend/app/models/entities.py`:
- `Dataset`: `project_id`, `area_id`, `storage_path`, `checksum_sha256`, `derived_from_dataset_id`, CRS/bounds/resolutie/banden, `dataset_role`, `source_name`, `reference_layer_name`, source- en provenance-metadata, tijdvelden, `source_version` en status;
- `DatasetVersion`: versie, storagepad, checksum, bronversie, observatietijd, validiteitsvenster en source/provenance-metadata;
- `VectorFeature`: `dataset_id`, bronfeature-id, klasse, properties en PostGIS-geometrie met SRID 4326;
- `AnalysisRun`: project/AOI/dataset/job, analysetype, status, modelnaam/-versie, parameters, resultaat en tijden;
- `Detection`/`Segmentation`: dataset/run/job, modelnaam/-versie, klasse, confidence, EPSG:4326-geometrie, pixelbbox, bron-tile en properties/provenance;
- `QualityCheck` en `Metric`: kandidaat-, referentie- en runkoppelingen, parameters, bevindingen en losse metricwaarden;
- `Export`: project, optionele run, type, storagepad en metadata.
Dat is een bruikbaar schema, maar de audit toont breuken tussen schema, invoercontrole en opgeslagen werkelijkheid.
## 3. Brondatasets en versies
### 3.1 Volledigheid van kernvelden
| Controle | Datasetresultaat | DatasetVersion-resultaat | Interpretatie |
|---|---:|---:|---|
| totaal | 3.377 | 1.671 | alle 3.377 datasets hebben status `ready` |
| checksum ontbreekt | 0 | 0 | bestandsidentiteit is op rijniveau aanwezig |
| CRS ontbreekt | 0 | n.v.t. | aanwezigheid bewijst nog niet dat coördinaten correct getransformeerd zijn |
| `imported_at` ontbreekt | 0 | n.v.t. | importtijd is gevuld |
| source metadata ontbreekt | 0 | 0 | metadataobject aanwezig; semantische authenticiteit niet bewezen |
| provenance metadata ontbreekt | 0 | 0 | provenanceobject aanwezig; inhoud kan caller-controlled zijn |
| storagepad ontbreekt | n.v.t. in deze query | 0 | 5.816 directe storagepaden apart fysiek gecontroleerd |
| `observed_at` ontbreekt | 2.377 | niet in deze gapquery | bronfamiliebeleid nodig: sommige vaste/scenariobronnen kunnen legitiem geen observatietijd hebben |
| `source_version` ontbreekt | 1.761 | niet in deze gapquery | geen betrouwbare freshness/reproduceerbaarheid zonder bronfamilieclassificatie |
“Metadata aanwezig” mag niet worden omgezet in “officiële lineage geldig”. `POST /datasets/upload` in `backend/app/api/routes/datasets.py:142-163` laat de caller `source_name`, `reference_layer_name`, source-/provenance-metadata en `source_version` invullen. De dekkingsresolver vertrouwt die waarden later als materialisatiebewijs. Een in-memory reproductie met een user-upload die zich als `grb`/`buildings` presenteerde, werd als operationele, gezaghebbende GRB-dekking teruggegeven. Er is dus een cryptografische bestandsketen, maar geen afgedwongen bronautoriteitsketen.
### 3.2 Waargenomen bronnen
De volgende rijen zijn feitelijk in de Tower-database waargenomen. `reference` is alleen de opgeslagen datasetrol/-metadata, geen onafhankelijke bevestiging door de genoemde instantie.
| `source_name` | datasets | reference |
|---|---:|---:|
| `manual` | 1.751 | 660 |
| `vmm_flood_hazard` | 339 | 0 |
| `vmm_vha_bathymetry_profiles` | 273 | 273 |
| `department_omgeving_thematic_raster` | 210 | 0 |
| `grb` | 179 | 179 |
| `digitaal_vlaanderen_orthophoto` | 168 | 0 |
| `spw_orthophoto` | 107 | 0 |
| `spw_picc` | 68 | 68 |
| `digitaal_vlaanderen_dhmv` | 60 | 0 |
| `urbis` | 36 | 36 |
| `urbis_orthophoto` | 26 | 0 |
| `waterinfo` | 26 | 26 |
| `department_omgeving_land_use` | 25 | 25 |
| `historical_landuse` | 21 | 21 |
| `agentschap_landbouw_zeevisserij_agricultural_parcels` | 18 | 18 |
| `statbel` | 15 | 15 |
| `dov_soil_map` | 13 | 13 |
| `inbo_bwk_natura2000` | 12 | 12 |
| `spw_walous_land_cover` | 7 | 0 |
| `vrbg` | 6 | 0 |
| `fixture` | 5 | 1 |
| `ngi_adminvector` | 4 | 4 |
| `map_selection` | 3 | 0 |
| `digitaal_vlaanderen_buildings_addresses_register` | 1 | 1 |
| `rbins_marine_reporting_units` | 1 | 1 |
| `rbins_msp_2026` | 1 | 1 |
| `spw_bathymetry` | 1 | 0 |
| `spw_terrain` | 1 | 0 |
Deze telling bewijst historische materialisatie, niet dat iedere externe endpoint op de peildatum bereikbaar of actueel was.
## 4. Vectorlineage
### 4.1 Bedoelde keten
`bron/upload -> Dataset + checksum/CRS/provenance -> VectorFeature rows -> spatial operation/AnalysisRun -> derived Dataset -> QA/export`.
De database bevat 6.689.447 vectorfeatures. De gebonden PostGIS-query vond 0 lege, 0 ongeldige, 0 niet-4326 en 0 buiten-4326-domeingeometrieën in de huidige tabel. Dat sluit drie concrete codeproblemen niet uit:
1. `backend/app/services/geojson_service.py:70-94` leest CRS-metadata, maar `vector_feature_service.py:271-299` krijgt die CRS niet mee en schrijft iedere feature met `srid=4326`. Een EPSG:31370-geometrie rond `(150000, 210000)` kon daardoor ongetransformeerd als 4326 worden opgebouwd. De huidige tabelquery bewijst alleen dat zo'n waarde nu niet in `vector_features` staat; het importpad blijft foutgevoelig.
2. `vector_operations_service.py:179-190` voert `geometry.buffer(distance_m)` uit op de geladen geometrie zonder metrische reprojection. De retained reproductie van een 100-meterbuffer leverde 200 graden span en bounds `[-95, -49, 105, 151]`.
3. `_persist_derived_dataset` heeft in `vector_operations_service.py:414-427` standaard `persist_vector_features=False`. Clip, buffer en intersect kunnen daardoor een `ready` GeoJSON-artifact en dataset opleveren zonder querybare `vector_features`, zonder AnalysisRun-koppeling en met een niet-atomische persistenceketen.
Lineageconclusie: validiteit van de huidige PostGIS-rijen is bewezen; een algemeen correct vectorimport-/operatiecontract is dat niet.
## 5. Raster- en tilelineage
Een rasterdataset bewaart bestandspad, checksum, CRS, bounds, resolutie en bandmetadata. Inference gebruikt een tegelmap onder `/app/storage/tiles/<project>/<dataset>/<run>/` met `manifest.json` en GeoTIFF-tiles.
De representatieve read-only smoke gebruikte:
| Artifact | Pad | SHA-256 |
|---|---|---|
| manifest | `/app/storage/tiles/cb80638d-dbef-48ac-b19c-cec7c3efc96e/ae0ff76d-70c0-404f-b777-54d14517179a/b191c968-7d56-4e0d-afbb-8b5baaa62470/manifest.json` | `6ab8a96bf2a1405e224932afb90255311a09bbeaed4e9a2fdcdf8b1bc2230abd` |
| tile | dezelfde map, `tile_0000.tif` | `134a9e86850c92c577c73bc6ee57a9df7d4c1c513ae6450263e800b6dd47b6ee` |
| actief model | `/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt` | `a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1` |
De tile is 669.227 bytes, 512×512, 3×`uint8`, CRS EPSG:31370, resolutie 0,9765625 meter/pixel en bounds `[193277.53584918313, 205708.27443503588, 193777.53584918313, 206208.27443503588]`. Dit is voldoende om deze ene inference-input exact te identificeren. Er is nog geen corpusbrede toets op duplicate/near-duplicate imagery, nodata, source-date/label-date, WMS-resampling, grenspadding of resolutieconsistentie.
## 6. Detectielineage
### 6.1 Persistente productieketen
`Dataset -> tile manifest -> AnalysisRun(parameters/model_name/model_version) -> Detection(source_tile_path/pixelbbox/confidence/EPSG:4326 geometry) -> QualityCheck/Metrics -> Export`.
Tower bevat 1.146 `analysis_runs`, alle type `detection` met modelnaam `yolo-configured`: 1.144 success en 2 failed. Daaruit zijn 299.233 detecties aanwezig. Geen confidence lag buiten `[0,1]`. De modelversie is bij alle 1.146 runs leeg. 1.143 runs bewaren wel een modelassethash; drie niet. Alle runs bewaren een tile-manifestpad, maar geen tile-manifest-SHA, runtime-/hardware-identiteit of seed. De drie hashloze runs zijn niet sluitend aan een checkpoint te koppelen; ook de overige missen het volledige reproduceerbare inferencecontract.
De read-only smoke riep de productieadapter rechtstreeks aan met seed `20260801`, deterministische algoritmen, confidence 0,5, `imgsz=640`, `max_det=1000` en `cuda:0`. Resultaat: 17 `building`-boxes, confidence 0,5008770,814435, gemiddelde 0,559481. De smoke schreef geen DB-records en testte dus niet de volledige georeference-/persistence-/QA-keten.
### 6.2 Bewezen historische lineagebreuk
Vier detecties in run `7ba34274-411d-45e3-8f54-c37baec598b1` hebben een PostGIS-geometrie met SRID 4326 maar numerieke Lambertwaarden rond X 193.278193.774 en Y 205.708206.054. Alle vier verwijzen naar:
- project `GeoIntel Real Data Validation 20260706T235847Z`;
- dataset `ae0ff76d-70c0-404f-b777-54d14517179a`, `geel_orthophoto_wms_512.tif`, bron `manual`;
- tile `/app/storage/tiles/cb80638d-dbef-48ac-b19c-cec7c3efc96e/ae0ff76d-70c0-404f-b777-54d14517179a/b191c968-7d56-4e0d-afbb-8b5baaa62470/tile_0000.tif`;
- creatietijd 6 juli 2026 en class `building`.
Dit zijn aantoonbaar buiten-domeindetecties; de collector markeerde de sectie desondanks generiek als `status: ok`, zodat consumenten de detailtellingen moeten gebruiken. De huidige `detection_georeferencing.py` bevat wel een pyproj-transformatie naar EPSG:4326; op basis van deze audit kan niet worden bewezen via welke historische codeversie de vier rijen ontstonden. Ze mogen niet in QA, kaarten of exports blijven zonder quarantaine/herberekening.
### 6.3 Scopebinding
`detection_service.py:223-232` laat configured YOLO alleen toe wanneer één van `Mol,Kempen` als substring in de muteerbare `Area.name` voorkomt. Een area met naam `Mol validation bypass` en een geometrie buiten de gevalideerde zone werd in de reproductie geaccepteerd. De model-toepassingsscope heeft daardoor geen onveranderlijke geometrische lineage.
## 7. QA/QC-, review- en exportlineage
Tower bevat 697 quality checks en 4.182 metrics. De metricgapquery toont:
| Metric | total checks | null |
|---|---:|---:|
| precision | 697 | 58 |
| recall | 697 | 1 |
| F1 | 697 | 58 |
| mean IoU | 697 | 90 |
| false positives | 697 | 0 |
| false negatives | 697 | 0 |
De golden fixturebenchmark koppelt kandidaat/referentie, QualityCheck en zes metrics correct in-memory. Bij het no-overlapscenario zijn precision en recall `0`, maar F1 en mean IoU `null`. Zolang het API-/metriccontract niet expliciet vastlegt of F1 daar `0` of “undefined” hoort te zijn, kunnen aggregaties dezelfde run verschillend behandelen.
Er staan 0 `detection_reviews` in de productiedatabase. De aanwezigheid van review-UI, contact sheets of reviewtabellen bewijst dus geen uitgevoerde menselijke productiereview. De 768 exportrecords verwijzen naar concrete storagepaden; dat bewijst artifactpersistentie, niet dat upstream geometrie/model/QA correct was.
## 8. Trainings- en corpuslineage
### 8.1 Bewezen v56-keten
De retained snapshot telt in totaal 26 modelassets, 229 trainingscheckpoints (28.512.052.142 bytes), 424 JSON-trainingsrapporten en 36 operator-manifests. Die inventarisclaim zegt niets over menselijke acceptatie of releasekwaliteit.
Het canonieke v56-manifest is `/app/storage/operator-data/building-be-v56-hard-negative-instance-roofs-r1/operator_samples_manifest.json`, 212.185 bytes, SHA-256 `27a9b09f184bb9a0cc4897f1cfab7f1e953af5715c1b97a5ec1a7b80a2d270f1`. De corpusaudit `/app/storage/training/building-be-v56-corpus-audit-r1/belgium-building-corpus-audit.json` (SHA-256 `cbb9802aec1b780b306318fc172e470d29166574e9bf13d28e3107ae42f4d63c`) rapporteert:
- datasetversie `building-be-v56-hard-negative-instance-roofs-r1`, immutable manifest;
- 180 AOI-samples: Vlaanderen 89 train/2 val/3 calibration/3 test/2 background-test; Wallonië 43/2/3/3/2; Brussel 18/2/3/3/2;
- 132 positieve samples en 48 `background_candidate`-samples;
- 31.452 inputfeatures, 30.662 geaccepteerd, 326 onder resolveerbare pixelgrootte en 464 na de beeldperiode;
- automated `spatial_leakage_status=ok` en 0 temporal-unknown samples;
- `status=needs_human_review`, `reviewed_sample_count=0`, `review_complete=false`, alle 180 in de reviewqueue.
Van de zes `background-test`-AOI's zijn er slechts drie aantoonbaar puur leeg: twee in Vlaanderen, één in Wallonië en nul in Brussel. De andere drie bevatten respectievelijk 2, 107 en 141 referentiefeatures en zijn moeilijke negatieven, geen pure backgrounds. Dit is te weinig om een regionale pure-background false-positiveclaim te dragen.
De bounded duplicaatscreen vond 0 exacte cross-split rasterhashduplicates en 0 hergebruikte raster-dataset-id's. Voor 4.833 cross-split dHash-paren was de minimum-Hammingafstand 17 en waren er geen paren op of onder 4. Dat sluit semantische, instance-level of flight-stripduplicatie niet uit. De AOI-bboxscreen vond een minimale cross-splitafstand van 95,7203 meter tussen `anderlecht-industry-train` en `jette-test`, 24 paren onder 2 km en 0 onder 64 meter. De auditstatus `ok` is daarom niet voldoende voor de strengere onafhankelijkheidseis; nabijheid is een risico dat instance-/imagerylineage vereist, geen bewijs op zichzelf van leakage.
De tile-qualityaudit `/app/storage/training/building-be-v56-rotated-quality-audit-r1/operator_yolo_dataset_quality_audit.json` (SHA-256 `da0d4648ee2365ab949f1b00518ddba0a17cb722a41cb228525b4d2e7657ce36`) rapporteert geautomatiseerd `status=ok`, 2.496 tiles, 60.229 labels, 0 ongeldige/missende labelfiles, 2.072 positieve en 424 negatieve tiles, waarvan 411 negatieve trainingtiles. Dat syntactische bewijs vervangt de ontbrekende menselijke semantische review niet.
De v58/v62-calibratierapporten verwijzen naar een afgeleide root `/app/storage/operator-data/building-be-v56-hard-negative-rotated-holdouts-r1`. Een releasebundel moet de transformatie van het canonieke instance-roofs-manifest naar deze rotated-holdouts-dataset expliciet, gehasht en reproduceerbaar vastleggen.
### 8.2 Bewezen v58/v62-preview, niet gepromoveerd
V58 `/app/storage/training/building-be-v58-v56-clean-pretrained-r1/preview-epoch-015/calibration.json` (SHA-256 `60538573a734d91107e25fb2a8a640ce45d90c482acfb65ed7110ff884f2f4c7`) en v62 `/app/storage/training/building-be-v62-v58-flanders-balanced-transfer-r1/preview-epoch-006/calibration-routed.json` (SHA-256 `05d8c8f0ade55907e37668758ac225aeff60cbb45604de662944fa447f6d590f`) zijn beide 144-tile, `split=val`, IoU-0,5 calibratiepreviews met 13 thresholds. Ze zijn geen protected-test-, unieke-gebouwen- of nationale releaseresultaten.
Bij threshold 0,15 rapporteren beide artifacts aggregate F1 0,512905, precision 0,545647 en recall 0,483871, maar Vlaanderen heeft true positives 0, recall 0 en F1 0. Bij threshold 0,02 heeft v58 aggregate F1 0,240667 en Vlaanderen F1 0,080402/recall 0,103380; v62 heeft aggregate F1 0,208283 en Vlaanderen F1 0,067111/recall 0,275348. Geen van deze observaties voldoet aan een geloofwaardige drie-regio-gate.
De v62-checkpointhash `889ee5…` verschilt van de actieve productiemodelhash `a9088b…`; de v62-run is dus niet het actieve model.
### 8.3 V66 is een gerichte trial
Het immutable manifest `/app/storage/operator-data/building-be-v66-lowrise-temporal-r1/operator_samples_manifest.json` (SHA-256 `d861da48aca40121e158e6ab243a6ccbe53e14d500a186af64b620f612935b07`) bevat drie positieve Vlaamse trainingssamples, nul andere splits/regio's en nul background-test-samples. Het kan v56 niet als nationale corpusbaseline vervangen.
### 8.4 Protected-testleakage in de loop
`scripts/run_belgium_building_training_loop.py:417-449` opent test en background nadat calibration slaagt. Wanneer de volledige assessment niet `training_complete` is, wordt die assessment op regels 477-498 als input gebruikt voor de volgende failure-driven trainingsset. `scripts/build_failure_driven_yolo_sampling.py:98-120` kiest expliciet `assessment.get("test") or assessment.get("calibration")` en gebruikt ook background-failures; regels 130-201 sturen vervolgens regionale/contextuele repeats van trainingtiles.
De sampler neemt geen protected tiles zelf op, maar de protected-testuitkomsten beïnvloeden wél de volgende trainingsverdeling. Daarmee is de testset een tuningbron geworden en kan zij niet langer als onafhankelijke, eenmalige protected test voor de uiteindelijke kandidaat gelden.
## 9. Lineagebreuken die Phase 2 moeten blokkeren
| ID | Breuk | Bewijs | Gevolg |
|---|---|---|---|
| LIN-01 | caller-controlled bronautoriteit | uploadvelden + coverage-reproductie | “officieel/operationeel” kan vals zijn |
| LIN-02 | cross-theme coverage-union | `coverage_registry_service.py:481-504` | kleine juiste laag kan volledige dekking erven van verkeerde laag |
| LIN-03 | niet-4326 import als SRID 4326 | `geojson_service.py` versus `_feature_row` | geometrie kan valide lijken maar ruimtelijk betekenisloos zijn |
| LIN-04 | meters als graden bij buffer | 100 m-reproductie | extreme, foutieve derived geometrieën |
| LIN-05 | derived artifact zonder querybare features/run | `persist_vector_features=False` | UI/artifact en PostGIS geven verschillende waarheid |
| LIN-06 | modelscope via naamsubstring | area-name-reproductie | inference buiten gevalideerde scope mogelijk |
| LIN-07 | lege modelversie bij 1.146 runs; 3 zonder modelhash; 1.146 zonder tile-manifest-SHA/runtime/seed | Tower-runtime-snapshot | historische run niet volledig reproduceerbaar; drie runs niet sluitend aan checkpoint te koppelen |
| LIN-08 | vier Lambert-detecties als SRID 4326 | gedetailleerde DB-snapshot | kaart, QA en export kunnen corrupte geometrie consumeren |
| LIN-09 | 0 menselijke productiereviews | `detection_reviews=0` | geen bewijs van representatieve operatoracceptatie |
| LIN-10 | v56 human review 0/180 | corpusaudit | label-/tijd-/contextkwaliteit niet semantisch vrijgegeven |
| LIN-11 | slechts 3 pure-empty backgrounds; 24 cross-split AOI-paren onder 2 km | retained v56-manifestanalyse | regionale background- en onafhankelijkheidsclaims zijn niet vrijgegeven |
| LIN-12 | v66 bevat alleen 3 Vlaamse training-AOI's | retained v66-manifest | gerichte trial mag niet als nationale opvolger worden behandeld |
| LIN-13 | protected test stuurt retraining | trainingloop + sampler | evaluatieleakage; releaseclaim ongeldig |
## 10. Reproduceerbare controles
Lokale evidence opnieuw opbouwen:
```powershell
cd C:\Projects\geointel
python scripts\run_accuracy_phase1_baseline.py --output-dir artifacts\evidence\accuracy\P1
$env:PYTHONPATH = "$(Resolve-Path backend);$(Resolve-Path .)"
python scripts\run_golden_qa_benchmark.py --json
```
Tower-artifactidentiteit read-only controleren:
```bash
sha256sum \
/mnt/user/appdata/geointel/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt \
/mnt/user/appdata/geointel/storage/operator-data/building-be-v56-hard-negative-instance-roofs-r1/operator_samples_manifest.json \
/mnt/user/appdata/geointel/storage/training/building-be-v56-corpus-audit-r1/belgium-building-corpus-audit.json \
/mnt/user/appdata/geointel/storage/training/building-be-v62-v58-flanders-balanced-transfer-r1/preview-epoch-006/calibration-routed.json
```
De runtimecollector is read-only en gebruikt een SQL-statementtimeout van 30 seconden. Een toekomstige releasegate moet dezelfde queries uitvoeren én detailtellingen laten falen; een generiek `status=ok` mag vier buiten-domeinrecords niet maskeren.
@@ -0,0 +1,229 @@
# GeoIntel Accuracy Improvement Program — 03 Baseline en gaps
## 1. Phase-1-oordeel
De technische foundation draait: frontend, API, PostGIS, jobs, datasetopslag, een echte Ultralytics/PyTorch-adapter en NVIDIA CUDA-inference zijn aantoonbaar operationeel. De accuracy-/releasebaseline is echter rood. Er is geen bewijs voor nationale gebouwdetectiekwaliteit, geen complete menselijke corpusreview, geen onafhankelijke protected-testcyclus en geen sluitende modelhash-per-run-lineage. Zeven kritieke/hoge correctheidsproblemen zijn deterministisch gereproduceerd en de canonieke backend-releasegate is niet groen.
Daarom gelden op 1 augustus 2026 de volgende harde uitspraken:
- “GeoIntel kan één bestaande tile met het actieve model op de RTX 4080 SUPER verwerken” is bewezen.
- “GeoIntel is 100% getraind”, “nationaal gevalideerd”, “productie-accurate” of “release ready” is niet bewezen en mag niet worden geclaimd.
- De bestaande v56/v58/v62/v66-artefacten zijn diagnostische/trainingsevidence; geen daarvan vormt een geldige nationale promotiebundel.
- Phase 2 mag remediëren en nieuwe evidence opbouwen, maar mag de protected test pas opnieuw bevriezen nadat de leakage-, corpus- en lineageproblemen zijn opgelost.
## 2. Reproduceerbare softwarebaseline
Alle hieronder genoemde logs zijn retained onder `C:\Projects\geointel\artifacts\evidence\accuracy\P1`.
| Check | Exact commando | Uitkomst | Evidence |
|---|---|---|---|
| volledige backendtestset vanuit repo-root | `python -m pytest backend/tests -q -p no:cacheprovider -W error::DeprecationWarning --junitxml=artifacts/evidence/accuracy/P1/backend-full-suite.junit.xml` | **fail**: 1.180 passed, 17 failed, 70,63 s | `backend-full-suite.txt`, `backend-full-suite.junit.xml` |
| canonieke CI/backend-entrypoint | vanuit `backend`: `python -m pytest -W error::DeprecationWarning` | **collection fail**: 1.194 items verzameld plus importerror `scripts.render_operator_polygon_label_qa` | `backend-ci-entrypoint.txt` |
| Phase-1 collectortests | `python -m pytest tests/test_accuracy_phase1_baseline.py -q -p no:cacheprovider` | **pass**: 4/4 | `phase1-tooling-tests.txt` |
| frontend unit | vanuit `frontend`: `npm run test:unit` | **pass**: 16 files, 51 tests | `frontend-vitest-unit.txt` |
| generiek frontendtestcommando | `npm test -- --run` | **fail**: script `test` ontbreekt | `frontend-vitest.txt` |
| frontend typecheck | `npm run typecheck` | **pass** | `frontend-typecheck.txt` |
| frontend build | `npm run build` | **pass**: 1.896 modules; Vite-build voltooid | `frontend-build.txt` |
| frontend lint | `npm run lint` | **fail**: script `lint` ontbreekt | `frontend-lint.txt` |
| Python lintbaseline | `python -m ruff check backend scripts tests --output-format json` | **fail**: 112 findings | `repository-ruff-baseline.json`, `.txt` |
| Alembic head | vanuit `backend`: `python -m alembic heads` | **pass**: één head `202607260001` | `alembic-heads.txt` |
| volledige offline migratieketen | `python -m alembic upgrade head --sql` | **pass**: alle 11 migraties renderen tot commit | `alembic-offline-upgrade.sql` |
| API-contractaudit | `python scripts/audit_api_contracts.py` | **pass**: 147 routes; 10 expliciete non-envelope endpoints | `openapi-contract-audit.txt` |
De 112 Ruff-bevindingen zijn: E402 13, E701 2, E702 69, F401 23, F403 1, F811 2 en F841 2. De nieuw toegevoegde Phase-1-audittools waren in de afzonderlijke check Ruff-clean; de telling is repositorybreed.
De 17 backendtestfailures zijn bron-/contractasserties tegen frontend-, README- en deployteksten/implementatiedetails. Dat maakt ze niet automatisch onbelangrijk of “alleen stale”: de verwachte automatische model/theme-selectie is bijvoorbeeld doelbewust gewijzigd naar gebruikersselectie, terwijl tests nog het oude contract eisen. Test en actueel productcontract moeten expliciet worden gereconcilieerd. Tot dat gebeurt is de releasegate rood.
De CI-entrypointfout heeft een afzonderlijke oorzaak: vanuit `backend` resolveert `scripts` naar `backend/scripts`, waardoor de rootmodule `scripts/render_operator_polygon_label_qa.py` niet importeerbaar is. Een root-run met expliciet importpad kan de tests wel verzamelen, maar repareert de feitelijke CI-opdracht niet.
## 3. Wat de huidige readinessgate niet uitvoert
`scripts/run_readiness_check.sh` compileert veel Python en draait backendtests, Alembic head, frontend unit/typecheck/build. De volgende checks zijn daar slechts syntaxcontroles:
- `node --check` voor de twee frontend-E2E-scripts;
- `bash -n` voor live migration, deploy, upgrade/fresh-install, browserruntime, demo, real-data detection/QA, calibratie, training en cleanupflows.
De gate voert dus geen volledige browserjourney, live PostGIS-migratie, externe provideracquisitie, echte CUDA-modelinference, protected-test-evaluatie of AI-imagebuild uit. De GitHub-/Gitea-build installeert standaard geen AI-dependencies. Een groene toekomstige unit/readinessgate blijft daarom onvoldoende zonder afzonderlijke live-, data- en modelgates.
## 4. Golden QA/QC-baseline
`python scripts/run_golden_qa_benchmark.py --json` slaagde twee keer semantisch met vier fixture-scenario's:
| Scenario | Precision | Recall | F1 | Mean IoU | FP | FN |
|---|---:|---:|---:|---:|---:|---:|
| partial match | 0,5 | 0,5 | 0,5 | 0,833976834 | 1 | 1 |
| perfect match | 1 | 1 | 1 | 1 | 0 | 0 |
| no overlap | 0 | 0 | `null` | `null` | 1 | 1 |
| exact MultiPolygon | 1 | 1 | 1 | 1 | 0 | 0 |
De twee retained JSON-runs zijn niet byte-identiek: SHA-256 `ec96862b…` tegenover `4f900711…`. De semantische resultaten zijn gelijk; UUID4-gegenereerde project/dataset/quality-check-id's maken de output nondeterministisch. Voor een reproduceerbare benchmarkbundel moeten ids deterministisch zijn of vóór hashing worden genormaliseerd.
Het no-overlapscenario legt daarnaast een metriccontractgap bloot: precision en recall zijn 0, maar F1 is `null`. Dit kan wiskundig als undefined worden verdedigd, maar aggregators en releasegates moeten één expliciete semantiek hanteren. De Tower-database bevestigt bredere nullvariatie: 58/697 F1, 58/697 precision, 1/697 recall en 90/697 mean IoU zijn null.
De golden benchmark gebruikt kleine checked-in fixtures. Hij bewijst rekenkundige regressiestabiliteit, niet de nauwkeurigheid van het actieve model op Belgische luchtbeelden.
## 5. Runtime-, database- en GPU-baseline
### 5.1 GPU-smoke
De retained smoke gebruikte het actieve model:
- model `/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt`;
- SHA-256 `a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1`;
- één 512×512 RGB-tile in EPSG:31370, tile-SHA `134a9e86850c92c577c73bc6ee57a9df7d4c1c513ae6450263e800b6dd47b6ee`;
- manifest-SHA `6ab8a96bf2a1405e224932afb90255311a09bbeaed4e9a2fdcdf8b1bc2230abd`;
- PyTorch `2.11.0+cu128`, Ultralytics `8.4.99`, NVIDIA GeForce RTX 4080 SUPER;
- seed `20260801`, deterministische algoritmen, `imgsz=640`, confidence 0,5, `max_det=1000`;
- 17 raw `building`-detecties; inference 0,8837 s, totale model-load plus inference 1,3689 s.
Deze smoke is read-only en passeert. Hij toetst één tile en schrijft geen AnalysisRun/Detection/QA/Export. Hij levert daarom geen accuracy-, calibratie-, georeference-persistence-, schaal- of generalisatieclaim.
### 5.2 Databaseintegriteit
Positieve baseline:
- runtime Alembic-head `202607260001`;
- 3.377 datasets en 1.671 versies hebben checksums en source/provenance-metadata;
- 5.816 directe dataset/version/export-storagepaden gecontroleerd, 0 ontbrekend;
- 387 area- en 6.689.447 vectorfeaturegeometrieën: 0 leeg, 0 ongeldig, 0 wrong-SRID, 0 buiten EPSG:4326-domein in de gebonden query;
- 299.233 detection-confidences: 0 buiten `[0,1]`.
Negatieve baseline:
- 2.377/3.377 datasets zonder `observed_at` en 1.761/3.377 zonder `source_version`; eerst per bronfamilie classificeren, niet blind invullen;
- vier detectiegeometrieën hebben SRID 4326 maar numerieke Lambertcoördinaten rond X 193k/Y 206k;
- alle 1.146 modelruns hebben een lege modelversie; 1.143 bewaren een modelassethash, drie niet; alle 1.146 missen tile-manifest-SHA, runtime/hardware en seed;
- 0 segmentaties en 0 detection reviews;
- 135/4.460 jobs en 2/1.146 analysis runs hebben status failed;
- PostGIS 3.6.4 meldt dat core/topology procedures uit 3.4.3 een upgrade nodig hebben.
De storagecheck dekt alleen directe DB-referenties. De volledige recursieve storageaudit eindigde in een time-out en is geen pass; daarmee is niet bewezen dat alle niet-gerefereerde caches, trainingoutputs of orphan artifacts bekend zijn.
## 6. Deterministisch gereproduceerde productfouten
`forensic-reproductions.json` bevat zeven read-only, deterministische reproducties; alle zeven zijn opnieuw waargenomen.
| ID | Ernst | Geobserveerd | Waarom blokkerend |
|---|---|---|---|
| P1-COV-001 | critical | een kleine buildings-partitie wordt `fully_covered=true` doordat een roads-bbox in dezelfde bounded union zit | coverage- en beschikbaarheidsclaims kunnen inhoudelijk fout zijn |
| P1-CRS-001 | critical | 100 “meter” buffer levert bounds `[-95,-49,105,151]`, 200 graden lengtespan | units/CRS worden verwisseld; derived geometrie is onbruikbaar |
| P1-CRS-002 | critical | EPSG:31370-coördinaten `(150000,210000)` worden ongewijzigd met SRID 4326 opgebouwd | valide SRID-label maskeert verkeerde werkelijkheid |
| P1-AUTH-001 | critical | caller-controlled upload `source_name=grb` wordt `operational`/`authoritative` | gebruikersmetadata kan officiële bronautoriteit spoofen |
| P1-AI-001 | critical | area `Mol validation bypass` met geometrie in Noord-Amerika passeert modelscope | model draait buiten de gevalideerde geografie |
| P1-COV-002 | high | dezelfde Vlaamse geometrie werkt als naam `Flanders`, maar wordt `outside=true` na rename naar `Vlaanderen` | wettelijke scope hangt van een muteerbare displaynaam af |
| P1-API-001 | high | Area PATCH accepteert payload met geometry maar negeert die stil | contract en opgeslagen AOI lopen uiteen |
Aanvullende statisch bewezen risico's staan nog buiten deze zeven reproductions:
- clip/buffer/intersect roepen `_persist_derived_dataset` standaard aan met `persist_vector_features=False`; een `ready` derived artifact hoeft dus niet PostGIS-querybaar te zijn;
- meerdere services slikken secundaire fouten of vallen stil terug, onder meer projectdetailbootstrap en cached rasterpreview; fallback moet expliciete status/provenance krijgen;
- het tracked mirror `geointel/` bevat 1.153 bestanden, waarvan 68 van de rootversie verschillen; Docker sluit de mirror uit, lokale tools niet noodzakelijk.
## 7. Corpus-, split- en trainingbaseline
### 7.1 Inventaris is geen kwaliteit
Tower bevat 26 modelassets, 229 trainingscheckpoints (28.512.052.142 bytes), 424 JSON-trainingsrapporten en 36 operator-manifests. De aantallen tonen veel experimenten, niet dat de beste kandidaat is gevonden of geldig vrijgegeven.
### 7.2 V56 is de breedste aangetroffen corpusbasis, maar niet vrijgegeven
De retained v56-evidence toont:
- 180 AOI's over Vlaanderen, Wallonië en Brussel;
- immutable manifest, 0 exacte cross-split rasterhashduplicates en 0 cross-split raster-dataset-id-duplicates;
- dHash-screen over 4.833 cross-split paren: minimumafstand 17, geen paren op/onder 4;
- bbox-afstandscreen: minimum 95,7203 m en 24 cross-split paren onder 2 km;
- 31.452 inputfeatures; 30.662 geaccepteerd, 326 onder pixel-resolutie, 464 na de imageryperiode;
- automated corpusstatus `needs_human_review`; 0/180 gereviewd, alle 180 pending;
- 48 background candidates, maar slechts drie pure-empty background-test-AOI's: Vlaanderen 2, Wallonië 1, Brussel 0;
- de drie overige background-test-AOI's bevatten 2, 107 en 141 referentiefeatures en zijn moeilijke negatieven.
De exact-hash- en dHashscreen zijn positief maar begrensd. Ze bewijzen geen gebouw-instance-, gemeente-, vluchtstrook-, seizoen- of bronopname-onafhankelijkheid. De 95,7-meter train/testnabijheid en 24 paren onder 2 km vereisen expliciete imagery-/instance-audits voordat een protected split wordt geaccepteerd.
De automated tile-qualityaudit meldt 2.496 tiles, 60.229 geldige labels, 0 invalid/missing labels, 2.072 positieve en 424 negatieve tiles. Dat valideert syntax en enkele pixelregels, niet of daken correct, volledig, tijdsconform of contextueel representatief zijn. Contact sheets bestaan, maar menselijke acceptatiebeslissingen ontbreken.
### 7.3 V58/V62 halen geen regionale gate
V58 en v62 hebben alleen 144-tile `val`/calibratiebewijs met IoU 0,5 en 13 confidence-sweeps. Bij threshold 0,15 rapporteren beide aggregate F1 0,512905, precision 0,545647 en recall 0,483871, terwijl Vlaanderen 0 true positives, recall 0 en F1 0 heeft. Bij threshold 0,02 blijft Vlaanderen zwak: v58 F1 0,080402/recall 0,103380; v62 F1 0,067111/recall 0,275348. Dit is geen protected test en geen nationale releaseprestatie.
V62 `best.pt` bestaat, maar de hash `889ee5…` is niet de actieve productiemodelhash `a9088b…`. Er is geen geldige promotie aangetoond.
### 7.4 V66 is geen nationale opvolger
Het v66-manifest heeft slechts drie positieve Vlaamse `train`-AOI's in twee low-rise-contexten, zonder val/calibration/test/background en zonder Wallonië/Brussel. Het is een gerichte trial.
### 7.5 Protected-testleakage
De trainingloop opent na een geslaagde calibration zowel test als background. Als de kandidaat vervolgens faalt, geeft de loop de volledige assessment aan de failure-driven sampler. Die kiest `assessment.get("test") or assessment.get("calibration")`, gebruikt regionale testmetrics en background-failures en verandert daarmee repeats van trainingtiles voor de volgende iteratie. Protected tiles worden niet letterlijk opgenomen, maar hun uitkomsten sturen training. De huidige testset is dus voor die loop niet langer onafhankelijk en moet na de correctie worden vervangen of aantoonbaar nooit eerder ingezien zijn.
## 8. Claim-matrix
| Claim | Status | Maximaal verdedigbare formulering |
|---|---|---|
| platform draait technisch | ondersteund | gezonde Tower-container, echte PostGIS-data en uitvoerbare frontend/API |
| NVIDIA/PyTorch wordt gebruikt | ondersteund | één actieve YOLO-adaptercall draaide op RTX 4080 SUPER/CUDA 12.8 |
| volledige inferenceketen is correct | niet ondersteund | smoke sloeg persistence/georeference/QA/export over; vier historische geometrieën zijn corrupt |
| actief model is reproduceerbaar | gedeeltelijk | actief bestand heeft SHA; historische runs missen volledige manifest/runtime-lineage en drie runs missen ook de modelassethash |
| labels zijn correct | niet ondersteund | automated syntax ok, maar v56 menselijke review 0/180 |
| splits zijn onafhankelijk | niet ondersteund | exact/dHashscreen positief, maar nabijheid onbeslist en testuitkomsten sturen retraining |
| building accuracy voor Mol/Kempen | niet vastgesteld in Phase 1 | er is geen retained protected Mol/Kempen-releasebenchmark voor actieve hash |
| building accuracy voor heel België | weerlegd als actuele releaseclaim | v58/v62 calibration heeft bij bruikbare aggregate threshold Vlaanderen recall/F1 0 |
| pure-backgroundrobustheid per regio | niet ondersteund | slechts 3 pure-empty AOI's, geen in Brussel |
| productie-segmentatie | niet ondersteund | abstraction/fixture aanwezig, 0 runtime segmentaties |
| officiële bronnen zijn authentiek | niet ondersteund | source identity kan via uploadmetadata worden gespooft |
| databasegeometrieën zijn integraal | gedeeltelijk | areas/vectorfeatures schoon in gebonden query; 4 detecties buiten domein |
| release ready | nee | backend/CI/lint rood plus kritieke data-/GIS-/ML-blockers |
## 9. Geprioriteerde gaps
### P0 — vóór nieuwe modeltraining of productclaim
1. Fix en regressietest de zeven gereproduceerde contract-/CRS-/scope-/authorityfouten; maak coverage unions themaspecifiek, source authority server-attested en modelscope geometrisch/checksumgebonden.
2. Quarantaineer de vier buiten-domeindetecties, identificeer hun producerende code/image/model, herbereken of verwijder ze via een gecontroleerde migratie en laat detailtellingen de gate falen.
3. Maak vector- en detectie-CRS-transformaties expliciet; verbied SRID relabeling; voer metric buffers in een geschikte lokale CRS uit; persisteer derived features en runlineage atomisch.
4. Verwijder test/background uit failure-driven sampling en checkpointpromotie. Bevries daarna een nieuwe, ongeziene protected test met hashes en éénmalige-openingspolicy.
5. Voer menselijke review uit op alle 180 v56-AOI's met beslissingen, reviewer, timestamp, label-/beeldversie en reden; herbouw contact sheets wanneer bron of label verandert.
6. Voeg onafhankelijke AOI's en pure-empty backgrounds toe per regio/context, in het bijzonder Brussel; audit de 24 cross-split paren onder 2 km op imagery-, vluchtstrook- en instance-overlap.
7. Herzie image/label time deltas, vooral de 464 uitgesloten features en dichte PICC/UrbIS-zones; definieer regels voor onzekere/occluded/nieuwe/verdwenen gebouwen.
8. Koppel iedere training, inference-run, detection, QA en export aan model-SHA, datasetmanifest-SHA, codecommit, containerdigest, seed, parameters en dependency lock.
### P1 — vóór releasecandidate
1. Maak de canonieke backend-CI-entrypoint verzamelbaar en reconcilieer de 17 contracttests met de actuele, handmatige modelselectie-UX.
2. Voeg Ruff en frontend lint toe aan de readinessgate; werk de 112 bestaande bevindingen gecontroleerd weg.
3. Bouw en test de echte AI/CUDA-image in CI; pin PyTorch/torchvision/Ultralytics en leg image digest/SBOM vast; hef cu128/cu130-drift op.
4. Draai echte browser-E2E, live PostGIS upgrade/fresh-install, externe-provider- en full inference/persistence/QA/exportjourneys.
5. Definieer null-/zero-/undefinedsemantiek voor precision, recall, F1 en IoU; maak golden outputs byte-reproduceerbaar of canonicaliseer ids.
6. Classificeer de 2.377 missende observatietijden en 1.761 missende bronversies per broncontract en maak onverklaarde gevallen fail-closed.
7. Verwijder bronambiguïteit door de tracked `geointel/`-mirror gecontroleerd te migreren; niet in Phase 1 destructief opruimen.
## 10. Wanneer Phase 2 veilig kan trainen
Nieuwe GPU-training is pas zinvol nadat de P0-datacontracten, human review en splitpolicy zijn opgelost. Anders optimaliseert een nieuwe run opnieuw tegen mogelijk foutieve labels, onvoldoende negatives en een gecontamineerde testlus. De veilige volgorde is:
`fix contracten -> nieuwe regressietests -> corpusbeslissingen -> onafhankelijke splits -> frozen manifests/hashes -> train/val/calibration -> kandidaatselectie -> éénmalige protected test -> onafhankelijke menselijke foutreview -> promotiebundel -> shadow deploy`.
Een trainingsloop mag itereren op train/validation/calibration. Hij mag de protected test niet opnieuw in de loop voeren. “100%” wordt niet als numerieke gate gebruikt; release vereist vooraf vastgelegde, context- en regiogebonden thresholds met confidence intervals, failure budgets en expliciete abstention/unsupported-statussen.
## 11. Evidence-integriteit
| Retained bestand | Bytes | SHA-256 |
|---|---:|---|
| `backend-full-suite.txt` | 23.972 | `8521ed48b17b382752418750b3ea374831958fb063e83ef87048212e0fd5ea69` |
| `backend-ci-entrypoint.txt` | 1.601 | `0ac774ec19b4ff0a15142aab5f1db68c2592a230401d794ae7d040320e3ac0c0` |
| `frontend-vitest-unit.txt` | 2.774 | `6341bfa51ca3f4fe5ec7d4af7239c3c5e1a29e6bfe8bdfae85e824a2a6482ad0` |
| `frontend-typecheck.txt` | 163 | `3891c85c77b5ff50a1eb6d27a2a65d40c2c05423768734efd9d980f3784d68fa` |
| `frontend-build.txt` | 2.356 | `c3b8e10ef177ac1c2dc045bd710df1caeb46f8922a721291be431b304abbc079` |
| `frontend-lint.txt` | 418 | `1115b013515c753e2dfb73abdad9024aec7b4c2337d117c7e1181341fef15c7f` |
| `repository-ruff-baseline.json` | 59.359 | `de6617f030e49550e714c49b6e14bf291bf85016fd58086e9ca38b33a52252e9` |
| `alembic-heads.txt` | 116 | `da4521233c6718fc7a5865c53904e73685fbdce65a1449b19cd0dc2e40d037ed` |
| `alembic-offline-upgrade.sql` | 20.199 | `e8905b881890cf95885a7515e3d9dcf2a7a0a24c4edbc57e98a363edcb22dd0e` |
| `openapi-contract-audit.txt` | 212 | `5af3d8f00be3f57fc309bc198fa3995d1eae7270a5f210b3e94d1aeeb653119d` |
| `forensic-reproductions.json` | 3.533 | `f6349199a15ae789092d3d65c17a39a9b32ea3ed557571c0c228c4be3cf7235e` |
| `golden-qa-reproducibility.json` | 505 | `576e5667a989c34086db3a2bc57003115a8a61b14e9f49f5407ee380baf0829d` |
| `phase1-tooling-tests.txt` | 250 | `a9bf261e1a811ad3e8bfa8edc439a11f00b46bc157f9ab6fc970033a87f45748` |
| `tower-gpu-inference-smoke.json` | 5.675 | `692a9fa193d589123b042110d7e80755f0c6634854f3134c291fc6083adb7b77` |
| `tower-runtime-database-snapshot-detailed.json` | 14.946 | `744389b6c384a9fb3a9e16e56f1477f3b752e11a5b98e1fad7df47b4703a9ca1` |
| `tower-ml-data-lineage-snapshot.json` | 71.659 | `d80275e8198ce63366d2a2d44eb8fba1f27c29d85aaaaedb94991d8c56febfb6` |
Deze hashes zijn van de retained Phase-1-bestanden op het moment van documentgeneratie. Als evidence opnieuw wordt gegenereerd, moet een nieuwe evidence-manifestversie de nieuwe hash, producerende commandoregel, timestamp en reden bewaren; oude evidence wordt niet overschreven of als identiek voorgesteld.
+89
View File
@@ -0,0 +1,89 @@
# GeoIntel Accuracy Improvement Program — risicoregister
- Status: **release blocked**
- Auditmoment: 2026-08-01
- Scope: Belgium land en Belgische Noordzee, met de actieve building-detector beperkt tot zijn bewezen Mol/Kempen-scope
## Beslissing en claimgrens
GeoIntel mag op basis van de huidige evidence niet als nationaal gevalideerd AI-systeem worden vrijgegeven en de scope van de actieve detector mag niet worden verruimd. De productie-adapter heeft op de Tower één echte tile op `cuda:0` verwerkt, maar die smoke bewijst uitsluitend dat het model technisch kan laden en infereren. Hij bewijst geen nauwkeurigheid, calibratie of geografische generalisatie.
De volgende regels gelden totdat alle bijbehorende exit-evidence is vastgelegd:
- geen nationale kwaliteitsclaim voor building detection;
- geen promotie van v56, v58, v62, v66 of een afgeleid checkpoint;
- geen wijziging van de actieve modelscope op basis van bestandsaanwezigheid of aggregate metrics;
- geen productieclaim voor segmentatie of zonnepaneeldetectie;
- geen verwijdering of overschrijving van bestaande datasets, checkpoints, evaluaties of auditbewijzen;
- elke herstelling begint met een regressietest die het bewezen defect op de oude implementatie reproduceert.
Ernstclassificatie:
- **Critical**: kan een plausibel maar ruimtelijk/inhoudelijk fout resultaat, een onterechte autoriteitsclaim of een besmet promotiebesluit veroorzaken;
- **High**: blokkeert de nationale claim, reproduceerbaarheid of een betrouwbare releasebeslissing;
- **Medium**: verzwakt assurance, onderhoudbaarheid of deploymentcontrole, maar is op zichzelf geen bewezen fout resultaat.
Alle risico's hieronder zijn open. Een controle telt pas als uitgevoerd wanneer de genoemde exit-evidence bestaat en door QA opnieuw is geverifieerd.
## A. Ruimtelijke juistheid en bronautoriteit
| ID | Ernst | Bewezen evidence | Impact | Owner | Vereiste controle | Exit-evidence |
| --- | --- | --- | --- | --- | --- | --- |
| ACC-R01 | Critical | `geojson_service.py` behoudt bron-CRS-informatie, terwijl `vector_feature_service.py` geometrieën zonder transformatie als SRID 4326 schrijft. De runtime-snapshot vindt 4 detections met Belgische Lambert-coördinaten rond x=193k/y=205k maar SRID 4326; zie `artifacts/evidence/accuracy/P1/tower-runtime-database-snapshot-detailed.json`. | Verkeerde kaartpositie, ruimtelijke predicates, clipping, export en coverage; resultaten kunnen geldig ogen maar buiten het EPSG:4326-domein liggen. | GIS + Architecture | Eén expliciet CRS-ingestcontract; gekende CRS transformeren naar canoniek 4326, onbekende CRS fail-closed weigeren; bestaande vier records quarantainen of herleiden vanuit tile-transform en bron-CRS. | Unit- en PostGIS-integratietests voor EPSG:31370, EPSG:3812 en EPSG:4326; round-trip/extent-asserties; migratierapport met voor/na-ID's en zero `outside_epsg4326_domain`; geen stille datawijziging. |
| ACC-R02 | Critical | `VectorOperationsService.buffer` voert `geometry.buffer(distance_m)` uit op ongetransformeerde GeoJSON. Een 100 m-reproductie nabij Mol levert een span van circa 200 graden. | Een meterparameter wordt als graden geïnterpreteerd; analysegebied en downstream-resultaten zijn materieel fout. | GIS | Bufferen in een geschikte metrische projectie/geography; bron- en doel-CRS en eenheden verplicht valideren; ongeschikte/ontbrekende CRS weigeren. | Regressietest met een gekend punt/polygoon in Mol en geodetische toleranties; API-test voor onbekende CRS; PostGIS/Shapely-pariteit; bewijs dat 100 m geen graadbuffer meer oplevert. |
| ACC-R03 | Critical | `CoverageRegistryService._matching_datasets` voegt bbox-scopes toe vóór `layer_matches` en `zone_matches`. Een kleine buildingscope plus een volledige roadscope resulteert voor buildings ten onrechte in `fully_covered=true`. | Verkeerde bron/thema-combinaties kunnen als operationeel en volledig gedekt verschijnen. | Architecture + GIS | Alleen scopes van datasets die bron, thema, layer en zone volledig matchen in de union opnemen; coverage per bronversie en thema isoleren. | Regressietest voor mixed-theme/mixed-zone datasets; property-based tests voor monotoniciteit en bronisolatie; API-evidence dat de buildingscope in de reproductie `fully_covered=false` blijft. |
| ACC-R04 | Critical | De uploadroute accepteert caller-controlled `source_name`, `reference_layer_name`, `source_metadata` en `provenance_metadata`; coverage vertrouwt deze velden. Een upload kan zich als GRB/authoritative voordoen. | Een handmatige upload kan de attributie, autoriteit en operationele dekking van een officiële bron erven. | Architecture + Data Governance | Server-owned source registry en trust class; user uploads altijd `manual/untrusted`; officiële identiteit uitsluitend via een governed acquisition adapter met immutable provider evidence; bestaande records classificeren. | Security/regressietest die een forged GRB-upload uitvoert en `manual/untrusted` verwacht; migratie-audit van bestaande bronnen; API-response toont bronvertrouwen en kan geen officiële attribution injecteren. |
| ACC-R05 | High | Juridische coveragezones worden via mutable displaynamen (`SCOPE_AREA_NAMES`) bepaald. Dezelfde geometrie met `Flanders` versus `Vlaanderen` verandert de zonematch. | Een hernoeming kan scope/coverage juridisch en operationeel veranderen zonder geometrische wijziging. | Architecture + GIS | Immutable area/zone code en geometry-backed containment gebruiken; displaynaam uitsluitend presentatie; grensgevallen expliciet modelleren. | Rename-regressietest met identieke geometrie; zone-ID-migratie; grens- en Noordzee-tests; coverage-uitkomst blijft naam-onafhankelijk. |
| ACC-R06 | Critical | De YOLO-validatiescope wordt afgedwongen via substringmatching op de mutable `Area.name`. Een gebied buiten Mol met naam `Mol validation bypass` wordt aanvaard. | Inferentie kan buiten de bewezen modelscope worden uitgevoerd en als ondersteund worden gepresenteerd. | AI + GIS + Architecture | Scope koppelen aan immutable geometry/zone-ID en model-card scope polygon; volledige containment/intersection-policy vastleggen; lege geometrie en naammatch nooit voldoende. | Negatieve API/service-tests voor spoofnaam, lege geometry en deels overlappende AOI; positieve Mol/Kempen-tests; scope-evidence bevat modelhash, scope-ID en spatial predicate. |
| ACC-R07 | High | `Area` PATCH documenteert geometry-wijziging, maar het schema verwerkt enkel naam/CRS en negeert extra geometry; CRS kan los van de geometry worden herschreven. | API kan 200 retourneren zonder gevraagde geometry-update of coördinaten fout labelen. | Architecture + GIS | Extra velden verbieden; contract en implementatie gelijkmaken; CRS alleen wijzigen via gevalideerde transformatie, niet via relabeling. | Contracttest met geometry PATCH; unknown-field 422-test; CRS-transformtest; OpenAPI-contract en implementatie tonen hetzelfde gedrag. |
| ACC-R08 | High | Clip/buffer/intersect roepen `_persist_derived_dataset` aan met `persist_vector_features=false`; een dataset kan `ready` zijn zonder querybare PostGIS-features, volledige provenance of atomaire AnalysisRun-koppeling. | Downstreamanalyse ziet een gereed artefact dat niet betrouwbaar querybaar/reproduceerbaar is; partial commits zijn mogelijk. | Architecture + GIS | Eén transactie voor AnalysisRun, DatasetVersion, vector_features en lineage; `processing` tot artifact én PostGIS-persist voltooid zijn; failure rollback/fail-closed. | Integratietests per operatie, inclusief geïnjecteerde write failure; zero ready-derived datasets zonder features; lineage-query van inputhash tot output; transactie-evidence. |
## B. Dataset-, split- en labelintegriteit
| ID | Ernst | Bewezen evidence | Impact | Owner | Vereiste controle | Exit-evidence |
| --- | --- | --- | --- | --- | --- | --- |
| ACC-R09 | Critical | De training-orchestrator evalueert iedere iteratie op calibration, test en background. `build_failure_driven_yolo_sampling.py` gebruikt vervolgens `assessment.get("test")` bij het wegen van de volgende training en registreert test als failure source. | De protected test stuurt training; daardoor is hij geen onafhankelijke eenmalige release-evaluatie meer. | AI + QA | Orchestrator in twee fysiek/logisch gescheiden paden splitsen: iteratie mag alleen train/val/calibration lezen; test/background-test blijven verzegeld tot kandidaat, threshold en gates zijn bevroren. Sampler moet ieder testartefact of protected sample weigeren. | Canary-tests met protected IDs; file-access/auditlog bewijst nul testreads vóór freeze; sampler-test faalt op test-input; één gesigneerd test-open event voor exact één modelhash/config; geen verdere training in die candidate family na opening. |
| ACC-R10 | High | V56 rapporteert een minimale cross-split AOI-afstand van 95,72 m en 24 cross-splitparen onder 2 km. Dat bewijst geen overlap, maar wel dat de vereiste buffer groter dan maximale tile-context nog niet is aangetoond. | Nabije context, dezelfde bebouwing of providerseams kunnen validatie/calibratie/test correleren met training. | GIS + AI + QA | Splittoewijzing op gebufferde AOI's vóór tiling; buffer afleiden uit tilecontext/resolutie; instance-, bronbeeld- en perceptuele near-duplicatecontrole. | Immutable split-audit met minimumafstand ≥ bevroren contextbuffer, zero intersecties, zero gedeelde feature-ID's/checksums/near-dupes en kaartbewijs per dichtste paar. |
| ACC-R11 | High | Het v56 operator-manifest heeft 180 AOI's maar `reviewed=0/180`; de productie-DB telt 0 `detection_reviews`. | Labelkwaliteit en representatieve fouten zijn niet door menselijke beoordeling afgesloten; automatische audits kunnen semantische mislabels niet bewijzen. | QA + Data Governance | Gestratificeerde menselijke review van label-contact-sheets door regio, context, provider, objectgrootte, dichte en moeilijke gevallen; beslissingen als immutable records met reviewer, tijd en reden. | 100% van de vooraf geselecteerde reviewqueue beslist; zero unresolved critical/high label findings; manifest met review-ID's en hashes; inter-reviewer steekproef en getekende QA-beslissing. |
| ACC-R12 | High | V56 bevat slechts 3 pure-empty background-test-AOI's (Flanders 2, Wallonia 1, Brussels 0). V66 bevat slechts 3 Vlaamse train-AOI's. | De achtergrond- en nationale claim is niet representatief voor alle regio's en contexten. | AI + GIS + QA | Nieuwe ruimtelijk onafhankelijke AOI's per regio/context; pure-empty én moeilijke negatieve voorbeelden voor alle regio's, waaronder Brussels; protected portfolios nooit naar training kopiëren. | Corpus-compositiematrix voldoet aan het vooraf bevroren contract uit `PYTORCH_TRAINING_ROADMAP_BELGIUM.md`; elk vereist stratum heeft positieve/negatieve coverage; zero protected-to-train overlap; kaart/contact-sheet evidence. |
| ACC-R13 | High | V56: 31.452 inputfeatures, 30.662 geaccepteerd, 326 onder minimumresolutie en 464 post-imagery; 521 kleine labels en maximale aspectratio 103,67. V66 gebruikt imagery/labels met circa 575 dagen verschil. | Onzichtbare, verouderde, te kleine of extreem gevormde labels veroorzaken fout-negatieven/positieven en onbetrouwbare boxtraining, met extra risico in dichte PICC/UrbIS-gebieden. | GIS + AI + QA | Per-provider tijdsrelatie en resolvability vastleggen; post-imagery/subpixel uitsluiten met reason code; extreme aspect/dense-cluster review; dak/footprint-displacement en PICC/UrbIS-regels bevriezen. | Label-audit met accepted/rejected reason per feature; temporal matrix per AOI; zero unknown silent inclusion; alle extreme/dense reviewqueues gesloten; opnieuw gerenderde contact sheets en corpus-SHA. |
| ACC-R14 | High | V56-tiling telt 2.496 tiles en 60.229 labels, maar dataset-YAML/class/tile/stride-velden zijn afwezig of null in de auditketen. | Het trainingsinputcontract kan niet exact worden gereconstrueerd of vergeleken; een checkpoint kan aan ambiguë preprocessing worden gekoppeld. | AI + Data Engineering | Eén verplicht manifest-schema met dataset-YAML SHA, class mapping, tile size, stride/overlap, nodata, normalization, augmentations, software/runtime en seed; schema fail-closed. | JSON-schema-tests; volledige immutable manifest; alle artifacts hash-bound; clean-room export produceert identieke samplelijst/labels en gedocumenteerde byteverschillen waar bestandmetadata varieert. |
| ACC-R15 | High | Van 3.377 datasets missen 2.377 `observed_at` en 1.761 `source_version`; checksums, CRS, imported_at en provenancevelden zijn wel volledig. | Tijd- en bronversievergelijkingen kunnen niet voor elke bronfamilie worden gereproduceerd; sommige misses kunnen legitiem zijn maar zijn nog niet geclassificeerd. | Data Governance + Architecture | Veldbeleid per bronfamilie: `required`, `not_applicable` of `unknown_with_reason`; backfill uitsluitend uit authoritative evidence; nooit downloadtijd als observatietijd invullen. | Audit per source family met zero ongeclassificeerde ontbrekende waarden; migratie/backfillrapport; API toont unknown/not-applicable expliciet; checksums blijven ongewijzigd. |
## C. Modelvalidatie, metrics en runtime-lineage
| ID | Ernst | Bewezen evidence | Impact | Owner | Vereiste controle | Exit-evidence |
| --- | --- | --- | --- | --- | --- | --- |
| ACC-R16 | Critical | V58-calibratie bij confidence 0,15 geeft aggregate F1 0,5129, Brussels 0,6944, Wallonia 0,4674 en Flanders 0; er is geen protected-test-, background-test- of promotion-evidence. De actieve modelkaart blijft `nationally_validated=false` en Mol/Kempen-scoped. | Aggregate score maskeert een volledige regionale failure; geen nationale kwaliteitsclaim of promotie is verdedigbaar. | AI + QA | Region/context macro- en worst-stratum gates vóór aggregate score; threshold alleen op calibration; actieve scope fail-closed behouden; geen status op basis van checkpointnaam. | Bevroren gateconfig vóór test; calibrationrapport per regio/context/objectgrootte; alle gates pass; daarna pas eenmalige protected-test en promotion report voor exact model-SHA. |
| ACC-R17 | Critical | 1.146 persisted detection runs en 299.233 detections hebben een lege `model_version`; 3 runs missen modelhash; alle missen tile-manifest-SHA, runtime/hardware en seed. | Resultaten kunnen niet eenduidig aan model, preprocessing en execution environment worden herleid; incidentanalyse en reproduceerbaarheid falen. | Architecture + AI | Verplicht immutable RunManifest vóór inference: model-ID/version/SHA, adapter/config, tile-manifest/SHA, dataset/version, CRS/transform, threshold/NMS/max_det, seed, runtime/container/GPU. Run weigeren als verplichte velden ontbreken. | DB-migratie en schema constraints; contracttests; nieuwe run round-trip; reproduction op dezelfde tile/model/config; legacy records expliciet `lineage_incomplete`, nooit stil aangevuld. |
| ACC-R18 | High | Van 697 QA-samenvattingen zijn F1 en precision 58 keer null, mean IoU 90 keer null en recall 1 keer null. De no-overlap golden case geeft precision/recall 0 maar F1 null. | Undefined, zero en not-computable worden door elkaar gebruikt; dashboards/gates kunnen failures overslaan. | QA + Architecture | Metricsemantiek per empty/empty, prediction-only, truth-only en no-match case bevriezen; status/reason naast waarde; gate-aggregator mag null niet negeren. | Truth-table unit tests; migratie/classificatie van historische nulls; golden benchmark per case; promotion fails bij onverwachte null of ontbrekend stratum. |
| ACC-R19 | High | De GPU-smoke op één bestaande EPSG:31370-tile slaagt met active model SHA `a9088…59c1`, 17 detections en RTX 4080 SUPER, maar heeft geen ground truth en verklaart zelf geen accuracy/generalization claim. | Technische beschikbaarheid kan onterecht als modelkwaliteit worden geïnterpreteerd. | AI + QA | Smoke strikt scheiden van evaluatie; UI/docs/status benoemen `runtime_ready` apart van `validated_scope`; geen promotie op smoke-resultaat. | Contracttests voor statussen; model card toont afzonderlijk runtime-, calibration-, test- en scopebewijs; promotion code vereist alle hashes en gates. |
| ACC-R20 | High | De runtime heeft 0 segmentations en het gevalideerde segmentatiemodel ontbreekt; aanwezige assets zijn geen configuratie- of kwaliteitsbewijs. | UI/API kan door bestandsaanwezigheid een niet-bewezen capability suggereren. | AI + Frontend + QA | Segmentatie en zonnepaneelmodellen blijven `not_configured` totdat hun eigen corpus, metrics, model card en releasepad slagen. | End-to-end `not_configured`-tests; capability registry is evidence-driven; geen persisted outputs vóór aparte validatie/promotion. |
## D. Verificatie, foutafhandeling en deployment
| ID | Ernst | Bewezen evidence | Impact | Owner | Vereiste controle | Exit-evidence |
| --- | --- | --- | --- | --- | --- | --- |
| ACC-R21 | High | Met root/backend `PYTHONPATH` worden 1.197 tests verzameld: 1.180 pass en 17 falen. De gedocumenteerde backend-CI-aanroep faalt al bij collection doordat `backend/scripts` het rootpakket `scripts` overschaduwt. | Een groene deelset kan een kapotte canonical testentrypoint of regressies verbergen. | QA + Architecture | Eén canonical testcommand vanuit repo-root; package shadowing verwijderen; stale source-text assertions vervangen door behavior tests; geen baseline-updating zonder contractreview. | Volledige suite pass op CI-runtime én Tower-compatible image; exact command/exitcode/log vastgelegd; 17 regressies inhoudelijk geclassificeerd en opgelost. |
| ACC-R22 | Medium | Ruff rapporteert 112 issues (69 E702, 23 F401, 13 E402, 2 E701, 2 F811, 2 F841 en 1 F403). Frontend heeft geen `lint` script ondanks de teststrategie; typecheck, 51 Vitest-tests en build slagen wel. | Dead imports/compact statements en ontbrekende frontendlint verhogen defectrisico; het formele QA-contract is niet uitvoerbaar zoals beschreven. | QA + Frontend + Architecture | Ruff-baseline gecontroleerd afbouwen; echte frontendlintconfig/script toevoegen; lint in CI; generated/legacy uitzonderingen expliciet en smal. | `ruff check` en `npm run lint` pass; CI voert beide uit; bestaande typecheck/Vitest/build blijven groen. |
| ACC-R23 | High | Frontend detail/bootstrapfouten worden stil omgezet naar fallback/null; raster cache-open kan ondanks fout `ready` opleveren; secundaire detection/segmentation DB-markering kan worden ingeslikt. | Gebruiker ziet oude, onvolledige of schijnbaar succesvolle output zonder zichtbare lineagebreuk. | Frontend + Architecture + QA | Error states expliciet en typed; stale/cache-status tonen; persistence-failure maakt run/dataset failed of incomplete; geen broad catch zonder telemetry en user-visible status. | Fault-injectiontests per fallback; API/UI tonen correlation/run ID en retry; zero `ready/success` na verplichte persistence failure; structured logs/evidence. |
| ACC-R24 | High | Golden benchmark gebruikt UUID4 en meerdere trainingsscripts hebben onvolledig bevroren seed/runtimegedrag. | Baselines en manifests zijn byte-onstabiel; reruns kunnen niet betrouwbaar worden vergeleken. | QA + AI | Deterministische IDs uit inhoud/hash, seed alle RNG's, deterministische algoritmen waar ondersteund, expliciete nondeterministische kernelregistratie en tolerantiebeleid. | Twee clean-room reruns met identieke manifest/sample/split hashes; metrische verschillen binnen vooraf bevroren tolerantie; runtime/seed in RunManifest. |
| ACC-R25 | High | Lokale audit draait op Python 3.13/Node 24, CI op Python 3.11/Node 20. Compose default verwijst naar cu130, deployment naar cu128; CI bouwt alleen AI-disabled en valideert CUDA/AI-dependencies niet. Tower gebruikt PyTorch 2.11.0+cu128 en Ultralytics 8.4.99. | Een image kan lokaal/CPU slagen en op de GPU-runtime falen of een andere dependencyset gebruiken. | DevOps + AI + QA | Eén pinned AI build path en lock/SBOM; CI bouwt CPU-contract én GPU-image; Tower preflight valideert exacte torch/CUDA/Ultralytics/modelhash; geen CPU-fallback voor training. | Image digest + SBOM; build/test logs voor AI-enabled image; `cuda:0` preflight en bounded inference op RTX 4080; runtimeversies exact in release manifest. |
| ACC-R26 | Medium | De repo bevat een tracked geneste `geointel/`-kopie met 1.153 files en 68 afwijkende root/mirror-paren, plus grote tracked audit/inputartefacten. `.dockerignore` sluit de geneste mirror wel uit. | Lokale tooling/imports kunnen de verkeerde kopie analyseren; review en branchgrootte worden onnodig ambigu. | Architecture + DevOps | Canonieke root expliciet afdwingen; mirror in P2 via afzonderlijke, reviewbare migratie inventariseren en pas na bewijs archiveren/verwijderen; artifact-retentiebeleid. | CI-test faalt bij nieuwe root/mirrorduplicatie; nul ambigue importpaden; migratiemanifest met hashes en herstelpad; runtime image blijft rootcode gebruiken. |
| ACC-R27 | High | Tower checkout staat op `main`, loopt 67 commits vóór de stale `origin/main` en bevat untracked runtime-evidence/cache. | Deploybron en remote history zijn niet eenduidig; rollback/restore kan een andere codebasis opleveren. | DevOps + Architecture | Deployment uitsluitend vanaf gepushte immutable commit/tag en image digest; servercheckout niet als bron van waarheid; untracked runtime data buiten source tree of expliciet gemount. | Push/commit ancestrybewijs; deployment manifest met commit+digest+config; clean source checkout; rollbacktest naar vorige digest zonder dataverlies. |
| ACC-R28 | Medium | PostGIS meldt versie 3.6.4, maar core/topology procedures uit 3.4.3 “need upgrade”. | Ruimtelijke functiegedrag/migraties kunnen per omgeving afwijken en upgrades kunnen onverwacht falen. | Architecture + GIS + DevOps | Extension-upgradepad in restorebare stagingdatabase; functieversie/preflight in deployment; geen in-place productie-upgrade zonder backup en dry-run. | Restoretest, `ALTER EXTENSION`-dry-run/resultaat, PostGIS regression suite en versie-evidence vóór/na; rollbackprocedure getest. |
| ACC-R29 | Medium | API-contract, implementation gap docs, preflightvoorbeelden en actieve runtimeversies spreken elkaar op meerdere plaatsen tegen. | Operators en agents kunnen verouderde instructies volgen of capabilities overclaimen. | Architecture + QA | Docs genereren/valideren tegen OpenAPI, migrations, capability registry en runtime manifest; stale claims verwijderen met behoud van historisch bewijs. | Doc-contracttests; alle versies/scopes uit één bron; accuracy-program en operator README verwijzen naar actuele model card en release status. |
## Positieve controles die behouden moeten blijven
Deze bevindingen verlagen de bovenstaande risico's niet, maar zijn bewezen onderdelen waarop Phase 2 kan voortbouwen:
- Tower ziet een NVIDIA GeForce RTX 4080 SUPER; PyTorch CUDA is beschikbaar en de productie-adapter heeft één bounded, read-only inference uitgevoerd.
- Alle 3.377 datasets hebben checksum, CRS, imported-at en source/provenance metadata; alle 1.671 DatasetVersions hebben checksum, storage path en metadata.
- De database-audit vond geen ontbrekende directe storage references in de gecontroleerde set.
- Areas en vector_features hebben in de runtime-snapshot geen invalid, empty of wrong-SRID geometrieën; de vier detection-afwijkingen blijven afzonderlijk releaseblokkerend.
- API/OpenAPI-, GIS-, AI-contract-, migratie-, frontend typecheck/Vitest/build- en gerichte testgroepen slaagden; de volledige canonical gate blijft desondanks rood door ACC-R21/22.
## Risicoacceptatie
Geen Critical-risico is accepteerbaar voor promotie of scopeverruiming. Een High-risico kan alleen door een expliciete, tijdgebonden operatorbeslissing worden geaccepteerd wanneer het geen accuracy-, bronautoriteits-, split- of lineagegate omzeilt. Zo'n uitzondering moet het risico-ID, bereik, eigenaar, rollback en bewijs bevatten. Ontbrekende evidence telt als een gefaalde gate, niet als “niet van toepassing”.
@@ -0,0 +1,631 @@
# GeoIntel Accuracy Improvement Program — metric framework
- Status: Phase 1 measurement contract
- Scope: Belgium and the Belgian North Sea, with task-specific claims only
- Evidence root: `artifacts/evidence/accuracy/P1/`
- Last updated: 2026-08-01
## 1. Purpose
This document defines how GeoIntel must measure, compare and communicate
accuracy. It is a measurement contract, not an assertion that any current model
meets a release threshold.
The framework has five non-negotiable goals:
1. every reported number is reproducible from checksum-bound inputs;
2. model-selection data stays separate from protected release evidence;
3. aggregate scores cannot hide a failing region or context;
4. missing, undefined or insufficient evidence fails closed;
5. claims never exceed the evaluated task, class, geography, source edition or
operating configuration.
This framework applies to learned detection and segmentation outputs and to
deterministic spatial QA where a candidate layer is compared with a reference.
It does not turn authoritative GIS functions into machine-learning tasks.
## 2. Current baseline versus future acceptance thresholds
### 2.1 Baselines that exist now
GeoIntel currently has a deterministic QA/QC code-regression baseline in
`fixtures/golden/golden_qa_benchmarks.json`. At IoU `0.5`, its frozen scenarios
include:
| Scenario | Precision | Recall | F1 | Mean matched IoU |
|---|---:|---:|---:|---:|
| partial match | 0.5 | 0.5 | 0.5 | 0.8339768339761133 |
| perfect match | 1.0 | 1.0 | 1.0 | 1.0 |
| no overlap | 0.0 | 0.0 | undefined (`null`) | undefined (`null`) |
| multipolygon match | 1.0 | 1.0 | 1.0 | 1.0 |
These values prove regression stability of the fixture matcher. They do **not**
measure production-model accuracy, Belgium-wide generalisation, source quality
or human acceptability.
The runtime GPU/model smoke evidence proves only that a particular asset can be
loaded and invoked under the recorded configuration. It is not an accuracy
benchmark. Historical calibration reports and persisted `Metric` rows are
evidence candidates only when their complete lineage, split role, evaluator
version and inputs can be reproduced.
Several scripts contain historical or provisional numeric defaults, including
the gate defaults in `scripts/assess_belgium_building_training_iteration.py`.
Those implementation defaults are not automatically approved Accuracy Program
release thresholds. Reports using them must label the gate source and version
as `legacy` or `provisional`; they cannot be relabelled as a Phase 1 contract.
At the Phase 1 evidence cut, a canonical release-grade national model-quality
baseline is therefore `not_established`. No existing number may be presented as
a Belgium-wide acceptance result until the data, split, evaluator and review
requirements below are satisfied.
### 2.2 Thresholds that must be frozen later
All new promotion floors, ceilings, non-inferiority margins, sample minima,
confidence levels and latency budgets are `TBD`. They must be proposed from a
reviewed incumbent baseline and operational requirements, approved and
versioned **before** the protected test set is opened.
The freeze record must identify:
- task, class ontology and claim scope;
- dataset and split-manifest hashes;
- incumbent and candidate model hashes;
- exact preprocessing, tile, overlap, confidence and NMS configuration;
- evaluator version and object-matching policy;
- metric, aggregation level, direction and threshold;
- required strata and minimum evaluable support;
- confidence-interval or paired-comparison rule;
- runtime hardware and latency measurement protocol;
- approver, timestamp and immutable gate-definition hash.
Seeing test results can never justify relaxing a threshold. A changed threshold
creates a new, independently approved evaluation protocol and requires evidence
that remains independent of the change.
## 3. Evaluation unit and identity contract
Every metric record must be traceable to the following identity tuple:
```text
task + class_ontology_version + model_sha256 + dataset_manifest_sha256
+ split_manifest_sha256 + evaluator_version + inference_config_sha256
+ reference_source_version + imagery_source_version + run_id
```
The evidence record must also contain the Git commit, container image digest,
Python/PyTorch/CUDA/driver versions, random seed, deterministic-mode flags,
hardware, timestamps and command arguments. A metric without this tuple is
`unverifiable`, not an approximate success.
All geometries must be validated and transformed to a declared common CRS
before matching. Distance, area and boundary metrics require a documented
metric/equal-area CRS appropriate to the AOI. CRS assumptions, geometry repair,
clipping and ignored boundary zones must be counted and reported.
## 4. Error taxonomy
Each reviewed error receives one primary code and any applicable secondary
codes. Counts must be available by sample, AOI, region and context. Free text
may explain an error but cannot replace a code.
### 4.1 Data and lineage errors (`D-*`)
- `D-MISSING-PROVENANCE`: missing source, edition, acquisition time, licence,
checksum or transformation lineage.
- `D-STALE-SOURCE`: imagery or reference is outside its declared validity
period.
- `D-TEMPORAL-MISMATCH`: the labelled object and image do not represent the
same observable time.
- `D-CRS-MISSING` / `D-CRS-WRONG`: CRS is absent, misdeclared or transformed
incorrectly.
- `D-ALIGNMENT`: systematic or local image/reference displacement.
- `D-INVALID-GEOMETRY`: empty, corrupt, self-intersecting or otherwise invalid
geometry, including an unrecorded repair.
- `D-RESOLUTION`: object is below the declared resolvable pixel/ground size or
input resolution differs from the model contract.
- `D-LICENCE-SCOPE`: source use or redistribution cannot be proven.
- `D-CACHE-STALE`: cached content does not match the requested source/version.
### 4.2 Label and ontology errors (`L-*`)
- `L-MISSING`: visible in-scope object has no label.
- `L-SPURIOUS`: label has no visible in-scope object.
- `L-CLASS`: wrong canonical class or source-to-canonical mapping.
- `L-EXTENT`: polygon/box extent is materially wrong.
- `L-INSTANCE-SPLIT`: one real instance is labelled as multiple instances.
- `L-INSTANCE-MERGE`: distinct instances are merged.
- `L-AMBIGUOUS`: imagery/reference cannot support a defensible decision.
- `L-EXCLUSION`: excluded ontology item was included, or a valid item was
excluded.
- `L-REVIEW-MISSING`: required human review or adjudication is absent.
### 4.3 Split and duplication errors (`S-*`)
- `S-EXACT-DUPLICATE`: byte/content-equivalent image, label or derived tile
crosses split boundaries.
- `S-NEAR-DUPLICATE`: materially identical view or transformed derivative
crosses split boundaries.
- `S-SPATIAL-OVERLAP`: AOIs, tile context or instances overlap across protected
split groups.
- `S-TEMPORAL-FAMILY`: repeated acquisitions of the same site leak site identity
or labels across splits without an approved temporal generalisation design.
- `S-SOURCE-FAMILY`: provider mosaics or parent rasters are divided in a way
that leaks shared context.
- `S-PROTECTED-USE`: calibration, test or background-test evidence influenced
fitting, sampling, augmentation, architecture, threshold or NMS selection.
- `S-MANIFEST-MUTATION`: an evaluated split or its role changed after freeze.
Any `S-*` error invalidates the affected comparative evaluation until a new
independent split is frozen.
### 4.4 Model-output errors (`M-*`)
- `M-FP-BACKGROUND`: detection on a pure-background sample.
- `M-FP-CONFUSER`: detection on a named hard-negative/confuser.
- `M-FP-DUPLICATE`: multiple retained predictions for one reference instance.
- `M-FP-LOCALISATION`: predicted instance overlaps a reference but misses the
frozen match criterion.
- `M-FN-MISSED`: resolvable in-scope reference instance is absent.
- `M-FN-SUPPRESSED`: valid instance is removed by confidence, NMS, containment
or post-processing.
- `M-CLASS`: prediction is assigned to the wrong class.
- `M-BOUNDARY`: segmentation boundary is materially displaced despite object
detection.
- `M-AREA-BIAS`: predicted geometry has systematic area over- or
under-estimation.
- `M-MISCALIBRATED`: confidence does not correspond to empirical correctness.
- `M-OOD`: sample is outside the declared training/evaluation domain.
False positives and false negatives must retain evidence links to prediction
and reference IDs, source tiles and review decisions.
### 4.5 Pipeline and claim errors (`P-*`, `C-*`)
- `P-FALLBACK`: mock, heuristic, alternate source or stale cache replaced the
declared path.
- `P-NONDETERMINISTIC`: rerun differences exceed the frozen reproducibility
policy without explanation.
- `P-PARTIAL`: missing tile/chunk/output was silently omitted.
- `P-UNIT`: metres, degrees, pixels, CRS units or resolution were confused.
- `P-RUNTIME`: OOM, timeout, truncation or worker failure changed the evaluated
output.
- `C-OVERCLAIM`: wording exceeds evaluated task/scope/data.
- `C-MISSING-DENOMINATOR`: a rate is published without counts/support.
- `C-UNSUPPORTED-AGGREGATE`: an aggregate hides missing or failed strata.
- `C-UNTRACEABLE`: result cannot be tied to the identity tuple in section 3.
## 5. Object-level metrics
### 5.1 Matching contract
Metrics are meaningless without a matching policy. Every report must specify:
- supported geometry types and class matching;
- IoU thresholds;
- confidence ordering;
- one-to-one matching algorithm and tie-breaking;
- boundary clipping/ignore policy;
- treatment of difficult, ambiguous and below-resolution labels;
- duplicate-suppression configuration.
The current `QaService` performs one-to-one greedy polygon matching at a
configurable IoU threshold (default `0.5`) and conditions mean IoU on matched
objects. This is a deterministic spatial QA baseline when input order and
evaluator version are fixed, but it is not automatically COCO AP.
For detector PR/AP evaluation, predictions must be sorted by confidence and
matched one-to-one at each IoU threshold using a frozen COCO-compatible policy.
For fixed-threshold spatial comparison, a separately named matcher may use a
deterministic maximum-weight one-to-one assignment. Results from different
matchers must never be combined under the same metric key.
### 5.2 Counts and rates
For an evaluable class and stratum:
```text
precision = TP / (TP + FP)
recall = TP / (TP + FN)
F1 = 2 * precision * recall / (precision + recall)
IoU = intersection_area / union_area
```
Always publish `TP`, `FP`, `FN`, prediction count and reference count beside the
rates. Undefined denominators produce `null`, never zero and never pass. In
particular:
- no predictions makes precision undefined;
- no references makes recall undefined;
- zero precision plus zero recall keeps F1 undefined under the current QA
service semantics;
- pure-background performance is evaluated with explicit FP/background metrics,
not fabricated precision or recall;
- mean matched IoU is conditional on matches and must not be used alone because
it excludes FP and FN.
### 5.3 Precision-recall and average precision
Report the full precision-recall curve and its confidence thresholds. Required
detector summaries are:
- `AP50`: area under the interpolated PR curve at IoU 0.50;
- `AP50-95`: mean AP across IoU 0.50 through 0.95 in steps of 0.05;
- per-class AP and macro AP for multi-class tasks;
- recall at the frozen operating point;
- precision at the frozen operating point.
AP must be calculated before choosing a production confidence threshold. A
single-threshold F1 value cannot be called AP. AP values from a training
framework are accepted only when evaluator version, input set, class mapping
and settings are recorded and independently reproducible.
### 5.4 Localisation and geometry
Report the distribution, not only the mean, of matched IoU: count, median,
quartiles and lower-tail quantiles. Segmentation candidates additionally need
pixel/geometry IoU, Dice, boundary distance/score, area bias and topology-error
counts under a frozen geometry protocol. Metric thresholds for those measures
remain `TBD` until the segmentation label contract exists.
Object results must be stratified by predeclared size bands derived from ground
area and/or source-image pixels. Band boundaries are `TBD` and must be frozen
from resolution and ontology rules before candidate results are inspected.
## 6. Tile-level metrics
Tile metrics expose operational failure modes that object-level micro-averages
can hide:
- tile count, evaluable tile count and excluded-tile reasons;
- positive/empty tile counts;
- fraction of pure-empty tiles with one or more predictions;
- FP count per pure-empty tile and per square kilometre;
- fraction of positive tiles with at least one FN;
- per-tile precision, recall and F1 where defined;
- detection/reference count error per tile;
- duplicate predictions created at tile overlaps;
- tile processing latency, throughput and peak memory;
- tile-level error/uncertainty score for review routing.
Report macro distributions across tiles as diagnostics. Do not average undefined
tile rates into zero, and do not let a large number of easy empty tiles dominate
the primary object metric.
## 7. AOI-level metrics
The AOI is the primary unit for paired operational comparison. For each AOI,
publish:
- TP, FP, FN, precision, recall, F1 and matched-IoU distribution;
- reference and predicted object counts and signed/absolute count error;
- reference and predicted geometry area and signed/absolute area bias when
geometry outputs support it;
- pure-background/context-negative errors;
- excluded area/objects and reasons;
- end-to-end runtime, failed/retried tiles and completeness;
- taxonomy counts and links to visual evidence.
Candidate-versus-incumbent deltas must be paired on the exact same AOIs,
references and inference contract. Macro AOI results give each AOI equal weight;
micro results pool TP/FP/FN. Both are reported and explicitly named. The primary
aggregation and any non-inferiority margin remain `TBD` until frozen in the
gate definition.
## 8. Region-, context- and portfolio-level metrics
At minimum, building-model evidence must report Flanders, Wallonia and Brussels
separately. The Belgian North Sea is not an implicit building-detector claim;
offshore tasks need their own ontology and evidence.
Required stratification dimensions, where applicable, are:
- region and provider/source edition;
- urban, suburban, ribbon development, rural/village, farm, industrial and
other frozen context families;
- dense/occluded/touching-object contexts;
- pure background and named hard-negative families;
- object-size band and input ground resolution;
- imagery period/season and reference-to-imagery time difference;
- AOI boundary/interior and tile-edge/interior;
- native class/source subtype and label-review state;
- normal domain versus declared OOD challenge set.
For every required stratum report support, micro metrics, macro AOI metrics and
uncertainty intervals. Also report the worst evaluable required stratum. A
portfolio aggregate is valid only if every mandatory stratum has sufficient
predeclared support; otherwise its state is `not_evaluable`.
Intersectional slices should be added where error evidence warrants them, but
post-hoc slices are diagnostic and cannot replace the frozen primary analysis.
Small groups stay visible with an `insufficient_support` label; they are not
silently pooled into a better-performing group.
## 9. Confidence calibration
Model confidence is not accuracy and is not a complete uncertainty estimate.
Calibration is evaluated on the frozen calibration split after one-to-one
matching and before protected-test access.
### 9.1 Expected calibration error
For prediction bins `B_m` frozen before evaluation:
```text
ECE = sum_m (|B_m| / n) * |accuracy(B_m) - confidence(B_m)|
```
The binning method, boundaries, empty-bin handling and minimum observations must
be recorded. Report a reliability diagram and per-bin counts. A predefined
fixed-bin ECE is the comparable metric; adaptive-bin ECE may be reported only as
a labelled diagnostic. ECE is paired with recall because missed references have
no prediction confidence and are invisible to prediction-only ECE.
### 9.2 Brier score
For each scored prediction, define `y=1` only when it is the retained match under
the frozen policy, otherwise `y=0`:
```text
Brier = mean((confidence - y)^2)
```
Report Brier overall and by required stratum. The exact construction of the
prediction set must be frozen; Brier does not replace FN/recall reporting.
Calibration fitting may use only calibration data. If temperature scaling,
isotonic regression or another mapping is used, its fitted parameters and code
version become part of the immutable inference configuration. Test data may
evaluate that mapping but may not refit it.
## 10. Latency, capacity and reliability
Accuracy promotion also requires a reproducible operational profile on the
declared server GPU. Record separately:
- cold-start/model-load time;
- warm model-only inference time per tile;
- preprocessing, inference, post-processing and persistence time;
- end-to-end AOI latency excluding and including queue wait;
- median, p95, p99 and maximum latency with sample counts;
- tiles/second and square kilometres/minute at the recorded resolution;
- peak allocated/reserved VRAM and host RAM;
- batch size, tile size, overlap, worker concurrency and retry count;
- OOM, timeout, truncation, partial-output and failed-job rates.
Warm-up count, timing clock, synchronisation method, hardware, driver and
background load must be fixed. A local CPU run and an RTX 4080 SUPER CUDA run
are different baselines. Latency budgets are `TBD`; current smoke timings may be
reported as observations only.
## 11. Uncertainty and selective review
Uncertainty evidence may include confidence, entropy/margin where meaningful,
test-time-augmentation disagreement, ensemble variance, spatial disagreement
and OOD scores. Every method must state what variation it measures; confidence
alone may not be labelled epistemic uncertainty.
For any abstain/review policy, report:
- retained coverage versus error/risk curve;
- error rate in auto-accepted, human-review and rejected bands;
- review volume per AOI/region/context;
- area under the risk-coverage curve as a diagnostic;
- failure cases where high-confidence predictions are wrong;
- threshold source and calibration-only selection evidence.
Uncertainty thresholds are `TBD`. Uncertainty may route work to a human; it must
not create self-training labels or silently suppress output.
## 12. Strict split and deduplication rules
### 12.1 Split roles
- `train`: fitting and training-time augmentation only;
- `val`: epoch/checkpoint/architecture selection only;
- `calibration`: confidence, tiling, overlap, NMS and calibration mapping only;
- `test`: sealed, one-time promotion evidence for a frozen candidate family;
- `background-test`: sealed, difficult and pure-background release evidence.
No test or background-test result may influence fitting, sampling weights,
label policy, architecture, hyperparameters, post-processing or gate values. If
it does, that split is retired from independent testing, the candidate becomes
a new family and a new untouched protected split is required.
### 12.2 Grouping before splitting
Deduplication and grouping happen before split assignment. The atomic group must
keep together:
- a parent AOI plus every derived/overlapping tile;
- all tiles sharing an object instance;
- exact image/label hashes and transformed copies;
- perceptual near-duplicate families;
- repeated views whose shared spatial context exceeds the declared independence
buffer;
- source-raster/mosaic or temporal families when they would leak scene identity.
Train, val, calibration, test and background-test AOIs must be spatially
disjoint, including a buffer larger than the maximum tile context used by any
candidate. The buffer value and near-duplicate thresholds are `TBD` but must be
frozen before corpus generation, not chosen after metric inspection.
### 12.3 Required leakage audits
The frozen split evidence must contain:
- exact SHA-256 duplicate matrix for raw image, processed image and label;
- perceptual-image and label-geometry near-duplicate audit;
- buffered AOI/tile intersection audit in a metric CRS;
- shared native feature/source identifier audit;
- parent raster, acquisition and temporal-family audit;
- tile-versus-manifest role consistency audit;
- counts and explicit disposition for every collision.
All cross-split collisions must be zero unless a written experimental design
defines a non-independence group and assigns that whole group to one split. A
leakage audit that did not run, timed out or lacks inputs is a failed gate.
## 13. Human-review metrics
Automated geometry checks do not replace representative human review. Corpus
labels and model outputs need separate review records.
### 13.1 Label-corpus review
Report:
- planned, rendered, reviewed and adjudicated sample/object counts;
- completion rate by region, context, provider, size band, split and label
source;
- contact-sheet/render failures and unreadable samples;
- prevalence of every `D-*` and `L-*` finding;
- accepted, corrected, excluded and ambiguous counts;
- single-review and independent double-review counts;
- raw reviewer agreement, class-wise agreement and an approved chance-corrected
agreement statistic such as Cohen's kappa or Krippendorff's alpha;
- disagreement/adjudication rate and unresolved critical findings.
Sampling quotas, double-review fraction, agreement statistic and acceptance
thresholds are `TBD` and must be frozen before reviewers see model results.
Protected test labels should be reviewed independently of candidate predictions.
### 13.2 Output-error review
For each candidate, use a frozen stratified sample that includes TP, FP, FN,
low-confidence, high-confidence, tile-edge, dense, temporal-mismatch and OOD
cases. Reviewers record taxonomy codes and severity, not only approve/reject.
Report reviewer agreement, adjudication, severe-error prevalence and error
distribution by stratum.
Review UI/version, imagery/reference layers, zoom/scale and reviewer identity or
pseudonymous ID must be retained. A generated contact sheet with zero completed
review decisions is `unreviewed`, not passed.
## 14. Statistical reporting
All rates include numerator, denominator and support. Confidence intervals use
a predeclared method appropriate to the unit: object-level bootstrap must not
pretend correlated tiles are independent. Prefer cluster bootstrap or paired
analysis at AOI/group level for model comparisons.
Report:
- point estimate and interval;
- number of independent AOIs/groups;
- micro and macro aggregation;
- paired candidate-minus-incumbent delta by AOI;
- worst required stratum and its support;
- missing/excluded evidence and reason;
- multiplicity policy for any formal multi-stratum hypothesis claims.
The confidence level, bootstrap seed/replicates and non-inferiority margins are
`TBD` until the gate specification is frozen. Descriptive diagnostics must not
be presented as confirmatory release evidence.
## 15. Acceptance-gate semantics
Every gate returns exactly one state:
- `pass`: complete evidence meets the frozen rule;
- `fail`: complete evidence violates the frozen rule;
- `not_evaluable`: evidence is missing, invalid, leaked or below frozen support.
`not_evaluable` blocks promotion exactly like `fail`; it is never coerced to
zero, ignored or averaged away.
A promotion decision is a logical AND across all mandatory gate families:
1. identity, provenance, licence and immutable manifest;
2. zero unresolved split leakage and duplicate-family violations;
3. automated data/label/geometry integrity;
4. completed representative human review and adjudication;
5. frozen object/AOI/region/context accuracy gates;
6. pure-background and hard-negative gates;
7. calibration and uncertainty/review-policy gates;
8. paired incumbent comparison and reproducibility rerun;
9. GPU latency, capacity and failure-rate gates;
10. model card, scope limitation, rollback and live shadow evidence.
No weighted composite score may compensate for a failed mandatory gate. The
aggregate and every mandatory region/context gate must pass independently.
Candidate selection and operating-point selection use calibration evidence
only. The protected test is evaluated once with the frozen configuration. A
rerun is allowed only to reproduce the same immutable computation or to resolve
a proven execution failure without inspecting/using partial results.
Gate output must include machine-readable rule IDs, observed values, expected
direction/range, support, evidence paths and hashes. Manual override cannot
change a metric result; an exceptional operational decision must remain a
separate signed record and cannot widen the accuracy claim.
## 16. No-claim rules
GeoIntel must not claim that a model is “fully trained”, “validated”,
“production accurate”, “Belgium-wide” or equivalent when any of the following
is true:
- only model-file presence, load success, a GPU smoke or output existence was
proven;
- results come from train/val data, an unfrozen calibration set or leaked test
evidence;
- the dataset, model, evaluator or inference configuration lacks hashes;
- required region/context support or representative human review is missing;
- a metric is undefined/null, a job is partial or a fallback/mock/heuristic was
used;
- only aggregate performance is shown while a required stratum is absent or
failing;
- precision/recall/AP is reported without match policy and TP/FP/FN/support;
- confidence is presented as probability of correctness without calibration
evidence;
- a deterministic fixture benchmark is presented as production-model quality;
- the claim is extrapolated to another class, source, imagery period,
resolution, region, segmentation task or deterministic GIS function;
- test results were used to choose the next training data or configuration.
Permitted wording must be evidence-bounded, for example: “candidate X achieved
the recorded metrics on frozen portfolio Y under configuration Z.” It must name
limitations and may not imply untested generalisation.
## 17. Required machine-readable outputs
Each future baseline or candidate evaluation must retain, without overwriting
earlier evidence:
```text
evaluation-contract.json
input-manifest.json
split-and-leakage-audit.json
object-metrics.json
tile-metrics.json
aoi-metrics.json
stratified-metrics.json
calibration-metrics.json
latency-and-reliability.json
human-review-summary.json
error-taxonomy.json
candidate-vs-incumbent.json
acceptance-gates.json
evidence-manifest.json
```
`evidence-manifest.json` must SHA-256 hash every retained input and output. Any
amendment is additive, versioned and linked to its predecessor. Raw records
must remain available so every aggregate can be recomputed.
## 18. Phase 2 metric implementation sequence
1. freeze evaluator schemas, taxonomy codes and undefined-value semantics;
2. implement strict group split and exact/near-duplicate audits;
3. validate CRS/alignment/label lineage and complete human corpus review;
4. freeze an incumbent portfolio and run a reproducible baseline twice;
5. derive and approve numeric thresholds from reviewed baseline distributions
and operational requirements without protected-test access;
6. calibrate candidate operating points on calibration data only;
7. execute the one-time protected test and background-test gates;
8. publish scope-bounded evidence or retain `not_configured`/current limited
scope when any gate is not evaluable.
Until those steps pass, this framework authorises measurement and remediation,
not a widened accuracy claim.
@@ -0,0 +1,342 @@
# GeoIntel Accuracy Improvement Program — uitvoerbare roadmap
- Status: **Phase 2 mag starten; release en nationale modelclaim blijven geblokkeerd**
- Bronnen: Phase-1 inventory, lineage, baseline/gaps, risicoregister en metric framework
- Runtime voor training: Tower NVIDIA GeForce RTX 4080 SUPER op `cuda:0`
## Doel en definitie van gereed
Deze roadmap herstelt eerst de bewijs- en vertrouwensketen en bouwt daarna pas een nieuw Belgisch building-corpus en model. “100% getraind” betekent hier: alle vooraf bevroren corpus-, split-, runtime-, metric-, review-, test-, promotion- en deploymentgates zijn aantoonbaar geslaagd voor één expliciete modelscope. Het betekent niet 100% precision/recall en geeft geen claim buiten de geëvalueerde regio's, contexts, imagery editions en objectgroottes.
De volgorde is verplicht. Een work package start pas wanneer zijn dependency-gate groen is. Bij een gefaalde gate blijft productie op de huidige beperkte, expliciet gecommuniceerde Mol/Kempen-scope of schakelt de betrokken capability fail-closed naar `not_configured`. Checkpoints, datasets en bewijs worden nooit overschreven.
## Niet-onderhandelbare regels
1. **Test-first:** ieder bewezen defect uit `04-risk-register.md` krijgt eerst een regressietest die op de huidige foutieve implementatie faalt.
2. **Protected-test isolation:** train, val en calibration mogen tijdens iteraties worden gelezen; test en background-test blijven verzegeld tot één kandidaat-SHA, preprocessingconfig, threshold en gates bevroren zijn.
3. **Geen testgestuurde retraining:** na openen van protected test volgt voor die candidate family geen training, thresholdwijziging, sampleweging of configuratiekeuze meer.
4. **Evidence of fail:** ontbrekend, null of niet-reproduceerbaar bewijs is een gefaalde gate.
5. **Immutable lineage:** iedere dataset-, run-, model- en releaseversie heeft een checksum-bound manifest; legacy gaps blijven zichtbaar als `lineage_incomplete`.
6. **GPU verplicht:** training gebruikt `cuda:0` op de Tower RTX 4080 SUPER met `TRAIN_REQUIRE_CUDA=true`; CPU-fallback is een failure.
7. **Geen claim op file presence:** een `.pt`-bestand of succesvolle smoke maakt een model niet gevalideerd.
8. **Menselijke review is echt menselijk:** automatische QA mag een ontbrekende review nooit als akkoord invullen. De finale productreview door de gebruiker volgt pas nadat alle objectieve gates groen zijn.
## Dependency-overzicht
| Volgorde | Work package | Depends on | Primaire output | Risico's gesloten |
| --- | --- | --- | --- | --- |
| P2-00 | Promotion lock en evidence freeze | Phase 1 | immutable baseline + release lock | claimgrens voor alle risico's |
| P2-01 | Canonical test harness | P2-00 | rode regressietests en uitvoerbare CI-matrix | ACC-R21, R22, basis voor alle fixes |
| P2-02 | CRS, units en geometry integrity | P2-01 | veilig ingest/transform/buffer + data repair | ACC-R01, R02, R07 |
| P2-03 | Coverage, authority en wettelijke scope | P2-02 | trusted source registry en geometry-backed scope | ACC-R03R06 |
| P2-04 | Transactionele lineage en foutzichtbaarheid | P2-02, P2-03 | complete Run/Dataset lineage en fail-closed persistence | ACC-R08, R15, R17, R23 |
| P2-05 | Protected-split redesign | P2-01, P2-04 | test vault, manifest schema en sampler firewall | ACC-R09, R10, R14, R24 |
| P2-06 | Menselijke labelreview en corpusrebuild | P2-02, P2-03, P2-05 | immutable reviewed `building-be-*` corpus | ACC-R10R15 |
| P2-07 | Metric framework en reproducible incumbent baseline | P2-04, P2-06 | frozen metrics/gates + paired baseline | ACC-R16, R18, R19 |
| P2-08 | Reproducible CUDA candidate training | P2-05, P2-06, P2-07 | immutable RTX 4080 candidate portfolio | ACC-R14, R16, R17, R24, R25 |
| P2-09 | Calibration-only improvement loop | P2-08 | fixed candidate that passes all pre-test gates | ACC-R12, R13, R16 |
| P2-10 | Eenmalige protected test | P2-09 | signed pass/fail promotion evidence | ACC-R09, R16, R18 |
| P2-11 | Guarded promotion, shadow en redeploy | P2-10 pass | model card, immutable image, rollback | ACC-R17, R19, R25, R27, R29 |
| P2-12 | Monitoring en controlled next cycle | P2-11 | reviewed drift queue zonder self-training | blijvende beheersing |
## P2-00 — Promotion lock en evidence freeze
### Uitvoering
- Leg current commit, server commit, container digest, DB migration head, active model path/SHA, modelscope en alle P1-evidencehashes vast.
- Zet `nationally_validated=false` en de actuele Mol/Kempen-scope expliciet in de capability/model-card response; voorkom scopeverruiming zonder promotion report.
- Markeer v56/v58/v62/v66 en andere checkpoints als `candidate/unpromoted`; verander of verwijder geen files.
- Maak een restorebare databaseback-up en inventory van storage references vóór migraties.
- Bewaar de succesvolle GPU-smoke als runtimebewijs met de expliciete claimgrens “geen accuracy-evidence”.
### Exit gate
- Evidence manifest is hash-compleet en read-only gekopieerd naar de release-auditlocatie.
- Production promotion endpoint/config weigert een kandidaat zonder signed promotion report.
- Rollbackdoel (huidige image digest + model SHA + config) is reproduceerbaar vastgelegd.
## P2-01 — Canonical test harness en rode regressies
### Uitvoering
- Maak één repo-root testentrypoint voor Python; verwijder de `backend/scripts` versus root `scripts` shadowing.
- Voeg vóór productcodewijzigingen regressies toe voor:
- EPSG:31370/3812 als 4326 gelabeld;
- 100 m buffer als graden;
- cross-theme coverage union;
- forged authoritative upload;
- area-name/YOLO-scope bypass;
- Area PATCH geometry/CRS;
- ready-derived dataset zonder PostGIS-features;
- protected test als samplerinput;
- metric null/empty truth table;
- persistence- en frontendfallbacks.
- Vervang stale broncode-stringasserties door behavior/contracttests; wijzig expected output alleen met een gedocumenteerde contractbeslissing.
- Voeg Ruff en een echte frontend `npm run lint` gate toe; behoud typecheck, Vitest en build.
- Laat CI dezelfde Python/Node-versies en commands gebruiken als de pinned build; maak een aparte AI-enabled image gate.
### Verificatiecontract
De CI-matrix bevat minimaal: volledige backend-Pytest, Alembic offline+live PostGIS, Ruff, frontend lint/typecheck/Vitest/build, OpenAPI-contractaudit en de gerichte GIS/AI regressies. De oude implementatie moet de nieuwe regressies aantoonbaar laten falen; pas daarna worden fixes geaccepteerd.
### Exit gate
- Eén canonical commandmatrix is volledig groen in een schone checkout.
- De 17 bestaande failures zijn per contract geclassificeerd en opgelost.
- Geen test wordt geskipt op basis van ontbrekende lokale AI/GIS-dependency zonder expliciete, afzonderlijk rode environment gate.
## P2-02 — CRS-, eenheden- en geometry-integriteit
### Uitvoering
1. Definieer per API/file-ingest het bron-CRS, canonical storage-CRS en output-CRS. Ontbrekende of ambigue CRS faalt met een typed fout.
2. Transformeer EPSG:31370 en EPSG:3812 met `pyproj`/GeoPandas/PostGIS naar EPSG:4326 vóór `from_shape(..., srid=4326)`.
3. Implementeer buffer via een geldige metrische projectie of PostGIS geography; log units en transform.
4. Maak Area PATCH exact conform contract: geometry wordt valide getransformeerd of extra input wordt geweigerd; CRS relabeling zonder transform is onmogelijk.
5. Herleid de vier buiten-domein detections uit originele tile, manifest, EPSG:31370-transform en modeloutput. Bewaar oude rijen/evidence; corrigeer via een auditabele migratie of markeer ze `invalid_legacy_geometry`.
6. Voeg DB constraints/checks toe waar die legitieme EPSG:4326-extents kunnen afdwingen zonder de Noordzee of grensgebieden fout af te wijzen.
### Exit gate
- CRS/units regressies en live PostGIS-tests slagen.
- Runtime-audit meldt zero ongeclassificeerde invalid/empty/wrong-SRID/out-of-domain geometry.
- De vier legacy detection-ID's zijn traceerbaar vóór en na migratie; geen stille overschrijving.
## P2-03 — Coverage-, authority- en scopevertrouwen
### Uitvoering
- Herstructureer coverage zodat alleen datasets die source, theme, layer én zone matchen aan de coverage union deelnemen.
- Introduceer immutable `source_registry_id`, trust class, provider adapter en bronversie. Caller metadata blijft descriptief en kan nooit official authority verlenen.
- Migreer user uploads naar `manual/untrusted` tenzij hun acquisition/job lineage een governed adapter bewijst.
- Vervang displaynaam-gebaseerde legal zones door stable codes en geometry-backed predicates.
- Vervang YOLO-name-substringcontrole door model-card scope geometry/zone IDs met expliciete containment-policy.
- Maak API/UI onderscheid tussen `available`, `covered`, `authoritative`, `runtime_ready` en `model_validated_for_scope`.
### Exit gate
- Forged-source-, mixed-theme-, rename- en scope-bypasstests slagen.
- Bestaande source records hebben een auditbare trust classification.
- Een AOI buiten Mol/Kempen kan het actieve model niet uitvoeren door naam of metadata te manipuleren.
## P2-04 — Transactionele lineage en zichtbare failures
### Uitvoering
- Maak een verplicht `RunManifest` met model-ID/version/SHA, dataset/version/SHA, tile-manifest/SHA, source imagery/reference versions, CRS/transform, preprocessing, threshold, NMS/max-det, seed, runtime/container/GPU en code commit.
- Maak derived vector persistence atomair: AnalysisRun, Dataset, DatasetVersion, artifact en vector_features gaan samen van `processing` naar `ready`; elke verplichte write failure maakt de run failed.
- Classificeer `observed_at` en `source_version` per bronfamilie als required/not-applicable/unknown-with-reason en voer een provenance-safe backfill uit.
- Markeer historische detection runs zonder volledige lineage als `lineage_incomplete`; vul modelversies niet afgeleid of op basis van huidige config in.
- Verwijder silent catches: UI krijgt een typed error/stale state met run ID; services mogen geen success/ready rapporteren na persistence failure.
### Exit gate
- Nieuwe analyses zijn van UI-resultaat tot tile, bron, model en container volledig traceerbaar.
- Fault-injection geeft failed/incomplete, nooit ready/success.
- Lineage-audit heeft zero ongeclassificeerde verplichte gaps voor nieuwe records en een expliciete legacybucket.
## P2-05 — Protected-split redesign en manifest firewall
### Uitvoering
1. Definieer één versioned corpusmanifest met immutable sample-ID, image/label SHA, bronfeature-ID, region/context, provider/edition, imagery/reference time, CRS/resolution, tile/stride/overlap, split en reviewstatus.
2. Bereken splits op buffered AOI's vóór tiles worden geëxporteerd. Controleer geometry overlap, contextbuffer, feature-ID's, image/label hashes en perceptuele near-dupes.
3. Verplaats protected test en background-test naar een afzonderlijke read-only locatie/credential die de training- en samplerprocessen niet kunnen lezen.
4. Splits de huidige orchestrator:
- `train/val/calibration loop`: fit, early stopping, threshold en error taxonomy;
- `release evaluation`: alleen frozen kandidaat/config en protected credentials.
5. Laat de failure-driven sampler uitsluitend calibration-aggregaten en train-only contextcatalogi lezen. Hij moet hard falen zodra een assessment testdata, test-ID's of een protected pad bevat.
6. Log iedere protected access met candidate SHA, config SHA, operator/runner, timestamp en output SHA.
### Exit gate
- Canary protected sample verschijnt in geen enkel train/val/calibration manifest, log, cache of sampleroutput.
- Minimum cross-splitafstand voldoet aan de vooraf vastgelegde contextbuffer; 24 huidige near-pairs zijn opgelost of met objectief geometrisch bewijs als onafhankelijk geclassificeerd.
- Testcredentials zijn tijdens training technisch niet beschikbaar.
## P2-06 — Menselijke review en immutable corpusrebuild
### 1. Review de bestaande kandidaatdata
- Genereer contact sheets/kaartoverlays voor een vooraf geregistreerde, gestratificeerde reviewqueue: regio, provider, dense urban, suburban, rural, industrial, coast, forest/heath, rail/port/quarry, pure-empty, hard negative, kleine objecten, extreme aspectratio en providerseams.
- Beoordeel expliciet de 521 kleine labels, extreme aspectgroepen, sub-resolution/post-imagery exclusions en zeer dichte PICC/UrbIS-labelgebieden.
- Sla accept/reject/repair/uncertain op met reviewer, reason code, native feature-ID, image/label version en checksum. `uncertain` blijft uitgesloten of in een afzonderlijke non-training queue.
### 2. Provision onafhankelijke AOI's
- Vul iedere vereiste region/context-cel uit het bevroren corpuscontract; voeg Brussels pure-background toe en breid moeilijke negatives uit zonder protected voorbeelden te kopiëren.
- Gebruik officiële imagery/reference adapters en leg acquisition edition/periode vast.
- Houd train-only uitbreidingen ruimtelijk onafhankelijk van val/calibration/test/background-test en van elkaar waar het contract dat vereist.
### 3. Herbouw en freeze
- Pas temporal/resolution/providersemantics toe op GRB, PICC en UrbIS; post-imagery en niet-resolveerbare features krijgen een expliciete rejection reason.
- Exporteer deterministisch met ingevulde dataset-YAML/class/tile/stride/overlap-velden.
- Run geometry, label, density, class, blank/variance, duplicate/near-duplicate, split-distance, temporal en provenance audits.
- Freeze een nieuwe corpusversie; verander v56/v66 niet.
### Exit gate
- Representatieve menselijke review is volledig; zero unresolved Critical/High findings.
- Elke verplichte region/context/background-cel voldoet aan het vooraf bevroren contract.
- Zero cross-split leakage/near-duplicate violations; timestamps en unknowns zijn expliciet.
- Corpus, reviewrecords, manifests, YAML en auditrapporten zijn SHA-bound en immutable.
## P2-07 — Metric framework en reproduceerbare incumbent baseline
### Uitvoering
- Implementeer de truth table uit `05-metric-framework.md` voor empty/no-match/undefined cases; null kan een gate nooit stil passeren.
- Meet detection precision, recall, F1 en AP op bevroren IoU-contracten; voeg objectgrootte, dichtheid, region, provider, context en pure-background strata toe.
- Behandel calibration en test afzonderlijk. Selecteer threshold/NMS/tile-overlap op calibration met worst-region/worst-context vóór aggregate.
- Evalueer het actieve model als incumbent op exact dezelfde niet-protected calibrationportfolio en bewaar paired AOI-resultaten.
- Maak FP/FN contact sheets en error taxonomy: label/temporal mismatch, tile-edge, small object, dense cluster, roof displacement, source seam, context confusion en model miss.
- Freeze alle numeric gates vóór protected test. Bestaande minimale gates mogen alleen vóór test en op basis van reviewed baseline distributions worden aangescherpt; nooit versoepeld na testinzage.
### Exit gate
- Twee baseline-runs met dezelfde inputs leveren dezelfde sample/split hashes en metrics binnen vooraf vastgelegde tolerantie.
- Alle strata hebben een waarde of expliciete failstatus; zero silently ignored nulls.
- Gateconfig, evaluator, incumbent SHA en calibrationresultaat zijn immutable.
## P2-08 — Reproduceerbare CUDA-training op RTX 4080 SUPER
### Uitvoering
- Bouw één pinned AI image voor PyTorch/CUDA/Ultralytics; leg image digest, SBOM, driver/runtime, GPU, peak VRAM en code commit vast.
- Voer VRAM-preflight uit voor iedere kandidaatconfig; OOM/failure blijft als artifact en mag niet stil naar CPU vallen.
- Train een vooraf begrensde matrix zoals vastgelegd in `PYTORCH_TRAINING_ROADMAP_BELGIUM.md`; wijzig matrix noch primary metric na resultaten te zien.
- Seed Python/NumPy/PyTorch/Ultralytics; gebruik deterministic algorithms waar ondersteund en registreer afwijkingen.
- Training leest uitsluitend train; val kiest epochs/checkpoint; calibration kiest threshold/NMS/tile policy. Protected testmount/credential ontbreekt.
- Sla per run config, stdout/stderr, curves, checkpoints, optimizer state, dataset/corpus SHA, seed, runtime en peak VRAM op. Kopieer checkpoints immutably; overschrijf active model nooit.
- Herhaal de winnende configuratie clean-room vanaf dezelfde base weights en corpus om reproduceerbaarheid te toetsen.
### Exit gate
- Alle geplande kandidaten hebben complete run manifests of expliciete failure artifacts.
- Minstens één kandidaat en zijn clean-room rerun voldoen aan vooraf bevroren reproducibilitytoleranties.
- GPU-evidence toont RTX 4080 SUPER/`cuda:0`; zero CPU fallback; protected-accesslog blijft leeg.
## P2-09 — Calibration-only verbeterloop
De loop mag worden herhaald, maar alleen binnen de volgende state machine:
```text
reviewed immutable corpus
-> CUDA train
-> validation checkpoint selection
-> calibration + error taxonomy
-> all pre-test gates pass?
no -> provision independent train-only AOIs / reviewed labels
-> freeze new corpus version -> CUDA train
yes -> freeze candidate SHA + preprocessing + threshold + gates
-> P2-10 protected test
```
### Regels
- Calibrationresultaten mogen aangeven welke regio/context faalt, maar nooit protected sample-ID's of testresultaten.
- Nieuwe voorbeelden komen uit onafhankelijk geprovisioneerde train-only AOI's en doorlopen dezelfde provenance, temporal en human-reviewgates.
- Een corpuswijziging maakt een nieuwe immutable corpusversie en een nieuwe run family; bestaande evidence blijft behouden.
- De loop stopt niet op aggregate F1 alleen. Iedere regionale/context-, background-, lineage-, runtime- en reviewgate moet groen zijn.
- Indien geen betrouwbare labels of onafhankelijke AOI's beschikbaar zijn, is de correcte status `blocked/not_validated`, niet een afgezwakte gate.
### Exit gate
- Eén candidate SHA passeert alle vooraf bevroren validation/calibration-, regional/context-, pure-background-calibration-, runtime-, lineage- en reviewgates.
- Candidate, threshold, NMS, tileconfig, corpus en evaluator zijn daarna read-only bevroren.
## P2-10 — Eenmalige protected-testbeslissing
### Voorwaarden vóór openen
- P2-00 tot P2-09 zijn groen.
- Candidate/model SHA, container digest, corpus SHA, preprocessing, threshold, evaluator en numeric gates zijn gesigneerd/bevroren.
- Test/background-test manifesthashes bestaan, maar hun inhoud was niet toegankelijk voor train/calibration runners.
- Promotion policy specificeert vooraf wat pass, fail en infrastructure-invalid betekent.
### Uitvoering
- Start één isolated release-evaluation job met read-only protected credentials op `cuda:0`.
- Bereken alle bevroren regionale/context/object-size en background-test metrics; produceer contact sheets en machine-readable gate decision.
- Een infrastructure-invalid run mag uitsluitend opnieuw worden uitgevoerd wanneer bewijs aantoont dat geen bruikbaar modelresultaat is vrijgegeven; de incidentbeslissing wordt gelogd.
### Beslissing
- **Pass:** ga naar P2-11; resultaten mogen niet worden gebruikt om alsnog threshold/config te wijzigen.
- **Fail:** release blijft blocked. Train deze candidate family niet verder op basis van het testresultaat. Archiveer de beslissing; een volgende poging vereist een nieuw vooraf geregistreerd ontwikkelprogramma en een nieuwe onaangeroerde protected portfolio.
### Exit gate
- Exact één geldig access event en één immutable report voor candidate SHA.
- Geen write naar corpus/training config na testopening.
- Alle gates zijn groen; anders is P2-11 niet bereikbaar.
## P2-11 — Guarded promotion, shadow en redeploy
### Uitvoering
1. Genereer model card en promotion report met task, class, scope, imagery/reference versions, known limitations, metrics per stratum, calibration, test, runtime, lineage en rollbackmodel.
2. Kopieer de kandidaat naar een immutable model-ID/version/SHA-pad; overschrijf het actieve `.pt`-bestand niet.
3. Bouw/push één immutable GPU image vanaf een gepushte commit/tag; leg image digest en SBOM vast.
4. Migreer DB/schema via backup, dry-run en restoretest; voer PostGIS extension-upgrade alleen volgens P2-evidence uit.
5. Draai production preflight: exact modelhash, CUDA required, bounded tile inference, CRS/georeferencing, persistence en restart.
6. Start shadowvergelijking binnen exact de gevalideerde scope; shadowoutput is niet publiek en kan de protected-testbeslissing niet aanpassen.
7. Laat UI/API alleen de bewezen scope/classes/status zien; segmentatie en solar blijven `not_configured`.
8. Activeer pas na shadow- en rollbackgate; monitor en behoud één-command rollback naar vorige image/model/config.
### Exit gate
- Commit/tag, image digest, model SHA, config SHA, migration head en promotion report verwijzen wederzijds naar elkaar.
- End-to-end selectie → inference → persisted result → uitschuifbare inzichten → export is getest met correcte lineage en zichtbare error states.
- Restart en rollback slagen zonder data- of evidenceverlies.
- De gedeclareerde scope is exact de geslaagde testscope, nooit “heel België” door implicatie.
## P2-12 — Monitoring en gecontroleerde volgende cyclus
### Uitvoering
- Monitor per region/context/provider/imagery edition/object size: input drift, confidence, density, QA mismatches, latency/VRAM en persistence failures.
- Maak een menselijke reviewqueue met FP/FN/uncertain voorbeelden; production outputs worden nooit automatisch training labels.
- Een volgende training gebruikt alleen een reviewed, opnieuw gefreezede labelrelease en herstart bij P2-05/P2-06.
- Bewaar oude datasets, modellen, run manifests, promotion reports en rollbackimages volgens retentiebeleid.
- Widening van scope of class is een nieuwe releaseclaim en doorloopt opnieuw P2-06 tot P2-11 met een onaangeroerde testportfolio.
### Exit gate
- Alerts, reviewqueue, ownership en rollbackrunbook zijn operationeel getest.
- Er bestaat geen automatische self-training of silent promotion path.
- Periodieke audits kunnen ieder publiek resultaat terugvoeren naar bron, tile, model, config en releasebeslissing.
## Verplichte release-evidence
P2-11 blijft geblokkeerd zolang één van deze artifacts ontbreekt:
- canonical CI commandmatrix en logs;
- CRS/coverage/authority/scope regressierapport;
- DB migration, backup/restore en legacy-quarantainerapport;
- trusted source registry en lineage completeness audit;
- immutable corpus, split/duplicate/temporal/label audits en human-reviewmanifest;
- pinned AI image digest/SBOM en RTX 4080 CUDA run manifests;
- reproducible incumbent/candidate calibrationrapporten;
- vooraf bevroren gateconfig;
- één protected-test/background-test accesslog en report;
- model card, signed promotion report, shadow report en rollbacktest;
- bijgewerkte API/contracts, limitations, execution log en TODO.
## Stop-the-line criteria
Stop de betrokken pipeline en behoud `release blocked` wanneer:
- protected data vóór de freeze wordt gelezen of in sampler/training evidence voorkomt;
- een geometry zonder betrouwbare CRS of een meteroperatie in graden wordt verwerkt;
- user metadata officiële authority kan verlenen;
- model/dataset/tile/config hashes ontbreken;
- een verplichte metric null/ontbrekend is;
- menselijke labelreview Critical/High findings openlaat;
- training niet aantoonbaar op de vereiste NVIDIA GPU draait;
- een regio/context/background-gate faalt;
- de protected test faalt;
- deploy commit, image, model, DB migration en promotion report niet exact aan elkaar gebonden zijn.
Alleen bewijs kan een gate openen. Een nieuwe training, hogere epoch count of gunstig aggregate cijfer kan een ontbrekende lineage-, split-, regionale, menselijke of deploymentgate niet compenseren.
+261
View File
@@ -0,0 +1,261 @@
{
"schema_version": 1,
"program": "GeoIntel Accuracy Improvement Program",
"phase": "P1",
"generated_at": "2026-08-01T18:40:00+02:00",
"scope": {
"product": "Belgium and the Belgian North Sea",
"active_building_model_claim": "Mol/Kempen only, operator review required",
"national_building_validation": false
},
"baseline": {
"branch": "codex/geointel-accuracy-program",
"repository_commit": "0c019bb22f816db1e4b7a68379bcad08924d9a21",
"database_migration_head": "202607260001",
"phase1_mutation_scope": "audit tooling, tests, documentation and retained evidence only"
},
"phase1": {
"status": "complete",
"meaning": "The forensic inventory, reproducible baseline, lineage assessment, risk register, metric contract, implementation roadmap and retained evidence exist.",
"does_not_mean": [
"release ready",
"nationally validated",
"human-reviewed corpus",
"strictly independent protected test",
"calibrated confidence",
"fully trained"
]
},
"release": {
"status": "blocked",
"promotion_allowed": false,
"scope_widening_allowed": false,
"training_allowed_now": false,
"training_unlock_gate": "P2-08 after P2-00 through P2-07 have passed",
"critical_risk_count": 8,
"high_risk_count": 17,
"medium_risk_count": 4
},
"phase2": {
"status": "ready_for_controlled_remediation",
"roadmap": "docs/accuracy-program/06-implementation-roadmap.md",
"first_work_package": "P2-00",
"required_order": [
"P2-00",
"P2-01",
"P2-02",
"P2-03",
"P2-04",
"P2-05",
"P2-06",
"P2-07",
"P2-08",
"P2-09",
"P2-10",
"P2-11",
"P2-12"
],
"protected_test_rule": "Open exactly once for a pre-registered immutable candidate after all pre-test gates pass; never feed its results back into that candidate family."
},
"runtime": {
"cuda_available": true,
"device": "NVIDIA GeForce RTX 4080 SUPER",
"configured_device": "cuda:0",
"python": "3.11.2",
"torch": "2.11.0+cu128",
"cuda_runtime": "12.8",
"ultralytics": "8.4.99",
"active_model": {
"model_id": "yolo-configured",
"model_version": "",
"path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt",
"sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1",
"size_bytes": 22516074,
"validated_area_names": [
"Mol",
"Kempen"
],
"nationally_validated": false
},
"gpu_smoke": {
"status": "passed",
"read_only": true,
"tile_crs": "EPSG:31370",
"tile_sha256": "134a9e86850c92c577c73bc6ee57a9df7d4c1c513ae6450263e800b6dd47b6ee",
"manifest_sha256": "6ab8a96bf2a1405e224932afb90255311a09bbeaed4e9a2fdcdf8b1bc2230abd",
"raw_detection_count": 17,
"inference_seconds": 0.8836944859940559,
"claim_boundary": "Runtime execution only; no accuracy, calibration, generalization or release claim."
}
},
"database": {
"mode": "read_only_audit",
"storage_references_checked": 5816,
"storage_references_missing": 0,
"table_counts": {
"projects": 1097,
"areas": 387,
"datasets": 3377,
"dataset_versions": 1671,
"vector_features": 6689447,
"analysis_runs": 1146,
"detections": 299233,
"segmentations": 0,
"detection_reviews": 0,
"quality_checks": 697,
"metrics": 4182,
"jobs": 4460,
"exports": 768,
"aoi_operations": 4,
"aoi_operation_partitions": 61
},
"lineage_gaps": {
"datasets_missing_observed_at": 2377,
"datasets_missing_source_version": 1761,
"detection_runs_with_empty_model_version": 1146,
"detections_with_empty_model_version": 299233,
"detection_runs_missing_model_hash": 3,
"detection_runs_missing_tile_manifest_hash": 1146
},
"geometry": {
"areas_invalid_or_wrong_srid": 0,
"vector_features_invalid_or_wrong_srid": 0,
"detections_outside_epsg4326_domain": 4,
"outside_domain_interpretation": "Four Geel detections contain Lambert-domain coordinates while persisted under SRID 4326."
}
},
"ml_data": {
"model_asset_count": 26,
"training_checkpoint_count": 229,
"training_json_report_count": 424,
"operator_manifest_count": 36,
"v56": {
"sample_count": 180,
"reviewed_sample_count": 0,
"review_complete": false,
"input_feature_count": 31452,
"accepted_feature_count": 30662,
"below_resolvable_pixel_size": 326,
"created_after_imagery_period": 464,
"pure_empty_background_count": 3,
"pure_empty_background_by_region": {
"flanders": 2,
"wallonia": 1,
"brussels": 0
},
"minimum_cross_split_aoi_distance_m": 95.72033647650719,
"cross_split_pairs_below_2000_m": 24,
"exact_cross_split_raster_hash_duplicates": 0,
"perceptual_pairs_hamming_at_or_below_4": 0,
"split_independence_proven": false
},
"candidate_evidence": {
"v58_v62_kind": "calibration-only tile-level bbox metrics",
"threshold_0_15_aggregate_f1": 0.512905360688286,
"threshold_0_15_flanders_recall": 0.0,
"protected_test_evidence": false,
"background_test_release_evidence": false,
"promotion_evidence": false
},
"protected_test_isolation": false,
"human_label_acceptance": false
},
"verification": {
"backend_full_suite": {
"status": "failed",
"passed": 1180,
"failed": 17,
"duration_seconds": 70.63,
"classification": "stale source/contract assertions; no new audit-tooling failures"
},
"backend_ci_entrypoint": {
"status": "collection_failed",
"collected": 1194,
"error": "ModuleNotFoundError: scripts.render_operator_polygon_label_qa"
},
"phase1_tooling_tests": {
"status": "passed",
"passed": 4
},
"new_phase1_code_ruff": {
"status": "passed"
},
"repository_ruff": {
"status": "failed",
"finding_count": 112,
"by_code": {
"E402": 13,
"E701": 2,
"E702": 69,
"F401": 23,
"F403": 1,
"F811": 2,
"F841": 2
}
},
"frontend_unit": {
"status": "passed",
"test_files": 16,
"tests": 51,
"command": "npm run test:unit"
},
"frontend_typecheck": {
"status": "passed"
},
"frontend_build": {
"status": "passed"
},
"frontend_lint": {
"status": "missing",
"error": "npm run lint: Missing script"
},
"openapi_contract": {
"status": "passed",
"implemented_routes": 147,
"explicit_non_envelope_endpoints": 10
},
"alembic": {
"status": "passed_offline",
"heads": [
"202607260001"
],
"offline_upgrade_sql_lines": 496,
"live_migration_tested_locally": false
},
"golden_qa": {
"semantic_results_stable": true,
"byte_identical": false,
"reason": "UUID4-backed run identity"
}
},
"reproduced_contract_violations": [
"P1-COV-001",
"P1-CRS-001",
"P1-CRS-002",
"P1-AUTH-001",
"P1-AI-001",
"P1-COV-002",
"P1-API-001"
],
"critical_blockers": [
"ACC-R01",
"ACC-R02",
"ACC-R03",
"ACC-R04",
"ACC-R06",
"ACC-R09",
"ACC-R16",
"ACC-R17"
],
"documents": [
"docs/accuracy-program/00-execution-contract.md",
"docs/accuracy-program/01-system-inventory.md",
"docs/accuracy-program/02-data-lineage.md",
"docs/accuracy-program/03-baseline-and-gaps.md",
"docs/accuracy-program/04-risk-register.md",
"docs/accuracy-program/05-metric-framework.md",
"docs/accuracy-program/06-implementation-roadmap.md"
],
"evidence_root": "artifacts/evidence/accuracy/P1",
"evidence_manifest": "artifacts/evidence/accuracy/P1/evidence-manifest.json"
}
@@ -0,0 +1,159 @@
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import subprocess
from typing import Any
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_EVIDENCE_ROOT = REPOSITORY_ROOT / "artifacts" / "evidence" / "accuracy" / "P1"
DEFAULT_OUTPUT = DEFAULT_EVIDENCE_ROOT / "evidence-manifest.json"
PROGRAM_PATHS = (
"docs/accuracy-program/00-execution-contract.md",
"docs/accuracy-program/01-system-inventory.md",
"docs/accuracy-program/02-data-lineage.md",
"docs/accuracy-program/03-baseline-and-gaps.md",
"docs/accuracy-program/04-risk-register.md",
"docs/accuracy-program/05-metric-framework.md",
"docs/accuracy-program/06-implementation-roadmap.md",
"docs/accuracy-program/status.json",
"scripts/build_accuracy_phase1_evidence_manifest.py",
"scripts/collect_accuracy_phase1_inference_smoke.py",
"scripts/collect_accuracy_phase1_ml_lineage.py",
"scripts/collect_accuracy_phase1_runtime.py",
"scripts/reproduce_accuracy_phase1_findings.py",
"scripts/run_accuracy_phase1_baseline.py",
"scripts/verify_accuracy_phase1_evidence.py",
"tests/test_accuracy_phase1_baseline.py",
)
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def role_for(path: Path) -> str:
name = path.name.lower()
if name.endswith(".junit.xml"):
return "test_report"
if name.endswith(".sql"):
return "migration_evidence"
if "runtime" in name or "gpu-inference" in name:
return "runtime_evidence"
if "lineage" in name:
return "lineage_evidence"
if "reproduction" in name:
return "defect_reproduction"
if "ruff" in name or "lint" in name:
return "lint_evidence"
if "test" in name or "vitest" in name or "golden-qa" in name:
return "test_evidence"
if path.suffix.lower() == ".json":
return "structured_inventory"
return "execution_log"
def record(path: Path, *, displayed_path: str, role: str) -> dict[str, Any]:
return {
"path": displayed_path,
"role": role,
"size_bytes": path.stat().st_size,
"sha256": sha256_file(path),
}
def git_head() -> str | None:
result = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=REPOSITORY_ROOT,
capture_output=True,
text=True,
check=False,
)
return result.stdout.strip() if result.returncode == 0 else None
def build_manifest(evidence_root: Path, output: Path) -> dict[str, Any]:
if not evidence_root.is_dir():
raise FileNotFoundError(f"Evidence root does not exist: {evidence_root}")
evidence_files = [
path
for path in evidence_root.rglob("*")
if path.is_file() and path.resolve() != output.resolve()
]
evidence_records = [
record(
path,
displayed_path=path.relative_to(REPOSITORY_ROOT).as_posix(),
role=role_for(path),
)
for path in sorted(evidence_files)
]
program_records = []
for relative in PROGRAM_PATHS:
path = REPOSITORY_ROOT / relative
if not path.is_file():
raise FileNotFoundError(f"Required Phase-1 program file is missing: {relative}")
program_records.append(record(path, displayed_path=relative, role="phase1_program"))
return {
"schema_version": 1,
"generated_at": datetime.now(timezone.utc).isoformat(),
"audited_repository_head": git_head(),
"claim_boundary": (
"This manifest proves retained-file identity and completeness. It does "
"not establish model accuracy, human label acceptance, split independence "
"or release readiness."
),
"evidence_root": evidence_root.relative_to(REPOSITORY_ROOT).as_posix(),
"evidence_file_count": len(evidence_records),
"evidence_total_bytes": sum(item["size_bytes"] for item in evidence_records),
"evidence_files": evidence_records,
"program_file_count": len(program_records),
"program_files": program_records,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Build the immutable GeoIntel Accuracy P1 evidence manifest.")
parser.add_argument("--evidence-root", type=Path, default=DEFAULT_EVIDENCE_ROOT)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
args = parser.parse_args()
evidence_root = args.evidence_root.expanduser().resolve()
output = args.output.expanduser().resolve()
if output.exists():
parser.error(f"refusing to overwrite existing evidence manifest: {output}")
if output.parent != evidence_root:
parser.error("--output must be directly inside --evidence-root")
payload = build_manifest(evidence_root, output)
output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(
json.dumps(
{
"status": "created",
"output": str(output),
"evidence_file_count": payload["evidence_file_count"],
"program_file_count": payload["program_file_count"],
},
indent=2,
sort_keys=True,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,181 @@
from __future__ import annotations
import argparse
from collections import Counter
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import random
import sys
from time import perf_counter
from typing import Any
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
BACKEND_ROOT = REPOSITORY_ROOT / "backend"
if str(BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(BACKEND_ROOT))
from app.core.config import Settings # noqa: E402
from app.services.yolo_adapter import YoloDetectionAdapter # noqa: E402
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _raster_metadata(path: Path) -> dict[str, Any]:
try:
import rasterio
except ImportError:
return {"available": False, "reason": "rasterio_not_installed"}
with rasterio.open(path) as dataset:
return {
"available": True,
"bounds": [float(value) for value in dataset.bounds],
"count": int(dataset.count),
"crs": str(dataset.crs) if dataset.crs else None,
"dtypes": list(dataset.dtypes),
"height": int(dataset.height),
"nodata": dataset.nodata,
"transform": [float(value) for value in dataset.transform],
"width": int(dataset.width),
}
def _summarize_detections(detections: list[dict[str, Any]]) -> dict[str, Any]:
confidences = [float(item["confidence"]) for item in detections]
class_counts = Counter(str(item["class_name"]) for item in detections)
return {
"count": len(detections),
"class_counts": dict(sorted(class_counts.items())),
"confidence": {
"minimum": min(confidences) if confidences else None,
"maximum": max(confidences) if confidences else None,
"mean": sum(confidences) / len(confidences) if confidences else None,
},
"sample": detections[:10],
}
def main() -> int:
parser = argparse.ArgumentParser(
description=(
"Run one read-only production-adapter inference and emit forensic JSON. "
"This proves runtime execution only; it does not establish model accuracy."
)
)
parser.add_argument("--model-path", required=True)
parser.add_argument("--tile-path", required=True)
parser.add_argument("--manifest-path")
parser.add_argument("--confidence", type=float, default=0.5)
parser.add_argument("--image-size", type=int, default=640)
parser.add_argument("--max-detections", type=int, default=1000)
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--seed", type=int, default=20260801)
args = parser.parse_args()
model_path = Path(args.model_path).expanduser().resolve()
tile_path = Path(args.tile_path).expanduser().resolve()
manifest_path = Path(args.manifest_path).expanduser().resolve() if args.manifest_path else None
for label, path in (("model", model_path), ("tile", tile_path)):
if not path.is_file():
parser.error(f"{label} path is not an existing file: {path}")
if manifest_path is not None and not manifest_path.is_file():
parser.error(f"manifest path is not an existing file: {manifest_path}")
if not 0.0 <= args.confidence <= 1.0:
parser.error("--confidence must be between 0 and 1")
import numpy as np
import torch
import ultralytics
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
torch.cuda.reset_peak_memory_stats()
torch.use_deterministic_algorithms(True, warn_only=True)
settings = Settings().model_copy(
update={
"yolo_enabled": True,
"yolo_model_path": str(model_path),
"yolo_device": args.device,
"yolo_require_cuda": True,
"yolo_image_size": args.image_size,
"yolo_max_detections": args.max_detections,
}
)
adapter = YoloDetectionAdapter(settings)
adapter.validate_runtime()
started = perf_counter()
model = adapter.load_model(model_path)
model_loaded = perf_counter()
detections = adapter.predict_tile(model, tile_path, args.confidence)
if torch.cuda.is_available():
torch.cuda.synchronize()
finished = perf_counter()
device_index = torch.cuda.current_device() if torch.cuda.is_available() else None
payload = {
"schema_version": 1,
"captured_at": datetime.now(timezone.utc).isoformat(),
"status": "passed",
"claim_boundary": (
"One production-adapter inference completed on one existing tile. "
"No accuracy, calibration, geographic-generalization, or release claim follows from this smoke."
),
"read_only": True,
"configuration": {
"confidence": args.confidence,
"device": args.device,
"image_size": args.image_size,
"max_detections": args.max_detections,
"seed": args.seed,
"deterministic_algorithms": True,
},
"model": {
"path": str(model_path),
"sha256": _sha256(model_path),
"size_bytes": model_path.stat().st_size,
},
"input": {
"tile_path": str(tile_path),
"tile_sha256": _sha256(tile_path),
"tile_size_bytes": tile_path.stat().st_size,
"manifest_path": str(manifest_path) if manifest_path else None,
"manifest_sha256": _sha256(manifest_path) if manifest_path else None,
"raster": _raster_metadata(tile_path),
},
"runtime": {
"python": sys.version,
"torch": torch.__version__,
"ultralytics": ultralytics.__version__,
"cuda_available": torch.cuda.is_available(),
"cuda_runtime": torch.version.cuda,
"cuda_device_index": device_index,
"cuda_device_name": torch.cuda.get_device_name(device_index) if device_index is not None else None,
"cuda_peak_memory_bytes": torch.cuda.max_memory_allocated() if torch.cuda.is_available() else None,
},
"timing_seconds": {
"model_load": model_loaded - started,
"inference": finished - model_loaded,
"total": finished - started,
},
"output": _summarize_detections(detections),
}
print(json.dumps(payload, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,364 @@
from __future__ import annotations
import argparse
from collections import Counter, defaultdict
from datetime import datetime, timezone
from itertools import combinations
import hashlib
import json
from pathlib import Path
from typing import Any
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def artifact_record(path: Path, *, include_hash: bool = True) -> dict[str, Any]:
stat = path.stat()
return {
"path": str(path),
"size_bytes": stat.st_size,
"modified_at": datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat(),
"sha256": sha256_file(path) if include_hash else None,
}
def manifest_summary(path: Path) -> dict[str, Any]:
payload = json.loads(path.read_text(encoding="utf-8"))
samples = payload.get("samples") if isinstance(payload.get("samples"), list) else []
region_split = Counter(
f"{sample.get('region')}/{sample.get('split')}" for sample in samples
)
contexts = Counter(str(sample.get("context")) for sample in samples)
roles = Counter(str(sample.get("sample_role")) for sample in samples)
background_samples = [
{
"sample_slug": sample.get("sample_slug"),
"region": sample.get("region"),
"reference_feature_count": sample.get("reference_feature_count"),
"require_empty": sample.get("require_empty"),
}
for sample in samples
if sample.get("split") == "background-test"
]
pure_empty = [
sample
for sample in background_samples
if sample.get("require_empty") is True
and int(sample.get("reference_feature_count") or 0) == 0
]
cross_split_raster_hashes: dict[str, set[str]] = defaultdict(set)
cross_split_dataset_ids: dict[str, set[str]] = defaultdict(set)
for sample in samples:
split = str(sample.get("split"))
if sample.get("raster_sha256"):
cross_split_raster_hashes[str(sample["raster_sha256"])].add(split)
if sample.get("raster_dataset_id"):
cross_split_dataset_ids[str(sample["raster_dataset_id"])].add(split)
spatial = cross_split_spatial_summary(samples)
perceptual = cross_split_dhash_summary(samples)
return {
**artifact_record(path),
"schema_version": payload.get("schema_version"),
"dataset_version": payload.get("dataset_version"),
"immutable": payload.get("immutable"),
"sample_count": len(samples),
"region_split_counts": dict(sorted(region_split.items())),
"context_counts": dict(sorted(contexts.items())),
"sample_role_counts": dict(sorted(roles.items())),
"background_test_samples": background_samples,
"pure_empty_background_count": len(pure_empty),
"pure_empty_background_by_region": dict(
sorted(Counter(str(item["region"]) for item in pure_empty).items())
),
"exact_cross_split_raster_hash_duplicate_count": sum(
1 for splits in cross_split_raster_hashes.values() if len(splits) > 1
),
"cross_split_raster_dataset_id_duplicate_count": sum(
1 for splits in cross_split_dataset_ids.values() if len(splits) > 1
),
"spatial_independence": spatial,
"perceptual_duplicate_screen": perceptual,
}
def cross_split_spatial_summary(samples: list[dict[str, Any]]) -> dict[str, Any]:
try:
from pyproj import Transformer
from shapely.geometry import box
from shapely.ops import transform
except ImportError:
return {"available": False, "reason": "geospatial_dependencies_unavailable"}
transformer = Transformer.from_crs(4326, 31370, always_xy=True)
records: list[tuple[str, str, Any]] = []
for sample in samples:
bbox_values = sample.get("bbox_epsg4326")
if not isinstance(bbox_values, list) or len(bbox_values) != 4:
continue
geometry = transform(transformer.transform, box(*[float(value) for value in bbox_values]))
records.append((str(sample.get("sample_slug")), str(sample.get("split")), geometry))
distances: list[tuple[float, str, str, str, str]] = []
for left, right in combinations(records, 2):
if left[1] == right[1]:
continue
distance = float(left[2].distance(right[2]))
distances.append((distance, left[0], left[1], right[0], right[1]))
distances.sort(key=lambda item: item[0])
return {
"available": True,
"crs": "EPSG:31370",
"cross_split_pair_count": len(distances),
"minimum_distance_m": distances[0][0] if distances else None,
"pairs_below_64_m": sum(1 for item in distances if item[0] < 64.0),
"pairs_below_2000_m": sum(1 for item in distances if item[0] < 2000.0),
"closest_pairs": [
{
"distance_m": item[0],
"left_sample": item[1],
"left_split": item[2],
"right_sample": item[3],
"right_split": item[4],
}
for item in distances[:20]
],
"claim_boundary": (
"AOI bounding-box distance is a screening check only; it does not prove "
"municipality, flight-strip, building-instance or imagery independence."
),
}
def _dhash(path: Path) -> int:
from PIL import Image
with Image.open(path) as image:
resized = image.convert("L").resize((9, 8))
pixel_source = (
resized.get_flattened_data()
if hasattr(resized, "get_flattened_data")
else resized.getdata()
)
pixels = list(pixel_source)
value = 0
for row in range(8):
offset = row * 9
for column in range(8):
value = (value << 1) | int(pixels[offset + column] > pixels[offset + column + 1])
return value
def cross_split_dhash_summary(samples: list[dict[str, Any]]) -> dict[str, Any]:
hashes: list[tuple[str, str, int]] = []
failures: list[dict[str, str]] = []
for sample in samples:
raster_path = Path(str(sample.get("raster_path") or ""))
if not raster_path.is_file():
failures.append({"sample_slug": str(sample.get("sample_slug")), "reason": "missing_raster"})
continue
try:
hashes.append(
(
str(sample.get("sample_slug")),
str(sample.get("split")),
_dhash(raster_path),
)
)
except Exception as exc:
failures.append({"sample_slug": str(sample.get("sample_slug")), "reason": str(exc)})
matches: list[dict[str, Any]] = []
minimum: int | None = None
compared = 0
for left, right in combinations(hashes, 2):
if left[1] == right[1]:
continue
compared += 1
distance = (left[2] ^ right[2]).bit_count()
minimum = distance if minimum is None else min(minimum, distance)
if distance <= 4:
matches.append(
{
"left_sample": left[0],
"left_split": left[1],
"right_sample": right[0],
"right_split": right[1],
"hamming_distance": distance,
}
)
return {
"algorithm": "64-bit difference hash over 9x8 grayscale resize",
"screened_raster_count": len(hashes),
"cross_split_pair_count": compared,
"minimum_hamming_distance": minimum,
"pairs_at_or_below_4": matches,
"failures": failures,
"claim_boundary": (
"This bounded perceptual screen is not semantic, instance-level or "
"flight-strip deduplication and cannot establish split independence."
),
}
def corpus_audit_summary(path: Path) -> dict[str, Any]:
payload = json.loads(path.read_text(encoding="utf-8"))
keys = (
"status",
"dataset_version",
"manifest_immutable",
"sample_count",
"split_counts",
"input_feature_count",
"accepted_feature_count",
"decision_counts",
"temporal_unknown_sample_count",
"spatial_leakage_status",
"reviewed_sample_count",
"review_complete",
"failures",
)
return {
**artifact_record(path),
**{key: payload.get(key) for key in keys},
"review_queue_count": len(payload.get("review_queue") or []),
"claim_boundary": (
"Automated status and an empty failure list do not substitute for "
"human label acceptance."
),
}
def calibration_summary(path: Path) -> dict[str, Any]:
payload = json.loads(path.read_text(encoding="utf-8"))
sweeps = payload.get("sweeps") if isinstance(payload.get("sweeps"), list) else []
def sweep_at(threshold: float) -> dict[str, Any] | None:
return next(
(
{
"threshold": item.get("threshold"),
"aggregate": item.get("aggregate"),
"regions": item.get("regions"),
"pure_empty_false_positives": item.get("pure_empty_false_positives"),
}
for item in sweeps
if abs(float(item.get("threshold")) - threshold) < 1e-12
),
None,
)
return {
**artifact_record(path),
"model": payload.get("model"),
"summary": payload.get("summary"),
"split": payload.get("split"),
"match_iou": payload.get("match_iou"),
"tile_count": payload.get("tile_count"),
"inference_imgsz": payload.get("inference_imgsz"),
"sweep_count": len(sweeps),
"threshold_0_15": sweep_at(0.15),
"threshold_0_02": sweep_at(0.02),
"claim_boundary": (
"This is calibration-split, tile-level bbox evidence; it is not a "
"protected-test, unique-building or national release result."
),
}
def inventory(root: Path) -> dict[str, Any]:
model_files = sorted(
path
for path in (root / "models").glob("*")
if path.is_file() and path.suffix.lower() in {".pt", ".pth", ".onnx", ".engine", ".ckpt", ".safetensors"}
)
training_root = root / "storage" / "training"
checkpoints = sorted(
(
path
for path in training_root.glob("building-be-*/**/*")
if path.is_file() and path.suffix.lower() in {".pt", ".pth", ".ckpt", ".onnx", ".engine", ".safetensors"}
),
key=lambda path: path.stat().st_mtime,
reverse=True,
)
reports = sorted(
(
path
for path in training_root.glob("building-be-*/**/*.json")
if path.is_file()
),
key=lambda path: path.stat().st_mtime,
reverse=True,
)
manifests = sorted(
(root / "storage" / "operator-data").glob("building-be-*/operator_samples_manifest.json"),
key=lambda path: path.stat().st_mtime,
reverse=True,
)
return {
"model_assets": [artifact_record(path) for path in model_files],
"training_checkpoint_count": len(checkpoints),
"training_checkpoint_bytes": sum(path.stat().st_size for path in checkpoints),
"newest_training_checkpoints": [
artifact_record(path, include_hash=False) for path in checkpoints[:30]
],
"training_json_report_count": len(reports),
"newest_training_json_reports": [
artifact_record(path, include_hash=False) for path in reports[:100]
],
"operator_manifest_count": len(manifests),
"operator_manifests": [artifact_record(path) for path in manifests],
}
def main() -> int:
parser = argparse.ArgumentParser(
description="Collect a bounded, read-only ML/data-lineage snapshot from the deployed GeoIntel volume."
)
parser.add_argument("--app-root", default="/app")
args = parser.parse_args()
root = Path(args.app_root).resolve()
paths = {
"v56_audit": root / "storage/training/building-be-v56-corpus-audit-r1/belgium-building-corpus-audit.json",
"v56_manifest": root / "storage/operator-data/building-be-v56-hard-negative-instance-roofs-r1/operator_samples_manifest.json",
"v58_calibration": root / "storage/training/building-be-v58-v56-clean-pretrained-r1/preview-epoch-015/calibration.json",
"v62_calibration": root / "storage/training/building-be-v62-v58-flanders-balanced-transfer-r1/preview-epoch-006/calibration-routed.json",
"v66_manifest": root / "storage/operator-data/building-be-v66-lowrise-temporal-r1/operator_samples_manifest.json",
}
missing = [str(path) for path in paths.values() if not path.is_file()]
if missing:
raise SystemExit("Required lineage artifacts are missing: " + ", ".join(missing))
payload = {
"schema_version": 1,
"captured_at": datetime.now(timezone.utc).isoformat(),
"read_only": True,
"inventory": inventory(root),
"v56": {
"corpus_audit": corpus_audit_summary(paths["v56_audit"]),
"manifest": manifest_summary(paths["v56_manifest"]),
},
"v58": {"calibration": calibration_summary(paths["v58_calibration"])},
"v62": {"calibration": calibration_summary(paths["v62_calibration"])},
"v66": {"manifest": manifest_summary(paths["v66_manifest"])},
"global_claim_boundary": (
"Inventory and historical calibration evidence do not prove human "
"label acceptance, strict split independence, calibration, protected-test "
"performance, national validity or release readiness."
),
}
print(json.dumps(payload, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+334
View File
@@ -0,0 +1,334 @@
#!/usr/bin/env python3
"""Emit a bounded read-only database/runtime integrity snapshot as JSON.
Run inside the GeoIntel application container. Every SQL statement has a
timeout; the script never writes application rows or storage artifacts.
"""
from __future__ import annotations
import hashlib
import importlib.metadata
import json
import os
import platform
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from sqlalchemy import text
from app.db.session import engine
TABLES = (
"projects",
"areas",
"datasets",
"dataset_versions",
"vector_features",
"jobs",
"analysis_runs",
"detections",
"segmentations",
"quality_checks",
"metrics",
"exports",
"detection_reviews",
"aoi_operations",
"aoi_operation_partitions",
)
PATH_QUERIES = {
"datasets": "SELECT id::text, storage_path FROM datasets WHERE storage_path IS NOT NULL",
"dataset_versions": "SELECT id::text, storage_path FROM dataset_versions WHERE storage_path IS NOT NULL",
"exports": "SELECT id::text, storage_path FROM exports WHERE storage_path IS NOT NULL",
}
def now() -> str:
return datetime.now(timezone.utc).isoformat()
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def package_version(name: str) -> str | None:
try:
return importlib.metadata.version(name)
except importlib.metadata.PackageNotFoundError:
return None
def normalize_path(raw: str, storage_root: Path) -> Path:
candidate = Path(raw)
if candidate.is_absolute():
return candidate
normalized = raw.replace("\\", "/")
if normalized.startswith("storage/"):
normalized = normalized.removeprefix("storage/")
return storage_root / normalized
def rows(connection: Any, statement: str, parameters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
result = connection.execute(text(statement), parameters or {})
return [dict(row._mapping) for row in result]
def scalar(connection: Any, statement: str) -> int:
return int(connection.execute(text(statement)).scalar() or 0)
def safe_query(connection: Any, name: str, statement: str) -> dict[str, Any]:
try:
return {"status": "ok", "rows": rows(connection, statement)}
except Exception as exc:
return {
"status": "error",
"error_type": type(exc).__name__,
"message": str(exc).splitlines()[0][:500],
"query_name": name,
}
def main() -> int:
storage_root = Path(os.environ.get("GEOINTEL_STORAGE_ROOT", "/app/storage")).resolve()
report: dict[str, Any] = {
"schema_version": 1,
"generated_at": now(),
"mode": "read_only",
"statement_timeout_ms": 30000,
"runtime": {
"python": platform.python_version(),
"platform": platform.platform(),
"packages": {
name: package_version(name)
for name in (
"geointel-backend",
"fastapi",
"sqlalchemy",
"geoalchemy2",
"shapely",
"pyproj",
"rasterio",
"geopandas",
"torch",
"torchvision",
"ultralytics",
)
},
},
"storage_root": str(storage_root),
}
try:
import torch
report["runtime"]["cuda"] = {
"available": torch.cuda.is_available(),
"runtime_version": torch.version.cuda,
"device_count": torch.cuda.device_count(),
"device_names": [
torch.cuda.get_device_name(index) for index in range(torch.cuda.device_count())
],
}
except Exception as exc:
report["runtime"]["cuda"] = {"available": False, "error": type(exc).__name__}
model_path = Path(os.environ.get("YOLO_MODEL_PATH", ""))
report["active_model"] = {
"configured_path": str(model_path) if str(model_path) else None,
"exists": model_path.is_file(),
"size_bytes": model_path.stat().st_size if model_path.is_file() else None,
"sha256": sha256_file(model_path) if model_path.is_file() else None,
"model_id": os.environ.get("YOLO_MODEL_ID"),
"model_version": os.environ.get("YOLO_MODEL_VERSION"),
"classes": os.environ.get("YOLO_MODEL_CLASSES"),
"device": os.environ.get("YOLO_DEVICE"),
"require_cuda": os.environ.get("YOLO_REQUIRE_CUDA"),
"validated_area_names": os.environ.get("YOLO_VALIDATED_AREA_NAMES"),
"validation_scope_enforced": os.environ.get("YOLO_ENFORCE_VALIDATION_SCOPE"),
}
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:
connection.execute(text("SET statement_timeout TO '30s'"))
report["database"] = {
"version": connection.execute(text("SELECT version()")).scalar(),
"postgis_version": connection.execute(text("SELECT PostGIS_Full_Version()")).scalar(),
"migration_heads": [row["version_num"] for row in rows(connection, "SELECT version_num FROM alembic_version")],
"table_counts": {table: scalar(connection, f'SELECT count(*) FROM "{table}"') for table in TABLES},
}
report["database"]["dataset_statuses"] = rows(
connection,
"SELECT status, count(*)::bigint AS count FROM datasets GROUP BY status ORDER BY status",
)
report["database"]["job_statuses"] = rows(
connection,
"SELECT status, count(*)::bigint AS count FROM jobs GROUP BY status ORDER BY status",
)
report["database"]["analysis_statuses"] = rows(
connection,
"SELECT status, count(*)::bigint AS count FROM analysis_runs GROUP BY status ORDER BY status",
)
report["database"]["dataset_lineage_gaps"] = rows(
connection,
"""
SELECT
count(*)::bigint AS total,
count(*) FILTER (WHERE crs IS NULL OR btrim(crs) = '')::bigint AS missing_crs,
count(*) FILTER (WHERE checksum_sha256 IS NULL OR btrim(checksum_sha256) = '')::bigint AS missing_checksum,
count(*) FILTER (WHERE source_version IS NULL OR btrim(source_version) = '')::bigint AS missing_source_version,
count(*) FILTER (WHERE source_metadata IS NULL OR source_metadata::text = '{}')::bigint AS missing_source_metadata,
count(*) FILTER (WHERE provenance_metadata IS NULL OR provenance_metadata::text = '{}')::bigint AS missing_provenance_metadata,
count(*) FILTER (WHERE imported_at IS NULL)::bigint AS missing_imported_at,
count(*) FILTER (WHERE observed_at IS NULL)::bigint AS missing_observed_at
FROM datasets
""",
)
report["database"]["dataset_version_gaps"] = rows(
connection,
"""
SELECT
count(*)::bigint AS total,
count(*) FILTER (WHERE checksum_sha256 IS NULL OR btrim(checksum_sha256) = '')::bigint AS missing_checksum,
count(*) FILTER (WHERE source_metadata IS NULL OR source_metadata::text = '{}')::bigint AS missing_source_metadata,
count(*) FILTER (WHERE provenance_metadata IS NULL OR provenance_metadata::text = '{}')::bigint AS missing_provenance_metadata,
count(*) FILTER (WHERE storage_path IS NULL OR btrim(storage_path) = '')::bigint AS missing_storage_path
FROM dataset_versions
""",
)
report["database"]["geometry_integrity"] = {}
for table, nullable in (
("areas", False),
("vector_features", False),
("detections", True),
("segmentations", False),
):
where = "WHERE geometry IS NOT NULL" if nullable else ""
statement = f"""
SELECT
count(*)::bigint AS populated,
count(*) FILTER (WHERE ST_IsEmpty(geometry))::bigint AS empty,
count(*) FILTER (WHERE NOT ST_IsValid(geometry))::bigint AS invalid,
count(*) FILTER (WHERE ST_SRID(geometry) <> 4326)::bigint AS wrong_srid,
count(*) FILTER (
WHERE ST_XMin(Box3D(geometry)) < -180
OR ST_XMax(Box3D(geometry)) > 180
OR ST_YMin(Box3D(geometry)) < -90
OR ST_YMax(Box3D(geometry)) > 90
)::bigint AS outside_epsg4326_domain
FROM {table} {where}
"""
report["database"]["geometry_integrity"][table] = safe_query(
connection, f"{table}_geometry_integrity", statement
)
report["database"]["outside_domain_detection_records"] = rows(
connection,
"""
SELECT
d.id::text AS detection_id,
d.analysis_run_id::text AS analysis_run_id,
d.dataset_id::text AS dataset_id,
p.name AS project_name,
ds.name AS dataset_name,
ds.source_name AS dataset_source_name,
ar.analysis_type,
ar.status AS analysis_status,
d.model_name,
d.model_version,
d.class_name,
d.confidence,
d.source_tile_path,
d.bbox_json,
ST_XMin(Box3D(d.geometry)) AS min_x,
ST_YMin(Box3D(d.geometry)) AS min_y,
ST_XMax(Box3D(d.geometry)) AS max_x,
ST_YMax(Box3D(d.geometry)) AS max_y,
d.created_at
FROM detections d
LEFT JOIN analysis_runs ar ON ar.id = d.analysis_run_id
LEFT JOIN datasets ds ON ds.id = d.dataset_id
LEFT JOIN projects p ON p.id = d.project_id
WHERE d.geometry IS NOT NULL
AND (
ST_XMin(Box3D(d.geometry)) < -180
OR ST_XMax(Box3D(d.geometry)) > 180
OR ST_YMin(Box3D(d.geometry)) < -90
OR ST_YMax(Box3D(d.geometry)) > 90
)
ORDER BY d.created_at, d.id
LIMIT 100
""",
)
report["database"]["confidence_integrity"] = rows(
connection,
"""
SELECT
(SELECT count(*) FROM detections WHERE confidence < 0 OR confidence > 1)::bigint
AS detections_outside_unit_interval,
(SELECT count(*) FROM segmentations
WHERE confidence IS NOT NULL AND (confidence < 0 OR confidence > 1))::bigint
AS segmentations_outside_unit_interval
""",
)
report["database"]["metric_nulls"] = rows(
connection,
"""
SELECT metric_key, count(*)::bigint AS total,
count(*) FILTER (WHERE metric_value IS NULL)::bigint AS null_values
FROM metrics GROUP BY metric_key ORDER BY metric_key
""",
)
report["database"]["model_run_summary"] = rows(
connection,
"""
SELECT analysis_type, status, coalesce(model_name, '<none>') AS model_name,
coalesce(model_version, '<none>') AS model_version, count(*)::bigint AS count
FROM analysis_runs
GROUP BY analysis_type, status, model_name, model_version
ORDER BY count(*) DESC, analysis_type
LIMIT 100
""",
)
report["database"]["source_summary"] = rows(
connection,
"""
SELECT coalesce(source_name, source, '<none>') AS source_name,
status, count(*)::bigint AS count,
count(*) FILTER (WHERE dataset_role = 'reference')::bigint AS reference_count
FROM datasets
GROUP BY coalesce(source_name, source, '<none>'), status
ORDER BY count(*) DESC, source_name
LIMIT 200
""",
)
path_records = []
for table, statement in PATH_QUERIES.items():
for row in rows(connection, statement):
path = normalize_path(row["storage_path"], storage_root)
path_records.append({
"table": table,
"id": row["id"],
"storage_path": row["storage_path"],
"resolved_path": str(path),
"exists": path.is_file() or path.is_dir(),
})
missing = [row for row in path_records if not row["exists"]]
report["storage_references"] = {
"checked_count": len(path_records),
"missing_count": len(missing),
"missing_records": missing[:500],
"records_truncated": len(missing) > 500,
"scope": "direct datasets, dataset_versions and exports storage_path columns",
}
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True, default=str))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,271 @@
from __future__ import annotations
import json
from pathlib import Path
from types import SimpleNamespace
from uuid import UUID
from geoalchemy2.shape import to_shape
from shapely.geometry import Point, box
import sys
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
BACKEND_ROOT = REPOSITORY_ROOT / "backend"
if str(BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(BACKEND_ROOT))
from app.core.config import Settings # noqa: E402
from app.schemas.area import AreaUpdate # noqa: E402
from app.services.coverage_registry_service import ( # noqa: E402
CoverageRegistryService,
SOURCE_DEFINITIONS,
)
from app.services.detection_service import DetectionService # noqa: E402
from app.services.vector_feature_service import VectorFeatureService # noqa: E402
FIXED_UUID = UUID("00000000-0000-4000-8000-000000000001")
def _dataset(*, layer: str, bbox_values: list[float], source_name: str = "spw_picc") -> SimpleNamespace:
return SimpleNamespace(
id=FIXED_UUID,
status="ready",
source_name=source_name,
reference_layer_name=layer,
source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": bbox_values},
provenance_metadata={},
observed_at=None,
source_version="forensic-reproduction",
resolution_json=None,
checksum_sha256="forensic-only",
)
def _coverage_cross_theme_contamination() -> dict:
definition = next(
item for item in SOURCE_DEFINITIONS if item.contract.source_name == "spw_geoportail"
)
selection = box(4.50, 50.50, 4.70, 50.60)
building = _dataset(layer="buildings", bbox_values=[4.55, 50.52, 4.56, 50.53])
road = _dataset(layer="roads", bbox_values=[4.50, 50.50, 4.70, 50.60])
matches, fully_covered = CoverageRegistryService._matching_datasets(
[building, road],
definition,
"buildings",
"wallonia",
selection,
)
observed_ids = [str(item.id) for item in matches]
reproduced = observed_ids == [str(building.id)] and fully_covered is True
return {
"id": "P1-COV-001",
"severity": "critical",
"source": "backend/app/services/coverage_registry_service.py:481-505",
"expected": "A small buildings partition remains partial; a roads bbox cannot complete buildings coverage.",
"observed": {"matched_dataset_ids": observed_ids, "fully_covered": fully_covered},
"reproduced": reproduced,
}
def _meter_buffer_as_degrees() -> dict:
geometry = Point(5.0, 51.0)
buffered = geometry.buffer(100.0)
bounds = [float(value) for value in buffered.bounds]
reproduced = round(bounds[2] - bounds[0], 6) == 200.0
return {
"id": "P1-CRS-001",
"severity": "critical",
"source": "backend/app/services/vector_operations_service.py:179-203",
"expected": "A 100 metre buffer is projected to a metric CRS and spans roughly hundreds of metres.",
"observed": {"bounds_epsg4326": bounds, "longitude_span_degrees": bounds[2] - bounds[0]},
"reproduced": reproduced,
}
def _lambert_feature_mislabeled() -> dict:
row = VectorFeatureService._feature_row(
FIXED_UUID,
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [150000.0, 210000.0]},
"properties": {},
},
0,
None,
)
assert row is not None
geometry = to_shape(row.geometry)
reproduced = (
int(row.geometry.srid) == 4326
and float(geometry.x) == 150000.0
and float(geometry.y) == 210000.0
)
return {
"id": "P1-CRS-002",
"severity": "critical",
"source": "backend/app/services/vector_feature_service.py:271-299",
"expected": "Non-WGS84 input is transformed to EPSG:4326 or rejected before persistence.",
"observed": {
"stored_srid": int(row.geometry.srid),
"stored_coordinates": [float(geometry.x), float(geometry.y)],
},
"reproduced": reproduced,
}
def _authority_spoof() -> dict:
selection = box(4.9, 50.9, 5.0, 51.0)
dataset = SimpleNamespace(
id=FIXED_UUID,
status="ready",
source_name="grb",
reference_layer_name="buildings",
source_metadata={
"coverage_zones": ["flanders"],
"bbox_epsg4326": [4.8, 50.8, 5.1, 51.1],
},
provenance_metadata={"provided_by": "caller"},
observed_at=None,
source_version="caller-provided",
resolution_json=None,
checksum_sha256="caller-provided",
crs="EPSG:4326",
)
item = CoverageRegistryService._resolve_item(
zone="flanders",
theme="buildings",
datasets=[dataset],
selection=selection,
)
authority = item.evidence[0].authority_level if item.evidence else None
reproduced = item.status == "operational" and authority == "authoritative"
return {
"id": "P1-AUTH-001",
"severity": "critical",
"source": (
"backend/app/api/routes/datasets.py:142-163; "
"backend/app/services/coverage_registry_service.py:463-559"
),
"expected": "Only server-attested source identities can produce authoritative operational coverage.",
"observed": {
"caller_controlled_source_name": dataset.source_name,
"coverage_status": item.status,
"reported_authority": authority,
},
"reproduced": reproduced,
}
def _mutable_name_model_scope() -> dict:
area = SimpleNamespace(name="Mol validation bypass", geometry=box(-75.0, 35.0, -74.9, 35.1))
dataset = SimpleNamespace(id=FIXED_UUID, area_id=FIXED_UUID)
class FakeSession:
@staticmethod
def get(_model, _identifier):
return area
settings = Settings().model_copy(
update={"yolo_validated_area_names": "Mol,Kempen"}
)
accepted = True
try:
DetectionService._validate_model_area_scope(FakeSession(), dataset, settings)
except Exception:
accepted = False
return {
"id": "P1-AI-001",
"severity": "critical",
"source": "backend/app/services/detection_service.py:223-232",
"expected": "Validation scope is bound to immutable geometry/source/checksum evidence.",
"observed": {
"area_name": area.name,
"geometry_bounds": [float(value) for value in area.geometry.bounds],
"accepted": accepted,
},
"reproduced": accepted,
}
def _mutable_name_legal_scope() -> dict:
selection = box(4.9, 50.9, 5.0, 51.0)
flemish_geometry = box(2.5, 50.7, 5.9, 51.5)
canonical = [SimpleNamespace(name="Flanders", geometry=flemish_geometry)]
renamed = [SimpleNamespace(name="Vlaanderen", geometry=flemish_geometry)]
canonical_zones, canonical_outside = CoverageRegistryService._intersected_zones(
canonical, selection
)
renamed_zones, renamed_outside = CoverageRegistryService._intersected_zones(
renamed, selection
)
reproduced = (
canonical_zones == ["flanders"]
and canonical_outside is False
and renamed_zones == []
and renamed_outside is True
)
return {
"id": "P1-COV-002",
"severity": "high",
"source": "backend/app/services/coverage_registry_service.py:56-65,425-447",
"expected": "Renaming an Area cannot change its legal coverage-zone identity.",
"observed": {
"canonical": {"zones": canonical_zones, "outside": canonical_outside},
"renamed_same_geometry": {"zones": renamed_zones, "outside": renamed_outside},
},
"reproduced": reproduced,
}
def _area_patch_ignores_geometry() -> dict:
payload = AreaUpdate.model_validate(
{
"name": "Renamed",
"geometry": {
"type": "Polygon",
"coordinates": [[[4.0, 50.0], [5.0, 50.0], [5.0, 51.0], [4.0, 50.0]]],
},
}
)
parsed = payload.model_dump()
reproduced = "geometry" not in parsed
return {
"id": "P1-API-001",
"severity": "high",
"source": "backend/app/schemas/area.py:15-17; backend/app/services/area_service.py:154-171",
"expected": "PATCH /areas/{area_id} either validates and applies geometry or rejects the field.",
"observed": {"parsed_payload": parsed, "geometry_silently_ignored": reproduced},
"reproduced": reproduced,
}
def main() -> int:
findings = [
_coverage_cross_theme_contamination(),
_meter_buffer_as_degrees(),
_lambert_feature_mislabeled(),
_authority_spoof(),
_mutable_name_model_scope(),
_mutable_name_legal_scope(),
_area_patch_ignores_geometry(),
]
reproduced_count = sum(bool(item["reproduced"]) for item in findings)
payload = {
"schema_version": 1,
"purpose": "Read-only deterministic reproductions of Phase-1 contract violations.",
"findings": findings,
"summary": {
"total": len(findings),
"reproduced": reproduced_count,
"all_reproduced": reproduced_count == len(findings),
},
}
print(json.dumps(payload, indent=2, sort_keys=True))
return 0 if reproduced_count == len(findings) else 1
if __name__ == "__main__":
raise SystemExit(main())
+365
View File
@@ -0,0 +1,365 @@
#!/usr/bin/env python3
"""Create a bounded, read-only GeoIntel Phase-1 accuracy baseline."""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import subprocess
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
SCHEMA_VERSION = 1
SKIP_DIRS = {".git", ".pytest_cache", ".ruff_cache", ".venv", "__pycache__", "dist", "node_modules"}
CODE_ROOTS = {
"backend": "backend/app",
"backend_tests": "backend/tests",
"frontend": "frontend/src",
"frontend_e2e": "frontend/e2e",
"root_tests": "tests",
"scripts": "scripts",
"migrations": "backend/alembic/versions",
}
CODE_SUFFIXES = {".py", ".ts", ".tsx", ".js", ".mjs", ".sh", ".ps1"}
ARTIFACT_ROOTS = ("models", "datasets", "data", "storage", "artifacts", "output")
MODEL_SUFFIXES = {".pt", ".pth", ".onnx", ".engine", ".safetensors"}
RASTER_SUFFIXES = {".tif", ".tiff", ".vrt", ".jp2"}
VECTOR_SUFFIXES = {".geojson", ".gpkg", ".shp", ".fgb"}
HASH_SUFFIXES = MODEL_SUFFIXES | RASTER_SUFFIXES | VECTOR_SUFFIXES | {
".json", ".yaml", ".yml", ".csv", ".txt", ".md", ".lock"
}
MARKERS = {
"fixture": re.compile(r"\bfixture\b", re.IGNORECASE),
"mock": re.compile(r"\bmock(?:ed|ing|s)?\b", re.IGNORECASE),
"placeholder": re.compile(r"\bplaceholder\b", re.IGNORECASE),
"heuristic": re.compile(r"\bheuristic(?:s)?\b", re.IGNORECASE),
"not_configured": re.compile(r"\bnot_configured\b", re.IGNORECASE),
"todo": re.compile(r"\bTODO\b"),
"fallback": re.compile(r"\b(?:fallback|fall back)\b", re.IGNORECASE),
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1])
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--max-hash-bytes", type=int, default=64 * 1024 * 1024)
return parser.parse_args()
def now() -> str:
return datetime.now(timezone.utc).isoformat()
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def write_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
def git(repo: Path, *arguments: str) -> str:
result = subprocess.run(
["git", *arguments],
cwd=repo,
check=True,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
)
return result.stdout
def iter_files(root: Path) -> Iterable[Path]:
if not root.is_dir():
return
for path in sorted(root.rglob("*")):
if path.is_file() and not any(part in SKIP_DIRS for part in path.parts):
yield path
def line_count(path: Path) -> int:
with path.open("r", encoding="utf-8", errors="replace") as handle:
return sum(1 for _ in handle)
def code_inventory(repo: Path) -> dict[str, Any]:
groups: dict[str, dict[str, int]] = {}
for name, relative in CODE_ROOTS.items():
files = [path for path in iter_files(repo / relative) if path.suffix.lower() in CODE_SUFFIXES]
groups[name] = {
"file_count": len(files),
"line_count": sum(line_count(path) for path in files),
}
pytest_pattern = re.compile(r"^\s*(?:async\s+)?def\s+test_", re.MULTILINE)
route_pattern = re.compile(r"@router\.(?:get|post|put|patch|delete)\s*\(")
pytest_count = 0
route_count = 0
frontend_test_count = 0
for base in (repo / "backend/tests", repo / "tests"):
for path in iter_files(base):
if path.suffix == ".py":
pytest_count += len(pytest_pattern.findall(path.read_text(encoding="utf-8", errors="replace")))
for path in iter_files(repo / "backend/app/api/routes"):
if path.suffix == ".py":
route_count += len(route_pattern.findall(path.read_text(encoding="utf-8", errors="replace")))
for path in iter_files(repo / "frontend"):
if ".test." in path.name.lower() or ".spec." in path.name.lower():
frontend_test_count += 1
return {
"groups": groups,
"pytest_test_function_count": pytest_count,
"frontend_test_file_count": frontend_test_count,
"api_route_decorator_count": route_count,
}
def migration_inventory(repo: Path) -> dict[str, Any]:
revision_re = re.compile(r'^revision\s*(?::[^=]+)?=\s*["\x27]([^"\x27]+)["\x27]', re.MULTILINE)
down_re = re.compile(
r'^down_revision\s*(?::[^=]+)?=\s*(?:["\x27]([^"\x27]+)["\x27]|None)',
re.MULTILINE,
)
rows = []
for path in iter_files(repo / "backend/alembic/versions"):
if path.suffix != ".py":
continue
text = path.read_text(encoding="utf-8", errors="replace")
revision = revision_re.search(text)
down = down_re.search(text)
rows.append({
"path": path.relative_to(repo).as_posix(),
"revision": revision.group(1) if revision else None,
"down_revision": down.group(1) if down and down.group(1) else None,
})
revisions = {row["revision"] for row in rows if row["revision"]}
parents = {row["down_revision"] for row in rows if row["down_revision"]}
return {
"count": len(rows),
"records": rows,
"heads_from_static_chain": sorted(revisions - parents),
"missing_revision_identifiers": sum(row["revision"] is None for row in rows),
}
def mirror_inventory(repo: Path, tracked: set[str]) -> dict[str, Any]:
identical = 0
different = []
for relative in sorted(item for item in tracked if not item.startswith("geointel/")):
mirror_relative = f"geointel/{relative}"
if mirror_relative not in tracked:
continue
source = repo / relative
mirror = repo / mirror_relative
if not source.is_file() or not mirror.is_file():
continue
source_hash = sha256_file(source)
mirror_hash = sha256_file(mirror)
if source_hash == mirror_hash:
identical += 1
else:
different.append({
"path": relative,
"root_sha256": source_hash,
"mirror_sha256": mirror_hash,
"root_size_bytes": source.stat().st_size,
"mirror_size_bytes": mirror.stat().st_size,
})
return {
"tracked_mirror_file_count": sum(item.startswith("geointel/") for item in tracked),
"paired_identical_file_count": identical,
"paired_different_file_count": len(different),
"different_files": different,
"risk": (
"The tracked geointel/ repository mirror can create ambiguous imports, stale tests "
"and local/deployment drift; Docker excludes it but local tools may not."
),
}
def marker_inventory(repo: Path) -> dict[str, Any]:
counts: Counter[str] = Counter()
examples: dict[str, list[dict[str, Any]]] = defaultdict(list)
for base in (repo / "backend/app", repo / "frontend/src", repo / "scripts"):
for path in iter_files(base):
if path.suffix.lower() not in CODE_SUFFIXES:
continue
relative = path.relative_to(repo).as_posix()
for number, text in enumerate(
path.read_text(encoding="utf-8", errors="replace").splitlines(), start=1
):
for name, pattern in MARKERS.items():
if pattern.search(text):
counts[name] += 1
if len(examples[name]) < 25:
examples[name].append({
"path": relative,
"line": number,
"text": text.strip()[:240],
})
return {
"counts": dict(sorted(counts.items())),
"examples": dict(sorted(examples.items())),
"interpretation": "Triage signals only; production impact requires a traced contract/runtime path.",
}
def artifact_role(path: Path) -> str:
suffix = path.suffix.lower()
name = path.name.lower()
if suffix in MODEL_SUFFIXES:
return "model_checkpoint"
if "manifest" in name or name in {"dataset.yaml", "data.yaml"}:
return "manifest"
if any(token in name for token in ("audit", "evaluation", "assessment", "metric", "report")):
return "evaluation_or_audit"
if any(token in name for token in ("contact_sheet", "review")) or suffix in {".png", ".jpg", ".jpeg"}:
return "visual_review"
if suffix in RASTER_SUFFIXES:
return "raster"
if suffix in VECTOR_SUFFIXES:
return "vector"
if suffix in {".db", ".sqlite", ".sqlite3", ".wal", ".shm"} or ".db-" in name:
return "database_runtime_state"
if suffix == ".txt" and "label" in path.as_posix().lower():
return "label"
return "other"
def artifact_inventory(
repo: Path,
tracked: set[str],
output: Path,
max_hash_bytes: int,
) -> dict[str, Any]:
records = []
output = output.resolve()
for root_name in ARTIFACT_ROOTS:
for path in iter_files(repo / root_name):
resolved = path.resolve()
if resolved == output or output in resolved.parents:
continue
stat = path.stat()
relative = path.relative_to(repo).as_posix()
can_hash = stat.st_size <= max_hash_bytes and path.suffix.lower() in HASH_SUFFIXES
records.append({
"path": relative,
"role": artifact_role(path),
"size_bytes": stat.st_size,
"modified_at": datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat(),
"tracked": relative in tracked,
"sha256": sha256_file(path) if can_hash else None,
"hash_omission_reason": None if can_hash else "suffix_or_size_limit",
})
roles = Counter(row["role"] for row in records)
roots = Counter(row["path"].split("/", 1)[0] for row in records)
return {
"roots": list(ARTIFACT_ROOTS),
"file_count": len(records),
"total_size_bytes": sum(row["size_bytes"] for row in records),
"role_counts": dict(sorted(roles.items())),
"root_counts": dict(sorted(roots.items())),
"model_checkpoint_count": roles.get("model_checkpoint", 0),
"records": records,
"limitations": [
"Ignored Tower corpora and mounted model volumes can be absent locally.",
"Large/non-evidence files are inventoried without a SHA-256 above the configured ceiling.",
"Near-duplicate imagery and semantic label quality need dedicated corpus checks.",
],
}
def main() -> int:
args = parse_args()
repo = args.repo_root.expanduser().resolve()
output = args.output_dir.expanduser()
output = output.resolve() if output.is_absolute() else (repo / output).resolve()
if not (repo / ".git").exists():
raise SystemExit(f"Not a Git repository root: {repo}")
if output == repo:
raise SystemExit("Output directory must not equal the repository root")
tracked = set(filter(None, git(repo, "ls-files").splitlines()))
repository = {
"schema_version": SCHEMA_VERSION,
"generated_at": now(),
"repo_root": str(repo),
"git": {
"branch": git(repo, "branch", "--show-current").strip(),
"head": git(repo, "rev-parse", "HEAD").strip(),
"status_porcelain": [line for line in git(repo, "status", "--short").splitlines() if line],
"tracked_file_count": len(tracked),
"top_level_tracked_counts": dict(sorted(Counter(
item.split("/", 1)[0] for item in tracked
).items())),
},
"code": code_inventory(repo),
"migrations": migration_inventory(repo),
"tracked_mirror": mirror_inventory(repo, tracked),
}
signals = marker_inventory(repo)
artifacts = artifact_inventory(repo, tracked, output, args.max_hash_bytes)
findings = []
mirror = repository["tracked_mirror"]
if mirror["tracked_mirror_file_count"]:
findings.append({
"id": "P1-REPO-001",
"severity": "high",
"title": "Tracked nested repository mirror creates ambiguous source state",
"evidence": {
"tracked_mirror_file_count": mirror["tracked_mirror_file_count"],
"paired_different_file_count": mirror["paired_different_file_count"],
},
})
if artifacts["model_checkpoint_count"] == 0:
findings.append({
"id": "P1-ML-LOCAL-001",
"severity": "info",
"title": "No local checkpoint is available in the repository checkout",
"interpretation": "Production model truth must be verified on the mounted Tower volume.",
})
summary = {
"schema_version": SCHEMA_VERSION,
"generated_at": now(),
"status": "findings_present" if findings else "no_static_findings",
"finding_count": len(findings),
"findings": findings,
"baseline_scope": [
"tracked repository state",
"static code/test/migration inventory",
"tracked mirror comparison",
"local artifact inventory",
"mock/fixture/placeholder/fallback triage signals",
],
"separate_required_evidence": [
"Tower database/storage audit",
"Tower CUDA/model preflight and representative inference",
"corpus leakage/duplicate/label/time review",
"independent human visual review",
],
}
write_json(output / "repository-inventory.json", repository)
write_json(output / "local-artifact-inventory.json", artifacts)
write_json(output / "static-risk-signals.json", signals)
write_json(output / "phase1-baseline-summary.json", summary)
print(json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+128
View File
@@ -0,0 +1,128 @@
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
from typing import Any
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_MANIFEST = (
REPOSITORY_ROOT / "artifacts" / "evidence" / "accuracy" / "P1" / "evidence-manifest.json"
)
REQUIRED_DOCUMENTS = tuple(
REPOSITORY_ROOT / "docs" / "accuracy-program" / f"{index:02d}-{name}.md"
for index, name in (
(0, "execution-contract"),
(1, "system-inventory"),
(2, "data-lineage"),
(3, "baseline-and-gaps"),
(4, "risk-register"),
(5, "metric-framework"),
(6, "implementation-roadmap"),
)
)
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def verify_record(item: dict[str, Any]) -> list[str]:
errors: list[str] = []
relative = item.get("path")
if not isinstance(relative, str):
return ["manifest record has no string path"]
path = REPOSITORY_ROOT / relative
if not path.is_file():
return [f"missing file: {relative}"]
expected_size = item.get("size_bytes")
if path.stat().st_size != expected_size:
errors.append(
f"size mismatch for {relative}: expected {expected_size}, got {path.stat().st_size}"
)
expected_hash = item.get("sha256")
actual_hash = sha256_file(path)
if actual_hash != expected_hash:
errors.append(
f"sha256 mismatch for {relative}: expected {expected_hash}, got {actual_hash}"
)
return errors
def main() -> int:
parser = argparse.ArgumentParser(description="Verify GeoIntel Accuracy P1 evidence and program hashes.")
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
args = parser.parse_args()
manifest_path = args.manifest.expanduser().resolve()
if not manifest_path.is_file():
parser.error(f"evidence manifest does not exist: {manifest_path}")
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
errors: list[str] = []
evidence_records = payload.get("evidence_files")
program_records = payload.get("program_files")
if not isinstance(evidence_records, list) or not isinstance(program_records, list):
errors.append("manifest must contain evidence_files and program_files arrays")
evidence_records = []
program_records = []
for item in [*evidence_records, *program_records]:
if not isinstance(item, dict):
errors.append("manifest file record is not an object")
continue
errors.extend(verify_record(item))
evidence_root = REPOSITORY_ROOT / str(payload.get("evidence_root") or "")
listed = {
str(item.get("path"))
for item in evidence_records
if isinstance(item, dict) and isinstance(item.get("path"), str)
}
current = {
path.relative_to(REPOSITORY_ROOT).as_posix()
for path in evidence_root.rglob("*")
if path.is_file() and path.resolve() != manifest_path
}
for relative in sorted(current - listed):
errors.append(f"unlisted evidence file: {relative}")
for relative in sorted(listed - current):
errors.append(f"listed evidence file no longer exists: {relative}")
for document in REQUIRED_DOCUMENTS:
if not document.is_file() or document.stat().st_size == 0:
errors.append(f"required document missing or empty: {document.relative_to(REPOSITORY_ROOT)}")
status_path = REPOSITORY_ROOT / "docs" / "accuracy-program" / "status.json"
if not status_path.is_file():
errors.append("required status.json is missing")
else:
status = json.loads(status_path.read_text(encoding="utf-8"))
if status.get("phase1", {}).get("status") != "complete":
errors.append("status.json must mark Phase 1 complete")
if status.get("release", {}).get("status") != "blocked":
errors.append("status.json must keep release blocked")
if status.get("release", {}).get("promotion_allowed") is not False:
errors.append("status.json must keep promotion disallowed")
if status.get("scope", {}).get("national_building_validation") is not False:
errors.append("status.json must not claim national building validation")
summary = {
"schema_version": 1,
"status": "passed" if not errors else "failed",
"manifest": manifest_path.relative_to(REPOSITORY_ROOT).as_posix(),
"evidence_files_checked": len(evidence_records),
"program_files_checked": len(program_records),
"errors": errors,
}
print(json.dumps(summary, indent=2, sort_keys=True))
return 0 if not errors else 1
if __name__ == "__main__":
raise SystemExit(main())
+72
View File
@@ -0,0 +1,72 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / "scripts" / "run_accuracy_phase1_baseline.py"
SPEC = importlib.util.spec_from_file_location("accuracy_phase1_baseline", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
INFERENCE_SCRIPT = ROOT / "scripts" / "collect_accuracy_phase1_inference_smoke.py"
INFERENCE_SPEC = importlib.util.spec_from_file_location("accuracy_phase1_inference", INFERENCE_SCRIPT)
assert INFERENCE_SPEC and INFERENCE_SPEC.loader
INFERENCE_MODULE = importlib.util.module_from_spec(INFERENCE_SPEC)
INFERENCE_SPEC.loader.exec_module(INFERENCE_MODULE)
def test_sha256_file_is_stable(tmp_path: Path) -> None:
artifact = tmp_path / "manifest.json"
artifact.write_bytes(b'{"version":1}\n')
assert MODULE.sha256_file(artifact) == "50208d78350a7a160dec59a82df1499b6ca7da33e54c5eb11c97e342118e68bb"
def test_mirror_inventory_distinguishes_identical_and_drifted_files(tmp_path: Path) -> None:
(tmp_path / "geointel").mkdir()
(tmp_path / "same.txt").write_text("same", encoding="utf-8")
(tmp_path / "geointel" / "same.txt").write_text("same", encoding="utf-8")
(tmp_path / "drift.txt").write_text("root", encoding="utf-8")
(tmp_path / "geointel" / "drift.txt").write_text("mirror", encoding="utf-8")
tracked = {"same.txt", "drift.txt", "geointel/same.txt", "geointel/drift.txt"}
report = MODULE.mirror_inventory(tmp_path, tracked)
assert report["tracked_mirror_file_count"] == 2
assert report["paired_identical_file_count"] == 1
assert report["paired_different_file_count"] == 1
assert report["different_files"][0]["path"] == "drift.txt"
def test_artifact_roles_do_not_claim_images_are_models() -> None:
assert MODULE.artifact_role(Path("candidate.pt")) == "model_checkpoint"
assert MODULE.artifact_role(Path("split-manifest.json")) == "manifest"
assert MODULE.artifact_role(Path("contact_sheet_001.png")) == "visual_review"
assert MODULE.artifact_role(Path("orthophoto.tif")) == "raster"
def test_inference_summary_is_explicit_for_empty_and_non_empty_outputs() -> None:
assert INFERENCE_MODULE._summarize_detections([]) == {
"count": 0,
"class_counts": {},
"confidence": {"minimum": None, "maximum": None, "mean": None},
"sample": [],
}
detections = [
{"class_name": "building", "confidence": 0.8, "bbox": [1, 2, 3, 4]},
{"class_name": "building", "confidence": 0.4, "bbox": [5, 6, 7, 8]},
{"class_name": "shed", "confidence": 0.6, "bbox": [9, 10, 11, 12]},
]
summary = INFERENCE_MODULE._summarize_detections(detections)
assert summary["count"] == 3
assert summary["class_counts"] == {"building": 2, "shed": 1}
assert summary["confidence"] == {
"minimum": 0.4,
"maximum": 0.8,
"mean": 0.6,
}
assert summary["sample"] == detections