audit: establish accuracy phase 1 baseline

This commit is contained in:
Jens
2026-08-01 18:57:38 +02:00
parent 0c019bb22f
commit 3d442ef43f
48 changed files with 15390 additions and 1 deletions
@@ -0,0 +1,663 @@
{
"counts": {
"fallback": 27,
"fixture": 94,
"heuristic": 1,
"mock": 31,
"not_configured": 24,
"placeholder": 39,
"todo": 2
},
"examples": {
"fallback": [
{
"line": 1046,
"path": "backend/app/services/export_service.py",
"text": "def _filename(name: str | None, fallback: str, suffix: str) -> str:"
},
{
"line": 1047,
"path": "backend/app/services/export_service.py",
"text": "raw_name = name or fallback"
},
{
"line": 1050,
"path": "backend/app/services/export_service.py",
"text": "cleaned = fallback"
},
{
"line": 29,
"path": "backend/app/services/mdk_bathymetry_acquisition_service.py",
"text": "No depth values are ever synthesized, no insecure TLS fallback exists and the"
},
{
"line": 173,
"path": "backend/app/services/mdk_bathymetry_probe_service.py",
"text": "message=\"MDK WCS TLS certificate validation failed; insecure fallback is prohibited.\","
},
{
"line": 186,
"path": "backend/app/services/mdk_bathymetry_probe_service.py",
"text": "\"MDK WCS TLS certificate validation failed; insecure fallback is prohibited.\""
},
{
"line": 440,
"path": "backend/app/services/raster_operations_service.py",
"text": "# best effort fallback; keep computed dimensions."
},
{
"line": 19,
"path": "backend/app/services/raster_service.py",
"text": "except Exception as exc: # pragma: no cover - exercised via API-level fallback tests"
},
{
"line": 26,
"path": "backend/app/services/storage_service.py",
"text": "fallback = \"upload\""
},
{
"line": 28,
"path": "backend/app/services/storage_service.py",
"text": "return fallback"
},
{
"line": 37,
"path": "backend/app/services/storage_service.py",
"text": "return cleaned or fallback"
},
{
"line": 76,
"path": "frontend/src/hooks/useGeoAssistant.ts",
"text": "const fallback = result.default_model && result.items.some((model) => model.name === result.default_model)"
},
{
"line": 79,
"path": "frontend/src/hooks/useGeoAssistant.ts",
"text": "setDefaultModel(fallback)"
},
{
"line": 9,
"path": "frontend/src/lib/authError.ts",
"text": "export function formatAuthError(error: unknown, fallback: string): string {"
},
{
"line": 10,
"path": "frontend/src/lib/authError.ts",
"text": "if (!(error instanceof Error)) return fallback"
},
{
"line": 21,
"path": "frontend/src/lib/authError.ts",
"text": "return fallback"
},
{
"line": 24,
"path": "frontend/src/lib/authError.ts",
"text": "return error.message || fallback"
},
{
"line": 1,
"path": "frontend/src/lib/formatError.ts",
"text": "export function formatError(error: unknown, fallback: string): string {"
},
{
"line": 6,
"path": "frontend/src/lib/formatError.ts",
"text": "return fallback"
},
{
"line": 501,
"path": "scripts/provision_belgium_north_sea_scope.py",
"text": "def _max_admin_modification(payloads: dict[str, dict[str, Any]], fallback: str) -> str:"
},
{
"line": 507,
"path": "scripts/provision_belgium_north_sea_scope.py",
"text": "value = max(values) if values else fallback"
},
{
"line": 111,
"path": "scripts/refine_yolo_labels_with_sam.py",
"text": "parser.add_argument(\"--fallback-policy\", choices=(\"retain\", \"drop\"), default=\"retain\")"
},
{
"line": 42,
"path": "scripts/run_accuracy_phase1_baseline.py",
"text": "\"fallback\": re.compile(r\"\\b(?:fallback|fall back)\\b\", re.IGNORECASE),"
},
{
"line": 347,
"path": "scripts/run_accuracy_phase1_baseline.py",
"text": "\"mock/fixture/placeholder/fallback triage signals\","
},
{
"line": 52,
"path": "scripts/run_golden_qa_benchmark.py",
"text": "# Backwards-compatible fallback for older checkouts and local smoke scripts."
}
],
"fixture": [
{
"line": 3,
"path": "backend/app/providers/__init__.py",
"text": "from app.providers import base, fixture, grb, manual, osm, registry"
},
{
"line": 5,
"path": "backend/app/providers/__init__.py",
"text": "__all__ = [\"base\", \"fixture\", \"grb\", \"manual\", \"osm\", \"registry\"]"
},
{
"line": 9,
"path": "backend/app/providers/fixture.py",
"text": "provider_name=\"fixture\","
},
{
"line": 10,
"path": "backend/app/providers/fixture.py",
"text": "display_name=\"Fixture data\","
},
{
"line": 11,
"path": "backend/app/providers/fixture.py",
"text": "authority_level=\"fixture\","
},
{
"line": 14,
"path": "backend/app/providers/fixture.py",
"text": "supported_query_modes=[\"fixture\"],"
},
{
"line": 15,
"path": "backend/app/providers/fixture.py",
"text": "fetch_signature=\"tests/fixtures and demo fixture upload flow\","
},
{
"line": 16,
"path": "backend/app/providers/fixture.py",
"text": "limitation_message=\"Fixture provider represents local test/demo fixtures only.\","
},
{
"line": 7,
"path": "backend/app/providers/registry.py",
"text": "from app.providers.fixture import FixtureProvider"
},
{
"line": 37,
"path": "backend/app/providers/registry.py",
"text": "\"fixture\": FixtureProvider(),"
},
{
"line": 112,
"path": "backend/app/providers/registry.py",
"text": "provider_name=\"fixture\","
},
{
"line": 114,
"path": "backend/app/providers/registry.py",
"text": "message=\"Fixture provider data must use checked-in demo/test fixture flows.\","
},
{
"line": 122,
"path": "backend/app/services/demo_workflow_service.py",
"text": ".filter(Dataset.source_name == \"fixture\")"
},
{
"line": 129,
"path": "backend/app/services/demo_workflow_service.py",
"text": ".filter(Dataset.source_name == \"fixture\")"
},
{
"line": 148,
"path": "backend/app/services/demo_workflow_service.py",
"text": ".filter(Dataset.source_name == \"fixture\")"
},
{
"line": 213,
"path": "backend/app/services/demo_workflow_service.py",
"text": "source=\"fixture\","
},
{
"line": 218,
"path": "backend/app/services/demo_workflow_service.py",
"text": "\"fixture\": True,"
},
{
"line": 298,
"path": "backend/app/services/demo_workflow_service.py",
"text": "source=\"fixture\","
},
{
"line": 300,
"path": "backend/app/services/demo_workflow_service.py",
"text": "source_name=\"fixture\","
},
{
"line": 303,
"path": "backend/app/services/demo_workflow_service.py",
"text": "\"fixture\": True,"
},
{
"line": 423,
"path": "backend/app/services/demo_workflow_service.py",
"text": ".filter(Dataset.source_name == \"fixture\")"
},
{
"line": 430,
"path": "backend/app/services/demo_workflow_service.py",
"text": ".filter(Dataset.source_name == \"fixture\")"
},
{
"line": 472,
"path": "backend/app/services/demo_workflow_service.py",
"text": "description=\"Offline fixture workflow: reference buildings, predicted buildings and persisted QA metrics.\","
},
{
"line": 497,
"path": "backend/app/services/demo_workflow_service.py",
"text": "source_name=\"fixture\","
},
{
"line": 509,
"path": "backend/app/services/demo_workflow_service.py",
"text": "source_name=\"fixture\","
}
],
"heuristic": [
{
"line": 39,
"path": "scripts/run_accuracy_phase1_baseline.py",
"text": "\"heuristic\": re.compile(r\"\\bheuristic(?:s)?\\b\", re.IGNORECASE),"
}
],
"mock": [
{
"line": 7,
"path": "frontend/src/components/auth/LandingPage.test.tsx",
"text": "vi.mock('../../services/api/auth', () => ({"
},
{
"line": 34,
"path": "frontend/src/components/auth/LandingPage.test.tsx",
"text": "vi.mocked(login).mockReset()"
},
{
"line": 35,
"path": "frontend/src/components/auth/LandingPage.test.tsx",
"text": "vi.mocked(loginAsGuest).mockReset()"
},
{
"line": 42,
"path": "frontend/src/components/auth/LandingPage.test.tsx",
"text": "vi.mocked(login).mockResolvedValue(operatorSession)"
},
{
"line": 57,
"path": "frontend/src/components/auth/LandingPage.test.tsx",
"text": "vi.mocked(loginAsGuest).mockResolvedValue(guestSession)"
},
{
"line": 75,
"path": "frontend/src/components/auth/LandingPage.test.tsx",
"text": "vi.mocked(login).mockRejectedValue(new Error('Gebruikersnaam of wachtwoord is onjuist.'))"
},
{
"line": 8,
"path": "frontend/src/components/map/MunicipalitySearch.test.tsx",
"text": "vi.mock('../../services/api/areas', () => ({"
},
{
"line": 14,
"path": "frontend/src/components/map/MunicipalitySearch.test.tsx",
"text": "vi.mocked(areasApi.searchMunicipalities).mockResolvedValue({"
},
{
"line": 5,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "const mocks = vi.hoisted(() => ({"
},
{
"line": 9,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "vi.mock('../services/api', () => ({"
},
{
"line": 11,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "resolveCoverage: mocks.resolveCoverage,"
},
{
"line": 35,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "mocks.resolveCoverage.mockResolvedValue(coverageResult)"
},
{
"line": 51,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "expect(mocks.resolveCoverage).not.toHaveBeenCalled()"
},
{
"line": 58,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "expect(mocks.resolveCoverage).not.toHaveBeenCalled()"
},
{
"line": 64,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "expect(mocks.resolveCoverage).toHaveBeenCalledWith({"
},
{
"line": 89,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "mocks.resolveCoverage.mockRejectedValueOnce(new Error('provider unavailable'))"
},
{
"line": 102,
"path": "frontend/src/hooks/useCoverageResolver.test.tsx",
"text": "mocks.resolveCoverage.mockImplementationOnce("
},
{
"line": 5,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "const mocks = vi.hoisted(() => ({"
},
{
"line": 9,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "vi.mock('../services/api/temporal', () => ({"
},
{
"line": 11,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "compare: mocks.compare,"
},
{
"line": 34,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "expect(mocks.compare).not.toHaveBeenCalled()"
},
{
"line": 43,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "expect(mocks.compare).not.toHaveBeenCalled()"
},
{
"line": 53,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "mocks.compare.mockResolvedValueOnce(comparison)"
},
{
"line": 64,
"path": "frontend/src/hooks/useTemporalComparison.test.tsx",
"text": "expect(mocks.compare).toHaveBeenCalledWith('project-1', {"
},
{
"line": 37,
"path": "scripts/run_accuracy_phase1_baseline.py",
"text": "\"mock\": re.compile(r\"\\bmock(?:ed|ing|s)?\\b\", re.IGNORECASE),"
}
],
"not_configured": [
{
"line": 166,
"path": "backend/app/api/routes/external.py",
"text": "status=response.get(\"status\", \"not_configured\"),"
},
{
"line": 149,
"path": "backend/app/api/routes/health.py",
"text": "yolo_status = configured_yolo.status if configured_yolo else \"not_configured\""
},
{
"line": 79,
"path": "backend/app/providers/base.py",
"text": "status=\"configured\" if self.is_configured else \"not_configured\","
},
{
"line": 94,
"path": "backend/app/providers/base.py",
"text": "\"status\": \"not_configured\","
},
{
"line": 96,
"path": "backend/app/providers/registry.py",
"text": "status=\"not_configured\","
},
{
"line": 9,
"path": "backend/app/schemas/coverage.py",
"text": "CoverageStatus = Literal[\"operational\", \"partial\", \"not_configured\", \"unsupported\"]"
},
{
"line": 53,
"path": "backend/app/services/coverage_registry_service.py",
"text": "STATUS_ORDER = (\"unsupported\", \"not_configured\", \"partial\", \"operational\")"
},
{
"line": 340,
"path": "backend/app/services/coverage_registry_service.py",
"text": "integration_status=\"not_configured\","
},
{
"line": 572,
"path": "backend/app/services/coverage_registry_service.py",
"text": "\"not_configured\""
},
{
"line": 266,
"path": "backend/app/services/geo_assistant_service.py",
"text": "status=\"not_configured\","
},
{
"line": 42,
"path": "backend/app/services/model_registry_service.py",
"text": "status=\"not_configured\","
},
{
"line": 98,
"path": "backend/app/services/model_registry_service.py",
"text": "status=\"not_configured\","
},
{
"line": 127,
"path": "backend/app/services/model_registry_service.py",
"text": "status = \"not_configured\""
},
{
"line": 169,
"path": "backend/app/services/model_registry_service.py",
"text": "status = \"not_configured\""
},
{
"line": 211,
"path": "backend/app/services/model_registry_service.py",
"text": "status = \"not_configured\""
},
{
"line": 36,
"path": "backend/app/services/yolo_preflight_service.py",
"text": "\"status\": \"not_configured\","
},
{
"line": 40,
"path": "frontend/src/components/detection/DetectionModelManagement.tsx",
"text": "if (value === 'not_configured') return 'niet geconfigureerd'"
},
{
"line": 295,
"path": "frontend/src/components/map/MapWorkspace.tsx",
"text": "not_configured: 'Niet gekoppeld',"
},
{
"line": 1050,
"path": "frontend/src/components/map/MapWorkspace.tsx",
"text": "{ operational: 0, partial: 0, not_configured: 0, unsupported: 0 },"
},
{
"line": 1051,
"path": "frontend/src/components/map/MapWorkspace.tsx",
"text": ") ?? { operational: 0, partial: 0, not_configured: 0, unsupported: 0 },"
},
{
"line": 3365,
"path": "frontend/src/components/map/MapWorkspace.tsx",
"text": "<span>{coverageCounts.not_configured} niet gekoppeld</span>"
},
{
"line": 6,
"path": "frontend/src/components/models/modelOptions.ts",
"text": "const configured = model.configured && model.status !== 'not_configured'"
},
{
"line": 1072,
"path": "frontend/src/types.ts",
"text": "export type CoverageStatus = 'operational' | 'partial' | 'not_configured' | 'unsupported'"
},
{
"line": 40,
"path": "scripts/run_accuracy_phase1_baseline.py",
"text": "\"not_configured\": re.compile(r\"\\bnot_configured\\b\", re.IGNORECASE),"
}
],
"placeholder": [
{
"line": 36,
"path": "backend/app/services/model_registry_service.py",
"text": "model_id=\"yolo-placeholder\","
},
{
"line": 37,
"path": "backend/app/services/model_registry_service.py",
"text": "display_name=\"YOLO detector placeholder\","
},
{
"line": 92,
"path": "backend/app/services/model_registry_service.py",
"text": "model_id=\"segmentation-placeholder\","
},
{
"line": 93,
"path": "backend/app/services/model_registry_service.py",
"text": "display_name=\"Segmentation placeholder\","
},
{
"line": 94,
"path": "backend/app/services/model_registry_service.py",
"text": "framework=\"placeholder\","
},
{
"line": 99,
"path": "backend/app/services/model_registry_service.py",
"text": "limitation_message=\"Segmentation inference is not configured for this placeholder; no model is downloaded or executed.\","
},
{
"line": 187,
"path": "frontend/src/components/assistant/GeoAssistantPanel.tsx",
"text": "placeholder=\"Bijvoorbeeld: hoeveel bos verdween er sinds 2013?\""
},
{
"line": 246,
"path": "frontend/src/components/datasets/DatasetPanel.tsx",
"text": "placeholder=\"bijvoorbeeld eigen luchtbeeld\""
},
{
"line": 306,
"path": "frontend/src/components/datasets/DatasetPanel.tsx",
"text": "placeholder=\"Naam, bron, gemeente of type\""
},
{
"line": 276,
"path": "frontend/src/components/datasets/RasterControls.tsx",
"text": "placeholder=\"EPSG:31370\""
},
{
"line": 345,
"path": "frontend/src/components/datasets/RasterControls.tsx",
"text": "placeholder=\"optioneel\""
},
{
"line": 44,
"path": "frontend/src/components/detection/DetectionLab.tsx",
"text": "if (modelName === 'yolo-placeholder') return 'Niet-geconfigureerd gebouwmodel'"
},
{
"line": 516,
"path": "frontend/src/components/detection/DetectionLab.tsx",
"text": "placeholder=\"Pad naar de aangemaakte beeldtegels\""
},
{
"line": 583,
"path": "frontend/src/components/detection/DetectionLab.tsx",
"text": "placeholder=\"0.50 0.25 0.15\""
},
{
"line": 715,
"path": "frontend/src/components/detection/DetectionLab.tsx",
"text": "placeholder=\"bijvoorbeeld gebouw\""
},
{
"line": 28,
"path": "frontend/src/components/detection/DetectionModelManagement.tsx",
"text": "if (model.model_id === 'yolo-placeholder') return 'Gebouwmodel nog niet geconfigureerd'"
},
{
"line": 402,
"path": "frontend/src/components/exports/ExportCenter.tsx",
"text": "placeholder=\"Type, id of pad\""
},
{
"line": 2312,
"path": "frontend/src/components/map/MapWorkspace.tsx",
"text": "placeholder=\"Zoek thema’s\""
},
{
"line": 82,
"path": "frontend/src/components/map/MunicipalitySearch.tsx",
"text": "placeholder=\"Gemeentenaam of NIS-code\""
},
{
"line": 98,
"path": "frontend/src/components/project/AreaPanel.tsx",
"text": "placeholder=\"Naam van het gebied\""
},
{
"line": 106,
"path": "frontend/src/components/project/AreaPanel.tsx",
"text": "placeholder=\"EPSG:4326\""
},
{
"line": 142,
"path": "frontend/src/components/project/AreaPanel.tsx",
"text": "placeholder=\"Bijvoorbeeld Brussel, Namen of Noordzee\""
},
{
"line": 119,
"path": "frontend/src/components/project/ProjectPanel.tsx",
"text": "placeholder=\"Naam van de werkruimte\""
},
{
"line": 127,
"path": "frontend/src/components/project/ProjectPanel.tsx",
"text": "placeholder=\"Korte beschrijving\""
},
{
"line": 135,
"path": "frontend/src/components/project/ProjectPanel.tsx",
"text": "placeholder=\"Regio\""
}
],
"todo": [
{
"line": 23,
"path": "scripts/codex_pass_end_check.sh",
"text": "\"TODO: implement later\\|placeholder only\\|fake completed\" \\"
},
{
"line": 6,
"path": "scripts/contract_drift_grep.sh",
"text": "if grep -R \"TODO: decide\\|TBD\\|placeholder only\\|fake real\" -n docs contracts backend frontend prompts 2>/dev/null; then"
}
]
},
"interpretation": "Triage signals only; production impact requires a traced contract/runtime path."
}