Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs(accuracy): refresh governed scan and training evidence

Browse Source
This commit is contained in:
Jens
2026-08-30 06:00:57 +02:00
parent c272220277
commit 0e3c1b20e9
27 changed files with 88787 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_accuracy_phase3_full_data_scan.py
+64 -3
View File
@@ -9,7 +9,13 @@ from pathlib import Path
SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "run_accuracy_phase3_full_data_scan.py"
def run_scan(repo: Path, output: Path, *, resume: bool = False) -> dict:
def run_scan(
repo: Path,
output: Path,
*,
resume: bool = False,
unreachable_scopes: tuple[str, ...] = (),
) -> dict:
command = [
sys.executable,
str(SCRIPT),
@@ -24,6 +30,8 @@ def run_scan(repo: Path, output: Path, *, resume: bool = False) -> dict:
]
if resume:
command.append("--resume")
for scope in unreachable_scopes:
command.extend(("--unreachable-scope", scope))
completed = subprocess.run(command, check=True, capture_output=True, text=True)
return json.loads(completed.stdout)
@@ -57,8 +65,18 @@ def test_phase3_scan_reconciles_and_resumes_deterministically(tmp_path: Path) ->
(data / "broken.tif").write_bytes(b"not a geotiff")
output = tmp_path / "evidence"
first = run_scan(tmp_path, output)
second = run_scan(tmp_path, output, resume=True)
unreachable_scopes = (
"external://tower-corpora=Tower corpora are not mounted in this fixture",
"external://mounted-model-volumes=Model volumes are not mounted in this fixture",
"external://production-postgis-or-api=Production database is not configured in this fixture",
)
first = run_scan(tmp_path, output, unreachable_scopes=unreachable_scopes)
second = run_scan(
tmp_path,
output,
resume=True,
unreachable_scopes=unreachable_scopes,
)
manifest = json.loads((output / "full-scan-manifest.json").read_text(encoding="utf-8"))
quarantine = json.loads((output / "quarantine-manifest.json").read_text(encoding="utf-8"))
@@ -68,3 +86,46 @@ def test_phase3_scan_reconciles_and_resumes_deterministically(tmp_path: Path) ->
assert any(item["path"] == "data/broken.tif" for item in quarantine["items"])
assert any(item["path"] == "data/invalid.geojson" for item in quarantine["items"])
assert len(manifest["duplicates"]["exact_duplicate_groups"]) == 1
def test_phase3_scan_does_not_invent_unreachable_production_boundaries(
tmp_path: Path,
) -> None:
data = tmp_path / "data"
data.mkdir()
(data / "present.json").write_text("{}", encoding="utf-8")
output = tmp_path / "evidence"
result = run_scan(tmp_path, output)
assert result["reconciliation"] == {
"examined": 1,
"skipped": 0,
"unreachable": 0,
"inventory_total": 1,
"reconciles": True,
}
def test_phase3_scan_records_a_missing_requested_root(tmp_path: Path) -> None:
output = tmp_path / "evidence"
completed = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--repo-root",
str(tmp_path),
"--output-dir",
str(output),
"--roots",
"missing-data",
],
check=True,
capture_output=True,
text=True,
)
result = json.loads(completed.stdout)
manifest = json.loads((output / "full-scan-manifest.json").read_text(encoding="utf-8"))
assert result["reconciliation"]["unreachable"] == 1
assert manifest["items"][0]["path"] == "root://missing-data"
backend/tests/test_belgium_training_loop.py
+25
View File
@@ -175,6 +175,31 @@ def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_pat
assert command[command.index("--output-dir") + 1].endswith("failure-driven-training")
def test_protected_assessment_uses_frozen_threshold_and_configured_gates(
tmp_path: Path,
) -> None:
command = MODULE.protected_assessment_command(
scripts_dir=tmp_path / "scripts",
calibration=tmp_path / "calibration.json",
test=tmp_path / "test.json",
background=tmp_path / "background.json",
output=tmp_path / "assessment.json",
selected_threshold=0.275,
min_aggregate_f1=0.71,
min_region_f1=0.62,
min_region_precision=0.73,
min_region_recall=0.58,
max_pure_empty_fp=1,
)
assert command[command.index("--selected-threshold") + 1] == "0.275"
assert command[command.index("--min-aggregate-f1") + 1] == "0.71"
assert command[command.index("--min-region-f1") + 1] == "0.62"
assert command[command.index("--min-region-precision") + 1] == "0.73"
assert command[command.index("--min-region-recall") + 1] == "0.58"
assert command[command.index("--max-pure-empty-fp") + 1] == "1"
def test_partial_iteration_resume_uses_exact_checkpoint_and_cuda(tmp_path: Path) -> None:
checkpoint = tmp_path / "runs" / "iteration-002" / "weights" / "last.pt"
assert MODULE.resumable_training_command("yolo", checkpoint) == [
backend/tests/test_sprint162_promoted_model_activation.py
+74 -2
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import hashlib
import json
import subprocess
from pathlib import Path
@@ -51,11 +52,27 @@ def _write_report(path: Path, *, promotion_status: str = "promote_candidate") ->
)
def _write_governed_release_gate(path: Path, model_file: Path, **overrides: object) -> None:
payload: dict[str, object] = {
"status": "pass",
"product_benchmark_status": "pass",
"promotion_allowed": True,
"phase_decision": "ready",
"candidate_key": CANDIDATE_KEY,
"candidate_model_sha256": hashlib.sha256(model_file.read_bytes()).hexdigest(),
"benchmark_manifest_sha256": "b" * 64,
}
payload.update(overrides)
path.write_text(json.dumps(payload), encoding="utf-8")
def _run_activation(tmp_path: Path, *extra_args: str) -> subprocess.CompletedProcess[str]:
models_dir = tmp_path / "models"
_write_model(models_dir)
model_file = _write_model(models_dir)
report_path = tmp_path / "promotion_report.json"
_write_report(report_path)
release_gate_path = tmp_path / "phase4-release-gate.json"
_write_governed_release_gate(release_gate_path, model_file)
env_file = tmp_path / ".env"
env_file.write_text("GEOINTEL_ENV=production\nYOLO_ENABLED=false\n", encoding="utf-8")
@@ -67,6 +84,8 @@ def _run_activation(tmp_path: Path, *extra_args: str) -> subprocess.CompletedPro
str(report_path),
"--candidate-key",
CANDIDATE_KEY,
"--phase4-release-gate-report",
str(release_gate_path),
"--models-dir",
str(models_dir),
"--container-model-dir",
@@ -94,6 +113,9 @@ def test_promoted_yolo_activation_dry_run_validates_report_and_model(tmp_path: P
assert payload["will_download_models"] is False
assert payload["candidate"]["candidate_key"] == CANDIDATE_KEY
assert payload["candidate"]["threshold"] == 0.35
assert payload["selected_model_sha256"] == hashlib.sha256(
(tmp_path / "models" / "geointel-building-yolov8s-aoi1024bg512r3e50.pt").read_bytes()
).hexdigest()
assert payload["env_updates"]["YOLO_ENABLED"] == "true"
assert payload["env_updates"]["YOLO_MODEL_PATH"] == "/app/models/geointel-building-yolov8s-aoi1024bg512r3e50.pt"
@@ -114,9 +136,11 @@ def test_promoted_yolo_activation_apply_updates_env_file(tmp_path: Path) -> None
def test_promoted_yolo_activation_rejects_non_promoted_report(tmp_path: Path) -> None:
models_dir = tmp_path / "models"
_write_model(models_dir)
model_file = _write_model(models_dir)
report_path = tmp_path / "promotion_report.json"
_write_report(report_path, promotion_status="reject")
release_gate_path = tmp_path / "phase4-release-gate.json"
_write_governed_release_gate(release_gate_path, model_file)
result = subprocess.run(
[
@@ -126,6 +150,8 @@ def test_promoted_yolo_activation_rejects_non_promoted_report(tmp_path: Path) ->
str(report_path),
"--candidate-key",
CANDIDATE_KEY,
"--phase4-release-gate-report",
str(release_gate_path),
"--models-dir",
str(models_dir),
"--env-file",
@@ -145,6 +171,52 @@ def test_promoted_yolo_activation_rejects_non_promoted_report(tmp_path: Path) ->
assert "rejection_reasons" in payload
def test_promoted_yolo_activation_rejects_failed_or_unbound_phase4_gate(
tmp_path: Path,
) -> None:
models_dir = tmp_path / "models"
model_file = _write_model(models_dir)
report_path = tmp_path / "promotion_report.json"
_write_report(report_path)
release_gate_path = tmp_path / "phase4-release-gate.json"
_write_governed_release_gate(
release_gate_path,
model_file,
status="fail",
promotion_allowed=False,
candidate_model_sha256="0" * 64,
)
result = subprocess.run(
[
"python",
str(SCRIPT),
"--promotion-report",
str(report_path),
"--phase4-release-gate-report",
str(release_gate_path),
"--candidate-key",
CANDIDATE_KEY,
"--models-dir",
str(models_dir),
"--env-file",
str(tmp_path / ".env"),
"--json",
],
cwd=ROOT,
capture_output=True,
text=True,
timeout=30,
check=False,
)
assert result.returncode == 3
payload = json.loads(result.stdout)
assert payload["status"] == "governed_release_not_approved"
assert "phase4_release_gate_not_passed" in payload["rejection_reasons"]
assert "governed_candidate_model_sha256_mismatch" in payload["rejection_reasons"]
def test_readiness_gate_compiles_promoted_activation_script() -> None:
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
backend/tests/test_sprint177_mol_primary_focus.py
+1 -1
View File
@@ -18,7 +18,7 @@ def test_frontend_declares_national_scope_as_primary_operating_focus() -> None:
navigation = read_feature("shell")
assert "NATIONAL_WORKSPACE_PROJECT_NAME = 'Belgium and North Sea Workbench'" in focus
assert "NATIONAL_WORKSPACE_REGION = 'Belgie en Belgische Noordzee'" in focus
assert "NATIONAL_WORKSPACE_REGION = 'België en Belgische Noordzee'" in focus
assert "NATIONAL_MAP_CENTER" in focus
assert "return nationalProject.id" in project_hook
assert "hasMappedAnalysisContext(data)" in project_hook
backend/tests/test_sprint189_kempen_scope.py
+2 -1
View File
@@ -137,7 +137,8 @@ def test_kempen_scope_operator_is_packaged_and_exposed_in_map_flow() -> None:
assert "Kempen (28 gemeenten)" in workspace
assert 'aria-label="Regio"' not in workspace
assert 'aria-label="Ingeladen regiobereik"' in workspace
assert "Zoek optioneel een gemeente" in workspace
assert 'aria-label="Optioneel een gemeente zoeken"' in workspace
assert 'placeholder="Gemeentenaam of NIS-code"' in workspace
assert "projects={projects}" in app
map_props = app.split("<MapWorkspace", maxsplit=1)[1].split("/>", maxsplit=1)[0]
assert "onSelectProject={selectProject}" not in map_props