Filter labels created after dated imagery
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Jens
2026-07-27 03:10:04 +02:00
parent a31dce325f
commit 52c4c4e709
6 changed files with 98 additions and 1 deletions
@@ -88,3 +88,47 @@ def test_spatial_leakage_audit_fails_cross_split_neighbors() -> None:
assert audit["status"] == "failed"
assert audit["findings"][0]["left"] == "train-a"
assert audit["findings"][0]["right"] == "val-a"
def test_normalizer_rejects_features_created_after_dated_imagery(tmp_path: Path) -> None:
raster_path = tmp_path / "image.tif"
with rasterio.open(
raster_path,
"w",
driver="GTiff",
width=100,
height=100,
count=3,
dtype="uint8",
crs="EPSG:4326",
transform=from_origin(4.0, 51.0, 0.001, 0.001),
) as dataset:
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
geometry = {
"type": "Polygon",
"coordinates": [[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]],
}
reference_path = tmp_path / "reference.geojson"
reference_path.write_text(
json.dumps(
{
"type": "FeatureCollection",
"features": [
{"type": "Feature", "id": "old", "properties": {"BEGINDATUM": "2024-01-01"}, "geometry": geometry},
{"type": "Feature", "id": "new", "properties": {"BEGINDATUM": "2026-01-01"}, "geometry": geometry},
],
}
),
encoding="utf-8",
)
normalized, audit = module.normalize(
reference_path=reference_path,
raster_path=raster_path,
source_name="grb",
min_label_px=3,
imagery_observed_at="2025-01-01T00:00:00Z",
imagery_valid_to="2025-12-31T23:59:59Z",
reference_observed_at="2026-07-01T00:00:00Z",
)
assert len(normalized["features"]) == 1
assert audit["decision_counts"] == {"accepted": 1, "created_after_imagery_period": 1}
+5
View File
@@ -81,3 +81,8 @@ Every failed assessment returns `continue_training_loop`. Only a report with
`training_complete` may proceed to final human review and guarded activation.
The orchestrator refuses to start unless the frozen dataset audit is `ok` and
contains zero blank/low-variance positive tiles.
For dated imagery, GRB `BEGINDATUM` and PICC `DATE_CREAT` are compared with the
end of the imagery period. A feature created afterward is retained in the
audit but excluded from training as `created_after_imagery_period`. UrbIS does
not expose an equivalent feature creation field in this acquisition contract,
so its remaining temporal relation stays an explicit sample-level limitation.
+15
View File
@@ -11587,3 +11587,18 @@ Next gate:
- The v5 tile audit passed with zero blank positive tiles. The automated CUDA
loop started from the strongest prior candidate and will checkpoint every
train/calibrate/test/background assessment without promoting failed models.
## 2026-07-27 - Feature-level temporal mismatch filtering
- Added provider-native creation-time filtering for dated training imagery:
GRB `BEGINDATUM` and PICC `DATE_CREAT` are parsed with explicit UTC handling.
Buildings created after the image period are audited and excluded rather
than taught as labels for structures absent from the image.
- Frozen corpus `building-be-v6-temporal-20260727-r1` excludes 246 such temporal
mismatches, accepts 13,524 labels and retains all 75 independent AOIs.
Manifest SHA-256 is
`973828b453e6fbeb5c04aa567ddb615566d92825d8698654f5056d2997d382eb`.
- Composition, spatial leakage, temporal identity and positive-imagery QA pass.
The v6 train/calibration/test/background exports are ready for the next loop
checkpoint; the running v5 iteration remains evidence but cannot supersede
the cleaner v6 corpus.
+1
View File
@@ -955,3 +955,4 @@ This file now starts with the current implementation status. Older preparation/b
- [ ] Resolve the v3 Flanders and Wallonia generalisation failures through additional training-only evidence and retraining.
- [x] Replace rolling-mosaic training inputs with governed dated 2025 Flanders/Brussels and complete 2023 SPW imagery; retain exact flight-day limitations.
- [x] Reject positive labels over blank/no-data imagery and replace partial SPW 2024 coverage with the complete dated SPW 2023 campaign.
- [x] Exclude GRB/PICC features created after the corresponding dated imagery period while retaining auditable rejection evidence.
@@ -149,6 +149,7 @@ def main() -> int:
else None
),
reference_observed_at=reference.observed_at.isoformat() if reference.observed_at else None,
imagery_valid_to=raster.valid_to.isoformat() if raster.valid_to else None,
)
normalized_target.write_text(json.dumps(normalized, ensure_ascii=False), encoding="utf-8")
audit_target.write_text(json.dumps(audit, ensure_ascii=False, indent=2), encoding="utf-8")
+32 -1
View File
@@ -12,7 +12,7 @@ import argparse
import hashlib
import json
from collections import Counter
from datetime import datetime
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
@@ -57,6 +57,26 @@ def _semantic_exclusion(properties: dict[str, Any]) -> str | None:
return None
def _source_creation_at(source_name: str, properties: dict[str, Any]) -> datetime | None:
raw: Any = None
if source_name == "grb":
raw = properties.get("BEGINDATUM")
elif source_name == "spw_picc":
raw = properties.get("DATE_CREAT")
if raw in (None, ""):
return None
try:
if isinstance(raw, (int, float)) or str(raw).isdigit():
value = float(raw)
if value > 10_000_000_000:
value /= 1000
return datetime.fromtimestamp(value, tz=UTC)
parsed = datetime.fromisoformat(str(raw).replace("Z", "+00:00"))
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
except (OSError, OverflowError, ValueError):
return None
def _polygonal(geometry: Any) -> Any | None:
if geometry.geom_type in {"Polygon", "MultiPolygon"}:
return geometry
@@ -78,6 +98,7 @@ def normalize(
min_label_px: float,
imagery_observed_at: str | None,
reference_observed_at: str | None,
imagery_valid_to: str | None = None,
) -> tuple[dict[str, Any], dict[str, Any]]:
if source_name not in SUPPORTED_SOURCES:
raise SystemExit(f"Unsupported governed building source: {source_name}")
@@ -89,6 +110,9 @@ def normalize(
decisions: list[dict[str, Any]] = []
seen: set[str] = set()
counts: Counter[str] = Counter()
imagery_cutoff = (
datetime.fromisoformat(imagery_valid_to.replace("Z", "+00:00")) if imagery_valid_to else None
)
with rasterio.open(raster_path) as raster:
if raster.crs is None:
raise SystemExit("Raster CRS is required")
@@ -104,6 +128,8 @@ def normalize(
"reason": None,
"geometry_repaired": False,
}
source_creation_at = _source_creation_at(source_name, properties)
decision["source_creation_at"] = source_creation_at.isoformat() if source_creation_at else None
try:
geometry = shape(feature.get("geometry"))
except Exception:
@@ -119,6 +145,8 @@ def normalize(
decision["reason"] = "invalid_geometry_unrepairable"
elif (reason := _semantic_exclusion(properties)) is not None:
decision["reason"] = reason
elif imagery_cutoff and source_creation_at and source_creation_at > imagery_cutoff:
decision["reason"] = "created_after_imagery_period"
else:
metric = shapely_transform(transformer.transform, geometry)
min_x, min_y, max_x, max_y = metric.bounds
@@ -179,6 +207,7 @@ def normalize(
"raster_path": str(raster_path),
"min_label_px": min_label_px,
"imagery_observed_at": imagery_observed_at,
"imagery_valid_to": imagery_valid_to,
"reference_observed_at": reference_observed_at,
"temporal_mismatch_days": temporal_mismatch_days,
"temporal_alignment_status": temporal_alignment_status,
@@ -200,6 +229,7 @@ def main() -> int:
parser.add_argument("--min-label-px", type=float, default=3.0)
parser.add_argument("--imagery-observed-at")
parser.add_argument("--reference-observed-at")
parser.add_argument("--imagery-valid-to")
args = parser.parse_args()
normalized, audit = normalize(
reference_path=args.reference,
@@ -208,6 +238,7 @@ def main() -> int:
min_label_px=args.min_label_px,
imagery_observed_at=args.imagery_observed_at,
reference_observed_at=args.reference_observed_at,
imagery_valid_to=args.imagery_valid_to,
)
args.output_reference.parent.mkdir(parents=True, exist_ok=True)
args.output_audit.parent.mkdir(parents=True, exist_ok=True)