from __future__ import annotations import csv import re import unicodedata from dataclasses import dataclass from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Any, Protocol from django.core.management.base import CommandError from django.db import transaction from apps.jobs.models import GeocodeLocationLookup class GeocodeProvider(Protocol): name: str version: str confidence: float metadata: dict[str, Any] def resolve(self, query: str) -> list[LocationMatch]: ... @dataclass(frozen=True) class GeoPoint: latitude: float longitude: float @dataclass(frozen=True) class LocationMatch: postal_code: str | None municipality: str | None region: str | None point: GeoPoint | None confidence: float source: str source_version: str metadata: dict[str, Any] @dataclass(frozen=True) class LocationMatchResult: query: str location: LocationMatch | None ambiguous: bool = False def _normalize_token(value: str) -> str: normalized = unicodedata.normalize("NFKD", (value or "").strip()) asciiish = "".join(ch for ch in normalized if not unicodedata.combining(ch)) return " ".join(ch.lower().strip() for ch in asciiish.split()) def parse_belgian_location_query(raw: str) -> tuple[str | None, str | None]: normalized = _normalize_token(raw) if not normalized: return None, None postal = None for chunk in re.findall(r"\b\d{4}\b", normalized): postal = chunk break municipality_part = normalized.split(",", 1)[0] if "," in normalized else normalized municipality_part = re.sub(r"\b\d{4}\b", " ", municipality_part) municipality_part = re.sub(r"[^a-z0-9 ]", " ", municipality_part) municipality = " ".join(municipality_part.split()) if not municipality: return postal, None if postal: check = re.sub(r"\b" + re.escape(postal) + r"\b", " ", municipality_part) municipality = _normalize_token(check) if not municipality: return postal, None return postal, municipality @dataclass(frozen=True) class _ParsedRow: postal_code: str municipality: str normalized_municipality: str region: str latitude: Decimal longitude: Decimal def _read_rows(path: str | Path) -> list[_ParsedRow]: csv_path = Path(path) if not csv_path.exists(): raise CommandError(f"Geodata-bestand niet gevonden: {csv_path}") rows: list[_ParsedRow] = [] seen: set[tuple[str, str]] = set() with csv_path.open("r", encoding="utf-8-sig", newline="") as handle: reader = csv.DictReader(handle) headers = set(reader.fieldnames or []) required = {"postal_code", "municipality", "region", "latitude", "longitude"} if not required.issubset(headers): raise CommandError( "Verplichte kolommen ontbreken: postal_code, municipality, region, " "latitude, longitude" ) for row_number, raw_row in enumerate(reader, start=2): postal_code = (raw_row.get("postal_code") or "").strip() municipality = (raw_row.get("municipality") or "").strip() region = (raw_row.get("region") or "").strip() normalized_municipality = _normalize_token(municipality) if not postal_code: raise CommandError(f"regel {row_number}: postal_code mag niet leeg zijn") if len(postal_code) != 4 or not postal_code.isdigit(): raise CommandError( f"regel {row_number}: ongeldige Belgische postcode {postal_code}" ) if not municipality: raise CommandError(f"regel {row_number}: municipality mag niet leeg zijn") try: latitude = Decimal((raw_row.get("latitude") or "").strip()) longitude = Decimal((raw_row.get("longitude") or "").strip()) except (TypeError, InvalidOperation) as exc: raise CommandError( f"regel {row_number}: latitude/longitude moet numeriek zijn" ) from exc if not (Decimal("-90") <= latitude <= Decimal("90")): raise CommandError(f"regel {row_number}: latitude buiten bereik") if not (Decimal("-180") <= longitude <= Decimal("180")): raise CommandError(f"regel {row_number}: longitude buiten bereik") row_key = (postal_code, normalized_municipality) if row_key in seen: raise CommandError( f"regel {row_number}: dubbel record in bestand voor " f"{postal_code} {municipality}" ) seen.add(row_key) rows.append( _ParsedRow( postal_code=postal_code, municipality=municipality, normalized_municipality=normalized_municipality, region=region, latitude=latitude, longitude=longitude, ) ) return rows class CsvGeocodeProvider: name = "csv" def __init__( self, *, source_name: str, source_version: str, confidence: float = 0.85, metadata: dict[str, Any] | None = None, ) -> None: self.source_name = source_name self.version = source_version self.confidence = float(confidence) self.metadata: dict[str, Any] = metadata or {} @staticmethod def _load_candidates( query_value: str, query_kind: str, *, source_name: str, source_version: str ): return GeocodeLocationLookup.objects.filter( source_name=source_name, source_version=source_version, query_kind=query_kind, query_value=query_value, ) @staticmethod def _to_match(row: GeocodeLocationLookup) -> LocationMatch: point = ( GeoPoint(latitude=float(row.latitude), longitude=float(row.longitude)) if row.latitude is not None and row.longitude is not None else None ) return LocationMatch( postal_code=row.postal_code or None, municipality=row.municipality or None, region=row.region or None, point=point, confidence=float(row.confidence), source=row.source_name, source_version=row.source_version, metadata={ "source": row.source_name, "version": row.source_version, "license_name": row.source_license_name, "license_url": row.source_license_url, }, ) def resolve(self, query: str) -> list[LocationMatch]: postal, municipality = parse_belgian_location_query(query) if not postal and not municipality: return [] candidates: list[GeocodeLocationLookup] = [] if postal: base = list( self._load_candidates( postal, "postal", source_name=self.source_name, source_version=self.version ) ) if municipality: normalized = _normalize_token(municipality) filtered = [ row for row in base if _normalize_token(row.municipality or "") == normalized ] if filtered: candidates = filtered elif base: candidates = [] else: candidates = base if not candidates and municipality: candidates = list( self._load_candidates( _normalize_token(municipality), "municipality", source_name=self.source_name, source_version=self.version, ) ) return [self._to_match(row) for row in candidates] def resolve_location(query: str, provider: GeocodeProvider) -> LocationMatchResult: candidates = provider.resolve(query) if not candidates: return LocationMatchResult(query=query, location=None) if len(candidates) > 1: return LocationMatchResult(query=query, location=None, ambiguous=True) return LocationMatchResult(query=query, location=candidates[0], ambiguous=False) def _latest_geocode_sources(limit: int = 5) -> list[tuple[str, str]]: rows = ( GeocodeLocationLookup.objects.order_by("-updated_at") .values_list("source_name", "source_version") .distinct()[:limit] ) return [(name, version) for name, version in rows] def resolve_cached_location( query: str, *, source_name: str | None = None, source_version: str | None = None, preferred_sources: list[tuple[str, str]] | None = None, ) -> LocationMatchResult: sources: list[tuple[str, str]] if source_name and source_version: sources = [(source_name, source_version)] elif preferred_sources: sources = preferred_sources else: sources = _latest_geocode_sources() if not sources: return LocationMatchResult(query=query, location=None) for source_name, source_version in sources: result = resolve_location( query, CsvGeocodeProvider( source_name=source_name, source_version=source_version, ), ) if result.location is not None or result.ambiguous: return result return LocationMatchResult(query=query, location=None) def validate_csv_geodata(path: str | Path) -> tuple[int, dict[str, Any]]: rows = _read_rows(path) return len(rows), {"rows": len(rows)} def import_csv_geodata( path: str | Path, *, source_name: str, source_version: str, source_license_name: str = "", source_license_url: str = "", source_metadata: dict[str, Any] | None = None, replace: bool = False, ) -> tuple[int, set[str], set[str]]: rows = _read_rows(path) if len(rows) > 50000: raise CommandError( "Importbestand bevat meer dan 50.000 records; import in delen aanbevolen." ) existing_rows = GeocodeLocationLookup.objects.filter( source_name=source_name, source_version=source_version, ) def _lookup_key( query_kind: str, query_value: str, postal_code: str, municipality: str ) -> tuple[str, str, str, str]: return ( query_kind, _normalize_token(query_value), postal_code, _normalize_token(municipality), ) with transaction.atomic(): existing_keys = { _lookup_key( row.query_kind, row.query_value, row.postal_code, row.municipality, ) for row in existing_rows } for row in rows: municipal_key = _lookup_key( "municipality", row.normalized_municipality, row.postal_code, row.municipality, ) postal_key = _lookup_key( "postal", row.postal_code, row.postal_code, row.municipality, ) if (not replace) and (municipal_key in existing_keys or postal_key in existing_keys): raise CommandError( "Import zou bestaande lookuprecords overschrijven zonder --replace." ) if replace: GeocodeLocationLookup.objects.filter( source_name=source_name, source_version=source_version, ).delete() batch: list[GeocodeLocationLookup] = [] metadata = dict(source_metadata or {}) metadata["license_name"] = source_license_name metadata["license_url"] = source_license_url for row in rows: batch.extend( [ GeocodeLocationLookup( source_name=source_name, source_version=source_version, source_license_name=source_license_name, source_license_url=source_license_url, source_metadata=metadata, query_kind="municipality", query_value=row.normalized_municipality, postal_code=row.postal_code, municipality=row.municipality, region=row.region, latitude=row.latitude, longitude=row.longitude, confidence=Decimal("1.0"), ), GeocodeLocationLookup( source_name=source_name, source_version=source_version, source_license_name=source_license_name, source_license_url=source_license_url, source_metadata=metadata, query_kind="postal", query_value=row.postal_code, postal_code=row.postal_code, municipality=row.municipality, region=row.region, latitude=row.latitude, longitude=row.longitude, confidence=Decimal("1.0"), ), ] ) created = GeocodeLocationLookup.objects.bulk_create(batch, ignore_conflicts=False) postalcodes = {row.postal_code for row in rows} municipalities = {row.municipality for row in rows} return len(created), postalcodes, municipalities