410 lines
14 KiB
Python
410 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import re
|
|
import unicodedata
|
|
from dataclasses import dataclass
|
|
from decimal import Decimal, InvalidOperation
|
|
from pathlib import Path
|
|
from typing import Any, Protocol
|
|
|
|
from django.core.management.base import CommandError
|
|
from django.db import transaction
|
|
|
|
from apps.jobs.models import GeocodeLocationLookup
|
|
|
|
|
|
class GeocodeProvider(Protocol):
|
|
name: str
|
|
version: str
|
|
confidence: float
|
|
metadata: dict[str, Any]
|
|
|
|
def resolve(self, query: str) -> list[LocationMatch]: ...
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GeoPoint:
|
|
latitude: float
|
|
longitude: float
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LocationMatch:
|
|
postal_code: str | None
|
|
municipality: str | None
|
|
region: str | None
|
|
point: GeoPoint | None
|
|
confidence: float
|
|
source: str
|
|
source_version: str
|
|
metadata: dict[str, Any]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LocationMatchResult:
|
|
query: str
|
|
location: LocationMatch | None
|
|
ambiguous: bool = False
|
|
|
|
|
|
def _normalize_token(value: str) -> str:
|
|
normalized = unicodedata.normalize("NFKD", (value or "").strip())
|
|
asciiish = "".join(ch for ch in normalized if not unicodedata.combining(ch))
|
|
return " ".join(ch.lower().strip() for ch in asciiish.split())
|
|
|
|
|
|
def parse_belgian_location_query(raw: str) -> tuple[str | None, str | None]:
|
|
normalized = _normalize_token(raw)
|
|
if not normalized:
|
|
return None, None
|
|
postal = None
|
|
for chunk in re.findall(r"\b\d{4}\b", normalized):
|
|
postal = chunk
|
|
break
|
|
municipality_part = normalized.split(",", 1)[0] if "," in normalized else normalized
|
|
municipality_part = re.sub(r"\b\d{4}\b", " ", municipality_part)
|
|
municipality_part = re.sub(r"[^a-z0-9 ]", " ", municipality_part)
|
|
municipality = " ".join(municipality_part.split())
|
|
if not municipality:
|
|
return postal, None
|
|
if postal:
|
|
check = re.sub(r"\b" + re.escape(postal) + r"\b", " ", municipality_part)
|
|
municipality = _normalize_token(check)
|
|
if not municipality:
|
|
return postal, None
|
|
return postal, municipality
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _ParsedRow:
|
|
postal_code: str
|
|
municipality: str
|
|
normalized_municipality: str
|
|
region: str
|
|
latitude: Decimal
|
|
longitude: Decimal
|
|
|
|
|
|
def _read_rows(path: str | Path) -> list[_ParsedRow]:
|
|
csv_path = Path(path)
|
|
if not csv_path.exists():
|
|
raise CommandError(f"Geodata-bestand niet gevonden: {csv_path}")
|
|
|
|
rows: list[_ParsedRow] = []
|
|
seen: set[tuple[str, str]] = set()
|
|
|
|
with csv_path.open("r", encoding="utf-8-sig", newline="") as handle:
|
|
reader = csv.DictReader(handle)
|
|
headers = set(reader.fieldnames or [])
|
|
required = {"postal_code", "municipality", "region", "latitude", "longitude"}
|
|
if not required.issubset(headers):
|
|
raise CommandError(
|
|
"Verplichte kolommen ontbreken: postal_code, municipality, region, "
|
|
"latitude, longitude"
|
|
)
|
|
|
|
for row_number, raw_row in enumerate(reader, start=2):
|
|
postal_code = (raw_row.get("postal_code") or "").strip()
|
|
municipality = (raw_row.get("municipality") or "").strip()
|
|
region = (raw_row.get("region") or "").strip()
|
|
normalized_municipality = _normalize_token(municipality)
|
|
|
|
if not postal_code:
|
|
raise CommandError(f"regel {row_number}: postal_code mag niet leeg zijn")
|
|
if len(postal_code) != 4 or not postal_code.isdigit():
|
|
raise CommandError(
|
|
f"regel {row_number}: ongeldige Belgische postcode {postal_code}"
|
|
)
|
|
if not municipality:
|
|
raise CommandError(f"regel {row_number}: municipality mag niet leeg zijn")
|
|
|
|
try:
|
|
latitude = Decimal((raw_row.get("latitude") or "").strip())
|
|
longitude = Decimal((raw_row.get("longitude") or "").strip())
|
|
except (TypeError, InvalidOperation) as exc:
|
|
raise CommandError(
|
|
f"regel {row_number}: latitude/longitude moet numeriek zijn"
|
|
) from exc
|
|
|
|
if not (Decimal("-90") <= latitude <= Decimal("90")):
|
|
raise CommandError(f"regel {row_number}: latitude buiten bereik")
|
|
if not (Decimal("-180") <= longitude <= Decimal("180")):
|
|
raise CommandError(f"regel {row_number}: longitude buiten bereik")
|
|
|
|
row_key = (postal_code, normalized_municipality)
|
|
if row_key in seen:
|
|
raise CommandError(
|
|
f"regel {row_number}: dubbel record in bestand voor "
|
|
f"{postal_code} {municipality}"
|
|
)
|
|
seen.add(row_key)
|
|
|
|
rows.append(
|
|
_ParsedRow(
|
|
postal_code=postal_code,
|
|
municipality=municipality,
|
|
normalized_municipality=normalized_municipality,
|
|
region=region,
|
|
latitude=latitude,
|
|
longitude=longitude,
|
|
)
|
|
)
|
|
|
|
return rows
|
|
|
|
|
|
class CsvGeocodeProvider:
|
|
name = "csv"
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
source_name: str,
|
|
source_version: str,
|
|
confidence: float = 0.85,
|
|
metadata: dict[str, Any] | None = None,
|
|
) -> None:
|
|
self.source_name = source_name
|
|
self.version = source_version
|
|
self.confidence = float(confidence)
|
|
self.metadata: dict[str, Any] = metadata or {}
|
|
|
|
@staticmethod
|
|
def _load_candidates(
|
|
query_value: str, query_kind: str, *, source_name: str, source_version: str
|
|
):
|
|
return GeocodeLocationLookup.objects.filter(
|
|
source_name=source_name,
|
|
source_version=source_version,
|
|
query_kind=query_kind,
|
|
query_value=query_value,
|
|
)
|
|
|
|
@staticmethod
|
|
def _to_match(row: GeocodeLocationLookup) -> LocationMatch:
|
|
point = (
|
|
GeoPoint(latitude=float(row.latitude), longitude=float(row.longitude))
|
|
if row.latitude is not None and row.longitude is not None
|
|
else None
|
|
)
|
|
return LocationMatch(
|
|
postal_code=row.postal_code or None,
|
|
municipality=row.municipality or None,
|
|
region=row.region or None,
|
|
point=point,
|
|
confidence=float(row.confidence),
|
|
source=row.source_name,
|
|
source_version=row.source_version,
|
|
metadata={
|
|
"source": row.source_name,
|
|
"version": row.source_version,
|
|
"license_name": row.source_license_name,
|
|
"license_url": row.source_license_url,
|
|
},
|
|
)
|
|
|
|
def resolve(self, query: str) -> list[LocationMatch]:
|
|
postal, municipality = parse_belgian_location_query(query)
|
|
if not postal and not municipality:
|
|
return []
|
|
|
|
candidates: list[GeocodeLocationLookup] = []
|
|
if postal:
|
|
base = list(
|
|
self._load_candidates(
|
|
postal, "postal", source_name=self.source_name, source_version=self.version
|
|
)
|
|
)
|
|
if municipality:
|
|
normalized = _normalize_token(municipality)
|
|
filtered = [
|
|
row for row in base if _normalize_token(row.municipality or "") == normalized
|
|
]
|
|
if filtered:
|
|
candidates = filtered
|
|
elif base:
|
|
candidates = []
|
|
else:
|
|
candidates = base
|
|
|
|
if not candidates and municipality:
|
|
candidates = list(
|
|
self._load_candidates(
|
|
_normalize_token(municipality),
|
|
"municipality",
|
|
source_name=self.source_name,
|
|
source_version=self.version,
|
|
)
|
|
)
|
|
|
|
return [self._to_match(row) for row in candidates]
|
|
|
|
|
|
def resolve_location(query: str, provider: GeocodeProvider) -> LocationMatchResult:
|
|
candidates = provider.resolve(query)
|
|
if not candidates:
|
|
return LocationMatchResult(query=query, location=None)
|
|
if len(candidates) > 1:
|
|
return LocationMatchResult(query=query, location=None, ambiguous=True)
|
|
return LocationMatchResult(query=query, location=candidates[0], ambiguous=False)
|
|
|
|
|
|
def _latest_geocode_sources(limit: int = 5) -> list[tuple[str, str]]:
|
|
rows = (
|
|
GeocodeLocationLookup.objects.order_by("-updated_at")
|
|
.values_list("source_name", "source_version")
|
|
.distinct()[:limit]
|
|
)
|
|
return [(name, version) for name, version in rows]
|
|
|
|
|
|
def resolve_cached_location(
|
|
query: str,
|
|
*,
|
|
source_name: str | None = None,
|
|
source_version: str | None = None,
|
|
preferred_sources: list[tuple[str, str]] | None = None,
|
|
) -> LocationMatchResult:
|
|
sources: list[tuple[str, str]]
|
|
if source_name and source_version:
|
|
sources = [(source_name, source_version)]
|
|
elif preferred_sources:
|
|
sources = preferred_sources
|
|
else:
|
|
sources = _latest_geocode_sources()
|
|
|
|
if not sources:
|
|
return LocationMatchResult(query=query, location=None)
|
|
|
|
for source_name, source_version in sources:
|
|
result = resolve_location(
|
|
query,
|
|
CsvGeocodeProvider(
|
|
source_name=source_name,
|
|
source_version=source_version,
|
|
),
|
|
)
|
|
if result.location is not None or result.ambiguous:
|
|
return result
|
|
|
|
return LocationMatchResult(query=query, location=None)
|
|
|
|
|
|
def validate_csv_geodata(path: str | Path) -> tuple[int, dict[str, Any]]:
|
|
rows = _read_rows(path)
|
|
return len(rows), {"rows": len(rows)}
|
|
|
|
|
|
def import_csv_geodata(
|
|
path: str | Path,
|
|
*,
|
|
source_name: str,
|
|
source_version: str,
|
|
source_license_name: str = "",
|
|
source_license_url: str = "",
|
|
source_metadata: dict[str, Any] | None = None,
|
|
replace: bool = False,
|
|
) -> tuple[int, set[str], set[str]]:
|
|
rows = _read_rows(path)
|
|
|
|
if len(rows) > 50000:
|
|
raise CommandError(
|
|
"Importbestand bevat meer dan 50.000 records; import in delen aanbevolen."
|
|
)
|
|
|
|
existing_rows = GeocodeLocationLookup.objects.filter(
|
|
source_name=source_name,
|
|
source_version=source_version,
|
|
)
|
|
|
|
def _lookup_key(
|
|
query_kind: str, query_value: str, postal_code: str, municipality: str
|
|
) -> tuple[str, str, str, str]:
|
|
return (
|
|
query_kind,
|
|
_normalize_token(query_value),
|
|
postal_code,
|
|
_normalize_token(municipality),
|
|
)
|
|
|
|
with transaction.atomic():
|
|
existing_keys = {
|
|
_lookup_key(
|
|
row.query_kind,
|
|
row.query_value,
|
|
row.postal_code,
|
|
row.municipality,
|
|
)
|
|
for row in existing_rows
|
|
}
|
|
for row in rows:
|
|
municipal_key = _lookup_key(
|
|
"municipality",
|
|
row.normalized_municipality,
|
|
row.postal_code,
|
|
row.municipality,
|
|
)
|
|
postal_key = _lookup_key(
|
|
"postal",
|
|
row.postal_code,
|
|
row.postal_code,
|
|
row.municipality,
|
|
)
|
|
if (not replace) and (municipal_key in existing_keys or postal_key in existing_keys):
|
|
raise CommandError(
|
|
"Import zou bestaande lookuprecords overschrijven zonder --replace."
|
|
)
|
|
|
|
if replace:
|
|
GeocodeLocationLookup.objects.filter(
|
|
source_name=source_name,
|
|
source_version=source_version,
|
|
).delete()
|
|
|
|
batch: list[GeocodeLocationLookup] = []
|
|
metadata = dict(source_metadata or {})
|
|
metadata["license_name"] = source_license_name
|
|
metadata["license_url"] = source_license_url
|
|
|
|
for row in rows:
|
|
batch.extend(
|
|
[
|
|
GeocodeLocationLookup(
|
|
source_name=source_name,
|
|
source_version=source_version,
|
|
source_license_name=source_license_name,
|
|
source_license_url=source_license_url,
|
|
source_metadata=metadata,
|
|
query_kind="municipality",
|
|
query_value=row.normalized_municipality,
|
|
postal_code=row.postal_code,
|
|
municipality=row.municipality,
|
|
region=row.region,
|
|
latitude=row.latitude,
|
|
longitude=row.longitude,
|
|
confidence=Decimal("1.0"),
|
|
),
|
|
GeocodeLocationLookup(
|
|
source_name=source_name,
|
|
source_version=source_version,
|
|
source_license_name=source_license_name,
|
|
source_license_url=source_license_url,
|
|
source_metadata=metadata,
|
|
query_kind="postal",
|
|
query_value=row.postal_code,
|
|
postal_code=row.postal_code,
|
|
municipality=row.municipality,
|
|
region=row.region,
|
|
latitude=row.latitude,
|
|
longitude=row.longitude,
|
|
confidence=Decimal("1.0"),
|
|
),
|
|
]
|
|
)
|
|
|
|
created = GeocodeLocationLookup.objects.bulk_create(batch, ignore_conflicts=False)
|
|
postalcodes = {row.postal_code for row in rows}
|
|
municipalities = {row.municipality for row in rows}
|
|
return len(created), postalcodes, municipalities
|