claim analysis jobs atomically and keep acquisition on its official host

Two defects of the same kind: work that is supposed to be bounded is not.

The analysis worker selected queued jobs and then set them to running in a
second statement. A restarted process overlapping the previous one, or a second
replica, could both select the same row and both start tiled GPU inference on
it — duplicate analysis runs and double the GPU load. The AOI worker beside it
already claims with FOR UPDATE SKIP LOCKED; this uses a conditional update,
which is the same guarantee in one statement. run_once now reports jobs it
actually claimed rather than jobs it looked at.

urlopen follows redirects, so although every acquisition URL is built from
settings and cannot be steered by a request payload, a misconfigured or
compromised upstream could send the runtime to the loopback interface, to
another container on the compose network, or to a cloud metadata endpoint — and
the bytes would then be persisted under an official provenance. That is exactly
the substitution the product forbids. All eight fetch sites now open through a
guard that refuses private, loopback and link-local destinations (resolving the
host first, so a DNS name cannot hide one) and refuses a redirect that leaves
the configured origin or downgrades from HTTPS.

The guard is proven by calling the services' own fetch paths, not by grepping
for the call: every existing acquisition test injects an opener, which bypasses
it by design.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Jens
2026-08-22 15:59:01 +02:00
co-authored by Claude Opus 5
parent 5b3839dc89
commit 16dedeb670
14 changed files with 493 additions and 16 deletions
@@ -0,0 +1,164 @@
"""Bounded acquisition must stay bounded to the official host.
Every acquisition service builds its URL from configured settings, so the
request payload cannot point the runtime anywhere. The redirect chain can:
``urlopen`` follows redirects by default, so a misconfigured or compromised
upstream can send the runtime to ``127.0.0.1``, to the container network, or to
a cloud metadata endpoint — and the response is then persisted as if it were
official source data.
The product's stated rule is that acquisition fails closed and never
substitutes fabricated data for official data. A redirect off the configured
host is exactly that substitution.
"""
from __future__ import annotations
import pytest
from app.core.errors import AppError
from app.services.outbound_request_guard import (
assert_public_http_url,
assert_same_origin_redirect,
)
class TestUrlShape:
def test_an_official_https_endpoint_is_accepted(self) -> None:
assert_public_http_url("https://geo.api.vlaanderen.be/dhmv/wcs?SERVICE=WCS")
def test_a_non_http_scheme_is_refused(self) -> None:
with pytest.raises(AppError) as exc_info:
assert_public_http_url("file:///etc/passwd")
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
@pytest.mark.parametrize(
"url",
[
"http://127.0.0.1:8000/internal",
"http://localhost/internal",
"http://10.1.2.3/internal",
"http://192.168.10.150/internal",
"http://172.16.0.9/internal",
"http://169.254.169.254/latest/meta-data/",
"http://[::1]/internal",
],
)
def test_private_and_loopback_destinations_are_refused(self, url: str) -> None:
with pytest.raises(AppError) as exc_info:
assert_public_http_url(url)
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
def test_a_url_without_a_host_is_refused(self) -> None:
with pytest.raises(AppError):
assert_public_http_url("https:///no-host")
class TestRedirects:
def test_a_redirect_within_the_same_origin_is_allowed(self) -> None:
assert_same_origin_redirect(
"https://geo.api.vlaanderen.be/dhmv/wcs",
"https://geo.api.vlaanderen.be/dhmv/wcs/v2?x=1",
)
def test_a_redirect_to_another_host_is_refused(self) -> None:
with pytest.raises(AppError) as exc_info:
assert_same_origin_redirect(
"https://geo.api.vlaanderen.be/dhmv/wcs",
"https://cdn.example.net/payload.tif",
)
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
assert "cdn.example.net" in str(exc_info.value.details)
def test_a_downgrade_to_plain_http_is_refused(self) -> None:
with pytest.raises(AppError) as exc_info:
assert_same_origin_redirect(
"https://geo.api.vlaanderen.be/wcs",
"http://geo.api.vlaanderen.be/wcs",
)
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
def test_a_redirect_to_the_loopback_is_refused_even_on_the_same_scheme(self) -> None:
with pytest.raises(AppError):
assert_same_origin_redirect("https://geo.api.vlaanderen.be/wcs", "https://127.0.0.1/wcs")
def test_an_upgrade_to_https_stays_allowed(self) -> None:
assert_same_origin_redirect("http://geo.example.be/wcs", "https://geo.example.be/wcs")
def test_the_guard_opener_refuses_a_cross_host_redirect() -> None:
"""The opener is what the acquisition services actually call."""
from app.services.outbound_request_guard import guarded_opener
opener = guarded_opener("https://geo.api.vlaanderen.be/wcs")
class _Redirecting:
def __init__(self, location: str) -> None:
self.url = location
def __enter__(self):
return self
def __exit__(self, *_args):
return False
with pytest.raises(AppError) as exc_info:
with opener(
type("Req", (), {"full_url": "https://geo.api.vlaanderen.be/wcs"})(),
timeout=1,
_transport=lambda *_a, **_k: _Redirecting("https://evil.example.net/x"),
):
pass
assert exc_info.value.code == "OUTBOUND_REDIRECT_NOT_ALLOWED"
class TestTheGuardIsWiredIntoAcquisition:
"""Behavioural, not a grep: each service is called on its real fetch path.
Every existing acquisition test injects an ``opener``, which bypasses the
guard by design — that is how those tests stub the network. These call the
production default instead.
"""
def _settings(self):
from app.core.config import Settings
return Settings(_env_file=None)
def test_dhmv_refuses_a_loopback_endpoint(self) -> None:
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
with pytest.raises(AppError) as exc_info:
DhmvAcquisitionService._fetch("http://127.0.0.1:9/wcs", self._settings())
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
def test_flood_hazard_refuses_a_link_local_endpoint(self) -> None:
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
with pytest.raises(AppError) as exc_info:
FloodHazardAcquisitionService._fetch("http://169.254.169.254/latest/", self._settings())
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
def test_thematic_raster_refuses_a_private_endpoint(self) -> None:
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
with pytest.raises(AppError) as exc_info:
ThematicRasterAcquisitionService._fetch("http://10.0.0.5/product.tif", self._settings())
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"
def test_orthophoto_refuses_a_private_endpoint(self) -> None:
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
with pytest.raises(AppError) as exc_info:
OrthophotoAcquisitionService._fetch("http://192.168.10.150/wms", self._settings())
assert exc_info.value.code == "OUTBOUND_URL_NOT_ALLOWED"