NuklearRabbit
ec8f809497
_claim_due_events flipped rows to 'delivering' and committed before the HTTP call; if the process died between that commit and the outcome- recording transaction, the row stayed 'delivering' forever with no reclaim path -- a real gap, not previously documented as an accepted limitation. Give each claim a lease deadline (reusing next_attempt_at, since it's only otherwise meaningful for pending-status backoff scheduling) and sweep expired leases back to pending at the start of every dispatch cycle, before claiming new work. attempts is preserved so the count still reflects true history. Only leases past their deadline are touched, so a still-alive worker mid-delivery is never disturbed or double-processed.
255 lines
8.3 KiB
Python
255 lines
8.3 KiB
Python
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import UTC, datetime, timedelta
|
|
from types import SimpleNamespace
|
|
|
|
from sqlalchemy import select
|
|
|
|
from app.core.config import get_settings
|
|
from app.core.db import SessionLocal
|
|
from app.models.booking import Booking
|
|
from app.models.outbox import OutboxEvent
|
|
from app.services import dispatcher
|
|
|
|
|
|
def _make_pending_event(vehicle_ref: str) -> uuid.UUID:
|
|
db = SessionLocal()
|
|
try:
|
|
booking = db.scalar(select(Booking).where(Booking.status == "returned").limit(1))
|
|
event = OutboxEvent(
|
|
event_id=uuid.uuid4(),
|
|
event_type="vehicle.returned.v1",
|
|
aggregate_type="booking",
|
|
aggregate_id=booking.id,
|
|
payload_json={
|
|
"correlation_id": str(uuid.uuid4()),
|
|
"aggregate": {
|
|
"type": "booking",
|
|
"id": str(booking.id),
|
|
"public_ref": booking.public_ref,
|
|
},
|
|
"data": {
|
|
"vehicle_ref": vehicle_ref,
|
|
"inspection_ref": "INSP-TEST",
|
|
"resulting_vehicle_status": "cleaning",
|
|
"attention_reasons": [],
|
|
},
|
|
"aggregate_ref": booking.public_ref,
|
|
},
|
|
occurred_at=datetime.now(UTC),
|
|
delivery_status="pending",
|
|
attempts=0,
|
|
)
|
|
db.add(event)
|
|
db.commit()
|
|
return event.event_id
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def _get_event(event_id: uuid.UUID) -> OutboxEvent:
|
|
db = SessionLocal()
|
|
try:
|
|
return db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def test_claim_marks_events_delivering():
|
|
event_id = _make_pending_event("MO-001")
|
|
claimed = dispatcher._claim_due_events()
|
|
assert event_id in claimed
|
|
assert _get_event(event_id).delivery_status == "delivering"
|
|
|
|
|
|
def test_deliver_one_success(monkeypatch):
|
|
event_id = _make_pending_event("MO-002")
|
|
dispatcher._claim_due_events()
|
|
|
|
def fake_post(url, json, timeout):
|
|
return SimpleNamespace(
|
|
raise_for_status=lambda: None,
|
|
json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
|
|
)
|
|
|
|
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
|
dispatcher._deliver_one(event_id)
|
|
|
|
event = _get_event(event_id)
|
|
assert event.delivery_status == "succeeded"
|
|
assert event.attempts == 1
|
|
assert event.external_run_id == str(event_id)
|
|
assert event.last_error is None
|
|
|
|
|
|
def test_deliver_one_failure_schedules_retry(monkeypatch):
|
|
event_id = _make_pending_event("MO-003")
|
|
dispatcher._claim_due_events()
|
|
|
|
def fake_post(url, json, timeout):
|
|
raise dispatcher.httpx.ConnectError("simulated connection failure")
|
|
|
|
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
|
dispatcher._deliver_one(event_id)
|
|
|
|
event = _get_event(event_id)
|
|
assert event.delivery_status == "pending"
|
|
assert event.attempts == 1
|
|
assert event.next_attempt_at is not None
|
|
assert "simulated connection failure" in event.last_error
|
|
|
|
|
|
def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
|
|
event_id = _make_pending_event("MO-004")
|
|
settings = get_settings()
|
|
|
|
def fake_post(url, json, timeout):
|
|
raise dispatcher.httpx.ConnectError("still down")
|
|
|
|
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
|
|
|
for _ in range(settings.n8n_max_attempts):
|
|
dispatcher._claim_due_events()
|
|
db = SessionLocal()
|
|
try:
|
|
event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
|
|
event.next_attempt_at = None
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
dispatcher._deliver_one(event_id)
|
|
|
|
event = _get_event(event_id)
|
|
assert event.delivery_status == "failed"
|
|
assert event.attempts == settings.n8n_max_attempts
|
|
|
|
|
|
def test_deliver_one_handles_malformed_payload_without_getting_stuck(monkeypatch):
|
|
# Regression test: seeded/legacy outbox rows may lack the full event envelope. Delivery
|
|
# must resolve the claimed "delivering" row to pending/failed, never leave it stuck.
|
|
db = SessionLocal()
|
|
try:
|
|
booking = db.scalar(select(Booking).limit(1))
|
|
event = OutboxEvent(
|
|
event_id=uuid.uuid4(),
|
|
event_type="vehicle.returned.v1",
|
|
aggregate_type="booking",
|
|
aggregate_id=booking.id,
|
|
payload_json={"aggregate_ref": booking.public_ref}, # missing correlation_id/etc.
|
|
occurred_at=datetime.now(UTC),
|
|
delivery_status="pending",
|
|
attempts=0,
|
|
)
|
|
db.add(event)
|
|
db.commit()
|
|
event_id = event.event_id
|
|
finally:
|
|
db.close()
|
|
|
|
def fake_post(url, json, timeout):
|
|
raise AssertionError("must not attempt delivery with a malformed payload")
|
|
|
|
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
|
|
|
dispatcher._claim_due_events()
|
|
dispatcher._deliver_one(event_id)
|
|
|
|
event = _get_event(event_id)
|
|
assert event.delivery_status in ("pending", "failed")
|
|
assert event.attempts == 1
|
|
assert "Malformed outbox payload" in event.last_error
|
|
|
|
|
|
def test_claim_sets_a_lease_deadline():
|
|
event_id = _make_pending_event("MO-006")
|
|
settings = get_settings()
|
|
before = datetime.now(UTC)
|
|
dispatcher._claim_due_events()
|
|
|
|
event = _get_event(event_id)
|
|
assert event.delivery_status == "delivering"
|
|
assert event.next_attempt_at is not None
|
|
lease = settings.n8n_delivery_lease_seconds
|
|
assert event.next_attempt_at > before + timedelta(seconds=lease - 5)
|
|
|
|
|
|
def test_reclaim_ignores_an_active_unexpired_lease():
|
|
# A worker that is still within its lease window must not be disturbed -- this is
|
|
# what prevents double delivery of an event another (still-alive) worker is handling.
|
|
event_id = _make_pending_event("MO-007")
|
|
dispatcher._claim_due_events()
|
|
|
|
reclaimed = dispatcher._reclaim_stale_deliveries()
|
|
assert reclaimed == 0
|
|
assert _get_event(event_id).delivery_status == "delivering"
|
|
|
|
|
|
def test_reclaim_recovers_an_expired_lease_and_preserves_attempts(monkeypatch):
|
|
# Simulates a process crash: the row was claimed (delivering) but no outcome was ever
|
|
# recorded, and its lease has since expired.
|
|
event_id = _make_pending_event("MO-008")
|
|
dispatcher._claim_due_events()
|
|
|
|
db = SessionLocal()
|
|
try:
|
|
event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
|
|
event.attempts = 2
|
|
event.next_attempt_at = datetime.now(UTC) - timedelta(seconds=1)
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
|
|
reclaimed = dispatcher._reclaim_stale_deliveries()
|
|
assert reclaimed == 1
|
|
|
|
event = _get_event(event_id)
|
|
assert event.delivery_status == "pending"
|
|
assert event.next_attempt_at is None
|
|
assert event.attempts == 2
|
|
assert "stale" in event.last_error.lower()
|
|
|
|
# The reclaimed event is now a normal pending event, immediately claimable again.
|
|
claimed = dispatcher._claim_due_events()
|
|
assert event_id in claimed
|
|
|
|
|
|
def test_run_dispatch_cycle_recovers_a_stale_lease_before_claiming(monkeypatch):
|
|
event_id = _make_pending_event("MO-009")
|
|
dispatcher._claim_due_events()
|
|
db = SessionLocal()
|
|
try:
|
|
event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
|
|
event.next_attempt_at = datetime.now(UTC) - timedelta(seconds=1)
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
|
|
def fake_post(url, json, timeout):
|
|
return SimpleNamespace(
|
|
raise_for_status=lambda: None,
|
|
json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
|
|
)
|
|
|
|
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
|
processed = dispatcher.run_dispatch_cycle()
|
|
|
|
assert processed >= 1
|
|
assert _get_event(event_id).delivery_status == "succeeded"
|
|
|
|
|
|
def test_run_dispatch_cycle_end_to_end(monkeypatch):
|
|
event_id = _make_pending_event("MO-005")
|
|
|
|
def fake_post(url, json, timeout):
|
|
return SimpleNamespace(
|
|
raise_for_status=lambda: None,
|
|
json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
|
|
)
|
|
|
|
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
|
processed = dispatcher.run_dispatch_cycle()
|
|
|
|
assert processed >= 1
|
|
assert _get_event(event_id).delivery_status == "succeeded"
|