feat: add governed nationwide AOI orchestration and CUDA enforcement

backend/app/services/runtime_reconciliation_service.py

@@ -5,13 +5,15 @@ from datetime import datetime, timezone
 
 from sqlalchemy.orm import Session
 
-from app.models import AnalysisRun, Job
+from app.models import AoiOperation, AoiOperationPartition, AnalysisRun, Job
 
 
 @dataclass(frozen=True)
 class ReconciliationResult:
     interrupted_jobs: int
     interrupted_analysis_runs: int
+    resumed_aoi_partitions: int
+    exhausted_aoi_partitions: int
 
 
 class RuntimeReconciliationService:
@@ -51,8 +53,43 @@ class RuntimeReconciliationService:
                 synchronize_session=False,
             )
         )
+        resumed_aoi_partitions = (
+            db.query(AoiOperationPartition)
+            .filter(
+                AoiOperationPartition.status == "running",
+                AoiOperationPartition.attempt_count < AoiOperationPartition.max_attempts,
+            )
+            .update(
+                {
+                    AoiOperationPartition.status: "queued",
+                    AoiOperationPartition.error_message: RuntimeReconciliationService.ERROR_MESSAGE,
+                    AoiOperationPartition.started_at: None,
+                },
+                synchronize_session=False,
+            )
+        )
+        exhausted_aoi_partitions = (
+            db.query(AoiOperationPartition)
+            .filter(
+                AoiOperationPartition.status == "running",
+                AoiOperationPartition.attempt_count >= AoiOperationPartition.max_attempts,
+            )
+            .update(
+                {
+                    AoiOperationPartition.status: "failed",
+                    AoiOperationPartition.finished_at: resolved_finished_at,
+                    AoiOperationPartition.error_message: RuntimeReconciliationService.ERROR_MESSAGE,
+                },
+                synchronize_session=False,
+            )
+        )
+        db.query(AoiOperation).filter(AoiOperation.status == "running").update(
+            {AoiOperation.status: "queued"}, synchronize_session=False
+        )
         db.commit()
         return ReconciliationResult(
             interrupted_jobs=interrupted_jobs,
             interrupted_analysis_runs=interrupted_analysis_runs,
+            resumed_aoi_partitions=resumed_aoi_partitions,
+            exhausted_aoi_partitions=exhausted_aoi_partitions,
         )