M45: authenticate Prometheus scraping

.env.example

@@ -31,7 +31,8 @@ OIDC_DEFAULT_ROLE=rental_employee
 # Observability: JSON logs are always enabled. Set a token only if /metrics is exposed
 # outside the private Compose network; Prometheus can send it as a bearer token.
 LOG_LEVEL=INFO
-METRICS_BEARER_TOKEN=
+# Required when the observability profile is enabled. Keep private and high entropy.
+METRICS_BEARER_TOKEN=replace-me-private-metrics-token
 GRAFANA_ADMIN_USER=admin
 GRAFANA_ADMIN_PASSWORD=change-me-before-start
 # Alertmanager sends every firing/resolved alert and the continuous watchdog to this

PROJECT_STATE.md

@@ -1,5 +1,14 @@
 # Project state
 
+## M45 — authenticate production metrics scraping (2026-08-21)
+
+- Pre-deployment inspection confirmed production protects `/metrics` with a non-empty
+  bearer token. Prometheus now renders that token into its private scrape authorization
+  config at container start instead of silently receiving HTTP 401.
+- Validation: the real pinned Prometheus and Alertmanager images started with rendered
+  configs; `promtool` accepted the configuration and all seven alert rules.
+- Exact next action: push M45, then execute the M44/M45 production release procedure.
+
 ## M44 — release integrity and assurance hardening (2026-08-21)
 
 - Replaced mutable archive overlays with checksum-verified, commit-named release staging,

compose.observability.yaml

@@ -2,12 +2,17 @@ services:
   prometheus:
     image: prom/prometheus:v3.7.1@sha256:ff7e389acbe064a4823212a500393d40a28a8f362e4b05cbf6742a9a3ef736b2
     profiles: ["observability"]
+    entrypoint: ["/bin/sh", "-ec"]
     command:
-      - --config.file=/etc/prometheus/prometheus.yml
-      - --storage.tsdb.retention.time=30d
-      - --web.enable-lifecycle
+      - >-
+        sed "s|__METRICS_BEARER_TOKEN__|$${METRICS_BEARER_TOKEN}|g"
+        /etc/prometheus/template.yml > /tmp/prometheus.yml
+        && exec /bin/prometheus --config.file=/tmp/prometheus.yml
+        --storage.tsdb.retention.time=30d --web.enable-lifecycle
+    environment:
+      METRICS_BEARER_TOKEN: ${METRICS_BEARER_TOKEN:?Set the private metrics bearer token}
     volumes:
-      - ./deploy/observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./deploy/observability/prometheus.yml:/etc/prometheus/template.yml:ro
       - ./deploy/observability/alerts.yml:/etc/prometheus/alerts.yml:ro
       - mobilityops-prometheus:/prometheus
     ports:

deploy/observability/prometheus.yml

@@ -13,5 +13,7 @@ alerting:
 scrape_configs:
   - job_name: mobilityops-api
     metrics_path: /metrics
+    authorization:
+      credentials: __METRICS_BEARER_TOKEN__
     static_configs:
       - targets: ["api:8000"]