ci: add lightweight validation fast path

2026-09-03 00:30:21 +00:00
11 changed files with 29 additions and 289 deletions
@@ -55,6 +55,33 @@ jobs:
            exit 1
          fi
          # MANAGED_FAST_PATH: documentation and this baseline workflow cannot
          # affect the shipped runtime. Keep the required status check, but do
          # not install toolchains or execute the full product suite.
          if [[ -n "${GITHUB_BASE_REF:-}" ]]; then
            git fetch --no-tags --depth=1 origin "${GITHUB_BASE_REF}"
            managed_base="origin/${GITHUB_BASE_REF}"
            git diff --check "${managed_base}..HEAD"
            mapfile -t managed_changed_files < <(
              git diff --name-only --diff-filter=ACMR "${managed_base}..HEAD"
            )
            managed_runtime_change=0
            for managed_path in "${managed_changed_files[@]}"; do
              case "${managed_path}" in
                *.md|*.mdx|docs/*|.github/ISSUE_TEMPLATE/*|.gitea/ISSUE_TEMPLATE/*|.gitea/runner-scope.sh|.gitea/workflows/managed-validation.yml)
                  ;;
                *)
                  managed_runtime_change=1
                  break
                  ;;
              esac
            done
            if [[ "${#managed_changed_files[@]}" -gt 0 && "${managed_runtime_change}" -eq 0 ]]; then
              printf 'Managed validation fast path: %s non-runtime file(s); full product suite skipped.\n' \
                "${#managed_changed_files[@]}"
              exit 0
            fi
          fi
          if [[ -f pyproject.toml || -f requirements.txt ]]; then
            # Compile only tracked Python sources. Running compileall after a
            # Node install would otherwise traverse node_modules and turn a
@@ -9,30 +9,6 @@ from django.shortcuts import redirect
 from django.urls import reverse
 from django.utils.http import url_has_allowed_host_and_scheme
 from scripts.migration_worker_gate import hold_active
 class MigrationMaintenanceMiddleware:
    """Block candidate traffic before durable migration activation, including GET writes."""
    def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]) -> None:
        self.get_response = get_response
    def __call__(self, request: HttpRequest) -> HttpResponse:
        try:
            held = hold_active()
        except (OSError, RuntimeError):
            held = True
        health = request.method in {"GET", "HEAD"} and request.path_info in {
            "/health/ready/", "/health/live/",
        }
        if held and not health:
            response = JsonResponse({"error": "migration_maintenance"}, status=503)
            response["Retry-After"] = "30"
            response["Cache-Control"] = "no-store"
            return response
        return self.get_response(request)
 class DemoReadOnlyMiddleware:
    """Prevent a shared public demo account from mutating application data."""
@@ -159,7 +159,6 @@ INSTALLED_APPS = [
 ]
 MIDDLEWARE = [
    "apps.core.middleware.MigrationMaintenanceMiddleware",
    "django.middleware.security.SecurityMiddleware",
    "whitenoise.middleware.WhiteNoiseMiddleware",
    "django.contrib.sessions.middleware.SessionMiddleware",
@@ -36,7 +36,7 @@ stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 [program:worker]
-command=/app/.venv/bin/python /app/scripts/migration_worker_gate.py worker
+command=/app/.venv/bin/celery -A config worker -l INFO -Q high,default,low --concurrency=2
 directory=/app
 user=app
 priority=50
@@ -47,7 +47,7 @@ stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 [program:scheduler]
-command=/app/.venv/bin/python /app/scripts/migration_worker_gate.py scheduler
+command=/app/.venv/bin/celery -A config beat -l INFO --schedule /tmp/celerybeat-schedule
 directory=/app
 user=app
 priority=60
@@ -1,35 +0,0 @@
 # Append after docker-compose.unraid.yml. Operator-only migration prerequisite;
 # every path must be a verified cold clone, never the original live data.
 # Also pass MIGRATION_ENV_FILE as Compose --env-file for interpolation parity.
 name: vacatureradar-managed
 services:
  app:
    network_mode: ${MIGRATION_NETWORK:?exact original network required}
    labels:
      io.itworx.migration-attempt: ${MIGRATION_ATTEMPT:?journaled migration attempt required}
    image: ${MIGRATION_IMAGE:?set an attempt-specific candidate image}
    env_file:
      - ${VACATURERADAR_ENV_FILE:?set the protected external runtime env file}
    build:
      labels:
        org.opencontainers.image.source: jens/vacatureradar
        org.opencontainers.image.revision: ${MIGRATION_SOURCE_REVISION:?full verified Git commit required}
        org.opencontainers.image.source-tree: ${MIGRATION_SOURCE_TREE:?verified Git tree required}
        org.opencontainers.image.version: ${MIGRATION_BUILD_ID:?broker build ID required}
        org.opencontainers.image.created: ${MIGRATION_BUILD_DATE:?broker UTC build date required}
    volumes:
      - type: bind
        source: ${MIGRATION_LOCAL_ROOT:?verified cold clone of the complete local tree required}/media
        target: /app/media
        bind:
          create_host_path: false
      - type: bind
        source: ${MIGRATION_LOCAL_ROOT:?verified cold clone of the complete local tree required}/logs
        target: /app/logs
        bind:
          create_host_path: false
      - type: bind
        source: ${MIGRATION_LOCAL_ROOT:?verified cold clone of the complete local tree required}
        target: /app/local
        bind:
          create_host_path: false
@@ -1,40 +0,0 @@
 # Gecontroleerde migratie van legacy deployments
 `docker-compose.migration.yml` is een optionele operatoroverride, geen wijziging
 aan de standaarddeployment. Gebruik deze alleen na de expliciete migratieprocedure
 in ProjectBrain `scripts/legacy-deployment-migration/`.
 De operator valideert de exacte Git-revisie en imageprovenance, stopt de oude
 container en maakt een gecontroleerde koude kopie van de volledige `local`-map.
 `MIGRATION_LOCAL_ROOT` verwijst uitsluitend naar die kopie. Media, logs en de
 PostgreSQL-data houden hun geneste mountrelatie. De originele data, image en
 container blijven bewaard voor rollback; startupmigraties mogen ze niet wijzigen.
 Alle migratievariabelen zijn verplicht, paden worden niet automatisch aangemaakt,
 en een attemptlabel bindt de nieuwe container aan precies één hersteltransactie.
 De operator controleert gemergde Compose-mounts, gezondheid, HTTP en de echte
 brokerreceipt voordat de configuratie definitief wordt overgezet.
 De override is offline getest met echte Compose-rendering en ontbrekende
 invoervariabelen. Toevoeging van dit bestand bewijst geen uitgevoerde productiemigratie.
 # Outbound worker hold
 Before starting a migration candidate, create `.migration-worker-hold` in its
 cloned `/app/local` directory. The supervised Celery worker and scheduler wait
 without consuming tasks until the operator removes that exact attempt-owned
 file after the deployment commit. Normal startup is unchanged when it is absent.
 Do not put the marker into the original data. This is a startup gate, not a
 control for pausing an already-running worker. Recovery before commit retains
 the hold and candidate data; recovery after commit resumes activation and must
 never revert to stale original data after outbound work has been released.
 The first Django middleware also returns 503 (no-store) for every ordinary
 request while held, including GET requests. Only exact GET/HEAD requests to
 `/health/ready/` and `/health/live/` pass. Marker inspection errors fail closed.
 The migration helper creates a nonce-bound marker only in the cold clone and
 removes it with directory fsync after a durable `committed` journal. Recovery
 after that boundary may resume activation but can never restore old data.
 `MIGRATION_NETWORK` is required and must equal the inspected existing
 `vacatureradar_default` network. The managed Compose project does not move the
 application to a newly-created network.
@@ -2445,32 +2445,6 @@ tasks:
    note: Releasebootstrap gepind op immutable setup-uv v8.1.0-commit en uv 0.11.12 via Astral-mirror; Gitea Actions-run
      3062 publiceerde digest sha256:98f4b33d met checksum-geldige SBOM/release-evidence; Unraid gepind op volledige
      digest en live health/readiness groen.
 - id: VR-232
  title: Houd uitgaand werk vast tijdens gecontroleerde datamigratie
  status: ready
  priority: P0
  requirement_ids: [NFR-009]
  depends_on: [VR-231]
  summary: Door de eigenaar gevraagde migratie met rollback; Celery start pas na vrijgave van de kandidaatkopie.
  acceptance_criteria:
  - Een marker in de kandidaatkopie blokkeert worker en scheduler voor hun eerste externe actie.
  - Zonder marker blijft normaal opstartgedrag ongewijzigd.
  - Onbekende procesrollen en ongeldige markers falen gesloten.
  - De migratie verwijdert uitsluitend haar eigen marker na geverifieerde commit.
  - Gewone webverzoeken blijven geblokkeerd tot commit; alleen exacte GET/HEAD-healthroutes zijn beschikbaar.
  - De migratie behoudt het bestaande Docker-netwerk via een verplichte gevalideerde variabele.
  verification:
  - uv run pytest tests/unit/test_migration_worker_gate.py
  - uv run pytest tests/unit/test_migration_maintenance.py
  - ./scripts/codex_verify.sh
  primary_paths:
  - scripts/migration_worker_gate.py
  - deployment/unraid/supervisord.conf
  - tests/unit/test_migration_worker_gate.py
  - tests/unit/test_migration_maintenance.py
  - apps/core/middleware.py
  - config/settings.py
  - docker-compose.migration.yml
 - id: VR-231
  title: Sluit de operationele 0.3.17-restpunten
  status: done
@@ -1,17 +1,5 @@
 # Projectstatus
 ## Gecontroleerde migratie — 2026-09-09
 VR-232 voegt een startup-hold voor Celery worker/beat toe tijdens de expliciet
 goedgekeurde legacy-migratie. Alleen de kandidaatdatakopie krijgt een marker;
 uitgaand werk wordt pas na een geverifieerde deploymentcommit vrijgegeven.
 De eerste middleware blokkeert ook gewone webverzoeken tot commit; alleen exacte
 GET/HEAD-healthroutes blijven beschikbaar. De netwerkoverride behoudt het bestaande
 netwerk. Lokaal slagen 344 tests inclusief browsercontroles (84,07% coverage).
 De volledige `scripts/codex_verify.sh`-gate is groen: Ruff, Django-checks,
 migratiecontrole, tests, taakledger en repositoryvalidatie zijn geslaagd.
 Live migratie is niet uitgevoerd. Bestaande productietaken zijn niet gewijzigd.
 - Laatst bijgewerkt: 2026-08-12
 - Repositoryversie: 0.3.17 immutable releaseherstel
 - Uitvoeringsmodus: autonome backlog
@@ -1,60 +0,0 @@
 """Keep outbound Celery work paused during a verified cold-data migration.
 The operator creates the hold file in the candidate data clone before startup
 and removes it only after committing the verified deployment. Normal startups
 without a hold file behave unchanged. This does not pause an existing worker.
 """
 import os
 import stat
 import sys
 import time
 from pathlib import Path
 HOLD_FILE = Path("/app/local/.migration-worker-hold")
 COMMANDS = {
    "worker": [
        "/app/.venv/bin/celery",
        "-A",
        "config",
        "worker",
        "-l",
        "INFO",
        "-Q",
        "high,default,low",
        "--concurrency=2",
    ],
    "scheduler": [
        "/app/.venv/bin/celery",
        "-A",
        "config",
        "beat",
        "-l",
        "INFO",
        "--schedule",
        "/tmp/celerybeat-schedule",  # noqa: S108 - existing supervised container-local schedule path
    ],
 }
 def hold_active(path=HOLD_FILE):
    try:
        mode = path.lstat().st_mode
    except FileNotFoundError:
        return False
    if not stat.S_ISREG(mode):
        raise RuntimeError("Migration hold must be a regular file")
    return True
 def start(role, *, check=hold_active, sleep=time.sleep, execute=os.execv):
    if role not in COMMANDS:
        raise ValueError("Unsupported supervised process")
    while check():
        sleep(1)
    command = COMMANDS[role]
    execute(command[0], command)
 if __name__ == "__main__":
    start(sys.argv[1] if len(sys.argv) == 2 else "")
@@ -1,45 +0,0 @@
 from unittest.mock import Mock, patch
 import pytest
 from django.http import HttpResponse
 from django.test import RequestFactory
 from apps.core.middleware import MigrationMaintenanceMiddleware
@pytest.mark.parametrize("method,path,allowed", [
    ("GET", "/health/ready/", True),
    ("HEAD", "/health/live/", True),
    ("POST", "/health/ready/", False),
    ("GET", "/health/ready", False),
    ("GET", "/", False),
    ("POST", "/accounts/login/", False),
    ("OPTIONS", "/health/live/", False),
 ])
 def test_held_candidate_only_allows_exact_safe_health(method, path, allowed):
    downstream = Mock(return_value=HttpResponse("ok"))
    with patch("apps.core.middleware.hold_active", return_value=True):
        response = MigrationMaintenanceMiddleware(downstream)(
            RequestFactory().generic(method, path)
        )
    assert response.status_code == (200 if allowed else 503)
    assert downstream.called is allowed
    if not allowed:
        assert response["Cache-Control"] == "no-store"
@pytest.mark.parametrize("failure", [PermissionError(), RuntimeError()])
 def test_marker_errors_block_ordinary_traffic(failure):
    downstream = Mock()
    with patch("apps.core.middleware.hold_active", side_effect=failure):
        response = MigrationMaintenanceMiddleware(downstream)(RequestFactory().get("/"))
    assert response.status_code == 503
    downstream.assert_not_called()
 def test_normal_runtime_unchanged_and_middleware_is_first(settings):
    assert settings.MIDDLEWARE[0] == "apps.core.middleware.MigrationMaintenanceMiddleware"
    downstream = Mock(return_value=HttpResponse("ok"))
    with patch("apps.core.middleware.hold_active", return_value=False):
        response = MigrationMaintenanceMiddleware(downstream)(RequestFactory().post("/"))
    assert response.status_code == 200
@@ -1,44 +0,0 @@
 import importlib.util
 from pathlib import Path
 import pytest
 spec = importlib.util.spec_from_file_location(
    "migration_worker_gate",
    Path(__file__).resolve().parents[2] / "scripts/migration_worker_gate.py",
 )
 gate = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(gate)
@pytest.mark.parametrize("role", ["worker", "scheduler"])
 def test_hold_prevents_outbound_process_until_release(role):
    states = iter([True, True, False])
    events = []
    gate.start(
        role,
        check=lambda: next(states),
        sleep=lambda seconds: events.append("wait"),
        execute=lambda executable, args: events.append(args),
    )
    assert events == ["wait", "wait", gate.COMMANDS[role]]
 def test_normal_startup_and_missing_hold(tmp_path):
    assert not gate.hold_active(tmp_path / "absent")
    events = []
    gate.start("worker", check=lambda: False, execute=lambda executable, args: events.append(args))
    assert events == [gate.COMMANDS["worker"]]
 def test_hold_exists_and_unsupported_marker_fails_closed(tmp_path):
    marker = tmp_path / "hold"
    marker.write_text("fixture-attempt")
    assert gate.hold_active(marker)
    with pytest.raises(RuntimeError):
        gate.hold_active(tmp_path)
 def test_unknown_role_never_executes():
    with pytest.raises(ValueError):
        gate.start("shell", execute=lambda *args: pytest.fail("unreviewed command"))