Publish ITWorx Pulse source

2026-09-03 02:09:19 +02:00
commit bd774932d5
614 changed files with 77116 additions and 0 deletions
@@ -0,0 +1,69 @@
+package alertworker
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/itworx/pulse/internal/alert"
+)
+
+type memoryLease struct {
+	lease      Lease
+	status     string
+	leaseUntil time.Time
+}
+
+type MemoryLeaseStore struct {
+	mu      sync.Mutex
+	records map[string]memoryLease
+}
+
+func NewMemoryLeaseStore() *MemoryLeaseStore {
+	return &MemoryLeaseStore{records: make(map[string]memoryLease)}
+}
+
+func (s *MemoryLeaseStore) Acquire(_ context.Context, jobType, jobKey string, scheduledAt time.Time, owner string, now time.Time, ttl time.Duration) (Lease, bool, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.records == nil {
+		s.records = make(map[string]memoryLease)
+	}
+	key := leaseKey(jobType, jobKey, scheduledAt)
+	if record, ok := s.records[key]; ok {
+		if record.status != "running" || record.leaseUntil.After(now) {
+			return Lease{}, false, nil
+		}
+	}
+	lease := Lease{ID: alert.NewID(), JobKey: jobKey, ScheduledAt: scheduledAt.UTC(), Owner: owner}
+	s.records[key] = memoryLease{lease: lease, status: "running", leaseUntil: now.Add(ttl)}
+	return lease, true, nil
+}
+
+func (s *MemoryLeaseStore) Complete(_ context.Context, lease Lease, status, _ string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	key := leaseKey(jobType, lease.JobKey, lease.ScheduledAt)
+	record, ok := s.records[key]
+	if !ok || record.lease.ID != lease.ID || record.lease.Owner != lease.Owner {
+		return ErrLeaseLost
+	}
+	record.status = status
+	record.leaseUntil = time.Time{}
+	s.records[key] = record
+	return nil
+}
+
+func (s *MemoryLeaseStore) Status(jobKey string, scheduledAt time.Time) string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	record, ok := s.records[leaseKey(jobType, jobKey, scheduledAt)]
+	if !ok {
+		return ""
+	}
+	return record.status
+}
+
+func leaseKey(jobType, jobKey string, scheduledAt time.Time) string {
+	return jobType + "|" + jobKey + "|" + scheduledAt.UTC().Format(time.RFC3339Nano)
+}
@@ -0,0 +1,89 @@
+package alertworker
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/itworx/pulse/internal/alert"
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+type PostgresLeaseStore struct {
+	Pool *pgxpool.Pool
+}
+
+func (s PostgresLeaseStore) Acquire(ctx context.Context, jobType, jobKey string, scheduledAt time.Time, owner string, now time.Time, ttl time.Duration) (Lease, bool, error) {
+	if s.Pool == nil {
+		return Lease{}, false, ErrInvalidConfig
+	}
+	tx, err := s.Pool.BeginTx(ctx, pgx.TxOptions{})
+	if err != nil {
+		return Lease{}, false, fmt.Errorf("begin evaluator lease: %w", err)
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+	leaseID := alert.NewID()
+	leaseUntil := now.Add(ttl)
+	var insertedID string
+	err = tx.QueryRow(ctx, `INSERT INTO job_runs (id,job_type,job_key,scheduled_at,started_at,status,lease_owner,lease_until) VALUES ($1,$2,$3,$4,$5,'running',$6,$7) ON CONFLICT (job_type,job_key,scheduled_at) DO NOTHING RETURNING id`, leaseID, jobType, jobKey, scheduledAt.UTC(), now.UTC(), owner, leaseUntil.UTC()).Scan(&insertedID)
+	if err == nil {
+		if err := tx.Commit(ctx); err != nil {
+			return Lease{}, false, fmt.Errorf("commit evaluator lease: %w", err)
+		}
+		return Lease{ID: insertedID, JobKey: jobKey, ScheduledAt: scheduledAt.UTC(), Owner: owner}, true, nil
+	}
+	if !errors.Is(err, pgx.ErrNoRows) {
+		return Lease{}, false, fmt.Errorf("insert evaluator lease: %w", err)
+	}
+	var status string
+	var existingUntil *time.Time
+	if err := tx.QueryRow(ctx, `SELECT status,lease_until FROM job_runs WHERE job_type=$1 AND job_key=$2 AND scheduled_at=$3 FOR UPDATE`, jobType, jobKey, scheduledAt.UTC()).Scan(&status, &existingUntil); err != nil {
+		return Lease{}, false, fmt.Errorf("read evaluator lease: %w", err)
+	}
+	if status != "running" || (existingUntil != nil && existingUntil.After(now)) {
+		if err := tx.Commit(ctx); err != nil {
+			return Lease{}, false, err
+		}
+		return Lease{}, false, nil
+	}
+	tag, err := tx.Exec(ctx, `UPDATE job_runs SET status='running',started_at=$1,completed_at=NULL,error_code=NULL,lease_owner=$2,lease_until=$3 WHERE job_type=$4 AND job_key=$5 AND scheduled_at=$6 AND status='running' AND (lease_until IS NULL OR lease_until <= $7)`, now.UTC(), owner, leaseUntil.UTC(), jobType, jobKey, scheduledAt.UTC(), now.UTC())
+	if err != nil {
+		return Lease{}, false, fmt.Errorf("renew evaluator lease: %w", err)
+	}
+	if tag.RowsAffected() != 1 {
+		if err := tx.Commit(ctx); err != nil {
+			return Lease{}, false, err
+		}
+		return Lease{}, false, nil
+	}
+	if err := tx.Commit(ctx); err != nil {
+		return Lease{}, false, fmt.Errorf("commit evaluator lease renewal: %w", err)
+	}
+	return Lease{ID: leaseID, JobKey: jobKey, ScheduledAt: scheduledAt.UTC(), Owner: owner}, true, nil
+}
+
+func (s PostgresLeaseStore) Complete(ctx context.Context, lease Lease, status, errorCode string) error {
+	if s.Pool == nil {
+		return ErrInvalidConfig
+	}
+	if status != "completed" && status != "failed" && status != "canceled" {
+		return errors.New("invalid evaluator job status")
+	}
+	errorCode = strings.TrimSpace(errorCode)
+	if len(errorCode) > 160 {
+		errorCode = errorCode[:160]
+	}
+	counts, _ := json.Marshal(map[string]string{"status": status})
+	tag, err := s.Pool.Exec(ctx, `UPDATE job_runs SET status=$1,completed_at=now(),counts=$2::jsonb,error_code=NULLIF($3,'') ,lease_owner=NULL,lease_until=NULL WHERE job_type=$4 AND job_key=$5 AND scheduled_at=$6 AND lease_owner=$7 AND status='running'`, status, counts, errorCode, jobType, lease.JobKey, lease.ScheduledAt.UTC(), lease.Owner)
+	if err != nil {
+		return fmt.Errorf("complete evaluator job: %w", err)
+	}
+	if tag.RowsAffected() != 1 {
+		return ErrLeaseLost
+	}
+	return nil
+}
@@ -0,0 +1,57 @@
+package alertworker
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/itworx/pulse/internal/alert"
+	"github.com/itworx/pulse/internal/database"
+)
+
+func TestPostgreSQLLeaseStoreCoordinatesAndReclaimsExpiredLease(t *testing.T) {
+	dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
+	if dsn == "" {
+		t.Skip("PULSE_TEST_DATABASE_URL is not set")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 4, MinConns: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pool.Close()
+	if err := database.Migrate(ctx, pool); err != nil {
+		t.Fatal(err)
+	}
+	store := PostgresLeaseStore{Pool: pool}
+	now := time.Date(2026, 8, 2, 5, 0, 0, 0, time.UTC)
+	key := alert.NewID()
+	scheduled := now
+	first, acquired, err := store.Acquire(ctx, jobType, key, scheduled, "worker-a", now, time.Minute)
+	if err != nil || !acquired {
+		t.Fatalf("first acquire lease=%#v acquired=%v err=%v", first, acquired, err)
+	}
+	if _, acquired, err := store.Acquire(ctx, jobType, key, scheduled, "worker-b", now, time.Minute); err != nil || acquired {
+		t.Fatalf("duplicate acquire acquired=%v err=%v", acquired, err)
+	}
+	if err := store.Complete(ctx, first, "completed", ""); err != nil {
+		t.Fatal(err)
+	}
+	if _, acquired, err := store.Acquire(ctx, jobType, key, scheduled, "worker-b", now, time.Minute); err != nil || acquired {
+		t.Fatalf("completed acquire acquired=%v err=%v", acquired, err)
+	}
+	expiredKey := alert.NewID()
+	expired, acquired, err := store.Acquire(ctx, jobType, expiredKey, scheduled, "worker-a", now, time.Second)
+	if err != nil || !acquired {
+		t.Fatalf("expired first acquire=%#v acquired=%v err=%v", expired, acquired, err)
+	}
+	reclaimed, acquired, err := store.Acquire(ctx, jobType, expiredKey, scheduled, "worker-b", now.Add(2*time.Second), time.Minute)
+	if err != nil || !acquired || reclaimed.JobKey != expiredKey {
+		t.Fatalf("reclaim lease=%#v acquired=%v err=%v", reclaimed, acquired, err)
+	}
+	if err := store.Complete(ctx, reclaimed, "failed", "timeout"); err != nil {
+		t.Fatal(err)
+	}
+}
@@ -0,0 +1,273 @@
+package alertworker
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/itworx/pulse/internal/alert"
+)
+
+const jobType = "alert-evaluation"
+
+var (
+	ErrInvalidConfig = errors.New("alert evaluator configuration is invalid")
+	ErrLeaseLost     = errors.New("alert evaluator lease was lost")
+)
+
+type RuleSource interface {
+	ListEnabled(context.Context, int) ([]alert.Rule, error)
+}
+
+type Evaluator interface {
+	Evaluate(context.Context, alert.Rule) error
+}
+
+type EvaluateFunc func(context.Context, alert.Rule) error
+
+func (f EvaluateFunc) Evaluate(ctx context.Context, rule alert.Rule) error {
+	return f(ctx, rule)
+}
+
+type Config struct {
+	MaxConcurrent  int
+	MaxBatch       int
+	AttemptTimeout time.Duration
+	LeaseTTL       time.Duration
+	Owner          string
+	Now            func() time.Time
+}
+
+func (c Config) validate() error {
+	if c.MaxConcurrent < 1 || c.MaxConcurrent > 64 || c.MaxBatch < 1 || c.MaxBatch > 100 || c.MaxConcurrent > c.MaxBatch {
+		return ErrInvalidConfig
+	}
+	if c.AttemptTimeout < time.Millisecond || c.AttemptTimeout > 2*time.Minute || c.LeaseTTL < c.AttemptTimeout || c.LeaseTTL > 10*time.Minute {
+		return ErrInvalidConfig
+	}
+	if c.Owner == "" || len(c.Owner) > 120 || c.Now == nil {
+		return ErrInvalidConfig
+	}
+	return nil
+}
+
+type Lease struct {
+	ID          string
+	JobKey      string
+	ScheduledAt time.Time
+	Owner       string
+}
+
+type LeaseStore interface {
+	Acquire(context.Context, string, string, time.Time, string, time.Time, time.Duration) (Lease, bool, error)
+	Complete(context.Context, Lease, string, string) error
+}
+
+type JobResult struct {
+	RuleID      string
+	JobKey      string
+	ScheduledAt time.Time
+	StartedAt   time.Time
+	CompletedAt time.Time
+	Status      string
+	ErrorCode   string
+}
+
+type RunReport struct {
+	Scheduled int
+	Started   int
+	Completed int
+	Skipped   int
+	Failed    int
+	Canceled  int
+	Jobs      []JobResult
+}
+
+type Metrics struct {
+	RunsStarted     uint64
+	RunsCompleted   uint64
+	JobsStarted     uint64
+	JobsCompleted   uint64
+	JobsFailed      uint64
+	JobsSkipped     uint64
+	LastRunDuration time.Duration
+}
+
+type Worker struct {
+	Source    RuleSource
+	Store     LeaseStore
+	Evaluator Evaluator
+	Config    Config
+	mu        sync.Mutex
+	metrics   Metrics
+}
+
+func New(source RuleSource, store LeaseStore, evaluator Evaluator, config Config) (Worker, error) {
+	if source == nil || store == nil || evaluator == nil {
+		return Worker{}, ErrInvalidConfig
+	}
+	if config.Owner == "" {
+		config.Owner = alert.NewID()
+	}
+	if config.Now == nil {
+		config.Now = time.Now
+	}
+	if err := config.validate(); err != nil {
+		return Worker{}, err
+	}
+	return Worker{Source: source, Store: store, Evaluator: evaluator, Config: config}, nil
+}
+
+func (w *Worker) RunOnce(ctx context.Context) (RunReport, error) {
+	if err := ctx.Err(); err != nil {
+		return RunReport{}, err
+	}
+	start := w.Config.Now().UTC()
+	w.mu.Lock()
+	w.metrics.RunsStarted++
+	w.mu.Unlock()
+	rules, err := w.Source.ListEnabled(ctx, w.Config.MaxBatch)
+	if err != nil {
+		return RunReport{}, fmt.Errorf("list enabled alert rules: %w", err)
+	}
+	report := RunReport{Scheduled: len(rules), Jobs: make([]JobResult, 0, len(rules))}
+	type job struct {
+		rule  alert.Rule
+		lease Lease
+	}
+	jobs := make([]job, 0, len(rules))
+	for _, rule := range rules {
+		interval := time.Duration(rule.EvaluationIntervalSeconds) * time.Second
+		scheduledAt := start.Truncate(interval)
+		key := rule.ID
+		lease, acquired, err := w.Store.Acquire(ctx, jobType, key, scheduledAt, w.Config.Owner, start, w.Config.LeaseTTL)
+		if err != nil {
+			return report, fmt.Errorf("acquire alert evaluation lease: %w", err)
+		}
+		if !acquired {
+			report.Skipped++
+			w.mu.Lock()
+			w.metrics.JobsSkipped++
+			w.mu.Unlock()
+			report.Jobs = append(report.Jobs, JobResult{RuleID: rule.ID, JobKey: key, ScheduledAt: scheduledAt, Status: "skipped"})
+			continue
+		}
+		jobs = append(jobs, job{rule: rule, lease: lease})
+	}
+	if len(jobs) == 0 {
+		w.finishRun(start)
+		return report, nil
+	}
+	sem := make(chan struct{}, w.Config.MaxConcurrent)
+	var wait sync.WaitGroup
+	var reportMu sync.Mutex
+	for _, item := range jobs {
+		select {
+		case <-ctx.Done():
+			report.Canceled++
+			report.Jobs = append(report.Jobs, JobResult{RuleID: item.rule.ID, JobKey: item.lease.JobKey, ScheduledAt: item.lease.ScheduledAt, Status: "canceled", ErrorCode: "shutdown"})
+			_ = w.completeLease(item.lease, "canceled", "shutdown")
+		case sem <- struct{}{}:
+			wait.Add(1)
+			report.Started++
+			w.mu.Lock()
+			w.metrics.JobsStarted++
+			w.mu.Unlock()
+			go func(item job) {
+				defer wait.Done()
+				defer func() { <-sem }()
+				jobResult := w.evaluate(ctx, item.rule, item.lease)
+				reportMu.Lock()
+				report.Jobs = append(report.Jobs, jobResult)
+				switch jobResult.Status {
+				case "completed":
+					report.Completed++
+				case "failed":
+					report.Failed++
+				case "canceled":
+					report.Canceled++
+				}
+				reportMu.Unlock()
+			}(item)
+		}
+	}
+	wait.Wait()
+	sort.Slice(report.Jobs, func(i, j int) bool {
+		if report.Jobs[i].ScheduledAt.Equal(report.Jobs[j].ScheduledAt) {
+			return report.Jobs[i].RuleID < report.Jobs[j].RuleID
+		}
+		return report.Jobs[i].ScheduledAt.Before(report.Jobs[j].ScheduledAt)
+	})
+	w.finishRun(start)
+	return report, nil
+}
+
+func (w *Worker) RunLoop(ctx context.Context, interval time.Duration) error {
+	if interval < time.Second || interval > time.Hour {
+		return ErrInvalidConfig
+	}
+	for {
+		_, err := w.RunOnce(ctx)
+		if err != nil && !errors.Is(err, context.Canceled) {
+			return err
+		}
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-time.After(interval):
+		}
+	}
+}
+
+func (w *Worker) evaluate(ctx context.Context, rule alert.Rule, lease Lease) JobResult {
+	started := w.Config.Now().UTC()
+	jobResult := JobResult{RuleID: rule.ID, JobKey: lease.JobKey, ScheduledAt: lease.ScheduledAt, StartedAt: started}
+	attemptCtx, cancel := context.WithTimeout(ctx, w.Config.AttemptTimeout)
+	err := w.Evaluator.Evaluate(attemptCtx, rule)
+	cancel()
+	status, errorCode := "completed", ""
+	if err != nil {
+		status = "failed"
+		errorCode = "evaluation_failed"
+		if errors.Is(err, context.DeadlineExceeded) || errors.Is(attemptCtx.Err(), context.DeadlineExceeded) {
+			errorCode = "timeout"
+		} else if errors.Is(err, context.Canceled) || errors.Is(ctx.Err(), context.Canceled) {
+			status, errorCode = "canceled", "shutdown"
+		}
+	}
+	jobResult.CompletedAt = w.Config.Now().UTC()
+	jobResult.Status = status
+	jobResult.ErrorCode = errorCode
+	_ = w.completeLease(lease, status, errorCode)
+	w.mu.Lock()
+	switch status {
+	case "completed":
+		w.metrics.JobsCompleted++
+	case "failed", "canceled":
+		w.metrics.JobsFailed++
+	}
+	w.mu.Unlock()
+	return jobResult
+}
+
+func (w *Worker) completeLease(lease Lease, status, errorCode string) error {
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(context.Background()), 2*time.Second)
+	defer cancel()
+	return w.Store.Complete(ctx, lease, status, errorCode)
+}
+
+func (w *Worker) finishRun(start time.Time) {
+	w.mu.Lock()
+	w.metrics.RunsCompleted++
+	w.metrics.LastRunDuration = w.Config.Now().UTC().Sub(start)
+	w.mu.Unlock()
+}
+
+func (w *Worker) Metrics() Metrics {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.metrics
+}
@@ -0,0 +1,188 @@
+package alertworker
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/itworx/pulse/internal/alert"
+)
+
+type fakeSource struct {
+	rules []alert.Rule
+}
+
+func (s fakeSource) ListEnabled(_ context.Context, limit int) ([]alert.Rule, error) {
+	if limit > len(s.rules) {
+		limit = len(s.rules)
+	}
+	return s.rules[:limit], nil
+}
+
+type trackingEvaluator struct {
+	active atomic.Int32
+	max    atomic.Int32
+	calls  atomic.Int32
+	wait   time.Duration
+	err    error
+}
+
+func (e *trackingEvaluator) Evaluate(ctx context.Context, _ alert.Rule) error {
+	e.calls.Add(1)
+	active := e.active.Add(1)
+	for {
+		current := e.max.Load()
+		if active <= current || e.max.CompareAndSwap(current, active) {
+			break
+		}
+	}
+	defer e.active.Add(-1)
+	if e.wait > 0 {
+		timer := time.NewTimer(e.wait)
+		defer timer.Stop()
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timer.C:
+		}
+	}
+	return e.err
+}
+
+func testRule(id string) alert.Rule {
+	return alert.Rule{Document: alert.Document{ID: id, EvaluationIntervalSeconds: 30}}
+}
+
+func testWorker(t *testing.T, source RuleSource, store LeaseStore, evaluator Evaluator, owner string, now time.Time, maxConcurrent int) *Worker {
+	t.Helper()
+	worker, err := New(source, store, evaluator, Config{MaxConcurrent: maxConcurrent, MaxBatch: 100, AttemptTimeout: 100 * time.Millisecond, LeaseTTL: time.Second, Owner: owner, Now: func() time.Time { return now }})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &worker
+}
+
+func TestDuplicateWorkersDoNotEvaluateSameSlotTwice(t *testing.T) {
+	source := fakeSource{rules: []alert.Rule{testRule("rule-1")}}
+	store := NewMemoryLeaseStore()
+	evaluator := &trackingEvaluator{wait: 20 * time.Millisecond}
+	now := time.Date(2026, 8, 2, 5, 0, 10, 0, time.UTC)
+	first := testWorker(t, source, store, evaluator, "worker-1", now, 1)
+	second := testWorker(t, source, store, evaluator, "worker-2", now, 1)
+	var reports [2]RunReport
+	var wait sync.WaitGroup
+	wait.Add(2)
+	go func() { defer wait.Done(); reports[0], _ = first.RunOnce(context.Background()) }()
+	go func() { defer wait.Done(); reports[1], _ = second.RunOnce(context.Background()) }()
+	wait.Wait()
+	if evaluator.calls.Load() != 1 {
+		t.Fatalf("evaluation calls = %d, want 1", evaluator.calls.Load())
+	}
+	if reports[0].Skipped+reports[1].Skipped != 1 {
+		t.Fatalf("skips = %d, want 1", reports[0].Skipped+reports[1].Skipped)
+	}
+	third, err := first.RunOnce(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if third.Skipped != 1 || evaluator.calls.Load() != 1 {
+		t.Fatalf("repeat report=%#v calls=%d", third, evaluator.calls.Load())
+	}
+}
+
+func TestTimeoutIsVisibleInJobResult(t *testing.T) {
+	source := fakeSource{rules: []alert.Rule{testRule("rule-timeout")}}
+	store := NewMemoryLeaseStore()
+	evaluator := &trackingEvaluator{wait: time.Second}
+	now := time.Date(2026, 8, 2, 5, 0, 10, 0, time.UTC)
+	worker, err := New(source, store, evaluator, Config{MaxConcurrent: 1, MaxBatch: 1, AttemptTimeout: 10 * time.Millisecond, LeaseTTL: time.Second, Owner: "worker-timeout", Now: func() time.Time { return now }})
+	if err != nil {
+		t.Fatal(err)
+	}
+	report, err := worker.RunOnce(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.Failed != 1 || len(report.Jobs) != 1 || report.Jobs[0].ErrorCode != "timeout" {
+		t.Fatalf("timeout report=%#v", report)
+	}
+}
+
+func TestShutdownCancelsOutstandingEvaluationSafely(t *testing.T) {
+	source := fakeSource{rules: []alert.Rule{testRule("rule-shutdown")}}
+	store := NewMemoryLeaseStore()
+	started := make(chan struct{})
+	evaluator := EvaluateFunc(func(ctx context.Context, _ alert.Rule) error {
+		close(started)
+		<-ctx.Done()
+		return ctx.Err()
+	})
+	now := time.Date(2026, 8, 2, 5, 0, 10, 0, time.UTC)
+	worker := testWorker(t, source, store, evaluator, "worker-shutdown", now, 1)
+	ctx, cancel := context.WithCancel(context.Background())
+	result := make(chan RunReport, 1)
+	go func() {
+		report, _ := worker.RunOnce(ctx)
+		result <- report
+	}()
+	<-started
+	cancel()
+	report := <-result
+	if report.Canceled != 1 || len(report.Jobs) != 1 || report.Jobs[0].ErrorCode != "shutdown" {
+		t.Fatalf("shutdown report=%#v", report)
+	}
+}
+
+func TestLoadIsBoundedByMaxConcurrentAndBatch(t *testing.T) {
+	rules := make([]alert.Rule, 20)
+	for i := range rules {
+		rules[i] = testRule(alert.NewID())
+	}
+	store := NewMemoryLeaseStore()
+	evaluator := &trackingEvaluator{wait: 2 * time.Millisecond}
+	now := time.Date(2026, 8, 2, 5, 0, 10, 0, time.UTC)
+	worker := testWorker(t, fakeSource{rules: rules}, store, evaluator, "worker-scale", now, 3)
+	report, err := worker.RunOnce(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.Scheduled != 20 || report.Started != 20 || report.Completed != 20 || evaluator.max.Load() > 3 {
+		t.Fatalf("bounded report=%#v max=%d", report, evaluator.max.Load())
+	}
+	if metrics := worker.Metrics(); metrics.JobsStarted != 20 || metrics.JobsCompleted != 20 {
+		t.Fatalf("metrics=%#v", metrics)
+	}
+}
+
+func TestInvalidWorkerConfigurationAndEvaluationError(t *testing.T) {
+	_, err := New(fakeSource{}, NewMemoryLeaseStore(), EvaluateFunc(func(context.Context, alert.Rule) error { return nil }), Config{MaxConcurrent: 0, MaxBatch: 1, AttemptTimeout: time.Second, LeaseTTL: time.Second, Owner: "x", Now: time.Now})
+	if !errors.Is(err, ErrInvalidConfig) {
+		t.Fatalf("config error=%v", err)
+	}
+	source := fakeSource{rules: []alert.Rule{testRule("rule-error")}}
+	store := NewMemoryLeaseStore()
+	worker := testWorker(t, source, store, EvaluateFunc(func(context.Context, alert.Rule) error { return errors.New("upstream failed") }), "worker-error", time.Now().UTC(), 1)
+	report, err := worker.RunOnce(context.Background())
+	if err != nil || report.Failed != 1 || report.Jobs[0].ErrorCode != "evaluation_failed" {
+		t.Fatalf("error report=%#v err=%v", report, err)
+	}
+}
+
+func TestFailedSlotIsVisibleAndNotRetried(t *testing.T) {
+	source := fakeSource{rules: []alert.Rule{testRule("rule-failed")}}
+	store := NewMemoryLeaseStore()
+	evaluator := &trackingEvaluator{err: errors.New("upstream failed")}
+	now := time.Date(2026, 8, 2, 5, 0, 10, 0, time.UTC)
+	worker := testWorker(t, source, store, evaluator, "worker-failed", now, 1)
+	first, err := worker.RunOnce(context.Background())
+	if err != nil || first.Failed != 1 || first.Jobs[0].ErrorCode != "evaluation_failed" {
+		t.Fatalf("first report=%#v err=%v", first, err)
+	}
+	second, err := worker.RunOnce(context.Background())
+	if err != nil || second.Skipped != 1 || evaluator.calls.Load() != 1 {
+		t.Fatalf("second report=%#v calls=%d err=%v", second, evaluator.calls.Load(), err)
+	}
+}