Publish ITWorx Pulse source

internal/workerruntime/runtime_test.go

@@ -0,0 +1,525 @@
+package workerruntime
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/itworx/pulse/internal/config"
+	"github.com/itworx/pulse/internal/systemstatus"
+)
+
+// configFixture is a minimal valid application configuration for status tests.
+func configFixture() config.Config { return config.Config{AuthMode: "mock"} }
+
+// fakeClock is a controllable clock. The scheduling loop uses it for due-ness
+// and for every recorded timestamp, so a test decides exactly when a job
+// becomes due instead of sleeping.
+type fakeClock struct {
+	mu  sync.Mutex
+	now time.Time
+}
+
+func newFakeClock() *fakeClock {
+	return &fakeClock{now: time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)}
+}
+func (c *fakeClock) Now() time.Time {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.now
+}
+func (c *fakeClock) Advance(d time.Duration) {
+	c.mu.Lock()
+	c.now = c.now.Add(d)
+	c.mu.Unlock()
+}
+
+func quietLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(nopWriter{}, &slog.HandlerOptions{Level: slog.LevelError + 1}))
+}
+
+type nopWriter struct{}
+
+func (nopWriter) Write(p []byte) (int, error) { return len(p), nil }
+
+func testConfig(t *testing.T, clock *fakeClock, store LeaseStore) Config {
+	t.Helper()
+	return Config{
+		Owner: "worker-test", Tick: 10 * time.Millisecond, HeartbeatFile: filepath.Join(t.TempDir(), "healthy"),
+		DrainTimeout: 2 * time.Second, Leases: store, Logger: quietLogger(), Now: clock.Now,
+	}
+}
+
+// waitFor polls until condition holds or the deadline passes.
+func waitFor(t *testing.T, timeout time.Duration, condition func() bool) bool {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if condition() {
+			return true
+		}
+		time.Sleep(time.Millisecond)
+	}
+	return condition()
+}
+
+func TestNewRejectsInvalidConfiguration(t *testing.T) {
+	clock := newFakeClock()
+	valid := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
+		Run: func(context.Context) (Outcome, error) { return Outcome{}, nil }}
+	for name, testCase := range map[string]struct {
+		mutate func(*Config)
+		jobs   []Job
+	}{
+		"no owner":         {mutate: func(c *Config) { c.Owner = "" }, jobs: []Job{valid}},
+		"no lease store":   {mutate: func(c *Config) { c.Leases = nil }, jobs: []Job{valid}},
+		"no jobs":          {mutate: func(*Config) {}, jobs: nil},
+		"tick above bound": {mutate: func(c *Config) { c.Tick = time.Minute }, jobs: []Job{valid}},
+		"duplicate job":    {mutate: func(*Config) {}, jobs: []Job{valid, valid}},
+		"unknown component": {mutate: func(*Config) {}, jobs: []Job{{Name: "x-job", Component: "storage", Interval: time.Minute,
+			Timeout: time.Second, Run: valid.Run}}},
+		"missing run": {mutate: func(*Config) {}, jobs: []Job{{Name: "discovery", Component: systemstatus.ComponentWorker,
+			Interval: time.Minute, Timeout: time.Second}}},
+		"timeout above bound": {mutate: func(*Config) {}, jobs: []Job{{Name: "discovery", Component: systemstatus.ComponentWorker,
+			Interval: time.Minute, Timeout: time.Hour, Run: valid.Run}}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			config := testConfig(t, clock, NewMemoryLeaseStore())
+			testCase.mutate(&config)
+			if _, err := New(config, testCase.jobs...); !errors.Is(err, ErrInvalidConfig) {
+				t.Fatalf("error = %v, want ErrInvalidConfig", err)
+			}
+		})
+	}
+}
+
+func TestJobRunsOncePerIntervalWindow(t *testing.T) {
+	clock := newFakeClock()
+	var runs atomic.Int64
+	job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 5 * time.Second,
+		Run: func(context.Context) (Outcome, error) {
+			runs.Add(1)
+			return Outcome{Counts: map[string]int64{"items": 1}}, nil
+		}}
+	runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), job)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- runtime.Run(ctx) }()
+
+	if !waitFor(t, time.Second, func() bool { return runs.Load() == 1 }) {
+		t.Fatalf("first run count = %d, want 1", runs.Load())
+	}
+	// Many loop iterations pass without the clock moving: the job must not run
+	// again inside its interval.
+	time.Sleep(60 * time.Millisecond)
+	if runs.Load() != 1 {
+		t.Fatalf("run count inside interval = %d, want 1", runs.Load())
+	}
+	clock.Advance(time.Minute)
+	if !waitFor(t, time.Second, func() bool { return runs.Load() == 2 }) {
+		t.Fatalf("run count after interval = %d, want 2", runs.Load())
+	}
+	cancel()
+	if err := <-done; err != nil {
+		t.Fatalf("run: %v", err)
+	}
+	status := runtime.Status()[0]
+	if status.LastStatus != StatusCompleted || status.Runs != 2 || status.Successes != 2 || status.LastSuccessAt.IsZero() {
+		t.Fatalf("status = %#v", status)
+	}
+}
+
+func TestSecondRuntimeIsBlockedByTheLeaseAndRepeatedRunsAreIdempotent(t *testing.T) {
+	clock := newFakeClock()
+	store := NewMemoryLeaseStore()
+	var first, second atomic.Int64
+	build := func(counter *atomic.Int64, owner string) *Runtime {
+		config := testConfig(t, clock, store)
+		config.Owner = owner
+		runtime, err := New(config, Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 5 * time.Second,
+			Run: func(context.Context) (Outcome, error) { counter.Add(1); return Outcome{}, nil }})
+		if err != nil {
+			t.Fatal(err)
+		}
+		return runtime
+	}
+	runtimeA, runtimeB := build(&first, "worker-a"), build(&second, "worker-b")
+	ctx, cancel := context.WithCancel(context.Background())
+	doneA, doneB := make(chan error, 1), make(chan error, 1)
+	go func() { doneA <- runtimeA.Run(ctx) }()
+	go func() { doneB <- runtimeB.Run(ctx) }()
+	if !waitFor(t, 2*time.Second, func() bool { return first.Load()+second.Load() >= 1 }) {
+		t.Fatal("no worker executed the job")
+	}
+	time.Sleep(80 * time.Millisecond)
+	if total := first.Load() + second.Load(); total != 1 {
+		t.Fatalf("executions for one window = %d, want 1", total)
+	}
+	// The next window is a different lease key, so exactly one worker runs again.
+	clock.Advance(time.Minute)
+	if !waitFor(t, 2*time.Second, func() bool { return first.Load()+second.Load() == 2 }) {
+		t.Fatalf("executions after second window = %d, want 2", first.Load()+second.Load())
+	}
+	cancel()
+	<-doneA
+	<-doneB
+	if store.Windows() != 2 {
+		t.Fatalf("claimed windows = %d, want 2", store.Windows())
+	}
+	skips := uint64(0)
+	for _, status := range append(runtimeA.Status(), runtimeB.Status()...) {
+		skips += status.Skips
+	}
+	if skips == 0 {
+		t.Fatal("the worker that lost a window did not record a lease contention skip")
+	}
+}
+
+func TestOneFailingJobDoesNotStopTheOthers(t *testing.T) {
+	clock := newFakeClock()
+	var healthy, panicking, failing atomic.Int64
+	jobs := []Job{
+		{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
+			Run: func(context.Context) (Outcome, error) {
+				failing.Add(1)
+				return Outcome{}, errors.New("database is unreachable")
+			}},
+		{Name: "probe-execution", Component: systemstatus.ComponentProbes, Interval: time.Minute, Timeout: time.Second,
+			Run: func(context.Context) (Outcome, error) { panicking.Add(1); panic("probe exploded") }},
+		{Name: "notification-drain", Component: systemstatus.ComponentNotifications, Interval: time.Minute, Timeout: time.Second,
+			Run: func(context.Context) (Outcome, error) { healthy.Add(1); return Outcome{}, nil }},
+	}
+	runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), jobs...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- runtime.Run(ctx) }()
+	if !waitFor(t, 2*time.Second, func() bool { return healthy.Load() == 1 && failing.Load() == 1 && panicking.Load() == 1 }) {
+		t.Fatalf("runs healthy=%d failing=%d panicking=%d", healthy.Load(), failing.Load(), panicking.Load())
+	}
+	clock.Advance(time.Minute)
+	if !waitFor(t, 2*time.Second, func() bool { return healthy.Load() == 2 && failing.Load() == 2 }) {
+		t.Fatalf("second window healthy=%d failing=%d", healthy.Load(), failing.Load())
+	}
+	cancel()
+	if err := <-done; err != nil {
+		t.Fatal(err)
+	}
+	byName := map[string]JobStatus{}
+	for _, status := range runtime.Status() {
+		byName[status.Name] = status
+	}
+	if byName["discovery"].LastStatus != StatusFailed || byName["discovery"].Failures == 0 || byName["discovery"].LastError == "" {
+		t.Fatalf("failing job status = %#v", byName["discovery"])
+	}
+	if !strings.Contains(byName["probe-execution"].LastError, "panicked") {
+		t.Fatalf("panicking job status = %#v", byName["probe-execution"])
+	}
+	if byName["notification-drain"].LastStatus != StatusCompleted || byName["notification-drain"].Successes == 0 {
+		t.Fatalf("healthy job status = %#v", byName["notification-drain"])
+	}
+}
+
+func TestJobTimeoutIsEnforcedAndRecorded(t *testing.T) {
+	clock := newFakeClock()
+	released := make(chan struct{})
+	job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
+		Run: func(ctx context.Context) (Outcome, error) {
+			<-ctx.Done()
+			close(released)
+			return Outcome{}, ctx.Err()
+		}}
+	runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), job)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- runtime.Run(ctx) }()
+	select {
+	case <-released:
+	case <-time.After(3 * time.Second):
+		t.Fatal("job was not cancelled by its timeout")
+	}
+	if !waitFor(t, 2*time.Second, func() bool { return runtime.Status()[0].LastStatus == StatusFailed }) {
+		t.Fatalf("status = %#v", runtime.Status()[0])
+	}
+	if reason := runtime.Status()[0].LastReason; reason != "timeout" {
+		t.Fatalf("reason = %q, want timeout", reason)
+	}
+	cancel()
+	<-done
+}
+
+func TestShutdownDrainsInFlightWork(t *testing.T) {
+	clock := newFakeClock()
+	started := make(chan struct{})
+	var finished atomic.Bool
+	job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 5 * time.Second,
+		Run: func(ctx context.Context) (Outcome, error) {
+			close(started)
+			select {
+			case <-time.After(150 * time.Millisecond):
+			case <-ctx.Done():
+				return Outcome{}, ctx.Err()
+			}
+			finished.Store(true)
+			return Outcome{}, nil
+		}}
+	runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), job)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- runtime.Run(ctx) }()
+	<-started
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("shutdown: %v", err)
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("shutdown did not complete")
+	}
+	if !finished.Load() {
+		t.Fatal("in-flight job was cancelled instead of drained")
+	}
+	if status := runtime.Status()[0]; status.LastStatus != StatusCompleted {
+		t.Fatalf("drained job status = %#v", status)
+	}
+}
+
+func TestShutdownCancelsWorkThatOutlastsTheDrainBudget(t *testing.T) {
+	clock := newFakeClock()
+	config := testConfig(t, clock, NewMemoryLeaseStore())
+	config.DrainTimeout = time.Second
+	started := make(chan struct{})
+	job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 4 * time.Minute,
+		Run: func(ctx context.Context) (Outcome, error) {
+			close(started)
+			<-ctx.Done()
+			return Outcome{}, ctx.Err()
+		}}
+	runtime, err := New(config, job)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- runtime.Run(ctx) }()
+	<-started
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("shutdown: %v", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("shutdown never terminated")
+	}
+}
+
+func TestHeartbeatIsWrittenByTheLoopAndSurvivesWriteFailure(t *testing.T) {
+	clock := newFakeClock()
+	directory := t.TempDir()
+	path := filepath.Join(directory, "healthy")
+	blocked := make(chan struct{})
+	job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 4 * time.Minute,
+		Run: func(ctx context.Context) (Outcome, error) {
+			// The loop must keep heartbeating while a job is stuck.
+			<-blocked
+			return Outcome{}, nil
+		}}
+	config := testConfig(t, clock, NewMemoryLeaseStore())
+	config.HeartbeatFile = path
+	runtime, err := New(config, job)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- runtime.Run(ctx) }()
+	if !waitFor(t, 2*time.Second, func() bool { return runtime.HeartbeatCount() >= 3 }) {
+		t.Fatalf("heartbeat count = %d while a job is stuck", runtime.HeartbeatCount())
+	}
+	close(blocked)
+	cancel()
+	if err := <-done; err != nil {
+		t.Fatalf("shutdown: %v", err)
+	}
+	content, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := time.Parse(time.RFC3339, strings.TrimSpace(string(content))); err != nil {
+		t.Fatalf("heartbeat content %q is not RFC3339: %v", content, err)
+	}
+}
+
+func TestHeartbeatFailureIsReportedWithoutStoppingTheLoop(t *testing.T) {
+	clock := newFakeClock()
+	config := testConfig(t, clock, NewMemoryLeaseStore())
+	// A directory can never be written as a file.
+	config.HeartbeatFile = t.TempDir()
+	var runs atomic.Int64
+	runtime, err := New(config, Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
+		Run: func(context.Context) (Outcome, error) { runs.Add(1); return Outcome{}, nil }})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- runtime.Run(ctx) }()
+	if !waitFor(t, 2*time.Second, func() bool { return runs.Load() == 1 }) {
+		t.Fatal("loop stopped after a heartbeat write failure")
+	}
+	if runtime.HeartbeatCount() != 0 {
+		t.Fatalf("heartbeat count = %d, want 0 after write failures", runtime.HeartbeatCount())
+	}
+	cancel()
+	<-done
+}
+
+func TestLeaseFailureIsReportedAsAFailedRun(t *testing.T) {
+	clock := newFakeClock()
+	config := testConfig(t, clock, failingLeaseStore{})
+	runtime, err := New(config, Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
+		Run: func(context.Context) (Outcome, error) { t.Error("job ran without a lease"); return Outcome{}, nil }})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- runtime.Run(ctx) }()
+	if !waitFor(t, 2*time.Second, func() bool { return runtime.Status()[0].LastStatus == StatusFailed }) {
+		t.Fatalf("status = %#v", runtime.Status()[0])
+	}
+	if reason := runtime.Status()[0].LastReason; reason != "lease_unavailable" {
+		t.Fatalf("reason = %q", reason)
+	}
+	cancel()
+	<-done
+}
+
+type failingLeaseStore struct{}
+
+func (failingLeaseStore) Acquire(context.Context, string, string, time.Time, string, time.Time, time.Duration) (Lease, bool, error) {
+	return Lease{}, false, errors.New("job_runs is unavailable")
+}
+func (failingLeaseStore) Complete(context.Context, Lease, string, string, map[string]int64) error {
+	return nil
+}
+
+func TestJobHealthReportsOnlyJobsThatRan(t *testing.T) {
+	clock := newFakeClock()
+	release := make(chan struct{})
+	jobs := []Job{
+		{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
+			Run: func(context.Context) (Outcome, error) { return Outcome{}, nil }},
+		{Name: "probe-execution", Component: systemstatus.ComponentProbes, Interval: time.Minute, Timeout: 4 * time.Minute,
+			Run: func(context.Context) (Outcome, error) { <-release; return Outcome{}, nil }},
+	}
+	runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), jobs...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- runtime.Run(ctx) }()
+	if !waitFor(t, 2*time.Second, func() bool { return len(runtime.JobHealth()) == 1 }) {
+		t.Fatalf("job health = %#v", runtime.JobHealth())
+	}
+	health := runtime.JobHealth()[0]
+	if health.Component != systemstatus.ComponentWorker || health.Status != systemstatus.JobCompleted || health.LastSuccessAt.IsZero() {
+		t.Fatalf("health = %#v", health)
+	}
+	// The still-running probe job has reported nothing, so its component keeps
+	// the Unknown "not recorded" state.
+	snapshot := systemstatus.Build(configFixture(), true, clock.Now(), nil, systemstatus.WithJobs(time.Minute, runtime.JobHealth()...))
+	for _, component := range snapshot.Components {
+		if component.ID == systemstatus.ComponentProbes && component.Reason != "probe_heartbeat_not_recorded" {
+			t.Fatalf("probes component = %#v", component)
+		}
+		if component.ID == systemstatus.ComponentWorker && component.State != systemstatus.StateHealthy {
+			t.Fatalf("worker component = %#v", component)
+		}
+	}
+	close(release)
+	cancel()
+	<-done
+}
+
+func TestScheduleUsesBoundedIntervalsAndDisablesMissingWork(t *testing.T) {
+	jobs := Schedule(ScheduleRuns{})
+	if len(jobs) != 4 {
+		t.Fatalf("job count = %d, want 4", len(jobs))
+	}
+	seen := map[string]Job{}
+	for _, job := range jobs {
+		if err := job.validate(); err != nil {
+			t.Fatalf("job %s: %v", job.Name, err)
+		}
+		if job.Timeout > 10*job.Interval {
+			t.Fatalf("job %s timeout %s is unreasonable for interval %s", job.Name, job.Timeout, job.Interval)
+		}
+		seen[job.Name] = job
+	}
+	for _, name := range []string{JobDiscovery, JobAlertEvaluation, JobProbeExecution, JobNotificationDrain} {
+		job, ok := seen[name]
+		if !ok {
+			t.Fatalf("job %s is not scheduled", name)
+		}
+		outcome, err := job.Run(context.Background())
+		if err != nil || !outcome.Disabled || outcome.Reason == "" {
+			t.Fatalf("unconfigured job %s outcome = %#v err = %v", name, outcome, err)
+		}
+	}
+}
+
+func TestASkippedRunNeverBecomesAHealthyComponent(t *testing.T) {
+	clock := newFakeClock()
+	store := NewMemoryLeaseStore()
+	job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
+		Run: func(context.Context) (Outcome, error) {
+			return Outcome{Skipped: true, Reason: "source_unavailable"}, nil
+		}}
+	runtime, err := New(testConfig(t, clock, store), job)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- runtime.Run(ctx) }()
+	if !waitFor(t, 2*time.Second, func() bool { return len(runtime.JobHealth()) == 1 }) {
+		t.Fatal("the skipped run was not reported")
+	}
+	health := runtime.JobHealth()
+	if !health[0].LastSuccessAt.IsZero() {
+		t.Fatalf("a skipped run recorded a success: %#v", health[0])
+	}
+	snapshot := systemstatus.Build(configFixture(), true, clock.Now(), nil, systemstatus.WithJobs(time.Minute, health...))
+	for _, component := range snapshot.Components {
+		if component.ID == systemstatus.ComponentWorker && component.State == systemstatus.StateHealthy {
+			t.Fatalf("a job that only skips reported healthy: %#v", component)
+		}
+	}
+	if recorded := store.Status(job.JobType(), job.Name, clock.Now().Truncate(job.Interval)); recorded != StatusSkipped {
+		t.Fatalf("recorded lease status = %q, want %q", recorded, StatusSkipped)
+	}
+	cancel()
+	<-done
+}