package workerruntime import ( "context" "errors" "log/slog" "os" "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" "github.com/itworx/pulse/internal/config" "github.com/itworx/pulse/internal/systemstatus" ) // configFixture is a minimal valid application configuration for status tests. func configFixture() config.Config { return config.Config{AuthMode: "mock"} } // fakeClock is a controllable clock. The scheduling loop uses it for due-ness // and for every recorded timestamp, so a test decides exactly when a job // becomes due instead of sleeping. type fakeClock struct { mu sync.Mutex now time.Time } func newFakeClock() *fakeClock { return &fakeClock{now: time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)} } func (c *fakeClock) Now() time.Time { c.mu.Lock() defer c.mu.Unlock() return c.now } func (c *fakeClock) Advance(d time.Duration) { c.mu.Lock() c.now = c.now.Add(d) c.mu.Unlock() } func quietLogger() *slog.Logger { return slog.New(slog.NewTextHandler(nopWriter{}, &slog.HandlerOptions{Level: slog.LevelError + 1})) } type nopWriter struct{} func (nopWriter) Write(p []byte) (int, error) { return len(p), nil } func testConfig(t *testing.T, clock *fakeClock, store LeaseStore) Config { t.Helper() return Config{ Owner: "worker-test", Tick: 10 * time.Millisecond, HeartbeatFile: filepath.Join(t.TempDir(), "healthy"), DrainTimeout: 2 * time.Second, Leases: store, Logger: quietLogger(), Now: clock.Now, } } // waitFor polls until condition holds or the deadline passes. func waitFor(t *testing.T, timeout time.Duration, condition func() bool) bool { t.Helper() deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { if condition() { return true } time.Sleep(time.Millisecond) } return condition() } func TestNewRejectsInvalidConfiguration(t *testing.T) { clock := newFakeClock() valid := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second, Run: func(context.Context) (Outcome, error) { return Outcome{}, nil }} for name, testCase := range map[string]struct { mutate func(*Config) jobs []Job }{ "no owner": {mutate: func(c *Config) { c.Owner = "" }, jobs: []Job{valid}}, "no lease store": {mutate: func(c *Config) { c.Leases = nil }, jobs: []Job{valid}}, "no jobs": {mutate: func(*Config) {}, jobs: nil}, "tick above bound": {mutate: func(c *Config) { c.Tick = time.Minute }, jobs: []Job{valid}}, "duplicate job": {mutate: func(*Config) {}, jobs: []Job{valid, valid}}, "unknown component": {mutate: func(*Config) {}, jobs: []Job{{Name: "x-job", Component: "storage", Interval: time.Minute, Timeout: time.Second, Run: valid.Run}}}, "missing run": {mutate: func(*Config) {}, jobs: []Job{{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second}}}, "timeout above bound": {mutate: func(*Config) {}, jobs: []Job{{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Hour, Run: valid.Run}}}, } { t.Run(name, func(t *testing.T) { config := testConfig(t, clock, NewMemoryLeaseStore()) testCase.mutate(&config) if _, err := New(config, testCase.jobs...); !errors.Is(err, ErrInvalidConfig) { t.Fatalf("error = %v, want ErrInvalidConfig", err) } }) } } func TestJobRunsOncePerIntervalWindow(t *testing.T) { clock := newFakeClock() var runs atomic.Int64 job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 5 * time.Second, Run: func(context.Context) (Outcome, error) { runs.Add(1) return Outcome{Counts: map[string]int64{"items": 1}}, nil }} runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), job) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- runtime.Run(ctx) }() if !waitFor(t, time.Second, func() bool { return runs.Load() == 1 }) { t.Fatalf("first run count = %d, want 1", runs.Load()) } // Many loop iterations pass without the clock moving: the job must not run // again inside its interval. time.Sleep(60 * time.Millisecond) if runs.Load() != 1 { t.Fatalf("run count inside interval = %d, want 1", runs.Load()) } clock.Advance(time.Minute) if !waitFor(t, time.Second, func() bool { return runs.Load() == 2 }) { t.Fatalf("run count after interval = %d, want 2", runs.Load()) } cancel() if err := <-done; err != nil { t.Fatalf("run: %v", err) } status := runtime.Status()[0] if status.LastStatus != StatusCompleted || status.Runs != 2 || status.Successes != 2 || status.LastSuccessAt.IsZero() { t.Fatalf("status = %#v", status) } } func TestSecondRuntimeIsBlockedByTheLeaseAndRepeatedRunsAreIdempotent(t *testing.T) { clock := newFakeClock() store := NewMemoryLeaseStore() var first, second atomic.Int64 build := func(counter *atomic.Int64, owner string) *Runtime { config := testConfig(t, clock, store) config.Owner = owner runtime, err := New(config, Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 5 * time.Second, Run: func(context.Context) (Outcome, error) { counter.Add(1); return Outcome{}, nil }}) if err != nil { t.Fatal(err) } return runtime } runtimeA, runtimeB := build(&first, "worker-a"), build(&second, "worker-b") ctx, cancel := context.WithCancel(context.Background()) doneA, doneB := make(chan error, 1), make(chan error, 1) go func() { doneA <- runtimeA.Run(ctx) }() go func() { doneB <- runtimeB.Run(ctx) }() if !waitFor(t, 2*time.Second, func() bool { return first.Load()+second.Load() >= 1 }) { t.Fatal("no worker executed the job") } time.Sleep(80 * time.Millisecond) if total := first.Load() + second.Load(); total != 1 { t.Fatalf("executions for one window = %d, want 1", total) } // The next window is a different lease key, so exactly one worker runs again. clock.Advance(time.Minute) if !waitFor(t, 2*time.Second, func() bool { return first.Load()+second.Load() == 2 }) { t.Fatalf("executions after second window = %d, want 2", first.Load()+second.Load()) } cancel() <-doneA <-doneB if store.Windows() != 2 { t.Fatalf("claimed windows = %d, want 2", store.Windows()) } skips := uint64(0) for _, status := range append(runtimeA.Status(), runtimeB.Status()...) { skips += status.Skips } if skips == 0 { t.Fatal("the worker that lost a window did not record a lease contention skip") } } func TestOneFailingJobDoesNotStopTheOthers(t *testing.T) { clock := newFakeClock() var healthy, panicking, failing atomic.Int64 jobs := []Job{ {Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second, Run: func(context.Context) (Outcome, error) { failing.Add(1) return Outcome{}, errors.New("database is unreachable") }}, {Name: "probe-execution", Component: systemstatus.ComponentProbes, Interval: time.Minute, Timeout: time.Second, Run: func(context.Context) (Outcome, error) { panicking.Add(1); panic("probe exploded") }}, {Name: "notification-drain", Component: systemstatus.ComponentNotifications, Interval: time.Minute, Timeout: time.Second, Run: func(context.Context) (Outcome, error) { healthy.Add(1); return Outcome{}, nil }}, } runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), jobs...) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- runtime.Run(ctx) }() if !waitFor(t, 2*time.Second, func() bool { return healthy.Load() == 1 && failing.Load() == 1 && panicking.Load() == 1 }) { t.Fatalf("runs healthy=%d failing=%d panicking=%d", healthy.Load(), failing.Load(), panicking.Load()) } clock.Advance(time.Minute) if !waitFor(t, 2*time.Second, func() bool { return healthy.Load() == 2 && failing.Load() == 2 }) { t.Fatalf("second window healthy=%d failing=%d", healthy.Load(), failing.Load()) } cancel() if err := <-done; err != nil { t.Fatal(err) } byName := map[string]JobStatus{} for _, status := range runtime.Status() { byName[status.Name] = status } if byName["discovery"].LastStatus != StatusFailed || byName["discovery"].Failures == 0 || byName["discovery"].LastError == "" { t.Fatalf("failing job status = %#v", byName["discovery"]) } if !strings.Contains(byName["probe-execution"].LastError, "panicked") { t.Fatalf("panicking job status = %#v", byName["probe-execution"]) } if byName["notification-drain"].LastStatus != StatusCompleted || byName["notification-drain"].Successes == 0 { t.Fatalf("healthy job status = %#v", byName["notification-drain"]) } } func TestJobTimeoutIsEnforcedAndRecorded(t *testing.T) { clock := newFakeClock() released := make(chan struct{}) job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second, Run: func(ctx context.Context) (Outcome, error) { <-ctx.Done() close(released) return Outcome{}, ctx.Err() }} runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), job) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- runtime.Run(ctx) }() select { case <-released: case <-time.After(3 * time.Second): t.Fatal("job was not cancelled by its timeout") } if !waitFor(t, 2*time.Second, func() bool { return runtime.Status()[0].LastStatus == StatusFailed }) { t.Fatalf("status = %#v", runtime.Status()[0]) } if reason := runtime.Status()[0].LastReason; reason != "timeout" { t.Fatalf("reason = %q, want timeout", reason) } cancel() <-done } func TestShutdownDrainsInFlightWork(t *testing.T) { clock := newFakeClock() started := make(chan struct{}) var finished atomic.Bool job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 5 * time.Second, Run: func(ctx context.Context) (Outcome, error) { close(started) select { case <-time.After(150 * time.Millisecond): case <-ctx.Done(): return Outcome{}, ctx.Err() } finished.Store(true) return Outcome{}, nil }} runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), job) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- runtime.Run(ctx) }() <-started cancel() select { case err := <-done: if err != nil { t.Fatalf("shutdown: %v", err) } case <-time.After(3 * time.Second): t.Fatal("shutdown did not complete") } if !finished.Load() { t.Fatal("in-flight job was cancelled instead of drained") } if status := runtime.Status()[0]; status.LastStatus != StatusCompleted { t.Fatalf("drained job status = %#v", status) } } func TestShutdownCancelsWorkThatOutlastsTheDrainBudget(t *testing.T) { clock := newFakeClock() config := testConfig(t, clock, NewMemoryLeaseStore()) config.DrainTimeout = time.Second started := make(chan struct{}) job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 4 * time.Minute, Run: func(ctx context.Context) (Outcome, error) { close(started) <-ctx.Done() return Outcome{}, ctx.Err() }} runtime, err := New(config, job) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- runtime.Run(ctx) }() <-started cancel() select { case err := <-done: if err != nil { t.Fatalf("shutdown: %v", err) } case <-time.After(5 * time.Second): t.Fatal("shutdown never terminated") } } func TestHeartbeatIsWrittenByTheLoopAndSurvivesWriteFailure(t *testing.T) { clock := newFakeClock() directory := t.TempDir() path := filepath.Join(directory, "healthy") blocked := make(chan struct{}) job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 4 * time.Minute, Run: func(ctx context.Context) (Outcome, error) { // The loop must keep heartbeating while a job is stuck. <-blocked return Outcome{}, nil }} config := testConfig(t, clock, NewMemoryLeaseStore()) config.HeartbeatFile = path runtime, err := New(config, job) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- runtime.Run(ctx) }() if !waitFor(t, 2*time.Second, func() bool { return runtime.HeartbeatCount() >= 3 }) { t.Fatalf("heartbeat count = %d while a job is stuck", runtime.HeartbeatCount()) } close(blocked) cancel() if err := <-done; err != nil { t.Fatalf("shutdown: %v", err) } content, err := os.ReadFile(path) if err != nil { t.Fatal(err) } if _, err := time.Parse(time.RFC3339, strings.TrimSpace(string(content))); err != nil { t.Fatalf("heartbeat content %q is not RFC3339: %v", content, err) } } func TestHeartbeatFailureIsReportedWithoutStoppingTheLoop(t *testing.T) { clock := newFakeClock() config := testConfig(t, clock, NewMemoryLeaseStore()) // A directory can never be written as a file. config.HeartbeatFile = t.TempDir() var runs atomic.Int64 runtime, err := New(config, Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second, Run: func(context.Context) (Outcome, error) { runs.Add(1); return Outcome{}, nil }}) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- runtime.Run(ctx) }() if !waitFor(t, 2*time.Second, func() bool { return runs.Load() == 1 }) { t.Fatal("loop stopped after a heartbeat write failure") } if runtime.HeartbeatCount() != 0 { t.Fatalf("heartbeat count = %d, want 0 after write failures", runtime.HeartbeatCount()) } cancel() <-done } func TestLeaseFailureIsReportedAsAFailedRun(t *testing.T) { clock := newFakeClock() config := testConfig(t, clock, failingLeaseStore{}) runtime, err := New(config, Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second, Run: func(context.Context) (Outcome, error) { t.Error("job ran without a lease"); return Outcome{}, nil }}) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- runtime.Run(ctx) }() if !waitFor(t, 2*time.Second, func() bool { return runtime.Status()[0].LastStatus == StatusFailed }) { t.Fatalf("status = %#v", runtime.Status()[0]) } if reason := runtime.Status()[0].LastReason; reason != "lease_unavailable" { t.Fatalf("reason = %q", reason) } cancel() <-done } type failingLeaseStore struct{} func (failingLeaseStore) Acquire(context.Context, string, string, time.Time, string, time.Time, time.Duration) (Lease, bool, error) { return Lease{}, false, errors.New("job_runs is unavailable") } func (failingLeaseStore) Complete(context.Context, Lease, string, string, map[string]int64) error { return nil } func TestJobHealthReportsOnlyJobsThatRan(t *testing.T) { clock := newFakeClock() release := make(chan struct{}) jobs := []Job{ {Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second, Run: func(context.Context) (Outcome, error) { return Outcome{}, nil }}, {Name: "probe-execution", Component: systemstatus.ComponentProbes, Interval: time.Minute, Timeout: 4 * time.Minute, Run: func(context.Context) (Outcome, error) { <-release; return Outcome{}, nil }}, } runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), jobs...) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- runtime.Run(ctx) }() if !waitFor(t, 2*time.Second, func() bool { return len(runtime.JobHealth()) == 1 }) { t.Fatalf("job health = %#v", runtime.JobHealth()) } health := runtime.JobHealth()[0] if health.Component != systemstatus.ComponentWorker || health.Status != systemstatus.JobCompleted || health.LastSuccessAt.IsZero() { t.Fatalf("health = %#v", health) } // The still-running probe job has reported nothing, so its component keeps // the Unknown "not recorded" state. snapshot := systemstatus.Build(configFixture(), true, clock.Now(), nil, systemstatus.WithJobs(time.Minute, runtime.JobHealth()...)) for _, component := range snapshot.Components { if component.ID == systemstatus.ComponentProbes && component.Reason != "probe_heartbeat_not_recorded" { t.Fatalf("probes component = %#v", component) } if component.ID == systemstatus.ComponentWorker && component.State != systemstatus.StateHealthy { t.Fatalf("worker component = %#v", component) } } close(release) cancel() <-done } func TestScheduleUsesBoundedIntervalsAndDisablesMissingWork(t *testing.T) { jobs := Schedule(ScheduleRuns{}) if len(jobs) != 4 { t.Fatalf("job count = %d, want 4", len(jobs)) } seen := map[string]Job{} for _, job := range jobs { if err := job.validate(); err != nil { t.Fatalf("job %s: %v", job.Name, err) } if job.Timeout > 10*job.Interval { t.Fatalf("job %s timeout %s is unreasonable for interval %s", job.Name, job.Timeout, job.Interval) } seen[job.Name] = job } for _, name := range []string{JobDiscovery, JobAlertEvaluation, JobProbeExecution, JobNotificationDrain} { job, ok := seen[name] if !ok { t.Fatalf("job %s is not scheduled", name) } outcome, err := job.Run(context.Background()) if err != nil || !outcome.Disabled || outcome.Reason == "" { t.Fatalf("unconfigured job %s outcome = %#v err = %v", name, outcome, err) } } } func TestASkippedRunNeverBecomesAHealthyComponent(t *testing.T) { clock := newFakeClock() store := NewMemoryLeaseStore() job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second, Run: func(context.Context) (Outcome, error) { return Outcome{Skipped: true, Reason: "source_unavailable"}, nil }} runtime, err := New(testConfig(t, clock, store), job) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- runtime.Run(ctx) }() if !waitFor(t, 2*time.Second, func() bool { return len(runtime.JobHealth()) == 1 }) { t.Fatal("the skipped run was not reported") } health := runtime.JobHealth() if !health[0].LastSuccessAt.IsZero() { t.Fatalf("a skipped run recorded a success: %#v", health[0]) } snapshot := systemstatus.Build(configFixture(), true, clock.Now(), nil, systemstatus.WithJobs(time.Minute, health...)) for _, component := range snapshot.Components { if component.ID == systemstatus.ComponentWorker && component.State == systemstatus.StateHealthy { t.Fatalf("a job that only skips reported healthy: %#v", component) } } if recorded := store.Status(job.JobType(), job.Name, clock.Now().Truncate(job.Interval)); recorded != StatusSkipped { t.Fatalf("recorded lease status = %q, want %q", recorded, StatusSkipped) } cancel() <-done }