ITWorx-Pulse-Public/internal/workerruntime/runtime_test.go

package workerruntime

import (
	"context"
	"errors"
	"log/slog"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/itworx/pulse/internal/config"
	"github.com/itworx/pulse/internal/systemstatus"
)

// configFixture is a minimal valid application configuration for status tests.
func configFixture() config.Config { return config.Config{AuthMode: "mock"} }

// fakeClock is a controllable clock. The scheduling loop uses it for due-ness
// and for every recorded timestamp, so a test decides exactly when a job
// becomes due instead of sleeping.
type fakeClock struct {
	mu  sync.Mutex
	now time.Time
}

func newFakeClock() *fakeClock {
	return &fakeClock{now: time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)}
}
func (c *fakeClock) Now() time.Time {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.now
}
func (c *fakeClock) Advance(d time.Duration) {
	c.mu.Lock()
	c.now = c.now.Add(d)
	c.mu.Unlock()
}

func quietLogger() *slog.Logger {
	return slog.New(slog.NewTextHandler(nopWriter{}, &slog.HandlerOptions{Level: slog.LevelError + 1}))
}

type nopWriter struct{}

func (nopWriter) Write(p []byte) (int, error) { return len(p), nil }

func testConfig(t *testing.T, clock *fakeClock, store LeaseStore) Config {
	t.Helper()
	return Config{
		Owner: "worker-test", Tick: 10 * time.Millisecond, HeartbeatFile: filepath.Join(t.TempDir(), "healthy"),
		DrainTimeout: 2 * time.Second, Leases: store, Logger: quietLogger(), Now: clock.Now,
	}
}

// waitFor polls until condition holds or the deadline passes.
func waitFor(t *testing.T, timeout time.Duration, condition func() bool) bool {
	t.Helper()
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		if condition() {
			return true
		}
		time.Sleep(time.Millisecond)
	}
	return condition()
}

func TestNewRejectsInvalidConfiguration(t *testing.T) {
	clock := newFakeClock()
	valid := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
		Run: func(context.Context) (Outcome, error) { return Outcome{}, nil }}
	for name, testCase := range map[string]struct {
		mutate func(*Config)
		jobs   []Job
	}{
		"no owner":         {mutate: func(c *Config) { c.Owner = "" }, jobs: []Job{valid}},
		"no lease store":   {mutate: func(c *Config) { c.Leases = nil }, jobs: []Job{valid}},
		"no jobs":          {mutate: func(*Config) {}, jobs: nil},
		"tick above bound": {mutate: func(c *Config) { c.Tick = time.Minute }, jobs: []Job{valid}},
		"duplicate job":    {mutate: func(*Config) {}, jobs: []Job{valid, valid}},
		"unknown component": {mutate: func(*Config) {}, jobs: []Job{{Name: "x-job", Component: "storage", Interval: time.Minute,
			Timeout: time.Second, Run: valid.Run}}},
		"missing run": {mutate: func(*Config) {}, jobs: []Job{{Name: "discovery", Component: systemstatus.ComponentWorker,
			Interval: time.Minute, Timeout: time.Second}}},
		"timeout above bound": {mutate: func(*Config) {}, jobs: []Job{{Name: "discovery", Component: systemstatus.ComponentWorker,
			Interval: time.Minute, Timeout: time.Hour, Run: valid.Run}}},
	} {
		t.Run(name, func(t *testing.T) {
			config := testConfig(t, clock, NewMemoryLeaseStore())
			testCase.mutate(&config)
			if _, err := New(config, testCase.jobs...); !errors.Is(err, ErrInvalidConfig) {
				t.Fatalf("error = %v, want ErrInvalidConfig", err)
			}
		})
	}
}

func TestJobRunsOncePerIntervalWindow(t *testing.T) {
	clock := newFakeClock()
	var runs atomic.Int64
	job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 5 * time.Second,
		Run: func(context.Context) (Outcome, error) {
			runs.Add(1)
			return Outcome{Counts: map[string]int64{"items": 1}}, nil
		}}
	runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), job)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- runtime.Run(ctx) }()

	if !waitFor(t, time.Second, func() bool { return runs.Load() == 1 }) {
		t.Fatalf("first run count = %d, want 1", runs.Load())
	}
	// Many loop iterations pass without the clock moving: the job must not run
	// again inside its interval.
	time.Sleep(60 * time.Millisecond)
	if runs.Load() != 1 {
		t.Fatalf("run count inside interval = %d, want 1", runs.Load())
	}
	clock.Advance(time.Minute)
	if !waitFor(t, time.Second, func() bool { return runs.Load() == 2 }) {
		t.Fatalf("run count after interval = %d, want 2", runs.Load())
	}
	cancel()
	if err := <-done; err != nil {
		t.Fatalf("run: %v", err)
	}
	status := runtime.Status()[0]
	if status.LastStatus != StatusCompleted || status.Runs != 2 || status.Successes != 2 || status.LastSuccessAt.IsZero() {
		t.Fatalf("status = %#v", status)
	}
}

func TestSecondRuntimeIsBlockedByTheLeaseAndRepeatedRunsAreIdempotent(t *testing.T) {
	clock := newFakeClock()
	store := NewMemoryLeaseStore()
	var first, second atomic.Int64
	build := func(counter *atomic.Int64, owner string) *Runtime {
		config := testConfig(t, clock, store)
		config.Owner = owner
		runtime, err := New(config, Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 5 * time.Second,
			Run: func(context.Context) (Outcome, error) { counter.Add(1); return Outcome{}, nil }})
		if err != nil {
			t.Fatal(err)
		}
		return runtime
	}
	runtimeA, runtimeB := build(&first, "worker-a"), build(&second, "worker-b")
	ctx, cancel := context.WithCancel(context.Background())
	doneA, doneB := make(chan error, 1), make(chan error, 1)
	go func() { doneA <- runtimeA.Run(ctx) }()
	go func() { doneB <- runtimeB.Run(ctx) }()
	if !waitFor(t, 2*time.Second, func() bool { return first.Load()+second.Load() >= 1 }) {
		t.Fatal("no worker executed the job")
	}
	time.Sleep(80 * time.Millisecond)
	if total := first.Load() + second.Load(); total != 1 {
		t.Fatalf("executions for one window = %d, want 1", total)
	}
	// The next window is a different lease key, so exactly one worker runs again.
	clock.Advance(time.Minute)
	if !waitFor(t, 2*time.Second, func() bool { return first.Load()+second.Load() == 2 }) {
		t.Fatalf("executions after second window = %d, want 2", first.Load()+second.Load())
	}
	cancel()
	<-doneA
	<-doneB
	if store.Windows() != 2 {
		t.Fatalf("claimed windows = %d, want 2", store.Windows())
	}
	skips := uint64(0)
	for _, status := range append(runtimeA.Status(), runtimeB.Status()...) {
		skips += status.Skips
	}
	if skips == 0 {
		t.Fatal("the worker that lost a window did not record a lease contention skip")
	}
}

func TestOneFailingJobDoesNotStopTheOthers(t *testing.T) {
	clock := newFakeClock()
	var healthy, panicking, failing atomic.Int64
	jobs := []Job{
		{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
			Run: func(context.Context) (Outcome, error) {
				failing.Add(1)
				return Outcome{}, errors.New("database is unreachable")
			}},
		{Name: "probe-execution", Component: systemstatus.ComponentProbes, Interval: time.Minute, Timeout: time.Second,
			Run: func(context.Context) (Outcome, error) { panicking.Add(1); panic("probe exploded") }},
		{Name: "notification-drain", Component: systemstatus.ComponentNotifications, Interval: time.Minute, Timeout: time.Second,
			Run: func(context.Context) (Outcome, error) { healthy.Add(1); return Outcome{}, nil }},
	}
	runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), jobs...)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- runtime.Run(ctx) }()
	if !waitFor(t, 2*time.Second, func() bool { return healthy.Load() == 1 && failing.Load() == 1 && panicking.Load() == 1 }) {
		t.Fatalf("runs healthy=%d failing=%d panicking=%d", healthy.Load(), failing.Load(), panicking.Load())
	}
	clock.Advance(time.Minute)
	if !waitFor(t, 2*time.Second, func() bool { return healthy.Load() == 2 && failing.Load() == 2 }) {
		t.Fatalf("second window healthy=%d failing=%d", healthy.Load(), failing.Load())
	}
	cancel()
	if err := <-done; err != nil {
		t.Fatal(err)
	}
	byName := map[string]JobStatus{}
	for _, status := range runtime.Status() {
		byName[status.Name] = status
	}
	if byName["discovery"].LastStatus != StatusFailed || byName["discovery"].Failures == 0 || byName["discovery"].LastError == "" {
		t.Fatalf("failing job status = %#v", byName["discovery"])
	}
	if !strings.Contains(byName["probe-execution"].LastError, "panicked") {
		t.Fatalf("panicking job status = %#v", byName["probe-execution"])
	}
	if byName["notification-drain"].LastStatus != StatusCompleted || byName["notification-drain"].Successes == 0 {
		t.Fatalf("healthy job status = %#v", byName["notification-drain"])
	}
}

func TestJobTimeoutIsEnforcedAndRecorded(t *testing.T) {
	clock := newFakeClock()
	released := make(chan struct{})
	job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
		Run: func(ctx context.Context) (Outcome, error) {
			<-ctx.Done()
			close(released)
			return Outcome{}, ctx.Err()
		}}
	runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), job)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- runtime.Run(ctx) }()
	select {
	case <-released:
	case <-time.After(3 * time.Second):
		t.Fatal("job was not cancelled by its timeout")
	}
	if !waitFor(t, 2*time.Second, func() bool { return runtime.Status()[0].LastStatus == StatusFailed }) {
		t.Fatalf("status = %#v", runtime.Status()[0])
	}
	if reason := runtime.Status()[0].LastReason; reason != "timeout" {
		t.Fatalf("reason = %q, want timeout", reason)
	}
	cancel()
	<-done
}

func TestShutdownDrainsInFlightWork(t *testing.T) {
	clock := newFakeClock()
	started := make(chan struct{})
	var finished atomic.Bool
	job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 5 * time.Second,
		Run: func(ctx context.Context) (Outcome, error) {
			close(started)
			select {
			case <-time.After(150 * time.Millisecond):
			case <-ctx.Done():
				return Outcome{}, ctx.Err()
			}
			finished.Store(true)
			return Outcome{}, nil
		}}
	runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), job)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- runtime.Run(ctx) }()
	<-started
	cancel()
	select {
	case err := <-done:
		if err != nil {
			t.Fatalf("shutdown: %v", err)
		}
	case <-time.After(3 * time.Second):
		t.Fatal("shutdown did not complete")
	}
	if !finished.Load() {
		t.Fatal("in-flight job was cancelled instead of drained")
	}
	if status := runtime.Status()[0]; status.LastStatus != StatusCompleted {
		t.Fatalf("drained job status = %#v", status)
	}
}

func TestShutdownCancelsWorkThatOutlastsTheDrainBudget(t *testing.T) {
	clock := newFakeClock()
	config := testConfig(t, clock, NewMemoryLeaseStore())
	config.DrainTimeout = time.Second
	started := make(chan struct{})
	job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 4 * time.Minute,
		Run: func(ctx context.Context) (Outcome, error) {
			close(started)
			<-ctx.Done()
			return Outcome{}, ctx.Err()
		}}
	runtime, err := New(config, job)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- runtime.Run(ctx) }()
	<-started
	cancel()
	select {
	case err := <-done:
		if err != nil {
			t.Fatalf("shutdown: %v", err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("shutdown never terminated")
	}
}

func TestHeartbeatIsWrittenByTheLoopAndSurvivesWriteFailure(t *testing.T) {
	clock := newFakeClock()
	directory := t.TempDir()
	path := filepath.Join(directory, "healthy")
	blocked := make(chan struct{})
	job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 4 * time.Minute,
		Run: func(ctx context.Context) (Outcome, error) {
			// The loop must keep heartbeating while a job is stuck.
			<-blocked
			return Outcome{}, nil
		}}
	config := testConfig(t, clock, NewMemoryLeaseStore())
	config.HeartbeatFile = path
	runtime, err := New(config, job)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- runtime.Run(ctx) }()
	if !waitFor(t, 2*time.Second, func() bool { return runtime.HeartbeatCount() >= 3 }) {
		t.Fatalf("heartbeat count = %d while a job is stuck", runtime.HeartbeatCount())
	}
	close(blocked)
	cancel()
	if err := <-done; err != nil {
		t.Fatalf("shutdown: %v", err)
	}
	content, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := time.Parse(time.RFC3339, strings.TrimSpace(string(content))); err != nil {
		t.Fatalf("heartbeat content %q is not RFC3339: %v", content, err)
	}
}

func TestHeartbeatFailureIsReportedWithoutStoppingTheLoop(t *testing.T) {
	clock := newFakeClock()
	config := testConfig(t, clock, NewMemoryLeaseStore())
	// A directory can never be written as a file.
	config.HeartbeatFile = t.TempDir()
	var runs atomic.Int64
	runtime, err := New(config, Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
		Run: func(context.Context) (Outcome, error) { runs.Add(1); return Outcome{}, nil }})
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- runtime.Run(ctx) }()
	if !waitFor(t, 2*time.Second, func() bool { return runs.Load() == 1 }) {
		t.Fatal("loop stopped after a heartbeat write failure")
	}
	if runtime.HeartbeatCount() != 0 {
		t.Fatalf("heartbeat count = %d, want 0 after write failures", runtime.HeartbeatCount())
	}
	cancel()
	<-done
}

func TestLeaseFailureIsReportedAsAFailedRun(t *testing.T) {
	clock := newFakeClock()
	config := testConfig(t, clock, failingLeaseStore{})
	runtime, err := New(config, Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
		Run: func(context.Context) (Outcome, error) { t.Error("job ran without a lease"); return Outcome{}, nil }})
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- runtime.Run(ctx) }()
	if !waitFor(t, 2*time.Second, func() bool { return runtime.Status()[0].LastStatus == StatusFailed }) {
		t.Fatalf("status = %#v", runtime.Status()[0])
	}
	if reason := runtime.Status()[0].LastReason; reason != "lease_unavailable" {
		t.Fatalf("reason = %q", reason)
	}
	cancel()
	<-done
}

type failingLeaseStore struct{}

func (failingLeaseStore) Acquire(context.Context, string, string, time.Time, string, time.Time, time.Duration) (Lease, bool, error) {
	return Lease{}, false, errors.New("job_runs is unavailable")
}
func (failingLeaseStore) Complete(context.Context, Lease, string, string, map[string]int64) error {
	return nil
}

func TestJobHealthReportsOnlyJobsThatRan(t *testing.T) {
	clock := newFakeClock()
	release := make(chan struct{})
	jobs := []Job{
		{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
			Run: func(context.Context) (Outcome, error) { return Outcome{}, nil }},
		{Name: "probe-execution", Component: systemstatus.ComponentProbes, Interval: time.Minute, Timeout: 4 * time.Minute,
			Run: func(context.Context) (Outcome, error) { <-release; return Outcome{}, nil }},
	}
	runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), jobs...)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- runtime.Run(ctx) }()
	if !waitFor(t, 2*time.Second, func() bool { return len(runtime.JobHealth()) == 1 }) {
		t.Fatalf("job health = %#v", runtime.JobHealth())
	}
	health := runtime.JobHealth()[0]
	if health.Component != systemstatus.ComponentWorker || health.Status != systemstatus.JobCompleted || health.LastSuccessAt.IsZero() {
		t.Fatalf("health = %#v", health)
	}
	// The still-running probe job has reported nothing, so its component keeps
	// the Unknown "not recorded" state.
	snapshot := systemstatus.Build(configFixture(), true, clock.Now(), nil, systemstatus.WithJobs(time.Minute, runtime.JobHealth()...))
	for _, component := range snapshot.Components {
		if component.ID == systemstatus.ComponentProbes && component.Reason != "probe_heartbeat_not_recorded" {
			t.Fatalf("probes component = %#v", component)
		}
		if component.ID == systemstatus.ComponentWorker && component.State != systemstatus.StateHealthy {
			t.Fatalf("worker component = %#v", component)
		}
	}
	close(release)
	cancel()
	<-done
}

func TestScheduleUsesBoundedIntervalsAndDisablesMissingWork(t *testing.T) {
	jobs := Schedule(ScheduleRuns{})
	if len(jobs) != 4 {
		t.Fatalf("job count = %d, want 4", len(jobs))
	}
	seen := map[string]Job{}
	for _, job := range jobs {
		if err := job.validate(); err != nil {
			t.Fatalf("job %s: %v", job.Name, err)
		}
		if job.Timeout > 10*job.Interval {
			t.Fatalf("job %s timeout %s is unreasonable for interval %s", job.Name, job.Timeout, job.Interval)
		}
		seen[job.Name] = job
	}
	for _, name := range []string{JobDiscovery, JobAlertEvaluation, JobProbeExecution, JobNotificationDrain} {
		job, ok := seen[name]
		if !ok {
			t.Fatalf("job %s is not scheduled", name)
		}
		outcome, err := job.Run(context.Background())
		if err != nil || !outcome.Disabled || outcome.Reason == "" {
			t.Fatalf("unconfigured job %s outcome = %#v err = %v", name, outcome, err)
		}
	}
}

func TestASkippedRunNeverBecomesAHealthyComponent(t *testing.T) {
	clock := newFakeClock()
	store := NewMemoryLeaseStore()
	job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
		Run: func(context.Context) (Outcome, error) {
			return Outcome{Skipped: true, Reason: "source_unavailable"}, nil
		}}
	runtime, err := New(testConfig(t, clock, store), job)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- runtime.Run(ctx) }()
	if !waitFor(t, 2*time.Second, func() bool { return len(runtime.JobHealth()) == 1 }) {
		t.Fatal("the skipped run was not reported")
	}
	health := runtime.JobHealth()
	if !health[0].LastSuccessAt.IsZero() {
		t.Fatalf("a skipped run recorded a success: %#v", health[0])
	}
	snapshot := systemstatus.Build(configFixture(), true, clock.Now(), nil, systemstatus.WithJobs(time.Minute, health...))
	for _, component := range snapshot.Components {
		if component.ID == systemstatus.ComponentWorker && component.State == systemstatus.StateHealthy {
			t.Fatalf("a job that only skips reported healthy: %#v", component)
		}
	}
	if recorded := store.Status(job.JobType(), job.Name, clock.Now().Truncate(job.Interval)); recorded != StatusSkipped {
		t.Fatalf("recorded lease status = %q, want %q", recorded, StatusSkipped)
	}
	cancel()
	<-done
}