// Package workerruntime is the supervised background runtime of pulse-worker. // // It owns one scheduling loop that starts a small, fixed set of jobs on // independent intervals. Every job is: // // - database-coordinated: a job run holds a lease row in job_runs keyed by // (job_type, job_key, scheduled_at), so two workers never run the same unit // of work twice and a worker that dies mid-run has its lease expire instead // of blocking the job forever; // - idempotent: the lease key is derived from the schedule, and each job's // writes use natural-key conflict handling, so repeating a run is safe; // - bounded: every run executes under a timeout and there is at most one // in-flight run per job, which caps concurrency at the number of jobs; // - isolated: a failing or panicking job is recorded, logged and counted, and // never stops the other jobs or the process; // - observable: per-job last run, duration and outcome are exposed through // Status and persisted in job_runs, which is what internal/systemstatus // reads instead of hardcoding "not recorded". // // The scheduling loop itself performs no blocking I/O: jobs run in their own // goroutines, so the loop keeps completing iterations and writing the heartbeat // file required by docs/operations/WORKER_AGENT_HEALTHCHECK_CONTRACT.md even // while a job is stuck against a slow database or network target. package workerruntime import ( "context" "errors" "fmt" "log/slog" "os" "regexp" "sort" "strings" "sync" "time" "github.com/itworx/pulse/internal/systemstatus" ) const ( // DefaultTick is the scheduling loop period. The healthcheck contract // requires a completed iteration at least every 10 seconds; 5 seconds // leaves room for a slow heartbeat write without going stale. DefaultTick = 5 * time.Second // MaxTick is the largest loop period the healthcheck contract tolerates. MaxTick = 10 * time.Second // DefaultDrainTimeout bounds how long shutdown waits for in-flight jobs. DefaultDrainTimeout = 20 * time.Second // DefaultHeartbeatFile matches the healthcheck script's default. DefaultHeartbeatFile = "/tmp/healthy" maxJobs = 16 maxErrorLength = 200 ) // Job outcome statuses recorded per run. const ( StatusCompleted = "completed" StatusFailed = "failed" StatusSkipped = "skipped" StatusDisabled = "disabled" StatusRunning = "running" ) var ( // ErrInvalidConfig reports a runtime or job definition outside safe bounds. ErrInvalidConfig = errors.New("worker runtime configuration is invalid") jobNamePattern = regexp.MustCompile(`^[a-z][a-z0-9-]{1,39}$`) ) // Outcome is the bounded, structured result of one job run. Counts are recorded // in job_runs so an operator can see what a run actually did. type Outcome struct { // Reason is a stable snake_case code explaining a skipped or disabled run. Reason string // Counts are small integer result counters, for example processed items. Counts map[string]int64 // Skipped marks a run that intentionally did nothing, for example because // another worker held the inner lease. Skipped bool // Disabled marks a run whose feature is not configured. It is reported as // Disabled rather than Healthy so an unconfigured feature never looks green. Disabled bool } // RunFunc executes one unit of work. It must respect the supplied context. type RunFunc func(context.Context) (Outcome, error) // Job is a scheduled unit of background work with a stable key. type Job struct { // Name is the stable job key. It becomes job_runs.job_key and, prefixed, // job_runs.job_type, so it must never change casually. Name string // Component is the internal/systemstatus component this job reports to. Component string // Interval is how often the job becomes due. It also defines the lease // window: two runs inside one interval are the same unit of work. Interval time.Duration // Timeout bounds a single run. Timeout time.Duration // Run performs the work. Run RunFunc } func (j Job) validate() error { if !jobNamePattern.MatchString(j.Name) { return fmt.Errorf("%w: job name %q must be a bounded lowercase key", ErrInvalidConfig, j.Name) } switch j.Component { case systemstatus.ComponentWorker, systemstatus.ComponentProbes, systemstatus.ComponentNotifications: default: return fmt.Errorf("%w: job %s has unknown component %q", ErrInvalidConfig, j.Name, j.Component) } if j.Interval < time.Second || j.Interval > time.Hour { return fmt.Errorf("%w: job %s interval must be between 1s and 1h", ErrInvalidConfig, j.Name) } if j.Timeout < time.Second || j.Timeout > 5*time.Minute { return fmt.Errorf("%w: job %s timeout must be between 1s and 5m", ErrInvalidConfig, j.Name) } if j.Run == nil { return fmt.Errorf("%w: job %s has no run function", ErrInvalidConfig, j.Name) } return nil } // JobType is the job_runs.job_type value used for a job's scheduling lease. func (j Job) JobType() string { return "worker-" + j.Name } // MetricSink receives in-process counters and gauges. *observability.Registry // satisfies it. The worker binds no network port by design, so these values are // a process-local aid; the durable, operator-visible record is job_runs. type MetricSink interface { IncCounter(name string) SetGauge(name string, value float64) } // Config configures the runtime loop. type Config struct { // Owner identifies this worker in job_runs.lease_owner. Owner string // Tick is the scheduling loop period; DefaultTick is used when zero. Tick time.Duration // HeartbeatFile is the path written after every completed loop iteration. // DefaultHeartbeatFile is used when empty. HeartbeatFile string // DrainTimeout bounds graceful shutdown; DefaultDrainTimeout when zero. DrainTimeout time.Duration // Leases coordinates duplicate workers. Leases LeaseStore // Logger receives structured, secret-free job events. Logger *slog.Logger // Metrics optionally receives counters and gauges. Metrics MetricSink // Now is injectable for tests; time.Now is used when nil. Now func() time.Time } // JobStatus is the observable state of one job. type JobStatus struct { Name string Component string Interval time.Duration LastStartedAt time.Time LastFinishedAt time.Time LastSuccessAt time.Time LastDuration time.Duration LastStatus string LastReason string LastError string Runs uint64 Successes uint64 Failures uint64 Skips uint64 Running bool } type jobState struct { job Job running bool // lastAttemptAt is when the job last became due, which drives scheduling // independently of how long the run itself took. lastAttemptAt time.Time status JobStatus } // Runtime supervises the scheduled jobs. type Runtime struct { config Config mu sync.Mutex states []*jobState wg sync.WaitGroup heartbeatMu sync.Mutex heartbeatCount uint64 heartbeatAt time.Time } // New validates the configuration and job set and returns a runtime. func New(config Config, jobs ...Job) (*Runtime, error) { if strings.TrimSpace(config.Owner) == "" || len(config.Owner) > 120 { return nil, fmt.Errorf("%w: owner must be 1-120 characters", ErrInvalidConfig) } if config.Leases == nil { return nil, fmt.Errorf("%w: a lease store is required", ErrInvalidConfig) } if len(jobs) == 0 || len(jobs) > maxJobs { return nil, fmt.Errorf("%w: between 1 and %d jobs are required", ErrInvalidConfig, maxJobs) } if config.Tick == 0 { config.Tick = DefaultTick } // The floor only guards against a busy loop; the ceiling is what the // healthcheck contract requires. if config.Tick < 10*time.Millisecond || config.Tick > MaxTick { return nil, fmt.Errorf("%w: tick must be between 10ms and %s", ErrInvalidConfig, MaxTick) } if config.DrainTimeout == 0 { config.DrainTimeout = DefaultDrainTimeout } if config.DrainTimeout < time.Second || config.DrainTimeout > 2*time.Minute { return nil, fmt.Errorf("%w: drain timeout must be between 1s and 2m", ErrInvalidConfig) } if config.HeartbeatFile == "" { config.HeartbeatFile = DefaultHeartbeatFile } if config.Logger == nil { config.Logger = slog.New(slog.NewJSONHandler(os.Stdout, nil)) } if config.Now == nil { config.Now = time.Now } seen := make(map[string]struct{}, len(jobs)) states := make([]*jobState, 0, len(jobs)) for _, job := range jobs { if err := job.validate(); err != nil { return nil, err } if _, duplicate := seen[job.Name]; duplicate { return nil, fmt.Errorf("%w: duplicate job name %q", ErrInvalidConfig, job.Name) } seen[job.Name] = struct{}{} states = append(states, &jobState{job: job, status: JobStatus{Name: job.Name, Component: job.Component, Interval: job.Interval}}) } sort.Slice(states, func(i, j int) bool { return states[i].job.Name < states[j].job.Name }) return &Runtime{config: config, states: states}, nil } // Run drives the scheduling loop until ctx is cancelled, then drains in-flight // jobs within the configured budget. It returns nil on a clean shutdown and an // error only when shutdown could not complete within that budget. func (r *Runtime) Run(ctx context.Context) error { // The first heartbeat is written before the first blocking call so a slow // but healthy cold start is not mistaken for a hang. r.writeHeartbeat() // In-flight jobs keep a context that outlives cancellation of ctx, so // shutdown can drain them instead of tearing them down mid-transaction. jobsCtx, cancelJobs := context.WithCancel(context.WithoutCancel(ctx)) defer cancelJobs() timer := time.NewTimer(r.config.Tick) defer timer.Stop() for { r.schedule(jobsCtx) // The heartbeat marks a completed loop iteration, including an // iteration in which nothing was due. r.writeHeartbeat() select { case <-ctx.Done(): return r.drain(cancelJobs) case <-timer.C: timer.Reset(r.config.Tick) } } } // schedule starts every job that is due and not already running. func (r *Runtime) schedule(jobsCtx context.Context) { now := r.now() r.mu.Lock() due := make([]*jobState, 0, len(r.states)) for _, state := range r.states { if state.running { continue } if !state.lastAttemptAt.IsZero() && now.Sub(state.lastAttemptAt) < state.job.Interval { continue } state.running = true state.lastAttemptAt = now state.status.Running = true due = append(due, state) } r.mu.Unlock() for _, state := range due { r.wg.Add(1) go func(state *jobState) { defer r.wg.Done() r.execute(jobsCtx, state) }(state) } } // drain stops scheduling and waits for in-flight jobs. Jobs that outlast the // drain budget are cancelled so shutdown always terminates. func (r *Runtime) drain(cancelJobs context.CancelFunc) error { done := make(chan struct{}) go func() { r.wg.Wait(); close(done) }() timer := time.NewTimer(r.config.DrainTimeout) defer timer.Stop() select { case <-done: r.config.Logger.Info("worker runtime drained", "in_flight", 0) return nil case <-timer.C: } r.config.Logger.Warn("worker runtime drain timeout, cancelling in-flight jobs", "drain_timeout", r.config.DrainTimeout.String()) cancelJobs() grace := time.NewTimer(5 * time.Second) defer grace.Stop() select { case <-done: return nil case <-grace.C: return errors.New("worker runtime shutdown left cancelled jobs running") } } // execute runs one job under its lease and timeout. It never returns an error: // a failure is recorded, logged and counted so it stays visible without // affecting any other job. func (r *Runtime) execute(parent context.Context, state *jobState) { job := state.job started := r.now() runCtx, cancel := context.WithTimeout(parent, job.Timeout) defer cancel() scheduledAt := started.Truncate(job.Interval) lease, acquired, err := r.config.Leases.Acquire(runCtx, job.JobType(), job.Name, scheduledAt, r.config.Owner, started, r.leaseTTL(job)) if err != nil { r.finish(state, started, StatusFailed, "lease_unavailable", err) return } if !acquired { // Another worker owns this window. That is coordination working, not a // failure, so the previous successful outcome is retained. r.finish(state, started, StatusSkipped, "lease_held_elsewhere", nil) return } outcome, runErr := safeRun(runCtx, job) status, reason := StatusCompleted, outcome.Reason switch { case runErr != nil: status = StatusFailed reason = classifyError(runCtx, runErr) case outcome.Disabled: status = StatusDisabled reason = defaultReason(reason, "not_configured") case outcome.Skipped: status = StatusSkipped reason = defaultReason(reason, "nothing_due") } // Lease completion must not be skipped because the run context expired, or // the row would stay "running" until its lease expires. completeCtx, completeCancel := context.WithTimeout(context.WithoutCancel(parent), 5*time.Second) completeErr := r.config.Leases.Complete(completeCtx, lease, status, reason, outcome.Counts) completeCancel() if completeErr != nil && runErr == nil { status = StatusFailed reason = "lease_completion_failed" runErr = completeErr } else if completeErr != nil { r.config.Logger.Error("worker job lease completion failed", "job", job.Name, "error", boundedError(completeErr)) } r.finish(state, started, status, reason, runErr) } // safeRun converts a panicking job into an ordinary error so one broken job // cannot take the process down. func safeRun(ctx context.Context, job Job) (outcome Outcome, err error) { defer func() { if recovered := recover(); recovered != nil { outcome = Outcome{} err = fmt.Errorf("job %s panicked: %v", job.Name, recovered) } }() return job.Run(ctx) } func (r *Runtime) finish(state *jobState, started time.Time, status, reason string, runErr error) { finished := r.now() duration := finished.Sub(started) message := boundedError(runErr) r.mu.Lock() state.running = false state.status.Running = false state.status.LastStartedAt = started state.status.LastFinishedAt = finished state.status.LastDuration = duration state.status.LastStatus = status state.status.LastReason = reason state.status.LastError = message state.status.Runs++ switch status { case StatusCompleted: state.status.Successes++ state.status.LastSuccessAt = finished case StatusFailed: state.status.Failures++ case StatusSkipped, StatusDisabled: state.status.Skips++ } name := state.job.Name component := state.job.Component r.mu.Unlock() r.record(name, status, duration) attributes := []any{"job", name, "component", component, "status", status, "duration_ms", duration.Milliseconds()} if reason != "" { attributes = append(attributes, "reason", reason) } if status == StatusFailed { r.config.Logger.Error("worker job failed", append(attributes, "error", message)...) return } r.config.Logger.Info("worker job finished", attributes...) } func (r *Runtime) record(name, status string, duration time.Duration) { if r.config.Metrics == nil { return } r.config.Metrics.IncCounter("pulse_worker_job_runs_total_" + metricSuffix(name) + "_" + status) r.config.Metrics.SetGauge("pulse_worker_job_duration_seconds_"+metricSuffix(name), duration.Seconds()) } // Status returns a copy of every job's observable state. func (r *Runtime) Status() []JobStatus { r.mu.Lock() defer r.mu.Unlock() result := make([]JobStatus, 0, len(r.states)) for _, state := range r.states { result = append(result, state.status) } return result } // JobHealth projects the in-process job status onto the system status model. // A job that has never reported is deliberately omitted so the component keeps // its "not recorded" Unknown state instead of being invented as healthy. func (r *Runtime) JobHealth() []systemstatus.JobHealth { statuses := r.Status() health := make([]systemstatus.JobHealth, 0, len(statuses)) for _, status := range statuses { if status.LastStatus == "" { continue } health = append(health, systemstatus.JobHealth{ Component: status.Component, JobKey: status.Name, Status: mapStatus(status.LastStatus), Reason: status.LastReason, ErrorCode: errorCodeFor(status), LastRunAt: status.LastFinishedAt, LastSuccessAt: status.LastSuccessAt, Duration: status.LastDuration, }) } return health } // HeartbeatCount reports how many loop iterations completed. Tests use it to // assert the heartbeat is driven by the loop rather than by a free-running // timer. func (r *Runtime) HeartbeatCount() uint64 { r.heartbeatMu.Lock() defer r.heartbeatMu.Unlock() return r.heartbeatCount } // LastHeartbeat reports when the loop last wrote the heartbeat file. func (r *Runtime) LastHeartbeat() time.Time { r.heartbeatMu.Lock() defer r.heartbeatMu.Unlock() return r.heartbeatAt } // writeHeartbeat records loop liveness. A write failure is logged and the loop // continues: the healthcheck notices sustained staleness on its own, and // crashing on a tmpfs hiccup would be worse than reporting it. func (r *Runtime) writeHeartbeat() { now := r.now() content := now.Format(time.RFC3339) + "\n" if err := os.WriteFile(r.config.HeartbeatFile, []byte(content), 0o600); err != nil { r.config.Logger.Error("worker heartbeat write failed", "path", r.config.HeartbeatFile, "error", boundedError(err)) return } r.heartbeatMu.Lock() r.heartbeatCount++ r.heartbeatAt = now r.heartbeatMu.Unlock() } func (r *Runtime) leaseTTL(job Job) time.Duration { ttl := 2 * job.Timeout if ttl < job.Interval { ttl = job.Interval + job.Timeout } if ttl > 10*time.Minute { ttl = 10 * time.Minute } return ttl } func (r *Runtime) now() time.Time { return r.config.Now().UTC() } // mapStatus projects a recorded run status onto the system status vocabulary. // A skipped run is reported as completed but never updates the last success, so // a job that only ever skips can reach Unknown but never Healthy. func mapStatus(status string) string { switch status { case StatusFailed: return systemstatus.JobFailed case StatusDisabled: return systemstatus.JobDisabled case StatusRunning: return systemstatus.JobRunning default: return systemstatus.JobCompleted } } func errorCodeFor(status JobStatus) string { if status.LastStatus != StatusFailed { return "" } return defaultReason(status.LastReason, "last_run_failed") } func defaultReason(reason, fallback string) string { if strings.TrimSpace(reason) == "" { return fallback } return reason } func classifyError(ctx context.Context, err error) string { switch { case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): return "timeout" case errors.Is(err, context.Canceled) || errors.Is(ctx.Err(), context.Canceled): return "canceled" default: return "run_failed" } } func boundedError(err error) string { if err == nil { return "" } message := strings.ReplaceAll(strings.ReplaceAll(err.Error(), "\n", " "), "\r", " ") if len(message) > maxErrorLength { return message[:maxErrorLength] } return message } func metricSuffix(name string) string { return strings.ReplaceAll(name, "-", "_") }