ITWorx Pulse release export
bd774932d5
Public source validation / validate (push) Failing after 3m8s
92 lines
3.3 KiB
Go
92 lines
3.3 KiB
Go
package workerruntime
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/itworx/pulse/internal/systemstatus"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
// ReadJobHealth reads the last recorded outcome of every supplied job from
|
|
// job_runs and projects it onto the system status model.
|
|
//
|
|
// This is the cross-process half of worker observability: the worker records
|
|
// each run in job_runs, and any reader (the API's system status endpoint, an
|
|
// operator, a second worker) can reconstruct what the background runtime is
|
|
// doing without talking to the worker process. A job with no recorded run is
|
|
// omitted, so its component keeps the Unknown "not recorded" state rather than
|
|
// being invented as healthy (ADR-0008).
|
|
func ReadJobHealth(ctx context.Context, pool *pgxpool.Pool, jobs []Job) ([]systemstatus.JobHealth, error) {
|
|
if pool == nil {
|
|
return nil, fmt.Errorf("%w: job health reader has no database pool", ErrInvalidConfig)
|
|
}
|
|
if len(jobs) == 0 || len(jobs) > maxJobs {
|
|
return nil, fmt.Errorf("%w: between 1 and %d jobs are required", ErrInvalidConfig, maxJobs)
|
|
}
|
|
types := make([]string, 0, len(jobs))
|
|
components := make(map[string]Job, len(jobs))
|
|
for _, job := range jobs {
|
|
types = append(types, job.JobType())
|
|
components[job.JobType()] = job
|
|
}
|
|
rows, err := pool.Query(ctx, `
|
|
SELECT DISTINCT ON (job_type) job_type, status, COALESCE(error_code,''), COALESCE(counts->>'reason',''), started_at, completed_at
|
|
FROM job_runs
|
|
WHERE job_type = ANY($1::text[])
|
|
ORDER BY job_type ASC, scheduled_at DESC, started_at DESC NULLS LAST`, types)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read worker job runs: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
health := make([]systemstatus.JobHealth, 0, len(jobs))
|
|
index := make(map[string]int, len(jobs))
|
|
for rows.Next() {
|
|
var jobType, status, errorCode, reason string
|
|
var startedAt, completedAt *time.Time
|
|
if err := rows.Scan(&jobType, &status, &errorCode, &reason, &startedAt, &completedAt); err != nil {
|
|
return nil, fmt.Errorf("scan worker job run: %w", err)
|
|
}
|
|
job, known := components[jobType]
|
|
if !known {
|
|
continue
|
|
}
|
|
item := systemstatus.JobHealth{Component: job.Component, JobKey: job.Name, Status: mapStatus(status), ErrorCode: errorCode, Reason: reason}
|
|
if completedAt != nil {
|
|
item.LastRunAt = completedAt.UTC()
|
|
} else if startedAt != nil {
|
|
item.LastRunAt = startedAt.UTC()
|
|
}
|
|
if startedAt != nil && completedAt != nil {
|
|
item.Duration = completedAt.Sub(*startedAt)
|
|
}
|
|
index[jobType] = len(health)
|
|
health = append(health, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("read worker job run rows: %w", err)
|
|
}
|
|
successRows, err := pool.Query(ctx, `SELECT job_type, max(completed_at) FROM job_runs WHERE job_type = ANY($1::text[]) AND status='completed' AND completed_at IS NOT NULL GROUP BY job_type`, types)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read worker job successes: %w", err)
|
|
}
|
|
defer successRows.Close()
|
|
for successRows.Next() {
|
|
var jobType string
|
|
var lastSuccess *time.Time
|
|
if err := successRows.Scan(&jobType, &lastSuccess); err != nil {
|
|
return nil, fmt.Errorf("scan worker job success: %w", err)
|
|
}
|
|
position, ok := index[jobType]
|
|
if !ok || lastSuccess == nil {
|
|
continue
|
|
}
|
|
health[position].LastSuccessAt = lastSuccess.UTC()
|
|
}
|
|
if err := successRows.Err(); err != nil {
|
|
return nil, fmt.Errorf("read worker job success rows: %w", err)
|
|
}
|
|
return health, nil
|
|
}
|