This commit is contained in:
ITWorx Pulse release export
@@ -0,0 +1,324 @@
|
||||
// Command worker is the ITWorx Pulse background runtime.
|
||||
//
|
||||
// It runs discovery/reconciliation, alert evaluation, service probes and the
|
||||
// notification outbox drain on independent schedules, coordinated with any
|
||||
// other worker through database leases. It is strictly observational
|
||||
// (ADR-0001): it reads sources and writes Pulse's own state, and never mutates
|
||||
// Unraid, Docker, the array or volumes.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"os"
|
||||
"os/signal"
|
||||
"reflect"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentsource"
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/alertworker"
|
||||
"github.com/itworx/pulse/internal/config"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
"github.com/itworx/pulse/internal/discovery"
|
||||
"github.com/itworx/pulse/internal/inventory"
|
||||
"github.com/itworx/pulse/internal/metriccatalog"
|
||||
"github.com/itworx/pulse/internal/metricquery"
|
||||
"github.com/itworx/pulse/internal/notification"
|
||||
"github.com/itworx/pulse/internal/observability"
|
||||
"github.com/itworx/pulse/internal/probe"
|
||||
"github.com/itworx/pulse/internal/prometheus"
|
||||
"github.com/itworx/pulse/internal/queryplan"
|
||||
"github.com/itworx/pulse/internal/runtimeconfig"
|
||||
"github.com/itworx/pulse/internal/servicedefaults"
|
||||
"github.com/itworx/pulse/internal/workerruntime"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// startupTimeout bounds every blocking call made before the scheduling loop
|
||||
// starts, so a slow database cannot hold the process before its first
|
||||
// heartbeat.
|
||||
const startupTimeout = 15 * time.Second
|
||||
|
||||
func main() {
|
||||
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
||||
if err := run(logger); err != nil {
|
||||
logger.Error("pulse worker failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(logger *slog.Logger) error {
|
||||
runtimeConfig, err := runtimeconfig.Load("worker")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
application, err := config.LoadWorker()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(application.DatabaseURL) == "" {
|
||||
return errors.New("PULSE_DATABASE_URL is required: every worker job is database-coordinated")
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
startupCtx, cancelStartup := context.WithTimeout(ctx, startupTimeout)
|
||||
pool, err := database.NewPool(startupCtx, database.Config{URL: application.DatabaseURL, MaxConns: 8, MinConns: 1})
|
||||
if err != nil {
|
||||
cancelStartup()
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Ping(startupCtx, pool); err != nil {
|
||||
cancelStartup()
|
||||
return err
|
||||
}
|
||||
cancelStartup()
|
||||
|
||||
owner := workerOwner()
|
||||
metrics := observability.NewRegistry(time.Now().UTC())
|
||||
jobs, probeJob, err := buildJobs(application, pool, owner, logger)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
runtime, err := workerruntime.New(workerruntime.Config{
|
||||
Owner: owner,
|
||||
Tick: workerruntime.DefaultTick,
|
||||
HeartbeatFile: runtimeConfig.HeartbeatFile,
|
||||
DrainTimeout: runtimeConfig.ShutdownAfter,
|
||||
Leases: workerruntime.PostgresLeaseStore{Pool: pool},
|
||||
Logger: logger,
|
||||
Metrics: metrics,
|
||||
}, jobs...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Info("pulse worker started",
|
||||
"owner", owner, "environment", application.Environment, "jobs", jobNames(jobs),
|
||||
"heartbeat_file", runtimeConfig.HeartbeatFile, "shutdown_timeout", runtimeConfig.ShutdownAfter.String(),
|
||||
"config", application.String())
|
||||
|
||||
runErr := runtime.Run(ctx)
|
||||
|
||||
// Probe execution owns goroutines of its own; give them the same bounded
|
||||
// grace as the scheduler before the process exits.
|
||||
shutdownCtx, cancelShutdown := context.WithTimeout(context.WithoutCancel(ctx), runtimeConfig.ShutdownAfter)
|
||||
defer cancelShutdown()
|
||||
if probeJob != nil {
|
||||
if err := probeJob.Shutdown(shutdownCtx); err != nil {
|
||||
logger.Warn("probe shutdown incomplete", "error", err.Error())
|
||||
}
|
||||
}
|
||||
for _, status := range runtime.Status() {
|
||||
logger.Info("worker job final state", "job", status.Name, "component", status.Component,
|
||||
"last_status", status.LastStatus, "runs", status.Runs, "failures", status.Failures, "skips", status.Skips)
|
||||
}
|
||||
logger.Info("pulse worker stopped")
|
||||
return runErr
|
||||
}
|
||||
|
||||
// buildJobs wires the repositories each job needs. A capability without its
|
||||
// dependencies is scheduled anyway and reports Disabled with a reason, so an
|
||||
// unconfigured feature is visible in system status instead of missing.
|
||||
func buildJobs(application config.Config, pool *pgxpool.Pool, owner string, logger *slog.Logger) ([]workerruntime.Job, *workerruntime.ProbeJob, error) {
|
||||
registry, err := metriccatalog.DefaultRegistry()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
inventoryRepo, err := inventory.NewRepository(pool)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
discoveryStore, err := discovery.NewPostgresStore(pool, owner)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
notificationRepo, err := notification.NewRepository(pool)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
configureCtx, cancelConfigure := context.WithTimeout(context.Background(), startupTimeout)
|
||||
defer cancelConfigure()
|
||||
notificationFactories, err := configureWebhookChannel(configureCtx, application, notificationRepo)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
discoveryJob := workerruntime.DiscoveryJob{
|
||||
SourceID: application.ContainerSourceID,
|
||||
// Reuse the same bounded agent snapshot transport as the public API. A
|
||||
// missing or stale snapshot resolves to Unknown and is skipped without
|
||||
// tombstoning inventory; a fresh snapshot drives idempotent reconciliation.
|
||||
Provider: containerDiscoveryProvider(pool),
|
||||
Aliases: workerruntime.PostgresContainerAliasStore{Pool: pool},
|
||||
Inventory: inventoryRepo,
|
||||
Runner: discovery.Runner{Store: discoveryStore, MaxAttempts: 2, BaseRetry: 250 * time.Millisecond},
|
||||
}
|
||||
|
||||
evaluator := &workerruntime.AlertEvaluator{
|
||||
States: alert.StateRepository{Pool: pool},
|
||||
Prior: workerruntime.PostgresAlertStateReader{Pool: pool},
|
||||
Versions: alert.Repository{Pool: pool, Registry: registry},
|
||||
Notifications: notificationRepo,
|
||||
Logger: logger,
|
||||
}
|
||||
alertJob := workerruntime.AlertEvaluationJob{Reason: "metric_source_not_configured"}
|
||||
if application.PrometheusURL != "" {
|
||||
source, sourceErr := prometheus.New(application.PrometheusURL, nil, prometheus.Limits{Timeout: application.PrometheusTimeout})
|
||||
if sourceErr != nil {
|
||||
return nil, nil, sourceErr
|
||||
}
|
||||
planner := queryplan.NewPlanner(registry, queryplan.Limits{})
|
||||
evaluator.Metrics = workerruntime.PrometheusMetricSource{Service: metricquery.NewService(planner, source, nil)}
|
||||
worker, workerErr := alertworker.New(alert.Repository{Pool: pool, Registry: registry}, alertworker.PostgresLeaseStore{Pool: pool}, evaluator, alertworker.Config{
|
||||
MaxConcurrent: 8, MaxBatch: workerruntime.MaxAlertRules, AttemptTimeout: 15 * time.Second, LeaseTTL: 2 * time.Minute, Owner: owner, Now: time.Now,
|
||||
})
|
||||
if workerErr != nil {
|
||||
return nil, nil, workerErr
|
||||
}
|
||||
alertJob = workerruntime.AlertEvaluationJob{Worker: &worker, Enabled: true}
|
||||
}
|
||||
|
||||
policy, err := probePolicy(application)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
serviceSummary, err := servicedefaults.Seed(configureCtx, pool, servicedefaults.Options{
|
||||
PublicURL: application.PublicURL, OIDCIssuer: application.OIDCIssuer,
|
||||
}, policy, probe.NetResolver{})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("configure system service monitoring: %w", err)
|
||||
}
|
||||
if serviceSummary.Services > 0 {
|
||||
logger.Info("system service monitoring configured", "services", serviceSummary.Services,
|
||||
"endpoints", serviceSummary.Endpoints, "probes", serviceSummary.Probes, "dependencies", serviceSummary.Dependencies)
|
||||
}
|
||||
probeJob, err := workerruntime.NewProbeJob(workerruntime.PostgresProbeStore{Pool: pool}, policy, logger)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
notificationJob := workerruntime.NotificationDrainJob{
|
||||
Store: notificationRepo,
|
||||
Channels: notificationRepo,
|
||||
Senders: map[string]notification.ChannelSender{},
|
||||
Factories: notificationFactories,
|
||||
Logger: logger,
|
||||
}
|
||||
|
||||
jobs := workerruntime.Schedule(workerruntime.ScheduleRuns{
|
||||
Discovery: discoveryJob.Run,
|
||||
AlertEvaluation: alertJob.Run,
|
||||
ProbeExecution: probeJob.Run,
|
||||
NotificationDrain: notificationJob.Run,
|
||||
})
|
||||
return jobs, probeJob, nil
|
||||
}
|
||||
|
||||
func containerDiscoveryProvider(pool *pgxpool.Pool) container.Provider {
|
||||
return agentsource.ContainerProvider{
|
||||
Reader: agentstore.PostgresStore{Pool: pool},
|
||||
Windows: agentsource.Windows{},
|
||||
}
|
||||
}
|
||||
|
||||
func configureWebhookChannel(ctx context.Context, application config.Config, repository notification.ChannelStore) (map[string]notification.ChannelSenderFactory, error) {
|
||||
if repository == nil {
|
||||
return nil, notification.ErrUnavailable
|
||||
}
|
||||
current, getErr := repository.GetChannel(ctx, notification.DefaultWebhookChannelID)
|
||||
if application.NotificationWebhookURL == "" {
|
||||
if errors.Is(getErr, notification.ErrNotFound) {
|
||||
return map[string]notification.ChannelSenderFactory{}, nil
|
||||
}
|
||||
if getErr != nil {
|
||||
return nil, fmt.Errorf("read system webhook channel: %w", getErr)
|
||||
}
|
||||
if current.Enabled {
|
||||
current.Enabled = false
|
||||
if _, err := repository.UpdateChannel(ctx, current, current.Revision); err != nil {
|
||||
return nil, fmt.Errorf("disable system webhook channel: %w", err)
|
||||
}
|
||||
}
|
||||
return map[string]notification.ChannelSenderFactory{}, nil
|
||||
}
|
||||
desired := notification.Channel{
|
||||
ID: notification.DefaultWebhookChannelID, Name: "Pulse webhook", Type: "webhook", Enabled: true,
|
||||
SecretRef: notification.SecretRef{ID: notification.WebhookSecretReference},
|
||||
Configuration: map[string]any{"url": application.NotificationWebhookURL, "timeoutSeconds": application.NotificationWebhookTimeout.Seconds()},
|
||||
Revision: 1,
|
||||
}
|
||||
if errors.Is(getErr, notification.ErrNotFound) {
|
||||
if _, err := repository.CreateChannel(ctx, desired); err != nil {
|
||||
return nil, fmt.Errorf("create system webhook channel: %w", err)
|
||||
}
|
||||
} else if getErr != nil {
|
||||
return nil, fmt.Errorf("read system webhook channel: %w", getErr)
|
||||
} else if current.Name != desired.Name || current.Type != desired.Type || !current.Enabled || current.SecretRef != desired.SecretRef || !reflect.DeepEqual(current.Configuration, desired.Configuration) {
|
||||
desired.Revision = current.Revision
|
||||
if _, err := repository.UpdateChannel(ctx, desired, current.Revision); err != nil {
|
||||
return nil, fmt.Errorf("update system webhook channel: %w", err)
|
||||
}
|
||||
}
|
||||
resolver := notification.SecretResolverFunc(func(_ context.Context, ref notification.SecretRef) (string, error) {
|
||||
if ref.ID != notification.WebhookSecretReference {
|
||||
return "", notification.ErrNotFound
|
||||
}
|
||||
return application.NotificationWebhookToken, nil
|
||||
})
|
||||
return map[string]notification.ChannelSenderFactory{
|
||||
"webhook": notification.WebhookFactory{
|
||||
Secrets: resolver,
|
||||
AllowHTTP: application.Environment != config.Production,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// probePolicy builds the probe network policy. Only administrator-configured
|
||||
// private ranges are added to the allowlist; every other protection in
|
||||
// internal/probe/policy.go keeps its default, so link-local, multicast, cloud
|
||||
// metadata and unlisted private addresses stay blocked.
|
||||
func probePolicy(application config.Config) (probe.NetworkPolicy, error) {
|
||||
policy := probe.NetworkPolicy{}
|
||||
for _, entry := range application.ProbeAllowedNetworks {
|
||||
prefix, err := netip.ParsePrefix(entry)
|
||||
if err != nil {
|
||||
return probe.NetworkPolicy{}, fmt.Errorf("probe allowlist entry %q is invalid", entry)
|
||||
}
|
||||
policy.AllowedNetworks = append(policy.AllowedNetworks, prefix)
|
||||
}
|
||||
if err := policy.Validate(); err != nil {
|
||||
return probe.NetworkPolicy{}, err
|
||||
}
|
||||
return policy, nil
|
||||
}
|
||||
|
||||
// workerOwner identifies this process in job_runs.lease_owner. It contains no
|
||||
// secret and stays stable for the lifetime of the process.
|
||||
func workerOwner() string {
|
||||
host, err := os.Hostname()
|
||||
if err != nil || strings.TrimSpace(host) == "" {
|
||||
host = "worker"
|
||||
}
|
||||
owner := fmt.Sprintf("%s/%d", host, os.Getpid())
|
||||
if len(owner) > 120 {
|
||||
owner = owner[:120]
|
||||
}
|
||||
return owner
|
||||
}
|
||||
|
||||
func jobNames(jobs []workerruntime.Job) []string {
|
||||
names := make([]string, 0, len(jobs))
|
||||
for _, job := range jobs {
|
||||
names = append(names, job.Name+"@"+job.Interval.String())
|
||||
}
|
||||
return names
|
||||
}
|
||||
Reference in New Issue
Block a user