ITWorx-Pulse-Public/cmd/worker/main.go

// Command worker is the ITWorx Pulse background runtime.
//
// It runs discovery/reconciliation, alert evaluation, service probes and the
// notification outbox drain on independent schedules, coordinated with any
// other worker through database leases. It is strictly observational
// (ADR-0001): it reads sources and writes Pulse's own state, and never mutates
// Unraid, Docker, the array or volumes.
package main

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"net/netip"
	"os"
	"os/signal"
	"reflect"
	"strings"
	"syscall"
	"time"

	"github.com/itworx/pulse/internal/agentsource"
	"github.com/itworx/pulse/internal/agentstore"
	"github.com/itworx/pulse/internal/alert"
	"github.com/itworx/pulse/internal/alertworker"
	"github.com/itworx/pulse/internal/config"
	"github.com/itworx/pulse/internal/container"
	"github.com/itworx/pulse/internal/database"
	"github.com/itworx/pulse/internal/discovery"
	"github.com/itworx/pulse/internal/inventory"
	"github.com/itworx/pulse/internal/metriccatalog"
	"github.com/itworx/pulse/internal/metricquery"
	"github.com/itworx/pulse/internal/notification"
	"github.com/itworx/pulse/internal/observability"
	"github.com/itworx/pulse/internal/probe"
	"github.com/itworx/pulse/internal/prometheus"
	"github.com/itworx/pulse/internal/queryplan"
	"github.com/itworx/pulse/internal/runtimeconfig"
	"github.com/itworx/pulse/internal/servicedefaults"
	"github.com/itworx/pulse/internal/workerruntime"
	"github.com/jackc/pgx/v5/pgxpool"
)

// startupTimeout bounds every blocking call made before the scheduling loop
// starts, so a slow database cannot hold the process before its first
// heartbeat.
const startupTimeout = 15 * time.Second

func main() {
	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
	if err := run(logger); err != nil {
		logger.Error("pulse worker failed", "error", err)
		os.Exit(1)
	}
}

func run(logger *slog.Logger) error {
	runtimeConfig, err := runtimeconfig.Load("worker")
	if err != nil {
		return err
	}
	application, err := config.LoadWorker()
	if err != nil {
		return err
	}
	if strings.TrimSpace(application.DatabaseURL) == "" {
		return errors.New("PULSE_DATABASE_URL is required: every worker job is database-coordinated")
	}
	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
	defer stop()

	startupCtx, cancelStartup := context.WithTimeout(ctx, startupTimeout)
	pool, err := database.NewPool(startupCtx, database.Config{URL: application.DatabaseURL, MaxConns: 8, MinConns: 1})
	if err != nil {
		cancelStartup()
		return err
	}
	defer pool.Close()
	if err := database.Ping(startupCtx, pool); err != nil {
		cancelStartup()
		return err
	}
	cancelStartup()

	owner := workerOwner()
	metrics := observability.NewRegistry(time.Now().UTC())
	jobs, probeJob, err := buildJobs(application, pool, owner, logger)
	if err != nil {
		return err
	}
	runtime, err := workerruntime.New(workerruntime.Config{
		Owner:         owner,
		Tick:          workerruntime.DefaultTick,
		HeartbeatFile: runtimeConfig.HeartbeatFile,
		DrainTimeout:  runtimeConfig.ShutdownAfter,
		Leases:        workerruntime.PostgresLeaseStore{Pool: pool},
		Logger:        logger,
		Metrics:       metrics,
	}, jobs...)
	if err != nil {
		return err
	}
	logger.Info("pulse worker started",
		"owner", owner, "environment", application.Environment, "jobs", jobNames(jobs),
		"heartbeat_file", runtimeConfig.HeartbeatFile, "shutdown_timeout", runtimeConfig.ShutdownAfter.String(),
		"config", application.String())

	runErr := runtime.Run(ctx)

	// Probe execution owns goroutines of its own; give them the same bounded
	// grace as the scheduler before the process exits.
	shutdownCtx, cancelShutdown := context.WithTimeout(context.WithoutCancel(ctx), runtimeConfig.ShutdownAfter)
	defer cancelShutdown()
	if probeJob != nil {
		if err := probeJob.Shutdown(shutdownCtx); err != nil {
			logger.Warn("probe shutdown incomplete", "error", err.Error())
		}
	}
	for _, status := range runtime.Status() {
		logger.Info("worker job final state", "job", status.Name, "component", status.Component,
			"last_status", status.LastStatus, "runs", status.Runs, "failures", status.Failures, "skips", status.Skips)
	}
	logger.Info("pulse worker stopped")
	return runErr
}

// buildJobs wires the repositories each job needs. A capability without its
// dependencies is scheduled anyway and reports Disabled with a reason, so an
// unconfigured feature is visible in system status instead of missing.
func buildJobs(application config.Config, pool *pgxpool.Pool, owner string, logger *slog.Logger) ([]workerruntime.Job, *workerruntime.ProbeJob, error) {
	registry, err := metriccatalog.DefaultRegistry()
	if err != nil {
		return nil, nil, err
	}
	inventoryRepo, err := inventory.NewRepository(pool)
	if err != nil {
		return nil, nil, err
	}
	discoveryStore, err := discovery.NewPostgresStore(pool, owner)
	if err != nil {
		return nil, nil, err
	}
	notificationRepo, err := notification.NewRepository(pool)
	if err != nil {
		return nil, nil, err
	}
	configureCtx, cancelConfigure := context.WithTimeout(context.Background(), startupTimeout)
	defer cancelConfigure()
	notificationFactories, err := configureWebhookChannel(configureCtx, application, notificationRepo)
	if err != nil {
		return nil, nil, err
	}

	discoveryJob := workerruntime.DiscoveryJob{
		SourceID: application.ContainerSourceID,
		// Reuse the same bounded agent snapshot transport as the public API. A
		// missing or stale snapshot resolves to Unknown and is skipped without
		// tombstoning inventory; a fresh snapshot drives idempotent reconciliation.
		Provider:  containerDiscoveryProvider(pool),
		Aliases:   workerruntime.PostgresContainerAliasStore{Pool: pool},
		Inventory: inventoryRepo,
		Runner:    discovery.Runner{Store: discoveryStore, MaxAttempts: 2, BaseRetry: 250 * time.Millisecond},
	}

	evaluator := &workerruntime.AlertEvaluator{
		States:        alert.StateRepository{Pool: pool},
		Prior:         workerruntime.PostgresAlertStateReader{Pool: pool},
		Versions:      alert.Repository{Pool: pool, Registry: registry},
		Notifications: notificationRepo,
		Logger:        logger,
	}
	alertJob := workerruntime.AlertEvaluationJob{Reason: "metric_source_not_configured"}
	if application.PrometheusURL != "" {
		source, sourceErr := prometheus.New(application.PrometheusURL, nil, prometheus.Limits{Timeout: application.PrometheusTimeout})
		if sourceErr != nil {
			return nil, nil, sourceErr
		}
		planner := queryplan.NewPlanner(registry, queryplan.Limits{})
		evaluator.Metrics = workerruntime.PrometheusMetricSource{Service: metricquery.NewService(planner, source, nil)}
		worker, workerErr := alertworker.New(alert.Repository{Pool: pool, Registry: registry}, alertworker.PostgresLeaseStore{Pool: pool}, evaluator, alertworker.Config{
			MaxConcurrent: 8, MaxBatch: workerruntime.MaxAlertRules, AttemptTimeout: 15 * time.Second, LeaseTTL: 2 * time.Minute, Owner: owner, Now: time.Now,
		})
		if workerErr != nil {
			return nil, nil, workerErr
		}
		alertJob = workerruntime.AlertEvaluationJob{Worker: &worker, Enabled: true}
	}

	policy, err := probePolicy(application)
	if err != nil {
		return nil, nil, err
	}
	serviceSummary, err := servicedefaults.Seed(configureCtx, pool, servicedefaults.Options{
		PublicURL: application.PublicURL, OIDCIssuer: application.OIDCIssuer,
	}, policy, probe.NetResolver{})
	if err != nil {
		return nil, nil, fmt.Errorf("configure system service monitoring: %w", err)
	}
	if serviceSummary.Services > 0 {
		logger.Info("system service monitoring configured", "services", serviceSummary.Services,
			"endpoints", serviceSummary.Endpoints, "probes", serviceSummary.Probes, "dependencies", serviceSummary.Dependencies)
	}
	probeJob, err := workerruntime.NewProbeJob(workerruntime.PostgresProbeStore{Pool: pool}, policy, logger)
	if err != nil {
		return nil, nil, err
	}

	notificationJob := workerruntime.NotificationDrainJob{
		Store:     notificationRepo,
		Channels:  notificationRepo,
		Senders:   map[string]notification.ChannelSender{},
		Factories: notificationFactories,
		Logger:    logger,
	}

	jobs := workerruntime.Schedule(workerruntime.ScheduleRuns{
		Discovery:         discoveryJob.Run,
		AlertEvaluation:   alertJob.Run,
		ProbeExecution:    probeJob.Run,
		NotificationDrain: notificationJob.Run,
	})
	return jobs, probeJob, nil
}

func containerDiscoveryProvider(pool *pgxpool.Pool) container.Provider {
	return agentsource.ContainerProvider{
		Reader:  agentstore.PostgresStore{Pool: pool},
		Windows: agentsource.Windows{},
	}
}

func configureWebhookChannel(ctx context.Context, application config.Config, repository notification.ChannelStore) (map[string]notification.ChannelSenderFactory, error) {
	if repository == nil {
		return nil, notification.ErrUnavailable
	}
	current, getErr := repository.GetChannel(ctx, notification.DefaultWebhookChannelID)
	if application.NotificationWebhookURL == "" {
		if errors.Is(getErr, notification.ErrNotFound) {
			return map[string]notification.ChannelSenderFactory{}, nil
		}
		if getErr != nil {
			return nil, fmt.Errorf("read system webhook channel: %w", getErr)
		}
		if current.Enabled {
			current.Enabled = false
			if _, err := repository.UpdateChannel(ctx, current, current.Revision); err != nil {
				return nil, fmt.Errorf("disable system webhook channel: %w", err)
			}
		}
		return map[string]notification.ChannelSenderFactory{}, nil
	}
	desired := notification.Channel{
		ID: notification.DefaultWebhookChannelID, Name: "Pulse webhook", Type: "webhook", Enabled: true,
		SecretRef:     notification.SecretRef{ID: notification.WebhookSecretReference},
		Configuration: map[string]any{"url": application.NotificationWebhookURL, "timeoutSeconds": application.NotificationWebhookTimeout.Seconds()},
		Revision:      1,
	}
	if errors.Is(getErr, notification.ErrNotFound) {
		if _, err := repository.CreateChannel(ctx, desired); err != nil {
			return nil, fmt.Errorf("create system webhook channel: %w", err)
		}
	} else if getErr != nil {
		return nil, fmt.Errorf("read system webhook channel: %w", getErr)
	} else if current.Name != desired.Name || current.Type != desired.Type || !current.Enabled || current.SecretRef != desired.SecretRef || !reflect.DeepEqual(current.Configuration, desired.Configuration) {
		desired.Revision = current.Revision
		if _, err := repository.UpdateChannel(ctx, desired, current.Revision); err != nil {
			return nil, fmt.Errorf("update system webhook channel: %w", err)
		}
	}
	resolver := notification.SecretResolverFunc(func(_ context.Context, ref notification.SecretRef) (string, error) {
		if ref.ID != notification.WebhookSecretReference {
			return "", notification.ErrNotFound
		}
		return application.NotificationWebhookToken, nil
	})
	return map[string]notification.ChannelSenderFactory{
		"webhook": notification.WebhookFactory{
			Secrets:   resolver,
			AllowHTTP: application.Environment != config.Production,
		},
	}, nil
}

// probePolicy builds the probe network policy. Only administrator-configured
// private ranges are added to the allowlist; every other protection in
// internal/probe/policy.go keeps its default, so link-local, multicast, cloud
// metadata and unlisted private addresses stay blocked.
func probePolicy(application config.Config) (probe.NetworkPolicy, error) {
	policy := probe.NetworkPolicy{}
	for _, entry := range application.ProbeAllowedNetworks {
		prefix, err := netip.ParsePrefix(entry)
		if err != nil {
			return probe.NetworkPolicy{}, fmt.Errorf("probe allowlist entry %q is invalid", entry)
		}
		policy.AllowedNetworks = append(policy.AllowedNetworks, prefix)
	}
	if err := policy.Validate(); err != nil {
		return probe.NetworkPolicy{}, err
	}
	return policy, nil
}

// workerOwner identifies this process in job_runs.lease_owner. It contains no
// secret and stays stable for the lifetime of the process.
func workerOwner() string {
	host, err := os.Hostname()
	if err != nil || strings.TrimSpace(host) == "" {
		host = "worker"
	}
	owner := fmt.Sprintf("%s/%d", host, os.Getpid())
	if len(owner) > 120 {
		owner = owner[:120]
	}
	return owner
}

func jobNames(jobs []workerruntime.Job) []string {
	names := make([]string, 0, len(jobs))
	for _, job := range jobs {
		names = append(names, job.Name+"@"+job.Interval.String())
	}
	return names
}