Publish ITWorx Pulse source

2026-09-03 02:09:19 +02:00
commit bd774932d5
614 changed files with 77116 additions and 0 deletions
@@ -0,0 +1,324 @@
+// Command worker is the ITWorx Pulse background runtime.
+//
+// It runs discovery/reconciliation, alert evaluation, service probes and the
+// notification outbox drain on independent schedules, coordinated with any
+// other worker through database leases. It is strictly observational
+// (ADR-0001): it reads sources and writes Pulse's own state, and never mutates
+// Unraid, Docker, the array or volumes.
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/netip"
+	"os"
+	"os/signal"
+	"reflect"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/itworx/pulse/internal/agentsource"
+	"github.com/itworx/pulse/internal/agentstore"
+	"github.com/itworx/pulse/internal/alert"
+	"github.com/itworx/pulse/internal/alertworker"
+	"github.com/itworx/pulse/internal/config"
+	"github.com/itworx/pulse/internal/container"
+	"github.com/itworx/pulse/internal/database"
+	"github.com/itworx/pulse/internal/discovery"
+	"github.com/itworx/pulse/internal/inventory"
+	"github.com/itworx/pulse/internal/metriccatalog"
+	"github.com/itworx/pulse/internal/metricquery"
+	"github.com/itworx/pulse/internal/notification"
+	"github.com/itworx/pulse/internal/observability"
+	"github.com/itworx/pulse/internal/probe"
+	"github.com/itworx/pulse/internal/prometheus"
+	"github.com/itworx/pulse/internal/queryplan"
+	"github.com/itworx/pulse/internal/runtimeconfig"
+	"github.com/itworx/pulse/internal/servicedefaults"
+	"github.com/itworx/pulse/internal/workerruntime"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+// startupTimeout bounds every blocking call made before the scheduling loop
+// starts, so a slow database cannot hold the process before its first
+// heartbeat.
+const startupTimeout = 15 * time.Second
+
+func main() {
+	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
+	if err := run(logger); err != nil {
+		logger.Error("pulse worker failed", "error", err)
+		os.Exit(1)
+	}
+}
+
+func run(logger *slog.Logger) error {
+	runtimeConfig, err := runtimeconfig.Load("worker")
+	if err != nil {
+		return err
+	}
+	application, err := config.LoadWorker()
+	if err != nil {
+		return err
+	}
+	if strings.TrimSpace(application.DatabaseURL) == "" {
+		return errors.New("PULSE_DATABASE_URL is required: every worker job is database-coordinated")
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	startupCtx, cancelStartup := context.WithTimeout(ctx, startupTimeout)
+	pool, err := database.NewPool(startupCtx, database.Config{URL: application.DatabaseURL, MaxConns: 8, MinConns: 1})
+	if err != nil {
+		cancelStartup()
+		return err
+	}
+	defer pool.Close()
+	if err := database.Ping(startupCtx, pool); err != nil {
+		cancelStartup()
+		return err
+	}
+	cancelStartup()
+
+	owner := workerOwner()
+	metrics := observability.NewRegistry(time.Now().UTC())
+	jobs, probeJob, err := buildJobs(application, pool, owner, logger)
+	if err != nil {
+		return err
+	}
+	runtime, err := workerruntime.New(workerruntime.Config{
+		Owner:         owner,
+		Tick:          workerruntime.DefaultTick,
+		HeartbeatFile: runtimeConfig.HeartbeatFile,
+		DrainTimeout:  runtimeConfig.ShutdownAfter,
+		Leases:        workerruntime.PostgresLeaseStore{Pool: pool},
+		Logger:        logger,
+		Metrics:       metrics,
+	}, jobs...)
+	if err != nil {
+		return err
+	}
+	logger.Info("pulse worker started",
+		"owner", owner, "environment", application.Environment, "jobs", jobNames(jobs),
+		"heartbeat_file", runtimeConfig.HeartbeatFile, "shutdown_timeout", runtimeConfig.ShutdownAfter.String(),
+		"config", application.String())
+
+	runErr := runtime.Run(ctx)
+
+	// Probe execution owns goroutines of its own; give them the same bounded
+	// grace as the scheduler before the process exits.
+	shutdownCtx, cancelShutdown := context.WithTimeout(context.WithoutCancel(ctx), runtimeConfig.ShutdownAfter)
+	defer cancelShutdown()
+	if probeJob != nil {
+		if err := probeJob.Shutdown(shutdownCtx); err != nil {
+			logger.Warn("probe shutdown incomplete", "error", err.Error())
+		}
+	}
+	for _, status := range runtime.Status() {
+		logger.Info("worker job final state", "job", status.Name, "component", status.Component,
+			"last_status", status.LastStatus, "runs", status.Runs, "failures", status.Failures, "skips", status.Skips)
+	}
+	logger.Info("pulse worker stopped")
+	return runErr
+}
+
+// buildJobs wires the repositories each job needs. A capability without its
+// dependencies is scheduled anyway and reports Disabled with a reason, so an
+// unconfigured feature is visible in system status instead of missing.
+func buildJobs(application config.Config, pool *pgxpool.Pool, owner string, logger *slog.Logger) ([]workerruntime.Job, *workerruntime.ProbeJob, error) {
+	registry, err := metriccatalog.DefaultRegistry()
+	if err != nil {
+		return nil, nil, err
+	}
+	inventoryRepo, err := inventory.NewRepository(pool)
+	if err != nil {
+		return nil, nil, err
+	}
+	discoveryStore, err := discovery.NewPostgresStore(pool, owner)
+	if err != nil {
+		return nil, nil, err
+	}
+	notificationRepo, err := notification.NewRepository(pool)
+	if err != nil {
+		return nil, nil, err
+	}
+	configureCtx, cancelConfigure := context.WithTimeout(context.Background(), startupTimeout)
+	defer cancelConfigure()
+	notificationFactories, err := configureWebhookChannel(configureCtx, application, notificationRepo)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	discoveryJob := workerruntime.DiscoveryJob{
+		SourceID: application.ContainerSourceID,
+		// Reuse the same bounded agent snapshot transport as the public API. A
+		// missing or stale snapshot resolves to Unknown and is skipped without
+		// tombstoning inventory; a fresh snapshot drives idempotent reconciliation.
+		Provider:  containerDiscoveryProvider(pool),
+		Aliases:   workerruntime.PostgresContainerAliasStore{Pool: pool},
+		Inventory: inventoryRepo,
+		Runner:    discovery.Runner{Store: discoveryStore, MaxAttempts: 2, BaseRetry: 250 * time.Millisecond},
+	}
+
+	evaluator := &workerruntime.AlertEvaluator{
+		States:        alert.StateRepository{Pool: pool},
+		Prior:         workerruntime.PostgresAlertStateReader{Pool: pool},
+		Versions:      alert.Repository{Pool: pool, Registry: registry},
+		Notifications: notificationRepo,
+		Logger:        logger,
+	}
+	alertJob := workerruntime.AlertEvaluationJob{Reason: "metric_source_not_configured"}
+	if application.PrometheusURL != "" {
+		source, sourceErr := prometheus.New(application.PrometheusURL, nil, prometheus.Limits{Timeout: application.PrometheusTimeout})
+		if sourceErr != nil {
+			return nil, nil, sourceErr
+		}
+		planner := queryplan.NewPlanner(registry, queryplan.Limits{})
+		evaluator.Metrics = workerruntime.PrometheusMetricSource{Service: metricquery.NewService(planner, source, nil)}
+		worker, workerErr := alertworker.New(alert.Repository{Pool: pool, Registry: registry}, alertworker.PostgresLeaseStore{Pool: pool}, evaluator, alertworker.Config{
+			MaxConcurrent: 8, MaxBatch: workerruntime.MaxAlertRules, AttemptTimeout: 15 * time.Second, LeaseTTL: 2 * time.Minute, Owner: owner, Now: time.Now,
+		})
+		if workerErr != nil {
+			return nil, nil, workerErr
+		}
+		alertJob = workerruntime.AlertEvaluationJob{Worker: &worker, Enabled: true}
+	}
+
+	policy, err := probePolicy(application)
+	if err != nil {
+		return nil, nil, err
+	}
+	serviceSummary, err := servicedefaults.Seed(configureCtx, pool, servicedefaults.Options{
+		PublicURL: application.PublicURL, OIDCIssuer: application.OIDCIssuer,
+	}, policy, probe.NetResolver{})
+	if err != nil {
+		return nil, nil, fmt.Errorf("configure system service monitoring: %w", err)
+	}
+	if serviceSummary.Services > 0 {
+		logger.Info("system service monitoring configured", "services", serviceSummary.Services,
+			"endpoints", serviceSummary.Endpoints, "probes", serviceSummary.Probes, "dependencies", serviceSummary.Dependencies)
+	}
+	probeJob, err := workerruntime.NewProbeJob(workerruntime.PostgresProbeStore{Pool: pool}, policy, logger)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	notificationJob := workerruntime.NotificationDrainJob{
+		Store:     notificationRepo,
+		Channels:  notificationRepo,
+		Senders:   map[string]notification.ChannelSender{},
+		Factories: notificationFactories,
+		Logger:    logger,
+	}
+
+	jobs := workerruntime.Schedule(workerruntime.ScheduleRuns{
+		Discovery:         discoveryJob.Run,
+		AlertEvaluation:   alertJob.Run,
+		ProbeExecution:    probeJob.Run,
+		NotificationDrain: notificationJob.Run,
+	})
+	return jobs, probeJob, nil
+}
+
+func containerDiscoveryProvider(pool *pgxpool.Pool) container.Provider {
+	return agentsource.ContainerProvider{
+		Reader:  agentstore.PostgresStore{Pool: pool},
+		Windows: agentsource.Windows{},
+	}
+}
+
+func configureWebhookChannel(ctx context.Context, application config.Config, repository notification.ChannelStore) (map[string]notification.ChannelSenderFactory, error) {
+	if repository == nil {
+		return nil, notification.ErrUnavailable
+	}
+	current, getErr := repository.GetChannel(ctx, notification.DefaultWebhookChannelID)
+	if application.NotificationWebhookURL == "" {
+		if errors.Is(getErr, notification.ErrNotFound) {
+			return map[string]notification.ChannelSenderFactory{}, nil
+		}
+		if getErr != nil {
+			return nil, fmt.Errorf("read system webhook channel: %w", getErr)
+		}
+		if current.Enabled {
+			current.Enabled = false
+			if _, err := repository.UpdateChannel(ctx, current, current.Revision); err != nil {
+				return nil, fmt.Errorf("disable system webhook channel: %w", err)
+			}
+		}
+		return map[string]notification.ChannelSenderFactory{}, nil
+	}
+	desired := notification.Channel{
+		ID: notification.DefaultWebhookChannelID, Name: "Pulse webhook", Type: "webhook", Enabled: true,
+		SecretRef:     notification.SecretRef{ID: notification.WebhookSecretReference},
+		Configuration: map[string]any{"url": application.NotificationWebhookURL, "timeoutSeconds": application.NotificationWebhookTimeout.Seconds()},
+		Revision:      1,
+	}
+	if errors.Is(getErr, notification.ErrNotFound) {
+		if _, err := repository.CreateChannel(ctx, desired); err != nil {
+			return nil, fmt.Errorf("create system webhook channel: %w", err)
+		}
+	} else if getErr != nil {
+		return nil, fmt.Errorf("read system webhook channel: %w", getErr)
+	} else if current.Name != desired.Name || current.Type != desired.Type || !current.Enabled || current.SecretRef != desired.SecretRef || !reflect.DeepEqual(current.Configuration, desired.Configuration) {
+		desired.Revision = current.Revision
+		if _, err := repository.UpdateChannel(ctx, desired, current.Revision); err != nil {
+			return nil, fmt.Errorf("update system webhook channel: %w", err)
+		}
+	}
+	resolver := notification.SecretResolverFunc(func(_ context.Context, ref notification.SecretRef) (string, error) {
+		if ref.ID != notification.WebhookSecretReference {
+			return "", notification.ErrNotFound
+		}
+		return application.NotificationWebhookToken, nil
+	})
+	return map[string]notification.ChannelSenderFactory{
+		"webhook": notification.WebhookFactory{
+			Secrets:   resolver,
+			AllowHTTP: application.Environment != config.Production,
+		},
+	}, nil
+}
+
+// probePolicy builds the probe network policy. Only administrator-configured
+// private ranges are added to the allowlist; every other protection in
+// internal/probe/policy.go keeps its default, so link-local, multicast, cloud
+// metadata and unlisted private addresses stay blocked.
+func probePolicy(application config.Config) (probe.NetworkPolicy, error) {
+	policy := probe.NetworkPolicy{}
+	for _, entry := range application.ProbeAllowedNetworks {
+		prefix, err := netip.ParsePrefix(entry)
+		if err != nil {
+			return probe.NetworkPolicy{}, fmt.Errorf("probe allowlist entry %q is invalid", entry)
+		}
+		policy.AllowedNetworks = append(policy.AllowedNetworks, prefix)
+	}
+	if err := policy.Validate(); err != nil {
+		return probe.NetworkPolicy{}, err
+	}
+	return policy, nil
+}
+
+// workerOwner identifies this process in job_runs.lease_owner. It contains no
+// secret and stays stable for the lifetime of the process.
+func workerOwner() string {
+	host, err := os.Hostname()
+	if err != nil || strings.TrimSpace(host) == "" {
+		host = "worker"
+	}
+	owner := fmt.Sprintf("%s/%d", host, os.Getpid())
+	if len(owner) > 120 {
+		owner = owner[:120]
+	}
+	return owner
+}
+
+func jobNames(jobs []workerruntime.Job) []string {
+	names := make([]string, 0, len(jobs))
+	for _, job := range jobs {
+		names = append(names, job.Name+"@"+job.Interval.String())
+	}
+	return names
+}
@@ -0,0 +1,123 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/itworx/pulse/internal/agentsource"
+	"github.com/itworx/pulse/internal/agentstore"
+	"github.com/itworx/pulse/internal/config"
+	"github.com/itworx/pulse/internal/notification"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+type channelStoreStub struct {
+	channels map[string]notification.Channel
+	creates  int
+	updates  int
+}
+
+func TestContainerDiscoveryProviderUsesAgentSnapshotStore(t *testing.T) {
+	pool := &pgxpool.Pool{}
+	provider, ok := containerDiscoveryProvider(pool).(agentsource.ContainerProvider)
+	if !ok {
+		t.Fatalf("container discovery provider = %T, want agentsource.ContainerProvider", containerDiscoveryProvider(pool))
+	}
+	store, ok := provider.Reader.(agentstore.PostgresStore)
+	if !ok || store.Pool != pool {
+		t.Fatalf("container discovery reader = %#v, want PostgresStore with worker pool", provider.Reader)
+	}
+}
+
+func (store *channelStoreStub) CreateChannel(_ context.Context, channel notification.Channel) (notification.Channel, error) {
+	if _, exists := store.channels[channel.ID]; exists {
+		return notification.Channel{}, notification.ErrConflict
+	}
+	store.creates++
+	channel.Revision = 1
+	store.channels[channel.ID] = channel
+	return channel, nil
+}
+func (store *channelStoreStub) GetChannel(_ context.Context, id string) (notification.Channel, error) {
+	channel, exists := store.channels[id]
+	if !exists {
+		return notification.Channel{}, notification.ErrNotFound
+	}
+	return channel, nil
+}
+func (store *channelStoreStub) ListChannels(context.Context, int) ([]notification.Channel, error) {
+	channels := make([]notification.Channel, 0, len(store.channels))
+	for _, channel := range store.channels {
+		channels = append(channels, channel)
+	}
+	return channels, nil
+}
+func (store *channelStoreStub) UpdateChannel(_ context.Context, channel notification.Channel, expected int64) (notification.Channel, error) {
+	current, exists := store.channels[channel.ID]
+	if !exists {
+		return notification.Channel{}, notification.ErrNotFound
+	}
+	if current.Revision != expected {
+		return notification.Channel{}, notification.ErrConflict
+	}
+	store.updates++
+	channel.Revision = expected + 1
+	store.channels[channel.ID] = channel
+	return channel, nil
+}
+func (store *channelStoreStub) DeleteChannel(_ context.Context, id string) error {
+	if _, exists := store.channels[id]; !exists {
+		return notification.ErrNotFound
+	}
+	delete(store.channels, id)
+	return nil
+}
+
+func TestConfigureWebhookChannelReconcilesWithoutRevisionChurn(t *testing.T) {
+	store := &channelStoreStub{channels: map[string]notification.Channel{}}
+	application := config.Config{
+		Environment: config.Development, NotificationWebhookURL: "http://127.0.0.1:18080/pulse",
+		NotificationWebhookToken: "runtime-only", NotificationWebhookTimeout: 3 * time.Second,
+	}
+	factories, err := configureWebhookChannel(context.Background(), application, store)
+	if err != nil {
+		t.Fatal(err)
+	}
+	channel := store.channels[notification.DefaultWebhookChannelID]
+	if store.creates != 1 || store.updates != 0 || !channel.Enabled || channel.SecretRef.ID != notification.WebhookSecretReference {
+		t.Fatalf("channel=%+v creates=%d updates=%d", channel, store.creates, store.updates)
+	}
+	if _, exists := channel.Configuration["token"]; exists {
+		t.Fatal("runtime credential entered persistent configuration")
+	}
+	if _, err := factories["webhook"].Sender(context.Background(), channel); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := configureWebhookChannel(context.Background(), application, store); err != nil {
+		t.Fatal(err)
+	}
+	if store.creates != 1 || store.updates != 0 {
+		t.Fatalf("idempotent reconciliation created=%d updated=%d", store.creates, store.updates)
+	}
+}
+
+func TestConfigureWebhookChannelDisablesRemovedRuntimeConfiguration(t *testing.T) {
+	store := &channelStoreStub{channels: map[string]notification.Channel{
+		notification.DefaultWebhookChannelID: {
+			ID: notification.DefaultWebhookChannelID, Name: "Pulse webhook", Type: "webhook", Enabled: true,
+			SecretRef: notification.SecretRef{ID: notification.WebhookSecretReference}, Revision: 4,
+		},
+	}}
+	factories, err := configureWebhookChannel(context.Background(), config.Config{}, store)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(factories) != 0 || store.updates != 1 || store.channels[notification.DefaultWebhookChannelID].Enabled {
+		t.Fatalf("factories=%v updates=%d channel=%+v", factories, store.updates, store.channels[notification.DefaultWebhookChannelID])
+	}
+	if _, err := configureWebhookChannel(context.Background(), config.Config{}, nil); !errors.Is(err, notification.ErrUnavailable) {
+		t.Fatalf("nil repository error=%v", err)
+	}
+}