Publish ITWorx Pulse source
Public source validation / validate (push) Failing after 3m8s

This commit is contained in:
ITWorx Pulse release export
2026-09-03 02:09:19 +02:00
commit bd774932d5
614 changed files with 77116 additions and 0 deletions
+372
View File
@@ -0,0 +1,372 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"os"
"time"
"github.com/itworx/pulse/internal/agentprotocol"
"github.com/itworx/pulse/internal/agentstore"
"github.com/itworx/pulse/internal/array"
"github.com/itworx/pulse/internal/container"
"github.com/itworx/pulse/internal/disk"
"github.com/itworx/pulse/internal/host"
"github.com/itworx/pulse/internal/pool"
"github.com/itworx/pulse/internal/process"
"github.com/itworx/pulse/internal/runtimeconfig"
"github.com/itworx/pulse/internal/share"
)
const (
// maxLoopInterval bounds one scheduling iteration so the heartbeat is refreshed at
// least every 10 seconds even when collection is configured to run far less often.
// See docs/operations/WORKER_AGENT_HEALTHCHECK_CONTRACT.md point 3.
maxLoopInterval = 5 * time.Second
// operationTimeout bounds every blocking call inside one iteration — collection and
// the database write. It sits well under the 10 second heartbeat window so a hung
// database cannot stall the loop into a false "unhealthy" restart, and equally
// cannot hide a real hang: the call is abandoned and reported.
operationTimeout = 4 * time.Second
// heartbeatFileMode keeps the liveness file readable only by the agent's own user;
// the healthcheck script runs as the same user.
heartbeatFileMode = 0o600
)
// snapshotSource is the narrow view of the collector the runtime needs. It keeps the
// loop testable without a procfs tree.
type snapshotSource interface {
Host(context.Context) (host.RawSnapshot, error)
Processes(context.Context) (process.RawSnapshot, error)
}
type containerSnapshotSource interface {
Containers(context.Context) (container.RawSnapshot, error)
}
type arraySnapshotSource interface {
Array(context.Context) (array.RawSnapshot, error)
}
type diskSnapshotSource interface {
Disks(context.Context) (disk.RawSnapshot, error)
}
type poolSnapshotSource interface {
Pools(context.Context) (pool.RawSnapshot, error)
}
type shareSnapshotSource interface {
Shares(context.Context) (share.RawSnapshot, error)
}
// capability binds one telemetry surface to the collection that produces it. Every
// capability the agent announces is read-only; there is no write path in this binary.
type capability struct {
id agentstore.Capability
version string
collect func(context.Context) (json.RawMessage, time.Time, error)
}
// tickerFactory produces the loop's tick channel. Tests replace it with a channel they
// drive by hand so loop behaviour is asserted without sleeping.
type tickerFactory func(time.Duration) (<-chan time.Time, func())
type agent struct {
agentID string
writer agentstore.Writer
logger *slog.Logger
now func() time.Time
collectInterval time.Duration
loopInterval time.Duration
operationTimeout time.Duration
heartbeatPath string
capabilities []capability
newTicker tickerFactory
// nextCollect is the earliest time the next collection pass may run. It is only
// touched from the loop goroutine.
nextCollect time.Time
}
func realTicker(interval time.Duration) (<-chan time.Time, func()) {
ticker := time.NewTicker(interval)
return ticker.C, ticker.Stop
}
func newAgent(config runtimeconfig.AgentConfig, source snapshotSource, writer agentstore.Writer, logger *slog.Logger) *agent {
loopInterval := config.CollectInterval
if loopInterval > maxLoopInterval {
loopInterval = maxLoopInterval
}
return &agent{
agentID: config.AgentID,
writer: writer,
logger: logger,
now: time.Now,
collectInterval: config.CollectInterval,
loopInterval: loopInterval,
operationTimeout: operationTimeout,
heartbeatPath: config.Service.HeartbeatFile,
capabilities: capabilities(source),
newTicker: realTicker,
}
}
// capabilities lists what this agent reports. Adding a capability here is the only way
// to widen what the agent reads, which keeps the surface auditable.
func capabilities(source snapshotSource) []capability {
result := []capability{
{
id: agentstore.CapabilityHost,
version: host.ContractVersion,
collect: func(ctx context.Context) (json.RawMessage, time.Time, error) {
snapshot, err := source.Host(ctx)
if err != nil {
return nil, time.Time{}, err
}
payload, err := json.Marshal(snapshot)
if err != nil {
return nil, time.Time{}, err
}
return payload, snapshot.ObservedAt, nil
},
},
{
id: agentstore.CapabilityProcesses,
version: process.ContractVersion,
collect: func(ctx context.Context) (json.RawMessage, time.Time, error) {
snapshot, err := source.Processes(ctx)
if err != nil {
return nil, time.Time{}, err
}
payload, err := json.Marshal(snapshot)
if err != nil {
return nil, time.Time{}, err
}
return payload, snapshot.ObservedAt, nil
},
},
}
if containers, ok := source.(containerSnapshotSource); ok {
result = append(result, capability{
id: agentstore.CapabilityContainers, version: container.ContractVersion,
collect: func(ctx context.Context) (json.RawMessage, time.Time, error) {
snapshot, err := containers.Containers(ctx)
if err != nil {
return nil, time.Time{}, err
}
payload, err := json.Marshal(snapshot)
if err != nil {
return nil, time.Time{}, err
}
return payload, snapshot.ObservedAt, nil
},
})
}
if source, ok := source.(arraySnapshotSource); ok {
result = append(result, rawCapability(agentstore.CapabilityArray, array.ContractVersion, source.Array))
}
if source, ok := source.(diskSnapshotSource); ok {
result = append(result, rawCapability(agentstore.CapabilityDisks, disk.ContractVersion, source.Disks))
}
if source, ok := source.(poolSnapshotSource); ok {
result = append(result, rawCapability(agentstore.CapabilityPools, pool.ContractVersion, source.Pools))
}
if source, ok := source.(shareSnapshotSource); ok {
result = append(result, rawCapability(agentstore.CapabilityShares, share.ContractVersion, source.Shares))
}
return result
}
func rawCapability[T any](id agentstore.Capability, version string, collect func(context.Context) (T, error)) capability {
return capability{id: id, version: version, collect: func(ctx context.Context) (json.RawMessage, time.Time, error) {
snapshot, err := collect(ctx)
if err != nil {
return nil, time.Time{}, err
}
payload, err := json.Marshal(snapshot)
if err != nil {
return nil, time.Time{}, err
}
observed := observedAt(snapshot)
return payload, observed, nil
}}
}
// All raw snapshot contracts carry ObservedAt. Keep this tiny type assertion local to
// the agent rather than introducing a repository-wide generic telemetry abstraction.
func observedAt(snapshot any) time.Time {
switch value := snapshot.(type) {
case array.RawSnapshot:
return value.ObservedAt
case disk.RawSnapshot:
return value.ObservedAt
case pool.RawSnapshot:
return value.ObservedAt
case share.RawSnapshot:
return value.ObservedAt
default:
return time.Time{}
}
}
// hello is the capability announcement. Every entry is read-only, which is not a
// decoration: agentprotocol.Hello.Validate rejects a hello that claims anything else,
// so the agent's own start-up self-check fails loudly if a mutating capability is ever
// added here by mistake.
func (a *agent) hello() agentprotocol.Hello {
announced := make([]agentprotocol.Capability, 0, len(a.capabilities))
for _, item := range a.capabilities {
announced = append(announced, agentprotocol.Capability{
ID: string(item.id),
Version: item.version,
ReadOnly: true,
})
}
return agentprotocol.Hello{
Protocol: agentprotocol.Version,
AgentID: a.agentID,
ObservedAt: a.now().UTC(),
Capabilities: announced,
}
}
// run drives the collection loop until the context is cancelled.
//
// The order is fixed by the healthcheck contract: validate, log, write one heartbeat
// before the first blocking call, then loop. Each iteration does its bounded work and
// ends by refreshing the heartbeat, so a stuck iteration stops the heartbeat instead of
// a background ticker papering over the stall.
func (a *agent) run(ctx context.Context) error {
if a.writer == nil {
return errNoWriter
}
hello := a.hello()
if err := hello.Validate(a.now().UTC()); err != nil {
return fmt.Errorf("agent hello failed its own read-only validation: %w", err)
}
announced := make([]string, 0, len(hello.Capabilities))
for _, item := range hello.Capabilities {
announced = append(announced, item.ID)
}
a.logger.Info("pulse agent started",
"agent_id", a.agentID,
"protocol", hello.Protocol,
"capabilities", announced,
"read_only", true,
"collect_interval", a.collectInterval.String(),
"loop_interval", a.loopInterval.String(),
"operation_timeout", a.operationTimeout.String(),
"heartbeat_file", a.heartbeatPath,
)
// Contract point 4: a slow-but-healthy cold start must not look like a hang.
a.heartbeat()
ticks, stop := a.newTicker(a.loopInterval)
defer stop()
loop:
for ctx.Err() == nil {
a.iterate(ctx)
select {
case <-ctx.Done():
break loop
case <-ticks:
}
}
a.logger.Info("pulse agent stopped", "agent_id", a.agentID)
return nil
}
// iterate is one unit of work: collect when due, then heartbeat. An empty iteration —
// nothing due yet — is still a completed iteration and still heartbeats.
func (a *agent) iterate(ctx context.Context) {
now := a.now()
if !now.Before(a.nextCollect) {
a.collectOnce(ctx)
a.nextCollect = a.now().Add(a.collectInterval)
}
a.heartbeat()
}
// collectOnce publishes every capability independently. One failing capability is
// logged and skipped; it neither aborts the pass nor causes a stale or empty snapshot
// to be written in its place. A missing snapshot is exactly what the reader turns into
// Unknown (ADR-0008), which is the honest outcome.
func (a *agent) collectOnce(ctx context.Context) {
for _, item := range a.capabilities {
if ctx.Err() != nil {
return
}
started := a.now()
bytesWritten, err := a.publish(ctx, item)
if err != nil {
a.logger.Error("agent capability collection failed",
"agent_id", a.agentID,
"capability", string(item.id),
"duration_ms", a.now().Sub(started).Milliseconds(),
"error", err.Error(),
)
continue
}
a.logger.Info("agent snapshot published",
"agent_id", a.agentID,
"capability", string(item.id),
"payload_bytes", bytesWritten,
"duration_ms", a.now().Sub(started).Milliseconds(),
)
}
}
// publish collects and writes one capability under a bounded context.
func (a *agent) publish(ctx context.Context, item capability) (int, error) {
operation, cancel := context.WithTimeout(ctx, a.operationTimeout)
defer cancel()
payload, observedAt, err := item.collect(operation)
if err != nil {
return 0, fmt.Errorf("collect %s: %w", item.id, err)
}
if len(payload) == 0 {
return 0, fmt.Errorf("collect %s: empty payload", item.id)
}
if len(payload) > agentstore.MaxPayloadBytes {
// Writing a truncated snapshot would be worse than writing none: the reader
// cannot tell a truncated payload from a complete one.
return 0, fmt.Errorf("collect %s: payload of %d bytes exceeds the %d byte limit", item.id, len(payload), agentstore.MaxPayloadBytes)
}
if observedAt.IsZero() {
observedAt = a.now().UTC()
}
if err := a.writer.Put(operation, agentstore.Snapshot{
AgentID: a.agentID,
Capability: item.id,
ObservedAt: observedAt.UTC(),
Payload: payload,
}); err != nil {
return 0, fmt.Errorf("publish %s: %w", item.id, err)
}
return len(payload), nil
}
// heartbeat refreshes the liveness file the compose healthcheck watches. Only the mtime
// matters to the script; the RFC 3339 body exists so `docker exec … cat /tmp/healthy`
// tells an operator something. A write failure is logged and the loop continues:
// crashing on a tmpfs hiccup would turn a cosmetic problem into an outage, and a
// sustained failure surfaces on its own as staleness.
func (a *agent) heartbeat() {
if a.heartbeatPath == "" {
return
}
content := a.now().UTC().Format(time.RFC3339) + "\n"
if err := os.WriteFile(a.heartbeatPath, []byte(content), heartbeatFileMode); err != nil {
a.logger.Error("agent heartbeat write failed",
"agent_id", a.agentID,
"heartbeat_file", a.heartbeatPath,
"error", err.Error(),
)
}
}
var errNoWriter = errors.New("agent snapshot writer is required")
+543
View File
@@ -0,0 +1,543 @@
package main
import (
"context"
"encoding/json"
"errors"
"io"
"log/slog"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/itworx/pulse/internal/agentstore"
"github.com/itworx/pulse/internal/array"
"github.com/itworx/pulse/internal/container"
"github.com/itworx/pulse/internal/disk"
"github.com/itworx/pulse/internal/host"
"github.com/itworx/pulse/internal/pool"
"github.com/itworx/pulse/internal/process"
"github.com/itworx/pulse/internal/runtimeconfig"
"github.com/itworx/pulse/internal/share"
)
// fakeWriter records what the agent publishes and can fail a chosen capability.
type fakeWriter struct {
mu sync.Mutex
puts []agentstore.Snapshot
deadlines []bool
failures map[agentstore.Capability]error
blockUntil chan struct{}
}
func newFakeWriter() *fakeWriter {
return &fakeWriter{failures: map[agentstore.Capability]error{}}
}
func (w *fakeWriter) Put(ctx context.Context, snapshot agentstore.Snapshot) error {
w.mu.Lock()
failure := w.failures[snapshot.Capability]
block := w.blockUntil
w.mu.Unlock()
if block != nil {
select {
case <-block:
case <-ctx.Done():
return ctx.Err()
}
}
_, hasDeadline := ctx.Deadline()
w.mu.Lock()
defer w.mu.Unlock()
w.deadlines = append(w.deadlines, hasDeadline)
if failure != nil {
return failure
}
w.puts = append(w.puts, snapshot)
return nil
}
func (w *fakeWriter) recorded() []agentstore.Snapshot {
w.mu.Lock()
defer w.mu.Unlock()
return append([]agentstore.Snapshot(nil), w.puts...)
}
func (w *fakeWriter) countFor(capability agentstore.Capability) int {
count := 0
for _, snapshot := range w.recorded() {
if snapshot.Capability == capability {
count++
}
}
return count
}
func (w *fakeWriter) failCapability(capability agentstore.Capability, err error) {
w.mu.Lock()
defer w.mu.Unlock()
w.failures[capability] = err
}
// fakeSource stands in for the procfs collector.
type fakeSource struct {
mu sync.Mutex
hostErr error
processErr error
hostCalls int
procCalls int
release chan struct{}
hugePayload bool
}
type fakeContainerSource struct{ *fakeSource }
func (s fakeContainerSource) Containers(context.Context) (container.RawSnapshot, error) {
return container.RawSnapshot{Source: container.Source{ID: "unraid", Type: "unraid"}, Containers: []container.RawContainer{{ID: "runtime-1", Name: "pulse-api", State: "running", Health: "healthy"}}, ObservedAt: time.Date(2026, 8, 10, 2, 0, 0, 0, time.UTC)}, nil
}
type fakeInventorySource struct{ *fakeSource }
func (s fakeInventorySource) Containers(context.Context) (container.RawSnapshot, error) {
return container.RawSnapshot{Source: container.Source{ID: "unraid", Type: "unraid"}, Containers: []container.RawContainer{}, ObservedAt: time.Date(2026, 8, 10, 2, 0, 0, 0, time.UTC)}, nil
}
func (s fakeInventorySource) Array(context.Context) (array.RawSnapshot, error) {
return array.RawSnapshot{Source: array.Source{ID: "unraid", Type: "unraid"}, State: array.StateOperational, Members: []array.RawMember{}, ObservedAt: time.Date(2026, 8, 10, 2, 0, 0, 0, time.UTC)}, nil
}
func (s fakeInventorySource) Disks(context.Context) (disk.RawSnapshot, error) {
return disk.RawSnapshot{Source: disk.Source{ID: "unraid", Type: "unraid"}, Disks: []disk.RawDisk{}, ObservedAt: time.Date(2026, 8, 10, 2, 0, 0, 0, time.UTC)}, nil
}
func (s fakeInventorySource) Pools(context.Context) (pool.RawSnapshot, error) {
return pool.RawSnapshot{Source: pool.Source{ID: "unraid", Type: "unraid"}, Pools: []pool.RawPool{}, ObservedAt: time.Date(2026, 8, 10, 2, 0, 0, 0, time.UTC)}, nil
}
func (s fakeInventorySource) Shares(context.Context) (share.RawSnapshot, error) {
return share.RawSnapshot{Source: share.Source{ID: "unraid", Type: "unraid"}, Shares: []share.RawShare{}, ObservedAt: time.Date(2026, 8, 10, 2, 0, 0, 0, time.UTC)}, nil
}
func (s *fakeSource) Host(ctx context.Context) (host.RawSnapshot, error) {
s.mu.Lock()
release := s.release
err := s.hostErr
s.hostCalls++
huge := s.hugePayload
s.mu.Unlock()
if release != nil {
select {
case <-release:
case <-ctx.Done():
return host.RawSnapshot{}, ctx.Err()
}
}
if err != nil {
return host.RawSnapshot{}, err
}
snapshot := host.RawSnapshot{
Identity: host.HostIdentity{Name: "tower"},
UptimeSeconds: 100,
Memory: host.RawMemory{TotalBytes: 1024, AvailableBytes: 512},
ObservedAt: time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC),
}
if huge {
warning := strings.Repeat("x", agentstore.MaxPayloadBytes)
snapshot.Warnings = []string{warning}
}
return snapshot, nil
}
func (s *fakeSource) Processes(context.Context) (process.RawSnapshot, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.procCalls++
if s.processErr != nil {
return process.RawSnapshot{}, s.processErr
}
return process.RawSnapshot{
Processes: []process.RawProcess{{PID: 1, Name: "init", State: "sleeping"}},
ObservedAt: time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC),
}, nil
}
func (s *fakeSource) calls() (int, int) {
s.mu.Lock()
defer s.mu.Unlock()
return s.hostCalls, s.procCalls
}
type testClock struct {
mu sync.Mutex
current time.Time
// step advances the clock on every read. A loop test needs strictly increasing
// timestamps, otherwise two iterations can observe the same instant and the second
// one is legitimately "not due yet".
step time.Duration
}
func (c *testClock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
current := c.current
c.current = c.current.Add(c.step)
return current
}
func (c *testClock) setStep(step time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.step = step
}
func (c *testClock) advance(d time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.current = c.current.Add(d)
}
func newTestAgent(t *testing.T, source snapshotSource, writer agentstore.Writer, collectInterval time.Duration) (*agent, *testClock, string) {
t.Helper()
heartbeat := filepath.Join(t.TempDir(), "healthy")
config := runtimeconfig.AgentConfig{
AgentID: "pulse-agent-test",
CollectInterval: collectInterval,
Service: runtimeconfig.ServiceConfig{ServiceName: "agent", HeartbeatFile: heartbeat},
}
logger := slog.New(slog.NewJSONHandler(io.Discard, nil))
instance := newAgent(config, source, writer, logger)
clock := &testClock{current: time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)}
instance.now = clock.now
return instance, clock, heartbeat
}
func readHeartbeat(t *testing.T, path string) string {
t.Helper()
data, err := os.ReadFile(path) //nolint:gosec // test-controlled path
if err != nil {
t.Fatalf("read heartbeat: %v", err)
}
return strings.TrimSpace(string(data))
}
func TestIterateCollectsOnTheIntervalAndHeartbeatsEveryIteration(t *testing.T) {
writer := newFakeWriter()
source := &fakeSource{}
// A 15 second collection interval must not stretch the loop: newAgent caps the
// loop at maxLoopInterval so the heartbeat stays inside the contract's 10s window.
instance, clock, heartbeat := newTestAgent(t, source, writer, 15*time.Second)
if instance.loopInterval != maxLoopInterval {
t.Fatalf("loop interval = %s, want %s", instance.loopInterval, maxLoopInterval)
}
ctx := context.Background()
for iteration := 0; iteration < 7; iteration++ {
instance.iterate(ctx)
if got := readHeartbeat(t, heartbeat); got != clock.now().UTC().Format(time.RFC3339) {
t.Fatalf("iteration %d heartbeat = %q, want the current time", iteration, got)
}
clock.advance(instance.loopInterval)
}
// Seven iterations five seconds apart cover t=0s..t=30s: collection is due at
// t=0s, t=15s and t=30s. The four iterations in between do no work, yet every one
// of them completed and refreshed the heartbeat, which is what the contract asks
// for ("an empty poll is still a completed iteration").
hostCalls, procCalls := source.calls()
if hostCalls != 3 || procCalls != 3 {
t.Fatalf("collections = host %d, processes %d; want 3 each over 30 seconds", hostCalls, procCalls)
}
if writer.countFor(agentstore.CapabilityHost) != 3 || writer.countFor(agentstore.CapabilityProcesses) != 3 {
t.Fatalf("unexpected publications: %+v", writer.recorded())
}
}
func TestOneFailingCapabilityDoesNotStopTheOthers(t *testing.T) {
writer := newFakeWriter()
source := &fakeSource{hostErr: errors.New("procfs read failed")}
instance, _, _ := newTestAgent(t, source, writer, time.Second)
instance.collectOnce(context.Background())
recorded := writer.recorded()
if len(recorded) != 1 || recorded[0].Capability != agentstore.CapabilityProcesses {
t.Fatalf("expected only the process snapshot, got %+v", recorded)
}
// The reverse case: the store rejects one capability.
writer.failCapability(agentstore.CapabilityProcesses, errors.New("store rejected"))
source.mu.Lock()
source.hostErr = nil
source.mu.Unlock()
instance.collectOnce(context.Background())
if writer.countFor(agentstore.CapabilityHost) != 1 {
t.Fatalf("host must still be published: %+v", writer.recorded())
}
if writer.countFor(agentstore.CapabilityProcesses) != 1 {
t.Fatalf("the rejected capability must not be recorded twice: %+v", writer.recorded())
}
}
func TestFailedCollectionWritesNothingRatherThanAnEmptySnapshot(t *testing.T) {
writer := newFakeWriter()
source := &fakeSource{hostErr: errors.New("boom"), processErr: errors.New("boom")}
instance, _, _ := newTestAgent(t, source, writer, time.Second)
instance.collectOnce(context.Background())
if recorded := writer.recorded(); len(recorded) != 0 {
t.Fatalf("a failed collection must publish nothing, got %+v", recorded)
}
}
func TestOversizedPayloadIsRefused(t *testing.T) {
writer := newFakeWriter()
source := &fakeSource{hugePayload: true}
instance, _, _ := newTestAgent(t, source, writer, time.Second)
if _, err := instance.publish(context.Background(), instance.capabilities[0]); err == nil {
t.Fatal("expected an oversized payload to be refused")
}
if writer.countFor(agentstore.CapabilityHost) != 0 {
t.Fatal("an oversized payload must never reach the store")
}
}
func TestPublishUsesABoundedContext(t *testing.T) {
writer := newFakeWriter()
instance, _, _ := newTestAgent(t, &fakeSource{}, writer, time.Second)
instance.collectOnce(context.Background())
writer.mu.Lock()
defer writer.mu.Unlock()
if len(writer.deadlines) == 0 {
t.Fatal("no writes recorded")
}
for index, bounded := range writer.deadlines {
if !bounded {
t.Fatalf("write %d ran without a deadline", index)
}
}
}
func TestPublishedSnapshotCarriesTheObservedTimeAndPayload(t *testing.T) {
writer := newFakeWriter()
instance, _, _ := newTestAgent(t, &fakeSource{}, writer, time.Second)
instance.collectOnce(context.Background())
for _, snapshot := range writer.recorded() {
if snapshot.AgentID != "pulse-agent-test" {
t.Fatalf("agent id = %q", snapshot.AgentID)
}
if !snapshot.Capability.Valid() {
t.Fatalf("unknown capability %q", snapshot.Capability)
}
if snapshot.ObservedAt.IsZero() || snapshot.ObservedAt.Location() != time.UTC {
t.Fatalf("observed at = %v", snapshot.ObservedAt)
}
var decoded map[string]any
if err := json.Unmarshal(snapshot.Payload, &decoded); err != nil {
t.Fatalf("payload is not a JSON object: %v", err)
}
}
}
func TestRunWritesTheFirstHeartbeatBeforeTheFirstBlockingCall(t *testing.T) {
writer := newFakeWriter()
release := make(chan struct{})
source := &fakeSource{release: release}
instance, clock, heartbeat := newTestAgent(t, source, writer, time.Second)
ticks := make(chan time.Time)
instance.newTicker = func(time.Duration) (<-chan time.Time, func()) { return ticks, func() {} }
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- instance.run(ctx) }()
// The collector is blocked inside the first iteration, yet the heartbeat must
// already exist: a slow cold start is not a hang.
deadline := time.Now().Add(2 * time.Second)
for {
if _, err := os.Stat(heartbeat); err == nil {
break
}
if time.Now().After(deadline) {
t.Fatal("no heartbeat was written before the first blocking call")
}
time.Sleep(time.Millisecond)
}
if got := readHeartbeat(t, heartbeat); got != clock.now().UTC().Format(time.RFC3339) {
t.Fatalf("heartbeat = %q", got)
}
close(release)
cancel()
select {
case err := <-done:
if err != nil {
t.Fatalf("run returned error: %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("run did not return after cancellation")
}
}
func TestRunHeartbeatsFromTheLoopAndShutsDownCleanly(t *testing.T) {
writer := newFakeWriter()
source := &fakeSource{}
instance, clock, heartbeat := newTestAgent(t, source, writer, time.Second)
clock.setStep(time.Second)
start := clock.now()
ticks := make(chan time.Time)
instance.newTicker = func(time.Duration) (<-chan time.Time, func()) { return ticks, func() {} }
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- instance.run(ctx) }()
// An unbuffered tick is only received once the previous iteration has finished, so
// each successful send proves one more completed loop iteration.
for iteration := 0; iteration < 3; iteration++ {
select {
case ticks <- time.Now():
case <-time.After(5 * time.Second):
t.Fatalf("iteration %d never completed", iteration)
}
}
if hostCalls, procCalls := source.calls(); hostCalls < 3 || procCalls < 3 {
t.Fatalf("collections = host %d, processes %d; want at least 3 each", hostCalls, procCalls)
}
cancel()
select {
case err := <-done:
if err != nil {
t.Fatalf("run returned error: %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("run did not stop on cancellation")
}
// Read the heartbeat only after the loop has stopped: the file is truncated and
// rewritten in place, so a concurrent reader can legitimately observe it empty.
// The healthcheck script reads only the mtime, which is why that is safe.
written, err := time.Parse(time.RFC3339, readHeartbeat(t, heartbeat))
if err != nil {
t.Fatalf("heartbeat content is not an RFC 3339 timestamp: %v", err)
}
if written.Before(start) {
t.Fatalf("heartbeat %s predates the loop start %s", written, start)
}
}
func TestRunDoesNotBlockShutdownOnAStuckWrite(t *testing.T) {
writer := newFakeWriter()
writer.blockUntil = make(chan struct{})
instance, _, _ := newTestAgent(t, &fakeSource{}, writer, time.Second)
ticks := make(chan time.Time)
instance.newTicker = func(time.Duration) (<-chan time.Time, func()) { return ticks, func() {} }
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- instance.run(ctx) }()
time.Sleep(20 * time.Millisecond)
cancel()
select {
case err := <-done:
if err != nil {
t.Fatalf("run returned error: %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("a blocked store write must not hold up shutdown")
}
}
func TestRunRefusesToStartWhenTheHelloIsNotValid(t *testing.T) {
writer := newFakeWriter()
instance, _, _ := newTestAgent(t, &fakeSource{}, writer, time.Second)
instance.agentID = ""
if err := instance.run(context.Background()); err == nil {
t.Fatal("an agent whose own hello fails validation must refuse to start")
}
if len(writer.recorded()) != 0 {
t.Fatal("nothing may be published before the self-check passes")
}
}
func TestRunRequiresAWriter(t *testing.T) {
instance, _, _ := newTestAgent(t, &fakeSource{}, nil, time.Second)
instance.writer = nil
if err := instance.run(context.Background()); !errors.Is(err, errNoWriter) {
t.Fatalf("error = %v, want errNoWriter", err)
}
}
func TestHelloAnnouncesOnlyReadOnlyCapabilities(t *testing.T) {
instance, clock, _ := newTestAgent(t, &fakeSource{}, newFakeWriter(), time.Second)
hello := instance.hello()
if err := hello.Validate(clock.now()); err != nil {
t.Fatalf("hello validation failed: %v", err)
}
if len(hello.Capabilities) != 2 {
t.Fatalf("capabilities = %+v", hello.Capabilities)
}
for _, item := range hello.Capabilities {
if !item.ReadOnly {
t.Fatalf("capability %q is not read-only", item.ID)
}
if !agentstore.Capability(item.ID).Valid() {
t.Fatalf("capability %q is not a recognised store capability", item.ID)
}
}
}
func TestContainerCapabilityIsPublishedOnlyWhenSourceSupportsIt(t *testing.T) {
writer := newFakeWriter()
instance, _, _ := newTestAgent(t, fakeContainerSource{fakeSource: &fakeSource{}}, writer, time.Second)
if len(instance.hello().Capabilities) != 3 {
t.Fatalf("capabilities = %+v, want host/processes/containers", instance.hello().Capabilities)
}
instance.collectOnce(context.Background())
if writer.countFor(agentstore.CapabilityContainers) != 1 {
t.Fatalf("container snapshot was not published: %+v", writer.recorded())
}
}
func TestInventoryCapabilitiesArePublishedOnlyForConfiguredUnraidSource(t *testing.T) {
writer := newFakeWriter()
instance, _, _ := newTestAgent(t, fakeInventorySource{fakeSource: &fakeSource{}}, writer, time.Second)
if got := len(instance.hello().Capabilities); got != 7 {
t.Fatalf("capabilities = %+v, want host/processes/containers/array/disks/pools/shares", instance.hello().Capabilities)
}
instance.collectOnce(context.Background())
for _, capability := range []agentstore.Capability{agentstore.CapabilityContainers, agentstore.CapabilityArray, agentstore.CapabilityDisks, agentstore.CapabilityPools, agentstore.CapabilityShares} {
if writer.countFor(capability) != 1 {
t.Fatalf("capability %q was not published: %+v", capability, writer.recorded())
}
}
}
func TestHeartbeatFailureIsNotFatal(t *testing.T) {
writer := newFakeWriter()
instance, _, _ := newTestAgent(t, &fakeSource{}, writer, time.Second)
instance.heartbeatPath = filepath.Join(t.TempDir(), "missing-directory", "healthy")
instance.iterate(context.Background())
if len(writer.recorded()) == 0 {
t.Fatal("a heartbeat write failure must not stop the collection loop")
}
}
func TestSnapshotWriterRefusesAMissingPool(t *testing.T) {
// A nil pool would let every publish fail deep in the driver. The agent must refuse
// to start instead, so an operator sees the misconfiguration rather than an agent
// that appears healthy while storing nothing.
if _, err := newSnapshotWriter(nil); !errors.Is(err, errStoreNotLinked) {
t.Fatalf("error = %v, want errStoreNotLinked", err)
}
}
+157
View File
@@ -0,0 +1,157 @@
// Command pulse-agent collects read-only host telemetry and publishes it as bounded
// snapshots for pulse-api to read.
//
// It exposes no network port, holds no Docker socket, and performs no mutation: it
// reads procfs/sysfs and writes one row per capability through agentstore.Writer. See
// docs/architecture/SYSTEM_ARCHITECTURE.md ("pulse-agent"), ADR-0005 and
// docs/operations/WORKER_AGENT_HEALTHCHECK_CONTRACT.md.
package main
import (
"context"
"errors"
"log/slog"
"os"
"os/signal"
"syscall"
"github.com/itworx/pulse/internal/array"
"github.com/itworx/pulse/internal/container"
"github.com/itworx/pulse/internal/disk"
"github.com/itworx/pulse/internal/host"
"github.com/itworx/pulse/internal/hostcollect"
"github.com/itworx/pulse/internal/pool"
"github.com/itworx/pulse/internal/process"
"github.com/itworx/pulse/internal/runtimeconfig"
"github.com/itworx/pulse/internal/share"
"github.com/itworx/pulse/internal/unraid"
)
type configuredSource struct {
host *hostcollect.Collector
}
func (s configuredSource) Host(ctx context.Context) (host.RawSnapshot, error) {
return s.host.Host(ctx)
}
func (s configuredSource) Processes(ctx context.Context) (process.RawSnapshot, error) {
return s.host.Processes(ctx)
}
type configuredUnraidSource struct {
configuredSource
containers interface {
Snapshot(context.Context) (container.RawSnapshot, error)
}
array interface {
Snapshot(context.Context) (array.RawSnapshot, error)
}
disks interface {
Snapshot(context.Context) (disk.RawSnapshot, error)
}
pools interface {
Snapshot(context.Context) (pool.RawSnapshot, error)
}
shares interface {
Snapshot(context.Context) (share.RawSnapshot, error)
}
}
func (s configuredUnraidSource) Containers(ctx context.Context) (container.RawSnapshot, error) {
if s.containers == nil {
return container.RawSnapshot{}, errUnraidSourceNotConfigured
}
return s.containers.Snapshot(ctx)
}
func (s configuredUnraidSource) Array(ctx context.Context) (array.RawSnapshot, error) {
if s.array == nil {
return array.RawSnapshot{}, errUnraidSourceNotConfigured
}
return s.array.Snapshot(ctx)
}
func (s configuredUnraidSource) Disks(ctx context.Context) (disk.RawSnapshot, error) {
if s.disks == nil {
return disk.RawSnapshot{}, errUnraidSourceNotConfigured
}
return s.disks.Snapshot(ctx)
}
func (s configuredUnraidSource) Pools(ctx context.Context) (pool.RawSnapshot, error) {
if s.pools == nil {
return pool.RawSnapshot{}, errUnraidSourceNotConfigured
}
return s.pools.Snapshot(ctx)
}
func (s configuredUnraidSource) Shares(ctx context.Context) (share.RawSnapshot, error) {
if s.shares == nil {
return share.RawSnapshot{}, errUnraidSourceNotConfigured
}
return s.shares.Snapshot(ctx)
}
var errUnraidSourceNotConfigured = errors.New("Unraid source is not configured")
func main() {
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
if err := run(logger); err != nil {
logger.Error("pulse agent failed", "error", err.Error())
os.Exit(1)
}
}
func run(logger *slog.Logger) error {
config, err := runtimeconfig.LoadAgent()
if err != nil {
return err
}
collector, err := hostcollect.New(hostcollect.Options{
ProcRoot: config.ProcRoot,
SysRoot: config.SysRoot,
FilesystemRoot: config.FilesystemRoot,
HostName: config.HostName,
SourceID: "host",
ProcessLimits: process.Limits{MaxRows: config.MaxProcesses},
})
if err != nil {
return err
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
writer, closeStore, err := openStore(ctx, config)
if err != nil {
return err
}
defer closeStore()
logger.Info("pulse agent configuration loaded",
"agent_id", config.AgentID,
"proc_root", config.ProcRoot,
"sys_root", config.SysRoot,
"filesystem_root", config.FilesystemRoot,
"collect_interval", config.CollectInterval.String(),
"shutdown_timeout", config.Service.ShutdownAfter.String(),
)
var source snapshotSource = configuredSource{host: collector}
if config.UnraidURL != "" {
var client *unraid.Client
if config.UnraidCAFile == "" {
client, err = unraid.New(config.UnraidURL, config.UnraidAPIToken, nil)
} else {
info, statErr := os.Stat(config.UnraidCAFile)
if statErr != nil || info.Size() <= 0 || info.Size() > 1<<20 {
return errors.New("PULSE_UNRAID_CA_FILE must be a readable certificate no larger than 1 MiB")
}
caPEM, readErr := os.ReadFile(config.UnraidCAFile)
if readErr != nil {
return errors.New("read PULSE_UNRAID_CA_FILE")
}
client, err = unraid.NewWithCAPEM(config.UnraidURL, config.UnraidAPIToken, caPEM)
}
if err != nil {
return err
}
source = configuredUnraidSource{configuredSource: configuredSource{host: collector}, containers: unraid.ContainerSource{Client: client}, array: unraid.ArraySource{Client: client}, disks: unraid.DiskSource{Client: client}, pools: unraid.PoolSource{Client: client}, shares: unraid.ShareSource{Client: client}}
}
return newAgent(config, source, writer, logger).run(ctx)
}
+70
View File
@@ -0,0 +1,70 @@
package main
import (
"context"
"errors"
"fmt"
"time"
"github.com/itworx/pulse/internal/agentstore"
"github.com/itworx/pulse/internal/database"
"github.com/itworx/pulse/internal/runtimeconfig"
"github.com/jackc/pgx/v5/pgxpool"
)
const (
// storeConnectTimeout bounds the start-up reachability check. The healthcheck
// contract wants the first heartbeat written after the database is reachable, so
// this must stay far below the 45 second staleness threshold.
storeConnectTimeout = 5 * time.Second
// agentPoolMaxConns keeps the agent's footprint on the shared database small: it
// issues one small write per capability per interval and never reads.
agentPoolMaxConns = 2
)
// errStoreNotLinked is returned by every write while no snapshot store implementation
// is compiled into this binary. It is deliberately an error on the write path rather
// than a silent drop: the reader turns a missing snapshot into Unknown, and the agent
// log states plainly why nothing arrives.
var errStoreNotLinked = errors.New("no agent snapshot store is linked into this build")
// openStore connects to PostgreSQL and returns the narrow Writer the agent publishes
// through, plus a close function.
//
// The agent depends on the agentstore.Writer interface only; the concrete PostgreSQL
// store owns migration 0016 and the agent_snapshots table.
func openStore(ctx context.Context, config runtimeconfig.AgentConfig) (agentstore.Writer, func(), error) {
pool, err := database.NewPool(ctx, database.Config{
URL: config.DatabaseURL,
MaxConns: agentPoolMaxConns,
MinConns: 1,
})
if err != nil {
// Never wrap the URL itself into the error: it carries the database password.
return nil, nil, errors.New("agent database pool could not be created")
}
reachable, cancel := context.WithTimeout(ctx, storeConnectTimeout)
defer cancel()
if err := database.Ping(reachable, pool); err != nil {
pool.Close()
return nil, nil, fmt.Errorf("agent database is unreachable: %w", err)
}
writer, err := newSnapshotWriter(pool)
if err != nil {
pool.Close()
return nil, nil, err
}
return writer, pool.Close, nil
}
// newSnapshotWriter builds the store the agent writes through. A nil pool would make
// every publish fail silently at the driver, so it is refused here instead: the agent
// must either have a real store or fail to start.
func newSnapshotWriter(pool *pgxpool.Pool) (agentstore.Writer, error) {
if pool == nil {
return nil, errStoreNotLinked
}
return agentstore.PostgresStore{Pool: pool}, nil
}
var _ agentstore.Writer = agentstore.PostgresStore{}
+569
View File
@@ -0,0 +1,569 @@
package main
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"os"
"os/signal"
"sync"
"syscall"
"time"
"github.com/itworx/pulse/internal/agentsource"
"github.com/itworx/pulse/internal/agentstore"
"github.com/itworx/pulse/internal/alert"
"github.com/itworx/pulse/internal/alertapi"
"github.com/itworx/pulse/internal/alertcontrol"
"github.com/itworx/pulse/internal/alertcontrolapi"
"github.com/itworx/pulse/internal/alertdefaults"
"github.com/itworx/pulse/internal/alertopsapi"
"github.com/itworx/pulse/internal/applicationapi"
"github.com/itworx/pulse/internal/array"
"github.com/itworx/pulse/internal/arrayapi"
"github.com/itworx/pulse/internal/audit"
"github.com/itworx/pulse/internal/auth"
"github.com/itworx/pulse/internal/authapi"
"github.com/itworx/pulse/internal/backup"
"github.com/itworx/pulse/internal/backupapi"
"github.com/itworx/pulse/internal/config"
"github.com/itworx/pulse/internal/container"
"github.com/itworx/pulse/internal/containerapi"
"github.com/itworx/pulse/internal/correlation"
"github.com/itworx/pulse/internal/dashboard"
"github.com/itworx/pulse/internal/dashboardapi"
"github.com/itworx/pulse/internal/database"
"github.com/itworx/pulse/internal/disk"
"github.com/itworx/pulse/internal/diskapi"
"github.com/itworx/pulse/internal/eventapi"
forecastdomain "github.com/itworx/pulse/internal/forecast"
"github.com/itworx/pulse/internal/forecastapi"
"github.com/itworx/pulse/internal/host"
"github.com/itworx/pulse/internal/hostapi"
"github.com/itworx/pulse/internal/incident"
"github.com/itworx/pulse/internal/incidentapi"
"github.com/itworx/pulse/internal/inventory"
"github.com/itworx/pulse/internal/inventoryapi"
"github.com/itworx/pulse/internal/live"
"github.com/itworx/pulse/internal/livesampler"
"github.com/itworx/pulse/internal/metriccatalog"
"github.com/itworx/pulse/internal/metricquery"
"github.com/itworx/pulse/internal/metricsapi"
"github.com/itworx/pulse/internal/network"
"github.com/itworx/pulse/internal/networkapi"
"github.com/itworx/pulse/internal/observability"
"github.com/itworx/pulse/internal/onboarding"
"github.com/itworx/pulse/internal/onboardingapi"
pooldomain "github.com/itworx/pulse/internal/pool"
"github.com/itworx/pulse/internal/poolapi"
"github.com/itworx/pulse/internal/problem"
"github.com/itworx/pulse/internal/process"
"github.com/itworx/pulse/internal/processapi"
"github.com/itworx/pulse/internal/prometheus"
"github.com/itworx/pulse/internal/queryplan"
"github.com/itworx/pulse/internal/reverseproxy"
"github.com/itworx/pulse/internal/reverseproxyapi"
"github.com/itworx/pulse/internal/runtimeconfig"
"github.com/itworx/pulse/internal/service"
"github.com/itworx/pulse/internal/serviceapi"
sharedomain "github.com/itworx/pulse/internal/share"
"github.com/itworx/pulse/internal/shareapi"
"github.com/itworx/pulse/internal/systemstatus"
"github.com/itworx/pulse/internal/systemstatusapi"
"github.com/itworx/pulse/internal/widget"
"github.com/itworx/pulse/internal/widgetapi"
"github.com/itworx/pulse/internal/workerruntime"
"github.com/jackc/pgx/v5/pgxpool"
)
func main() {
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
if err := run(logger); err != nil {
logger.Error("pulse api failed", "error", err)
os.Exit(1)
}
}
func run(logger *slog.Logger) error {
runtime, err := runtimeconfig.Load("api")
if err != nil {
return err
}
application, err := config.Load()
if err != nil {
return err
}
registry, err := metriccatalog.DefaultRegistry()
if err != nil {
return err
}
ctx := context.Background()
internalMetrics := observability.NewRegistry(time.Now().UTC())
var pool *pgxpool.Pool
var inventoryRepo *inventory.Repository
var dashboardRepo dashboard.Repository
var alertRepo alert.Store
var alertControlStore alertcontrol.Store
var alertOperationsStore alertopsapi.Store
var incidentStore incident.Store
var serviceProvider service.Provider = service.UnknownProvider{Reason: "source_unavailable"}
var reverseProxyProvider reverseproxy.Provider = reverseproxy.DisabledProvider{SourceID: "reverse-proxy", SourceType: "connector", Reason: "connector_disabled"}
var dependencyRepo *service.DependencyRepository
var onboardingService onboarding.Service
// agentReader is the read half of the pulse-agent telemetry transport. It stays nil
// without a database, which keeps every monitoring surface Unknown instead of
// inventing state (ADR-0008).
var agentReader agentstore.Reader
databaseReady := false
if application.DatabaseURL != "" {
pool, err = database.NewPool(ctx, database.Config{URL: application.DatabaseURL})
if err != nil {
return err
}
defer pool.Close()
if err := database.Ping(ctx, pool); err != nil {
return err
}
databaseReady = true
agentReader = agentstore.PostgresStore{Pool: pool}
dashboardRepo = dashboard.Repository{Pool: pool}
alertRepo = alert.Repository{Pool: pool, Registry: registry}
if report, seedErr := alertdefaults.Seed(ctx, alertRepo, registry, "system-defaults"); seedErr != nil {
return fmt.Errorf("seed alert defaults: %w", seedErr)
} else {
logger.Info("alert defaults reconciled", "added", report.Added, "existing", report.Existing)
}
alertControlStore = alertcontrol.Repository{Pool: pool}
alertOperationsStore = alert.StateRepository{Pool: pool}
incidentStore, err = incident.NewRepository(pool)
if err != nil {
return err
}
serviceProvider, err = service.NewPostgresProvider(pool, service.StatusPolicy{})
if err != nil {
return err
}
dependencyRepo, err = service.NewDependencyRepository(pool, audit.PostgresStore{Pool: pool})
if err != nil {
return err
}
inventoryRepo, err = inventory.NewRepository(pool)
if err != nil {
return err
}
onboardingService = onboarding.Service{State: onboarding.StateStore{Pool: pool}, Pool: pool, Dashboards: dashboardRepo, Alerts: alertRepo, AuthMode: application.AuthMode, OIDCIssuer: application.OIDCIssuer, OIDCClient: application.OIDCClientID, OIDCRedirect: application.OIDCRedirectURL, Prometheus: application.PrometheusURL != "", Unraid: application.UnraidURL != "" && application.UnraidAPIToken != ""}
}
var queryService *metricquery.Service
var liveSampler live.Sampler
var promSource *prometheus.Client
if application.PrometheusURL != "" {
source, sourceErr := prometheus.New(application.PrometheusURL, nil, prometheus.Limits{Timeout: application.PrometheusTimeout})
if sourceErr != nil {
return sourceErr
}
promSource = source
queryService = metricquery.NewService(queryplan.NewPlanner(registry, queryplan.Limits{}), source, nil)
sampler, samplerErr := livesampler.New(registry, source, livesampler.Options{})
if samplerErr != nil {
return samplerErr
}
liveSampler = sampler
}
if err != nil {
return err
}
// publishSourceMetrics refreshes adapter counters into the internal registry just
// before it is read, so operators see current Prometheus latency and error counts
// rather than the values captured at process start.
publishSourceMetrics := func() {
if promSource != nil {
promSource.PublishMetrics(internalMetrics)
}
}
sessions := auth.NewSlidingSessionManager("pulse_session", application.SessionIdleTTL, application.SessionAbsoluteTTL, application.Environment == config.Production)
backupManager := &backup.Manager{Pool: pool, Directory: application.BackupDirectory, Retention: application.BackupRetention}
var backupObservation struct {
sync.Mutex
checkedAt time.Time
latest time.Time
verified time.Time
err error
}
readBackupObservation := func(ctx context.Context, now time.Time) (time.Time, time.Time, error) {
backupObservation.Lock()
defer backupObservation.Unlock()
if !backupObservation.checkedAt.IsZero() && now.Sub(backupObservation.checkedAt) < 5*time.Minute {
return backupObservation.latest, backupObservation.verified, backupObservation.err
}
results, err := backupManager.List(ctx)
latest := time.Time{}
if len(results) > 0 {
latest = results[0].Created
}
verified := time.Time{}
if err == nil {
verified = now
}
backupObservation.checkedAt, backupObservation.latest, backupObservation.verified, backupObservation.err = now, latest, verified, err
return latest, verified, err
}
invalidateBackupObservation := func() {
backupObservation.Lock()
defer backupObservation.Unlock()
backupObservation.checkedAt = time.Time{}
}
mux := service.HealthMuxWithReadiness(func() bool { return databaseReady })
mux.HandleFunc("/auth/test-login", func(response http.ResponseWriter, request *http.Request) {
if application.Environment == config.Production || application.AuthMode != "mock" {
problem.Write(response, request, http.StatusNotFound, "NOT_FOUND", "Not found", "The requested resource does not exist.", nil)
return
}
principal := auth.Principal{Subject: "development-user", Role: auth.RoleAdministrator}
if err := sessions.Issue(response, principal, time.Now().UTC()); err != nil {
problem.Write(response, request, http.StatusInternalServerError, "SESSION_ERROR", "Session unavailable", "The session could not be created.", nil)
return
}
if pool != nil {
if err := audit.RecordSecurityAction(request.Context(), audit.PostgresStore{Pool: pool}, principal.Subject, "auth.test_login", "success", correlation.FromContext(request.Context())); err != nil {
problem.Write(response, request, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "Authentication unavailable", "The authentication event could not be recorded.", nil)
return
}
}
response.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(response).Encode(map[string]string{"status": "authenticated", "mode": "mock-development"})
})
mux.HandleFunc("/session/logout", func(response http.ResponseWriter, request *http.Request) {
sessions.Clear(response, request)
response.WriteHeader(http.StatusNoContent)
})
if application.AuthMode == "oidc" && application.OIDCIssuer != "" {
oidcLogin, loginErr := authapi.New(authapi.Options{
OIDC: auth.OIDCConfig{
Issuer: application.OIDCIssuer,
ClientID: application.OIDCClientID,
ClientSecret: application.OIDCClientSecret,
RedirectURL: application.OIDCRedirectURL,
},
RoleMapping: roleMapping(application.OIDCRoleMapping),
GroupsClaim: application.OIDCGroupsClaim,
Sessions: sessions,
Secure: application.Environment == config.Production,
Logger: logger,
// Failures land on the overview route, where the web app reads the reason
// code from the query string, shows a localized notice and strips it from
// the URL. There is deliberately no dedicated error page to maintain.
ErrorPath: "/",
Audit: func(ctx context.Context, actor, result string) error {
if pool == nil {
return nil
}
return audit.RecordSecurityAction(ctx, audit.PostgresStore{Pool: pool}, actor, "auth.login", result, correlation.FromContext(ctx))
},
})
if loginErr != nil {
return loginErr
}
mux.Handle("/auth/login", oidcLogin.LoginHandler())
mux.Handle("/auth/callback", oidcLogin.CallbackHandler())
logger.Info("oidc login enabled", "mapped_claims", len(application.OIDCRoleMapping))
}
// reportedJobs is the metadata-only view of the worker schedule. The API never runs
// these jobs; it reads their recorded outcomes from job_runs so the status surface
// reflects what the worker actually did instead of the hardcoded "not recorded"
// placeholders it used before the worker runtime existed.
reportedJobs := workerruntime.Schedule(workerruntime.ScheduleRuns{})
snapshot := func(requestContext context.Context) (systemstatus.Snapshot, error) {
publishSourceMetrics()
var auditEvents *int64
var statusOptions []systemstatus.Option
now := time.Now().UTC()
_, authenticated := auth.PrincipalFromContext(requestContext)
statusOptions = append(statusOptions, systemstatus.WithAuthenticatedSession(authenticated))
sourceHealth := make([]systemstatus.SourceHealth, 0, 3)
if promSource != nil {
sourceHealth = append(sourceHealth, systemstatus.FromDatasource("prometheus", promSource.Health(requestContext), now))
}
if agentReader != nil {
unraidHealth, storageHealth, healthErr := readAgentSourceHealth(requestContext, agentReader, now)
if healthErr != nil {
return systemstatus.Snapshot{}, fmt.Errorf("read agent source health: %w", healthErr)
}
if unraidHealth.ReasonCode != agentsource.ReasonUnavailable || storageHealth.ReasonCode != agentsource.ReasonUnavailable {
internalMetrics.SetGauge("pulse_unraid_configured", 1)
}
sourceHealth = append(sourceHealth, systemstatus.FromDatasource("unraid", unraidHealth, now), systemstatus.FromDatasource("storage", storageHealth, now))
}
if len(sourceHealth) > 0 {
statusOptions = append(statusOptions, systemstatus.WithSources(sourceHealth...))
}
if application.BackupDirectory != "" {
latestBackup, verifiedAt, backupErr := readBackupObservation(requestContext, now)
if backupErr != nil {
statusOptions = append(statusOptions, systemstatus.WithBackupObservation(time.Time{}, time.Time{}, backupErr))
} else if !latestBackup.IsZero() {
statusOptions = append(statusOptions, systemstatus.WithBackupObservation(latestBackup, verifiedAt, nil))
}
}
if pool != nil && databaseReady {
var count int64
if err := pool.QueryRow(requestContext, `SELECT count(*) FROM audit_events`).Scan(&count); err != nil {
return systemstatus.Snapshot{}, fmt.Errorf("read audit event count: %w", err)
}
auditEvents = &count
var migrationVersion string
if err := pool.QueryRow(requestContext, `SELECT id FROM schema_migrations ORDER BY applied_at DESC, id DESC LIMIT 1`).Scan(&migrationVersion); err != nil {
return systemstatus.Snapshot{}, fmt.Errorf("read migration version: %w", err)
}
statusOptions = append(statusOptions, systemstatus.WithMigrationVersion(migrationVersion))
jobs, jobsErr := workerruntime.ReadJobHealth(requestContext, pool, reportedJobs)
if jobsErr != nil {
// A failed read must not be reported as healthy. Omitting the option
// leaves every job component Unknown, which is the honest answer when
// the worker's recorded state cannot be established (ADR-0008).
logger.Error("read worker job health", "error", jobsErr)
} else {
statusOptions = append(statusOptions, systemstatus.WithJobs(0, jobs...))
}
}
return systemstatus.Build(application, databaseReady, now, auditEvents, statusOptions...), nil
}
internalMetrics.SetGauge("pulse_database_ready", boolMetric(databaseReady))
internalMetrics.SetGauge("pulse_prometheus_configured", boolMetric(application.PrometheusURL != ""))
internalMetrics.SetGauge("pulse_unraid_configured", boolMetric(application.UnraidURL != "" && application.UnraidAPIToken != ""))
if onboardingService.State.Pool != nil {
onboardingService.RuntimeCapabilities = func(requestContext context.Context) ([]onboarding.Capability, error) {
status, statusErr := snapshot(requestContext)
if statusErr != nil {
return nil, statusErr
}
capabilities := make([]onboarding.Capability, 0, 2)
for _, component := range status.Components {
if component.ID != "prometheus" && component.ID != "unraid" {
continue
}
state, detail := "unknown", "Geen actuele runtimewaarneming beschikbaar."
switch component.State {
case systemstatus.StateHealthy:
state, detail = "ready", "Actuele telemetrie wordt ontvangen via de veilige runtimebron."
case systemstatus.StateDegraded:
state, detail = "incomplete", "De runtimebron vraagt aandacht."
case systemstatus.StateDisabled:
state, detail = "not-ready", "De runtimebron is niet geconfigureerd."
}
capabilities = append(capabilities, onboarding.Capability{ID: component.ID, State: state, Detail: detail})
}
return capabilities, nil
}
}
statusHandler := systemstatusapi.Handler{
Snapshot: snapshot,
Diagnostics: func(requestContext context.Context) (systemstatusapi.Diagnostics, error) {
status, err := snapshot(requestContext)
if err != nil {
return systemstatusapi.Diagnostics{}, err
}
return systemstatusapi.Diagnostics{
Status: status,
Config: systemstatusapi.ConfigSummary{
Environment: string(application.Environment), Timezone: application.Timezone, Locale: application.DefaultLocale, AuthMode: application.AuthMode,
PublicURLConfigured: application.PublicURL != "", DatabaseConfigured: application.DatabaseURL != "", PrometheusConfigured: application.PrometheusURL != "",
UnraidConfigured: application.UnraidURL != "" && application.UnraidAPIToken != "", OIDCConfigured: application.OIDCIssuer != "" && application.OIDCClientID != "",
},
Runtime: systemstatusapi.Runtime(), Metrics: internalMetrics.Exposition(time.Now().UTC()),
}, nil
},
}
protectedStatus := withSession(sessions, auth.Require(auth.PermissionView, statusHandler))
protectedDiagnostics := withSession(sessions, auth.Require(auth.PermissionOperate, statusHandler))
mux.Handle("/api/v1/system/status", protectedStatus)
mux.Handle("/api/v1/system/diagnostics", protectedDiagnostics)
metricsExposition := internalMetrics.Handler()
mux.Handle("/api/v1/system/metrics", withSession(sessions, auth.Require(auth.PermissionOperate, http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
publishSourceMetrics()
metricsExposition.ServeHTTP(response, request)
}))))
mux.Handle("/api/v1/system/backups", withSession(sessions, auth.Require(auth.PermissionAdmin, backupapi.Handler{Manager: backupManager, OnCreated: invalidateBackupObservation, Audit: func(ctx context.Context, actor, result string) error {
return audit.RecordSecurityAction(ctx, audit.PostgresStore{Pool: pool}, actor, "backup.create", result, correlation.FromContext(ctx))
}})))
onboardingHandler := onboardingapi.Handler{Service: onboardingService, Audit: audit.PostgresStore{Pool: pool}}
mux.Handle("/api/v1/onboarding", withSession(sessions, auth.Require(auth.PermissionView, onboardingHandler)))
metricsHandler := metricsapi.Handler{Registry: registry}
widgetRegistry, err := widget.NewRegistry(widget.DefaultDefinitions())
if err != nil {
return err
}
widgetHandler := widgetapi.Handler{Registry: widgetRegistry}
mux.Handle("/api/v1/widgets/catalog", withSession(sessions, auth.Require(auth.PermissionView, widgetHandler)))
mux.Handle("/api/v1/widgets/preview", withSession(sessions, auth.Require(auth.PermissionEdit, widgetHandler)))
mux.Handle("/api/v1/metrics/catalog", withSession(sessions, auth.Require(auth.PermissionView, metricsHandler)))
queryHandler := metricquery.Handler{Service: queryService}
mux.Handle("/api/v1/metrics/query", withSession(sessions, auth.Require(auth.PermissionView, queryHandler)))
mux.Handle("/api/v1/metrics/query-range", withSession(sessions, auth.Require(auth.PermissionView, queryHandler)))
mux.Handle("/api/v1/metrics/inspect", withSession(sessions, auth.Require(auth.PermissionOperate, queryHandler)))
livePlanner := queryplan.NewPlanner(registry, queryplan.Limits{})
liveRegistry := live.NewRegistry(liveSampler, live.RegistryOptions{})
liveHandler := live.Handler{Planner: &livePlanner, Registry: liveRegistry}
mux.Handle("/api/v1/live", withSession(sessions, auth.Require(auth.PermissionView, liveHandler)))
if alertRepo != nil {
alertHandler := alertapi.Handler{Repository: alertRepo, Registry: registry, Audit: audit.PostgresStore{Pool: pool}}
mux.Handle("/api/v1/alert-rules", withSession(sessions, alertHandler))
mux.Handle("/api/v1/alert-rules/", withSession(sessions, alertHandler))
operationsHandler := alertopsapi.Handler{Store: alertOperationsStore, Audit: audit.PostgresStore{Pool: pool}}
protectedOperations := withSession(sessions, auth.Require(auth.PermissionView, operationsHandler))
mux.Handle("/api/v1/alerts", protectedOperations)
mux.Handle("/api/v1/alerts/", protectedOperations)
controlHandler := alertcontrolapi.Handler{Store: alertControlStore, Audit: audit.PostgresStore{Pool: pool}}
protectedControls := withSession(sessions, auth.Require(auth.PermissionView, controlHandler))
mux.Handle("/api/v1/alert-silences", protectedControls)
mux.Handle("/api/v1/alert-silences/", protectedControls)
mux.Handle("/api/v1/maintenance-windows", protectedControls)
mux.Handle("/api/v1/maintenance-windows/", protectedControls)
}
if incidentStore != nil {
incidentHandler := incidentapi.Handler{Store: incidentStore, Audit: audit.PostgresStore{Pool: pool}}
protectedIncidents := withSession(sessions, auth.Require(auth.PermissionView, incidentHandler))
mux.Handle("/api/v1/incidents", protectedIncidents)
mux.Handle("/api/v1/incidents/", protectedIncidents)
}
if dashboardRepo.Pool != nil {
dashboardHandler := dashboardapi.Handler{Repository: dashboardRepo, Audit: audit.PostgresStore{Pool: pool}}
mux.Handle("/api/v1/dashboards", withSession(sessions, dashboardHandler))
mux.Handle("/api/v1/dashboards/", withSession(sessions, dashboardHandler))
}
if inventoryRepo != nil {
inventoryHandler := inventoryapi.Handler{Repository: inventoryRepo}
protectedInventory := withSession(sessions, auth.Require(auth.PermissionView, inventoryHandler))
mux.Handle("/api/v1/entities", protectedInventory)
mux.Handle("/api/v1/entities/", protectedInventory)
}
if pool != nil {
eventsHandler := eventapi.Handler{Store: eventapi.PostgresStore{Pool: pool}}
mux.Handle("/api/v1/events", withSession(sessions, auth.Require(auth.PermissionView, eventsHandler)))
}
// Monitoring surfaces are served from the bounded snapshots pulse-agent writes into
// PostgreSQL. Without a database there is no transport at all, so each surface keeps
// the empty adapter it had before, which resolves to Unknown rather than Healthy.
agentWindows := agentsource.Windows{}
if err := agentWindows.Validate(); err != nil {
return err
}
var hostProvider host.Provider = host.UnknownProvider{SourceID: "host", SourceType: "agent", Reason: "source_unavailable"}
var processProvider interface {
Snapshot(context.Context) (process.Snapshot, error)
} = process.Adapter{}
var containerProvider container.Provider = container.Adapter{}
var arrayProvider array.Provider = array.Adapter{}
var diskProvider disk.Provider = disk.Adapter{}
var poolProvider pooldomain.Provider = pooldomain.Adapter{}
var shareProvider sharedomain.Provider = sharedomain.Adapter{}
if agentReader != nil {
hostProvider = agentsource.HostProvider{Reader: agentReader, Windows: agentWindows}
processProvider = agentsource.ProcessProvider{Reader: agentReader, Windows: agentWindows}
containerProvider = agentsource.ContainerProvider{Reader: agentReader, Windows: agentWindows}
arrayProvider = agentsource.ArrayProvider{Reader: agentReader, Windows: agentWindows}
diskProvider = agentsource.DiskProvider{Reader: agentReader, Windows: agentWindows}
poolProvider = agentsource.PoolProvider{Reader: agentReader, Windows: agentWindows}
shareProvider = agentsource.ShareProvider{Reader: agentReader, Windows: agentWindows}
}
// Applications have no capability of their own: they aggregate the container
// inventory with the service probe results, and report Unknown when either input is
// missing or stale.
applicationProvider := agentsource.ApplicationProvider{Containers: containerProvider, Services: serviceProvider}
hostHandler := hostapi.Handler{Provider: hostProvider}
mux.Handle("/api/v1/host", withSession(sessions, auth.Require(auth.PermissionView, hostHandler)))
processHandler := processapi.Handler{Provider: processProvider}
mux.Handle("/api/v1/processes", withSession(sessions, auth.Require(auth.PermissionView, processHandler)))
containerHandler := containerapi.Handler{Provider: containerProvider}
mux.Handle("/api/v1/containers", withSession(sessions, auth.Require(auth.PermissionView, containerHandler)))
mux.Handle("/api/v1/containers/", withSession(sessions, auth.Require(auth.PermissionView, containerHandler)))
applicationHandler := applicationapi.Handler{Provider: applicationProvider}
mux.Handle("/api/v1/applications", withSession(sessions, auth.Require(auth.PermissionView, applicationHandler)))
mux.Handle("/api/v1/applications/", withSession(sessions, auth.Require(auth.PermissionView, applicationHandler)))
arrayHandler := arrayapi.Handler{Provider: arrayProvider}
mux.Handle("/api/v1/array", withSession(sessions, auth.Require(auth.PermissionView, arrayHandler)))
diskHandler := diskapi.Handler{Provider: diskProvider}
mux.Handle("/api/v1/disks", withSession(sessions, auth.Require(auth.PermissionView, diskHandler)))
mux.Handle("/api/v1/disks/", withSession(sessions, auth.Require(auth.PermissionView, diskHandler)))
poolHandler := poolapi.Handler{Provider: poolProvider}
mux.Handle("/api/v1/pools", withSession(sessions, auth.Require(auth.PermissionView, poolHandler)))
mux.Handle("/api/v1/pools/", withSession(sessions, auth.Require(auth.PermissionView, poolHandler)))
shareHandler := shareapi.Handler{Provider: shareProvider}
mux.Handle("/api/v1/shares", withSession(sessions, auth.Require(auth.PermissionView, shareHandler)))
mux.Handle("/api/v1/shares/", withSession(sessions, auth.Require(auth.PermissionView, shareHandler)))
forecastHandler := forecastapi.Handler{Provider: forecastdomain.StorageProvider{Shares: shareProvider, Pools: poolProvider, History: forecastdomain.PostgresHistory{Pool: pool}, Policy: forecastdomain.Policy{Enabled: true}}}
mux.Handle("/api/v1/forecasts", withSession(sessions, auth.Require(auth.PermissionView, forecastHandler)))
serviceHandler := serviceapi.Handler{Provider: serviceProvider, Dependencies: dependencyRepo, ReverseProxy: reverseProxyProvider}
networkHandler := networkapi.Handler{Provider: network.Aggregator{Host: hostProvider, Services: serviceProvider}}
mux.Handle("/api/v1/services", withSession(sessions, auth.Require(auth.PermissionView, serviceHandler)))
mux.Handle("/api/v1/services/", withSession(sessions, auth.Require(auth.PermissionView, serviceHandler)))
mux.Handle("/api/v1/topology", withSession(sessions, auth.Require(auth.PermissionView, serviceHandler)))
mux.Handle("/api/v1/network", withSession(sessions, auth.Require(auth.PermissionView, networkHandler)))
reverseProxyHandler := reverseproxyapi.Handler{Provider: reverseProxyProvider}
mux.Handle("/api/v1/reverse-proxy", withSession(sessions, auth.Require(auth.PermissionView, reverseProxyHandler)))
server := &http.Server{Addr: runtime.ListenAddress, Handler: observability.Middleware(internalMetrics, correlation.Middleware(mux)), ReadHeaderTimeout: 5 * time.Second}
go func() {
logger.Info("pulse api listening", "addr", runtime.ListenAddress, "environment", application.Environment)
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
logger.Error("pulse api stopped", "error", err)
}
}()
stopContext, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
go alertcontrol.RunExpiryLoop(stopContext, alertControlStore, time.Minute, logger)
service.WaitForStop(stopContext, nil)
shutdownContext, cancel := context.WithTimeout(context.Background(), runtime.ShutdownAfter)
defer cancel()
if err := server.Shutdown(shutdownContext); err != nil {
return err
}
logger.Info("pulse api stopped")
return nil
}
func boolMetric(value bool) float64 {
if value {
return 1
}
return 0
}
// roleMapping converts the validated configuration mapping of identity provider
// group claims onto the typed roles used by the authorization layer. Configuration
// already rejects unknown role names, so no further validation is needed here.
func roleMapping(configured map[string]string) map[string]auth.Role {
if len(configured) == 0 {
return nil
}
mapping := make(map[string]auth.Role, len(configured))
for claim, role := range configured {
mapping[claim] = auth.Role(role)
}
return mapping
}
func withSession(manager *auth.SessionManager, next http.Handler) http.Handler {
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
authentication, ok := manager.AuthenticateSession(response, request, time.Now().UTC())
if !ok {
next.ServeHTTP(response, request)
return
}
ctx, cancel := context.WithCancel(request.Context())
stop := context.AfterFunc(authentication.Context, cancel)
defer func() {
stop()
cancel()
}()
next.ServeHTTP(response, request.WithContext(auth.WithPrincipal(ctx, authentication.Principal)))
})
}
+38
View File
@@ -0,0 +1,38 @@
package main
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/coder/websocket"
"github.com/itworx/pulse/internal/auth"
"github.com/itworx/pulse/internal/live"
)
func TestClearingSessionClosesAuthenticatedLiveConnection(t *testing.T) {
manager := auth.NewSlidingSessionManager("pulse_test_session", time.Minute, time.Hour, false)
issued := httptest.NewRecorder()
if err := manager.Issue(issued, auth.Principal{Subject: "viewer", Role: auth.RoleViewer}, time.Now().UTC()); err != nil {
t.Fatal(err)
}
cookie := issued.Result().Cookies()[0]
server := httptest.NewServer(withSession(manager, auth.Require(auth.PermissionView, live.Handler{})))
defer server.Close()
options := &websocket.DialOptions{HTTPHeader: http.Header{"Cookie": []string{cookie.String()}}}
conn, _, err := websocket.Dial(context.Background(), "ws"+server.URL[4:]+"/api/v1/live", options)
if err != nil {
t.Fatal(err)
}
defer conn.CloseNow()
clearRequest := httptest.NewRequest(http.MethodPost, "/auth/logout", nil)
clearRequest.AddCookie(cookie)
manager.Clear(httptest.NewRecorder(), clearRequest)
readCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if _, _, err := conn.Read(readCtx); err == nil {
t.Fatal("live connection survived session revocation")
}
}
+40
View File
@@ -0,0 +1,40 @@
package main
import (
"context"
"fmt"
"time"
"github.com/itworx/pulse/internal/agentsource"
"github.com/itworx/pulse/internal/agentstore"
"github.com/itworx/pulse/internal/datasource"
)
var requiredUnraidCapabilities = []agentstore.Capability{
agentstore.CapabilityHost,
agentstore.CapabilityProcesses,
agentstore.CapabilityContainers,
agentstore.CapabilityArray,
agentstore.CapabilityDisks,
agentstore.CapabilityPools,
agentstore.CapabilityShares,
}
var requiredStorageCapabilities = []agentstore.Capability{
agentstore.CapabilityArray,
agentstore.CapabilityDisks,
agentstore.CapabilityPools,
agentstore.CapabilityShares,
}
func readAgentSourceHealth(ctx context.Context, reader agentstore.Reader, now time.Time) (datasource.SourceHealth, datasource.SourceHealth, error) {
unraidHealth, err := agentsource.Health(ctx, reader, requiredUnraidCapabilities, agentsource.Windows{}, now)
if err != nil {
return datasource.SourceHealth{}, datasource.SourceHealth{}, fmt.Errorf("summarize Unraid capabilities: %w", err)
}
storageHealth, err := agentsource.Health(ctx, reader, requiredStorageCapabilities, agentsource.Windows{}, now)
if err != nil {
return datasource.SourceHealth{}, datasource.SourceHealth{}, fmt.Errorf("summarize storage capabilities: %w", err)
}
return unraidHealth, storageHealth, nil
}
+61
View File
@@ -0,0 +1,61 @@
package main
import (
"context"
"errors"
"testing"
"time"
"github.com/itworx/pulse/internal/agentsource"
"github.com/itworx/pulse/internal/agentstore"
"github.com/itworx/pulse/internal/datasource"
)
type sourceHealthReader map[agentstore.Capability]agentstore.Snapshot
func (reader sourceHealthReader) Latest(_ context.Context, capability agentstore.Capability) (agentstore.Snapshot, error) {
snapshot, ok := reader[capability]
if !ok {
return agentstore.Snapshot{}, agentstore.ErrNoSnapshot
}
return snapshot, nil
}
func TestReadAgentSourceHealthRequiresContainersForHolisticUnraidHealth(t *testing.T) {
now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC)
reader := sourceHealthReader{}
for _, capability := range requiredUnraidCapabilities {
reader[capability] = agentstore.Snapshot{Capability: capability, ObservedAt: now.Add(-time.Second), ReceivedAt: now}
}
stale := reader[agentstore.CapabilityContainers]
stale.ObservedAt = now.Add(-time.Hour)
reader[agentstore.CapabilityContainers] = stale
unraidHealth, storageHealth, err := readAgentSourceHealth(context.Background(), reader, now)
if err != nil {
t.Fatal(err)
}
if unraidHealth.State != datasource.HealthUnknown || unraidHealth.ReasonCode != agentsource.ReasonStale {
t.Fatalf("partially stale Unraid health = %#v", unraidHealth)
}
if storageHealth.State != datasource.HealthHealthy {
t.Fatalf("fresh storage health = %#v", storageHealth)
}
fresh := reader[agentstore.CapabilityContainers]
fresh.ObservedAt = now.Add(-time.Second)
reader[agentstore.CapabilityContainers] = fresh
unraidHealth, storageHealth, err = readAgentSourceHealth(context.Background(), reader, now)
if err != nil || unraidHealth.State != datasource.HealthHealthy || storageHealth.State != datasource.HealthHealthy {
t.Fatalf("fully fresh health unraid=%#v storage=%#v err=%v", unraidHealth, storageHealth, err)
}
}
func TestReadAgentSourceHealthPropagatesCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, _, err := readAgentSourceHealth(ctx, sourceHealthReader{}, time.Now().UTC())
if !errors.Is(err, context.Canceled) {
t.Fatalf("cancellation error = %v", err)
}
}
+36
View File
@@ -0,0 +1,36 @@
package main
import (
"context"
"log/slog"
"os"
"time"
"github.com/itworx/pulse/internal/database"
)
func main() {
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
databaseURL := os.Getenv("PULSE_DATABASE_URL")
if databaseURL == "" {
logger.Error("migration failed", "error", "PULSE_DATABASE_URL is required")
os.Exit(1)
}
pool, err := database.NewPool(ctx, database.Config{URL: databaseURL})
if err == nil {
err = database.Ping(ctx, pool)
}
if err == nil {
err = database.Migrate(ctx, pool)
}
if pool != nil {
pool.Close()
}
if err != nil {
logger.Error("migration failed", "error", err)
os.Exit(1)
}
logger.Info("database migrations complete")
}
+324
View File
@@ -0,0 +1,324 @@
// Command worker is the ITWorx Pulse background runtime.
//
// It runs discovery/reconciliation, alert evaluation, service probes and the
// notification outbox drain on independent schedules, coordinated with any
// other worker through database leases. It is strictly observational
// (ADR-0001): it reads sources and writes Pulse's own state, and never mutates
// Unraid, Docker, the array or volumes.
package main
import (
"context"
"errors"
"fmt"
"log/slog"
"net/netip"
"os"
"os/signal"
"reflect"
"strings"
"syscall"
"time"
"github.com/itworx/pulse/internal/agentsource"
"github.com/itworx/pulse/internal/agentstore"
"github.com/itworx/pulse/internal/alert"
"github.com/itworx/pulse/internal/alertworker"
"github.com/itworx/pulse/internal/config"
"github.com/itworx/pulse/internal/container"
"github.com/itworx/pulse/internal/database"
"github.com/itworx/pulse/internal/discovery"
"github.com/itworx/pulse/internal/inventory"
"github.com/itworx/pulse/internal/metriccatalog"
"github.com/itworx/pulse/internal/metricquery"
"github.com/itworx/pulse/internal/notification"
"github.com/itworx/pulse/internal/observability"
"github.com/itworx/pulse/internal/probe"
"github.com/itworx/pulse/internal/prometheus"
"github.com/itworx/pulse/internal/queryplan"
"github.com/itworx/pulse/internal/runtimeconfig"
"github.com/itworx/pulse/internal/servicedefaults"
"github.com/itworx/pulse/internal/workerruntime"
"github.com/jackc/pgx/v5/pgxpool"
)
// startupTimeout bounds every blocking call made before the scheduling loop
// starts, so a slow database cannot hold the process before its first
// heartbeat.
const startupTimeout = 15 * time.Second
func main() {
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
if err := run(logger); err != nil {
logger.Error("pulse worker failed", "error", err)
os.Exit(1)
}
}
func run(logger *slog.Logger) error {
runtimeConfig, err := runtimeconfig.Load("worker")
if err != nil {
return err
}
application, err := config.LoadWorker()
if err != nil {
return err
}
if strings.TrimSpace(application.DatabaseURL) == "" {
return errors.New("PULSE_DATABASE_URL is required: every worker job is database-coordinated")
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
startupCtx, cancelStartup := context.WithTimeout(ctx, startupTimeout)
pool, err := database.NewPool(startupCtx, database.Config{URL: application.DatabaseURL, MaxConns: 8, MinConns: 1})
if err != nil {
cancelStartup()
return err
}
defer pool.Close()
if err := database.Ping(startupCtx, pool); err != nil {
cancelStartup()
return err
}
cancelStartup()
owner := workerOwner()
metrics := observability.NewRegistry(time.Now().UTC())
jobs, probeJob, err := buildJobs(application, pool, owner, logger)
if err != nil {
return err
}
runtime, err := workerruntime.New(workerruntime.Config{
Owner: owner,
Tick: workerruntime.DefaultTick,
HeartbeatFile: runtimeConfig.HeartbeatFile,
DrainTimeout: runtimeConfig.ShutdownAfter,
Leases: workerruntime.PostgresLeaseStore{Pool: pool},
Logger: logger,
Metrics: metrics,
}, jobs...)
if err != nil {
return err
}
logger.Info("pulse worker started",
"owner", owner, "environment", application.Environment, "jobs", jobNames(jobs),
"heartbeat_file", runtimeConfig.HeartbeatFile, "shutdown_timeout", runtimeConfig.ShutdownAfter.String(),
"config", application.String())
runErr := runtime.Run(ctx)
// Probe execution owns goroutines of its own; give them the same bounded
// grace as the scheduler before the process exits.
shutdownCtx, cancelShutdown := context.WithTimeout(context.WithoutCancel(ctx), runtimeConfig.ShutdownAfter)
defer cancelShutdown()
if probeJob != nil {
if err := probeJob.Shutdown(shutdownCtx); err != nil {
logger.Warn("probe shutdown incomplete", "error", err.Error())
}
}
for _, status := range runtime.Status() {
logger.Info("worker job final state", "job", status.Name, "component", status.Component,
"last_status", status.LastStatus, "runs", status.Runs, "failures", status.Failures, "skips", status.Skips)
}
logger.Info("pulse worker stopped")
return runErr
}
// buildJobs wires the repositories each job needs. A capability without its
// dependencies is scheduled anyway and reports Disabled with a reason, so an
// unconfigured feature is visible in system status instead of missing.
func buildJobs(application config.Config, pool *pgxpool.Pool, owner string, logger *slog.Logger) ([]workerruntime.Job, *workerruntime.ProbeJob, error) {
registry, err := metriccatalog.DefaultRegistry()
if err != nil {
return nil, nil, err
}
inventoryRepo, err := inventory.NewRepository(pool)
if err != nil {
return nil, nil, err
}
discoveryStore, err := discovery.NewPostgresStore(pool, owner)
if err != nil {
return nil, nil, err
}
notificationRepo, err := notification.NewRepository(pool)
if err != nil {
return nil, nil, err
}
configureCtx, cancelConfigure := context.WithTimeout(context.Background(), startupTimeout)
defer cancelConfigure()
notificationFactories, err := configureWebhookChannel(configureCtx, application, notificationRepo)
if err != nil {
return nil, nil, err
}
discoveryJob := workerruntime.DiscoveryJob{
SourceID: application.ContainerSourceID,
// Reuse the same bounded agent snapshot transport as the public API. A
// missing or stale snapshot resolves to Unknown and is skipped without
// tombstoning inventory; a fresh snapshot drives idempotent reconciliation.
Provider: containerDiscoveryProvider(pool),
Aliases: workerruntime.PostgresContainerAliasStore{Pool: pool},
Inventory: inventoryRepo,
Runner: discovery.Runner{Store: discoveryStore, MaxAttempts: 2, BaseRetry: 250 * time.Millisecond},
}
evaluator := &workerruntime.AlertEvaluator{
States: alert.StateRepository{Pool: pool},
Prior: workerruntime.PostgresAlertStateReader{Pool: pool},
Versions: alert.Repository{Pool: pool, Registry: registry},
Notifications: notificationRepo,
Logger: logger,
}
alertJob := workerruntime.AlertEvaluationJob{Reason: "metric_source_not_configured"}
if application.PrometheusURL != "" {
source, sourceErr := prometheus.New(application.PrometheusURL, nil, prometheus.Limits{Timeout: application.PrometheusTimeout})
if sourceErr != nil {
return nil, nil, sourceErr
}
planner := queryplan.NewPlanner(registry, queryplan.Limits{})
evaluator.Metrics = workerruntime.PrometheusMetricSource{Service: metricquery.NewService(planner, source, nil)}
worker, workerErr := alertworker.New(alert.Repository{Pool: pool, Registry: registry}, alertworker.PostgresLeaseStore{Pool: pool}, evaluator, alertworker.Config{
MaxConcurrent: 8, MaxBatch: workerruntime.MaxAlertRules, AttemptTimeout: 15 * time.Second, LeaseTTL: 2 * time.Minute, Owner: owner, Now: time.Now,
})
if workerErr != nil {
return nil, nil, workerErr
}
alertJob = workerruntime.AlertEvaluationJob{Worker: &worker, Enabled: true}
}
policy, err := probePolicy(application)
if err != nil {
return nil, nil, err
}
serviceSummary, err := servicedefaults.Seed(configureCtx, pool, servicedefaults.Options{
PublicURL: application.PublicURL, OIDCIssuer: application.OIDCIssuer,
}, policy, probe.NetResolver{})
if err != nil {
return nil, nil, fmt.Errorf("configure system service monitoring: %w", err)
}
if serviceSummary.Services > 0 {
logger.Info("system service monitoring configured", "services", serviceSummary.Services,
"endpoints", serviceSummary.Endpoints, "probes", serviceSummary.Probes, "dependencies", serviceSummary.Dependencies)
}
probeJob, err := workerruntime.NewProbeJob(workerruntime.PostgresProbeStore{Pool: pool}, policy, logger)
if err != nil {
return nil, nil, err
}
notificationJob := workerruntime.NotificationDrainJob{
Store: notificationRepo,
Channels: notificationRepo,
Senders: map[string]notification.ChannelSender{},
Factories: notificationFactories,
Logger: logger,
}
jobs := workerruntime.Schedule(workerruntime.ScheduleRuns{
Discovery: discoveryJob.Run,
AlertEvaluation: alertJob.Run,
ProbeExecution: probeJob.Run,
NotificationDrain: notificationJob.Run,
})
return jobs, probeJob, nil
}
func containerDiscoveryProvider(pool *pgxpool.Pool) container.Provider {
return agentsource.ContainerProvider{
Reader: agentstore.PostgresStore{Pool: pool},
Windows: agentsource.Windows{},
}
}
func configureWebhookChannel(ctx context.Context, application config.Config, repository notification.ChannelStore) (map[string]notification.ChannelSenderFactory, error) {
if repository == nil {
return nil, notification.ErrUnavailable
}
current, getErr := repository.GetChannel(ctx, notification.DefaultWebhookChannelID)
if application.NotificationWebhookURL == "" {
if errors.Is(getErr, notification.ErrNotFound) {
return map[string]notification.ChannelSenderFactory{}, nil
}
if getErr != nil {
return nil, fmt.Errorf("read system webhook channel: %w", getErr)
}
if current.Enabled {
current.Enabled = false
if _, err := repository.UpdateChannel(ctx, current, current.Revision); err != nil {
return nil, fmt.Errorf("disable system webhook channel: %w", err)
}
}
return map[string]notification.ChannelSenderFactory{}, nil
}
desired := notification.Channel{
ID: notification.DefaultWebhookChannelID, Name: "Pulse webhook", Type: "webhook", Enabled: true,
SecretRef: notification.SecretRef{ID: notification.WebhookSecretReference},
Configuration: map[string]any{"url": application.NotificationWebhookURL, "timeoutSeconds": application.NotificationWebhookTimeout.Seconds()},
Revision: 1,
}
if errors.Is(getErr, notification.ErrNotFound) {
if _, err := repository.CreateChannel(ctx, desired); err != nil {
return nil, fmt.Errorf("create system webhook channel: %w", err)
}
} else if getErr != nil {
return nil, fmt.Errorf("read system webhook channel: %w", getErr)
} else if current.Name != desired.Name || current.Type != desired.Type || !current.Enabled || current.SecretRef != desired.SecretRef || !reflect.DeepEqual(current.Configuration, desired.Configuration) {
desired.Revision = current.Revision
if _, err := repository.UpdateChannel(ctx, desired, current.Revision); err != nil {
return nil, fmt.Errorf("update system webhook channel: %w", err)
}
}
resolver := notification.SecretResolverFunc(func(_ context.Context, ref notification.SecretRef) (string, error) {
if ref.ID != notification.WebhookSecretReference {
return "", notification.ErrNotFound
}
return application.NotificationWebhookToken, nil
})
return map[string]notification.ChannelSenderFactory{
"webhook": notification.WebhookFactory{
Secrets: resolver,
AllowHTTP: application.Environment != config.Production,
},
}, nil
}
// probePolicy builds the probe network policy. Only administrator-configured
// private ranges are added to the allowlist; every other protection in
// internal/probe/policy.go keeps its default, so link-local, multicast, cloud
// metadata and unlisted private addresses stay blocked.
func probePolicy(application config.Config) (probe.NetworkPolicy, error) {
policy := probe.NetworkPolicy{}
for _, entry := range application.ProbeAllowedNetworks {
prefix, err := netip.ParsePrefix(entry)
if err != nil {
return probe.NetworkPolicy{}, fmt.Errorf("probe allowlist entry %q is invalid", entry)
}
policy.AllowedNetworks = append(policy.AllowedNetworks, prefix)
}
if err := policy.Validate(); err != nil {
return probe.NetworkPolicy{}, err
}
return policy, nil
}
// workerOwner identifies this process in job_runs.lease_owner. It contains no
// secret and stays stable for the lifetime of the process.
func workerOwner() string {
host, err := os.Hostname()
if err != nil || strings.TrimSpace(host) == "" {
host = "worker"
}
owner := fmt.Sprintf("%s/%d", host, os.Getpid())
if len(owner) > 120 {
owner = owner[:120]
}
return owner
}
func jobNames(jobs []workerruntime.Job) []string {
names := make([]string, 0, len(jobs))
for _, job := range jobs {
names = append(names, job.Name+"@"+job.Interval.String())
}
return names
}
+123
View File
@@ -0,0 +1,123 @@
package main
import (
"context"
"errors"
"testing"
"time"
"github.com/itworx/pulse/internal/agentsource"
"github.com/itworx/pulse/internal/agentstore"
"github.com/itworx/pulse/internal/config"
"github.com/itworx/pulse/internal/notification"
"github.com/jackc/pgx/v5/pgxpool"
)
type channelStoreStub struct {
channels map[string]notification.Channel
creates int
updates int
}
func TestContainerDiscoveryProviderUsesAgentSnapshotStore(t *testing.T) {
pool := &pgxpool.Pool{}
provider, ok := containerDiscoveryProvider(pool).(agentsource.ContainerProvider)
if !ok {
t.Fatalf("container discovery provider = %T, want agentsource.ContainerProvider", containerDiscoveryProvider(pool))
}
store, ok := provider.Reader.(agentstore.PostgresStore)
if !ok || store.Pool != pool {
t.Fatalf("container discovery reader = %#v, want PostgresStore with worker pool", provider.Reader)
}
}
func (store *channelStoreStub) CreateChannel(_ context.Context, channel notification.Channel) (notification.Channel, error) {
if _, exists := store.channels[channel.ID]; exists {
return notification.Channel{}, notification.ErrConflict
}
store.creates++
channel.Revision = 1
store.channels[channel.ID] = channel
return channel, nil
}
func (store *channelStoreStub) GetChannel(_ context.Context, id string) (notification.Channel, error) {
channel, exists := store.channels[id]
if !exists {
return notification.Channel{}, notification.ErrNotFound
}
return channel, nil
}
func (store *channelStoreStub) ListChannels(context.Context, int) ([]notification.Channel, error) {
channels := make([]notification.Channel, 0, len(store.channels))
for _, channel := range store.channels {
channels = append(channels, channel)
}
return channels, nil
}
func (store *channelStoreStub) UpdateChannel(_ context.Context, channel notification.Channel, expected int64) (notification.Channel, error) {
current, exists := store.channels[channel.ID]
if !exists {
return notification.Channel{}, notification.ErrNotFound
}
if current.Revision != expected {
return notification.Channel{}, notification.ErrConflict
}
store.updates++
channel.Revision = expected + 1
store.channels[channel.ID] = channel
return channel, nil
}
func (store *channelStoreStub) DeleteChannel(_ context.Context, id string) error {
if _, exists := store.channels[id]; !exists {
return notification.ErrNotFound
}
delete(store.channels, id)
return nil
}
func TestConfigureWebhookChannelReconcilesWithoutRevisionChurn(t *testing.T) {
store := &channelStoreStub{channels: map[string]notification.Channel{}}
application := config.Config{
Environment: config.Development, NotificationWebhookURL: "http://127.0.0.1:18080/pulse",
NotificationWebhookToken: "runtime-only", NotificationWebhookTimeout: 3 * time.Second,
}
factories, err := configureWebhookChannel(context.Background(), application, store)
if err != nil {
t.Fatal(err)
}
channel := store.channels[notification.DefaultWebhookChannelID]
if store.creates != 1 || store.updates != 0 || !channel.Enabled || channel.SecretRef.ID != notification.WebhookSecretReference {
t.Fatalf("channel=%+v creates=%d updates=%d", channel, store.creates, store.updates)
}
if _, exists := channel.Configuration["token"]; exists {
t.Fatal("runtime credential entered persistent configuration")
}
if _, err := factories["webhook"].Sender(context.Background(), channel); err != nil {
t.Fatal(err)
}
if _, err := configureWebhookChannel(context.Background(), application, store); err != nil {
t.Fatal(err)
}
if store.creates != 1 || store.updates != 0 {
t.Fatalf("idempotent reconciliation created=%d updated=%d", store.creates, store.updates)
}
}
func TestConfigureWebhookChannelDisablesRemovedRuntimeConfiguration(t *testing.T) {
store := &channelStoreStub{channels: map[string]notification.Channel{
notification.DefaultWebhookChannelID: {
ID: notification.DefaultWebhookChannelID, Name: "Pulse webhook", Type: "webhook", Enabled: true,
SecretRef: notification.SecretRef{ID: notification.WebhookSecretReference}, Revision: 4,
},
}}
factories, err := configureWebhookChannel(context.Background(), config.Config{}, store)
if err != nil {
t.Fatal(err)
}
if len(factories) != 0 || store.updates != 1 || store.channels[notification.DefaultWebhookChannelID].Enabled {
t.Fatalf("factories=%v updates=%d channel=%+v", factories, store.updates, store.channels[notification.DefaultWebhookChannelID])
}
if _, err := configureWebhookChannel(context.Background(), config.Config{}, nil); !errors.Is(err, notification.ErrUnavailable) {
t.Fatalf("nil repository error=%v", err)
}
}