This commit is contained in:
@@ -0,0 +1,372 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentprotocol"
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/array"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/disk"
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
"github.com/itworx/pulse/internal/pool"
|
||||
"github.com/itworx/pulse/internal/process"
|
||||
"github.com/itworx/pulse/internal/runtimeconfig"
|
||||
"github.com/itworx/pulse/internal/share"
|
||||
)
|
||||
|
||||
const (
|
||||
// maxLoopInterval bounds one scheduling iteration so the heartbeat is refreshed at
|
||||
// least every 10 seconds even when collection is configured to run far less often.
|
||||
// See docs/operations/WORKER_AGENT_HEALTHCHECK_CONTRACT.md point 3.
|
||||
maxLoopInterval = 5 * time.Second
|
||||
// operationTimeout bounds every blocking call inside one iteration — collection and
|
||||
// the database write. It sits well under the 10 second heartbeat window so a hung
|
||||
// database cannot stall the loop into a false "unhealthy" restart, and equally
|
||||
// cannot hide a real hang: the call is abandoned and reported.
|
||||
operationTimeout = 4 * time.Second
|
||||
// heartbeatFileMode keeps the liveness file readable only by the agent's own user;
|
||||
// the healthcheck script runs as the same user.
|
||||
heartbeatFileMode = 0o600
|
||||
)
|
||||
|
||||
// snapshotSource is the narrow view of the collector the runtime needs. It keeps the
|
||||
// loop testable without a procfs tree.
|
||||
type snapshotSource interface {
|
||||
Host(context.Context) (host.RawSnapshot, error)
|
||||
Processes(context.Context) (process.RawSnapshot, error)
|
||||
}
|
||||
|
||||
type containerSnapshotSource interface {
|
||||
Containers(context.Context) (container.RawSnapshot, error)
|
||||
}
|
||||
type arraySnapshotSource interface {
|
||||
Array(context.Context) (array.RawSnapshot, error)
|
||||
}
|
||||
type diskSnapshotSource interface {
|
||||
Disks(context.Context) (disk.RawSnapshot, error)
|
||||
}
|
||||
type poolSnapshotSource interface {
|
||||
Pools(context.Context) (pool.RawSnapshot, error)
|
||||
}
|
||||
type shareSnapshotSource interface {
|
||||
Shares(context.Context) (share.RawSnapshot, error)
|
||||
}
|
||||
|
||||
// capability binds one telemetry surface to the collection that produces it. Every
|
||||
// capability the agent announces is read-only; there is no write path in this binary.
|
||||
type capability struct {
|
||||
id agentstore.Capability
|
||||
version string
|
||||
collect func(context.Context) (json.RawMessage, time.Time, error)
|
||||
}
|
||||
|
||||
// tickerFactory produces the loop's tick channel. Tests replace it with a channel they
|
||||
// drive by hand so loop behaviour is asserted without sleeping.
|
||||
type tickerFactory func(time.Duration) (<-chan time.Time, func())
|
||||
|
||||
type agent struct {
|
||||
agentID string
|
||||
writer agentstore.Writer
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
collectInterval time.Duration
|
||||
loopInterval time.Duration
|
||||
operationTimeout time.Duration
|
||||
heartbeatPath string
|
||||
capabilities []capability
|
||||
newTicker tickerFactory
|
||||
|
||||
// nextCollect is the earliest time the next collection pass may run. It is only
|
||||
// touched from the loop goroutine.
|
||||
nextCollect time.Time
|
||||
}
|
||||
|
||||
func realTicker(interval time.Duration) (<-chan time.Time, func()) {
|
||||
ticker := time.NewTicker(interval)
|
||||
return ticker.C, ticker.Stop
|
||||
}
|
||||
|
||||
func newAgent(config runtimeconfig.AgentConfig, source snapshotSource, writer agentstore.Writer, logger *slog.Logger) *agent {
|
||||
loopInterval := config.CollectInterval
|
||||
if loopInterval > maxLoopInterval {
|
||||
loopInterval = maxLoopInterval
|
||||
}
|
||||
return &agent{
|
||||
agentID: config.AgentID,
|
||||
writer: writer,
|
||||
logger: logger,
|
||||
now: time.Now,
|
||||
collectInterval: config.CollectInterval,
|
||||
loopInterval: loopInterval,
|
||||
operationTimeout: operationTimeout,
|
||||
heartbeatPath: config.Service.HeartbeatFile,
|
||||
capabilities: capabilities(source),
|
||||
newTicker: realTicker,
|
||||
}
|
||||
}
|
||||
|
||||
// capabilities lists what this agent reports. Adding a capability here is the only way
|
||||
// to widen what the agent reads, which keeps the surface auditable.
|
||||
func capabilities(source snapshotSource) []capability {
|
||||
result := []capability{
|
||||
{
|
||||
id: agentstore.CapabilityHost,
|
||||
version: host.ContractVersion,
|
||||
collect: func(ctx context.Context) (json.RawMessage, time.Time, error) {
|
||||
snapshot, err := source.Host(ctx)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
payload, err := json.Marshal(snapshot)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
return payload, snapshot.ObservedAt, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
id: agentstore.CapabilityProcesses,
|
||||
version: process.ContractVersion,
|
||||
collect: func(ctx context.Context) (json.RawMessage, time.Time, error) {
|
||||
snapshot, err := source.Processes(ctx)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
payload, err := json.Marshal(snapshot)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
return payload, snapshot.ObservedAt, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
if containers, ok := source.(containerSnapshotSource); ok {
|
||||
result = append(result, capability{
|
||||
id: agentstore.CapabilityContainers, version: container.ContractVersion,
|
||||
collect: func(ctx context.Context) (json.RawMessage, time.Time, error) {
|
||||
snapshot, err := containers.Containers(ctx)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
payload, err := json.Marshal(snapshot)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
return payload, snapshot.ObservedAt, nil
|
||||
},
|
||||
})
|
||||
}
|
||||
if source, ok := source.(arraySnapshotSource); ok {
|
||||
result = append(result, rawCapability(agentstore.CapabilityArray, array.ContractVersion, source.Array))
|
||||
}
|
||||
if source, ok := source.(diskSnapshotSource); ok {
|
||||
result = append(result, rawCapability(agentstore.CapabilityDisks, disk.ContractVersion, source.Disks))
|
||||
}
|
||||
if source, ok := source.(poolSnapshotSource); ok {
|
||||
result = append(result, rawCapability(agentstore.CapabilityPools, pool.ContractVersion, source.Pools))
|
||||
}
|
||||
if source, ok := source.(shareSnapshotSource); ok {
|
||||
result = append(result, rawCapability(agentstore.CapabilityShares, share.ContractVersion, source.Shares))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func rawCapability[T any](id agentstore.Capability, version string, collect func(context.Context) (T, error)) capability {
|
||||
return capability{id: id, version: version, collect: func(ctx context.Context) (json.RawMessage, time.Time, error) {
|
||||
snapshot, err := collect(ctx)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
payload, err := json.Marshal(snapshot)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
observed := observedAt(snapshot)
|
||||
return payload, observed, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// All raw snapshot contracts carry ObservedAt. Keep this tiny type assertion local to
|
||||
// the agent rather than introducing a repository-wide generic telemetry abstraction.
|
||||
func observedAt(snapshot any) time.Time {
|
||||
switch value := snapshot.(type) {
|
||||
case array.RawSnapshot:
|
||||
return value.ObservedAt
|
||||
case disk.RawSnapshot:
|
||||
return value.ObservedAt
|
||||
case pool.RawSnapshot:
|
||||
return value.ObservedAt
|
||||
case share.RawSnapshot:
|
||||
return value.ObservedAt
|
||||
default:
|
||||
return time.Time{}
|
||||
}
|
||||
}
|
||||
|
||||
// hello is the capability announcement. Every entry is read-only, which is not a
|
||||
// decoration: agentprotocol.Hello.Validate rejects a hello that claims anything else,
|
||||
// so the agent's own start-up self-check fails loudly if a mutating capability is ever
|
||||
// added here by mistake.
|
||||
func (a *agent) hello() agentprotocol.Hello {
|
||||
announced := make([]agentprotocol.Capability, 0, len(a.capabilities))
|
||||
for _, item := range a.capabilities {
|
||||
announced = append(announced, agentprotocol.Capability{
|
||||
ID: string(item.id),
|
||||
Version: item.version,
|
||||
ReadOnly: true,
|
||||
})
|
||||
}
|
||||
return agentprotocol.Hello{
|
||||
Protocol: agentprotocol.Version,
|
||||
AgentID: a.agentID,
|
||||
ObservedAt: a.now().UTC(),
|
||||
Capabilities: announced,
|
||||
}
|
||||
}
|
||||
|
||||
// run drives the collection loop until the context is cancelled.
|
||||
//
|
||||
// The order is fixed by the healthcheck contract: validate, log, write one heartbeat
|
||||
// before the first blocking call, then loop. Each iteration does its bounded work and
|
||||
// ends by refreshing the heartbeat, so a stuck iteration stops the heartbeat instead of
|
||||
// a background ticker papering over the stall.
|
||||
func (a *agent) run(ctx context.Context) error {
|
||||
if a.writer == nil {
|
||||
return errNoWriter
|
||||
}
|
||||
hello := a.hello()
|
||||
if err := hello.Validate(a.now().UTC()); err != nil {
|
||||
return fmt.Errorf("agent hello failed its own read-only validation: %w", err)
|
||||
}
|
||||
announced := make([]string, 0, len(hello.Capabilities))
|
||||
for _, item := range hello.Capabilities {
|
||||
announced = append(announced, item.ID)
|
||||
}
|
||||
a.logger.Info("pulse agent started",
|
||||
"agent_id", a.agentID,
|
||||
"protocol", hello.Protocol,
|
||||
"capabilities", announced,
|
||||
"read_only", true,
|
||||
"collect_interval", a.collectInterval.String(),
|
||||
"loop_interval", a.loopInterval.String(),
|
||||
"operation_timeout", a.operationTimeout.String(),
|
||||
"heartbeat_file", a.heartbeatPath,
|
||||
)
|
||||
|
||||
// Contract point 4: a slow-but-healthy cold start must not look like a hang.
|
||||
a.heartbeat()
|
||||
|
||||
ticks, stop := a.newTicker(a.loopInterval)
|
||||
defer stop()
|
||||
|
||||
loop:
|
||||
for ctx.Err() == nil {
|
||||
a.iterate(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break loop
|
||||
case <-ticks:
|
||||
}
|
||||
}
|
||||
a.logger.Info("pulse agent stopped", "agent_id", a.agentID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// iterate is one unit of work: collect when due, then heartbeat. An empty iteration —
|
||||
// nothing due yet — is still a completed iteration and still heartbeats.
|
||||
func (a *agent) iterate(ctx context.Context) {
|
||||
now := a.now()
|
||||
if !now.Before(a.nextCollect) {
|
||||
a.collectOnce(ctx)
|
||||
a.nextCollect = a.now().Add(a.collectInterval)
|
||||
}
|
||||
a.heartbeat()
|
||||
}
|
||||
|
||||
// collectOnce publishes every capability independently. One failing capability is
|
||||
// logged and skipped; it neither aborts the pass nor causes a stale or empty snapshot
|
||||
// to be written in its place. A missing snapshot is exactly what the reader turns into
|
||||
// Unknown (ADR-0008), which is the honest outcome.
|
||||
func (a *agent) collectOnce(ctx context.Context) {
|
||||
for _, item := range a.capabilities {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
started := a.now()
|
||||
bytesWritten, err := a.publish(ctx, item)
|
||||
if err != nil {
|
||||
a.logger.Error("agent capability collection failed",
|
||||
"agent_id", a.agentID,
|
||||
"capability", string(item.id),
|
||||
"duration_ms", a.now().Sub(started).Milliseconds(),
|
||||
"error", err.Error(),
|
||||
)
|
||||
continue
|
||||
}
|
||||
a.logger.Info("agent snapshot published",
|
||||
"agent_id", a.agentID,
|
||||
"capability", string(item.id),
|
||||
"payload_bytes", bytesWritten,
|
||||
"duration_ms", a.now().Sub(started).Milliseconds(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// publish collects and writes one capability under a bounded context.
|
||||
func (a *agent) publish(ctx context.Context, item capability) (int, error) {
|
||||
operation, cancel := context.WithTimeout(ctx, a.operationTimeout)
|
||||
defer cancel()
|
||||
|
||||
payload, observedAt, err := item.collect(operation)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("collect %s: %w", item.id, err)
|
||||
}
|
||||
if len(payload) == 0 {
|
||||
return 0, fmt.Errorf("collect %s: empty payload", item.id)
|
||||
}
|
||||
if len(payload) > agentstore.MaxPayloadBytes {
|
||||
// Writing a truncated snapshot would be worse than writing none: the reader
|
||||
// cannot tell a truncated payload from a complete one.
|
||||
return 0, fmt.Errorf("collect %s: payload of %d bytes exceeds the %d byte limit", item.id, len(payload), agentstore.MaxPayloadBytes)
|
||||
}
|
||||
if observedAt.IsZero() {
|
||||
observedAt = a.now().UTC()
|
||||
}
|
||||
if err := a.writer.Put(operation, agentstore.Snapshot{
|
||||
AgentID: a.agentID,
|
||||
Capability: item.id,
|
||||
ObservedAt: observedAt.UTC(),
|
||||
Payload: payload,
|
||||
}); err != nil {
|
||||
return 0, fmt.Errorf("publish %s: %w", item.id, err)
|
||||
}
|
||||
return len(payload), nil
|
||||
}
|
||||
|
||||
// heartbeat refreshes the liveness file the compose healthcheck watches. Only the mtime
|
||||
// matters to the script; the RFC 3339 body exists so `docker exec … cat /tmp/healthy`
|
||||
// tells an operator something. A write failure is logged and the loop continues:
|
||||
// crashing on a tmpfs hiccup would turn a cosmetic problem into an outage, and a
|
||||
// sustained failure surfaces on its own as staleness.
|
||||
func (a *agent) heartbeat() {
|
||||
if a.heartbeatPath == "" {
|
||||
return
|
||||
}
|
||||
content := a.now().UTC().Format(time.RFC3339) + "\n"
|
||||
if err := os.WriteFile(a.heartbeatPath, []byte(content), heartbeatFileMode); err != nil {
|
||||
a.logger.Error("agent heartbeat write failed",
|
||||
"agent_id", a.agentID,
|
||||
"heartbeat_file", a.heartbeatPath,
|
||||
"error", err.Error(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
var errNoWriter = errors.New("agent snapshot writer is required")
|
||||
@@ -0,0 +1,543 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/array"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/disk"
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
"github.com/itworx/pulse/internal/pool"
|
||||
"github.com/itworx/pulse/internal/process"
|
||||
"github.com/itworx/pulse/internal/runtimeconfig"
|
||||
"github.com/itworx/pulse/internal/share"
|
||||
)
|
||||
|
||||
// fakeWriter records what the agent publishes and can fail a chosen capability.
|
||||
type fakeWriter struct {
|
||||
mu sync.Mutex
|
||||
puts []agentstore.Snapshot
|
||||
deadlines []bool
|
||||
failures map[agentstore.Capability]error
|
||||
blockUntil chan struct{}
|
||||
}
|
||||
|
||||
func newFakeWriter() *fakeWriter {
|
||||
return &fakeWriter{failures: map[agentstore.Capability]error{}}
|
||||
}
|
||||
|
||||
func (w *fakeWriter) Put(ctx context.Context, snapshot agentstore.Snapshot) error {
|
||||
w.mu.Lock()
|
||||
failure := w.failures[snapshot.Capability]
|
||||
block := w.blockUntil
|
||||
w.mu.Unlock()
|
||||
if block != nil {
|
||||
select {
|
||||
case <-block:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
_, hasDeadline := ctx.Deadline()
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.deadlines = append(w.deadlines, hasDeadline)
|
||||
if failure != nil {
|
||||
return failure
|
||||
}
|
||||
w.puts = append(w.puts, snapshot)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *fakeWriter) recorded() []agentstore.Snapshot {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return append([]agentstore.Snapshot(nil), w.puts...)
|
||||
}
|
||||
|
||||
func (w *fakeWriter) countFor(capability agentstore.Capability) int {
|
||||
count := 0
|
||||
for _, snapshot := range w.recorded() {
|
||||
if snapshot.Capability == capability {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (w *fakeWriter) failCapability(capability agentstore.Capability, err error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.failures[capability] = err
|
||||
}
|
||||
|
||||
// fakeSource stands in for the procfs collector.
|
||||
type fakeSource struct {
|
||||
mu sync.Mutex
|
||||
hostErr error
|
||||
processErr error
|
||||
hostCalls int
|
||||
procCalls int
|
||||
release chan struct{}
|
||||
hugePayload bool
|
||||
}
|
||||
|
||||
type fakeContainerSource struct{ *fakeSource }
|
||||
|
||||
func (s fakeContainerSource) Containers(context.Context) (container.RawSnapshot, error) {
|
||||
return container.RawSnapshot{Source: container.Source{ID: "unraid", Type: "unraid"}, Containers: []container.RawContainer{{ID: "runtime-1", Name: "pulse-api", State: "running", Health: "healthy"}}, ObservedAt: time.Date(2026, 8, 10, 2, 0, 0, 0, time.UTC)}, nil
|
||||
}
|
||||
|
||||
type fakeInventorySource struct{ *fakeSource }
|
||||
|
||||
func (s fakeInventorySource) Containers(context.Context) (container.RawSnapshot, error) {
|
||||
return container.RawSnapshot{Source: container.Source{ID: "unraid", Type: "unraid"}, Containers: []container.RawContainer{}, ObservedAt: time.Date(2026, 8, 10, 2, 0, 0, 0, time.UTC)}, nil
|
||||
}
|
||||
func (s fakeInventorySource) Array(context.Context) (array.RawSnapshot, error) {
|
||||
return array.RawSnapshot{Source: array.Source{ID: "unraid", Type: "unraid"}, State: array.StateOperational, Members: []array.RawMember{}, ObservedAt: time.Date(2026, 8, 10, 2, 0, 0, 0, time.UTC)}, nil
|
||||
}
|
||||
func (s fakeInventorySource) Disks(context.Context) (disk.RawSnapshot, error) {
|
||||
return disk.RawSnapshot{Source: disk.Source{ID: "unraid", Type: "unraid"}, Disks: []disk.RawDisk{}, ObservedAt: time.Date(2026, 8, 10, 2, 0, 0, 0, time.UTC)}, nil
|
||||
}
|
||||
func (s fakeInventorySource) Pools(context.Context) (pool.RawSnapshot, error) {
|
||||
return pool.RawSnapshot{Source: pool.Source{ID: "unraid", Type: "unraid"}, Pools: []pool.RawPool{}, ObservedAt: time.Date(2026, 8, 10, 2, 0, 0, 0, time.UTC)}, nil
|
||||
}
|
||||
func (s fakeInventorySource) Shares(context.Context) (share.RawSnapshot, error) {
|
||||
return share.RawSnapshot{Source: share.Source{ID: "unraid", Type: "unraid"}, Shares: []share.RawShare{}, ObservedAt: time.Date(2026, 8, 10, 2, 0, 0, 0, time.UTC)}, nil
|
||||
}
|
||||
|
||||
func (s *fakeSource) Host(ctx context.Context) (host.RawSnapshot, error) {
|
||||
s.mu.Lock()
|
||||
release := s.release
|
||||
err := s.hostErr
|
||||
s.hostCalls++
|
||||
huge := s.hugePayload
|
||||
s.mu.Unlock()
|
||||
if release != nil {
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
return host.RawSnapshot{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return host.RawSnapshot{}, err
|
||||
}
|
||||
snapshot := host.RawSnapshot{
|
||||
Identity: host.HostIdentity{Name: "tower"},
|
||||
UptimeSeconds: 100,
|
||||
Memory: host.RawMemory{TotalBytes: 1024, AvailableBytes: 512},
|
||||
ObservedAt: time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC),
|
||||
}
|
||||
if huge {
|
||||
warning := strings.Repeat("x", agentstore.MaxPayloadBytes)
|
||||
snapshot.Warnings = []string{warning}
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func (s *fakeSource) Processes(context.Context) (process.RawSnapshot, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.procCalls++
|
||||
if s.processErr != nil {
|
||||
return process.RawSnapshot{}, s.processErr
|
||||
}
|
||||
return process.RawSnapshot{
|
||||
Processes: []process.RawProcess{{PID: 1, Name: "init", State: "sleeping"}},
|
||||
ObservedAt: time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *fakeSource) calls() (int, int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.hostCalls, s.procCalls
|
||||
}
|
||||
|
||||
type testClock struct {
|
||||
mu sync.Mutex
|
||||
current time.Time
|
||||
// step advances the clock on every read. A loop test needs strictly increasing
|
||||
// timestamps, otherwise two iterations can observe the same instant and the second
|
||||
// one is legitimately "not due yet".
|
||||
step time.Duration
|
||||
}
|
||||
|
||||
func (c *testClock) now() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
current := c.current
|
||||
c.current = c.current.Add(c.step)
|
||||
return current
|
||||
}
|
||||
|
||||
func (c *testClock) setStep(step time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.step = step
|
||||
}
|
||||
|
||||
func (c *testClock) advance(d time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.current = c.current.Add(d)
|
||||
}
|
||||
|
||||
func newTestAgent(t *testing.T, source snapshotSource, writer agentstore.Writer, collectInterval time.Duration) (*agent, *testClock, string) {
|
||||
t.Helper()
|
||||
heartbeat := filepath.Join(t.TempDir(), "healthy")
|
||||
config := runtimeconfig.AgentConfig{
|
||||
AgentID: "pulse-agent-test",
|
||||
CollectInterval: collectInterval,
|
||||
Service: runtimeconfig.ServiceConfig{ServiceName: "agent", HeartbeatFile: heartbeat},
|
||||
}
|
||||
logger := slog.New(slog.NewJSONHandler(io.Discard, nil))
|
||||
instance := newAgent(config, source, writer, logger)
|
||||
clock := &testClock{current: time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)}
|
||||
instance.now = clock.now
|
||||
return instance, clock, heartbeat
|
||||
}
|
||||
|
||||
func readHeartbeat(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path) //nolint:gosec // test-controlled path
|
||||
if err != nil {
|
||||
t.Fatalf("read heartbeat: %v", err)
|
||||
}
|
||||
return strings.TrimSpace(string(data))
|
||||
}
|
||||
|
||||
func TestIterateCollectsOnTheIntervalAndHeartbeatsEveryIteration(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
source := &fakeSource{}
|
||||
// A 15 second collection interval must not stretch the loop: newAgent caps the
|
||||
// loop at maxLoopInterval so the heartbeat stays inside the contract's 10s window.
|
||||
instance, clock, heartbeat := newTestAgent(t, source, writer, 15*time.Second)
|
||||
if instance.loopInterval != maxLoopInterval {
|
||||
t.Fatalf("loop interval = %s, want %s", instance.loopInterval, maxLoopInterval)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
for iteration := 0; iteration < 7; iteration++ {
|
||||
instance.iterate(ctx)
|
||||
if got := readHeartbeat(t, heartbeat); got != clock.now().UTC().Format(time.RFC3339) {
|
||||
t.Fatalf("iteration %d heartbeat = %q, want the current time", iteration, got)
|
||||
}
|
||||
clock.advance(instance.loopInterval)
|
||||
}
|
||||
|
||||
// Seven iterations five seconds apart cover t=0s..t=30s: collection is due at
|
||||
// t=0s, t=15s and t=30s. The four iterations in between do no work, yet every one
|
||||
// of them completed and refreshed the heartbeat, which is what the contract asks
|
||||
// for ("an empty poll is still a completed iteration").
|
||||
hostCalls, procCalls := source.calls()
|
||||
if hostCalls != 3 || procCalls != 3 {
|
||||
t.Fatalf("collections = host %d, processes %d; want 3 each over 30 seconds", hostCalls, procCalls)
|
||||
}
|
||||
if writer.countFor(agentstore.CapabilityHost) != 3 || writer.countFor(agentstore.CapabilityProcesses) != 3 {
|
||||
t.Fatalf("unexpected publications: %+v", writer.recorded())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneFailingCapabilityDoesNotStopTheOthers(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
source := &fakeSource{hostErr: errors.New("procfs read failed")}
|
||||
instance, _, _ := newTestAgent(t, source, writer, time.Second)
|
||||
|
||||
instance.collectOnce(context.Background())
|
||||
|
||||
recorded := writer.recorded()
|
||||
if len(recorded) != 1 || recorded[0].Capability != agentstore.CapabilityProcesses {
|
||||
t.Fatalf("expected only the process snapshot, got %+v", recorded)
|
||||
}
|
||||
|
||||
// The reverse case: the store rejects one capability.
|
||||
writer.failCapability(agentstore.CapabilityProcesses, errors.New("store rejected"))
|
||||
source.mu.Lock()
|
||||
source.hostErr = nil
|
||||
source.mu.Unlock()
|
||||
instance.collectOnce(context.Background())
|
||||
|
||||
if writer.countFor(agentstore.CapabilityHost) != 1 {
|
||||
t.Fatalf("host must still be published: %+v", writer.recorded())
|
||||
}
|
||||
if writer.countFor(agentstore.CapabilityProcesses) != 1 {
|
||||
t.Fatalf("the rejected capability must not be recorded twice: %+v", writer.recorded())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedCollectionWritesNothingRatherThanAnEmptySnapshot(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
source := &fakeSource{hostErr: errors.New("boom"), processErr: errors.New("boom")}
|
||||
instance, _, _ := newTestAgent(t, source, writer, time.Second)
|
||||
|
||||
instance.collectOnce(context.Background())
|
||||
|
||||
if recorded := writer.recorded(); len(recorded) != 0 {
|
||||
t.Fatalf("a failed collection must publish nothing, got %+v", recorded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOversizedPayloadIsRefused(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
source := &fakeSource{hugePayload: true}
|
||||
instance, _, _ := newTestAgent(t, source, writer, time.Second)
|
||||
|
||||
if _, err := instance.publish(context.Background(), instance.capabilities[0]); err == nil {
|
||||
t.Fatal("expected an oversized payload to be refused")
|
||||
}
|
||||
if writer.countFor(agentstore.CapabilityHost) != 0 {
|
||||
t.Fatal("an oversized payload must never reach the store")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishUsesABoundedContext(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
instance, _, _ := newTestAgent(t, &fakeSource{}, writer, time.Second)
|
||||
|
||||
instance.collectOnce(context.Background())
|
||||
|
||||
writer.mu.Lock()
|
||||
defer writer.mu.Unlock()
|
||||
if len(writer.deadlines) == 0 {
|
||||
t.Fatal("no writes recorded")
|
||||
}
|
||||
for index, bounded := range writer.deadlines {
|
||||
if !bounded {
|
||||
t.Fatalf("write %d ran without a deadline", index)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishedSnapshotCarriesTheObservedTimeAndPayload(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
instance, _, _ := newTestAgent(t, &fakeSource{}, writer, time.Second)
|
||||
|
||||
instance.collectOnce(context.Background())
|
||||
|
||||
for _, snapshot := range writer.recorded() {
|
||||
if snapshot.AgentID != "pulse-agent-test" {
|
||||
t.Fatalf("agent id = %q", snapshot.AgentID)
|
||||
}
|
||||
if !snapshot.Capability.Valid() {
|
||||
t.Fatalf("unknown capability %q", snapshot.Capability)
|
||||
}
|
||||
if snapshot.ObservedAt.IsZero() || snapshot.ObservedAt.Location() != time.UTC {
|
||||
t.Fatalf("observed at = %v", snapshot.ObservedAt)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(snapshot.Payload, &decoded); err != nil {
|
||||
t.Fatalf("payload is not a JSON object: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunWritesTheFirstHeartbeatBeforeTheFirstBlockingCall(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
release := make(chan struct{})
|
||||
source := &fakeSource{release: release}
|
||||
instance, clock, heartbeat := newTestAgent(t, source, writer, time.Second)
|
||||
ticks := make(chan time.Time)
|
||||
instance.newTicker = func(time.Duration) (<-chan time.Time, func()) { return ticks, func() {} }
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- instance.run(ctx) }()
|
||||
|
||||
// The collector is blocked inside the first iteration, yet the heartbeat must
|
||||
// already exist: a slow cold start is not a hang.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
if _, err := os.Stat(heartbeat); err == nil {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no heartbeat was written before the first blocking call")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if got := readHeartbeat(t, heartbeat); got != clock.now().UTC().Format(time.RFC3339) {
|
||||
t.Fatalf("heartbeat = %q", got)
|
||||
}
|
||||
|
||||
close(release)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("run returned error: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("run did not return after cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHeartbeatsFromTheLoopAndShutsDownCleanly(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
source := &fakeSource{}
|
||||
instance, clock, heartbeat := newTestAgent(t, source, writer, time.Second)
|
||||
clock.setStep(time.Second)
|
||||
start := clock.now()
|
||||
ticks := make(chan time.Time)
|
||||
instance.newTicker = func(time.Duration) (<-chan time.Time, func()) { return ticks, func() {} }
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- instance.run(ctx) }()
|
||||
|
||||
// An unbuffered tick is only received once the previous iteration has finished, so
|
||||
// each successful send proves one more completed loop iteration.
|
||||
for iteration := 0; iteration < 3; iteration++ {
|
||||
select {
|
||||
case ticks <- time.Now():
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("iteration %d never completed", iteration)
|
||||
}
|
||||
}
|
||||
if hostCalls, procCalls := source.calls(); hostCalls < 3 || procCalls < 3 {
|
||||
t.Fatalf("collections = host %d, processes %d; want at least 3 each", hostCalls, procCalls)
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("run returned error: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("run did not stop on cancellation")
|
||||
}
|
||||
|
||||
// Read the heartbeat only after the loop has stopped: the file is truncated and
|
||||
// rewritten in place, so a concurrent reader can legitimately observe it empty.
|
||||
// The healthcheck script reads only the mtime, which is why that is safe.
|
||||
written, err := time.Parse(time.RFC3339, readHeartbeat(t, heartbeat))
|
||||
if err != nil {
|
||||
t.Fatalf("heartbeat content is not an RFC 3339 timestamp: %v", err)
|
||||
}
|
||||
if written.Before(start) {
|
||||
t.Fatalf("heartbeat %s predates the loop start %s", written, start)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDoesNotBlockShutdownOnAStuckWrite(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
writer.blockUntil = make(chan struct{})
|
||||
instance, _, _ := newTestAgent(t, &fakeSource{}, writer, time.Second)
|
||||
ticks := make(chan time.Time)
|
||||
instance.newTicker = func(time.Duration) (<-chan time.Time, func()) { return ticks, func() {} }
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- instance.run(ctx) }()
|
||||
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("run returned error: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a blocked store write must not hold up shutdown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRefusesToStartWhenTheHelloIsNotValid(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
instance, _, _ := newTestAgent(t, &fakeSource{}, writer, time.Second)
|
||||
instance.agentID = ""
|
||||
|
||||
if err := instance.run(context.Background()); err == nil {
|
||||
t.Fatal("an agent whose own hello fails validation must refuse to start")
|
||||
}
|
||||
if len(writer.recorded()) != 0 {
|
||||
t.Fatal("nothing may be published before the self-check passes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRequiresAWriter(t *testing.T) {
|
||||
instance, _, _ := newTestAgent(t, &fakeSource{}, nil, time.Second)
|
||||
instance.writer = nil
|
||||
if err := instance.run(context.Background()); !errors.Is(err, errNoWriter) {
|
||||
t.Fatalf("error = %v, want errNoWriter", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelloAnnouncesOnlyReadOnlyCapabilities(t *testing.T) {
|
||||
instance, clock, _ := newTestAgent(t, &fakeSource{}, newFakeWriter(), time.Second)
|
||||
hello := instance.hello()
|
||||
if err := hello.Validate(clock.now()); err != nil {
|
||||
t.Fatalf("hello validation failed: %v", err)
|
||||
}
|
||||
if len(hello.Capabilities) != 2 {
|
||||
t.Fatalf("capabilities = %+v", hello.Capabilities)
|
||||
}
|
||||
for _, item := range hello.Capabilities {
|
||||
if !item.ReadOnly {
|
||||
t.Fatalf("capability %q is not read-only", item.ID)
|
||||
}
|
||||
if !agentstore.Capability(item.ID).Valid() {
|
||||
t.Fatalf("capability %q is not a recognised store capability", item.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerCapabilityIsPublishedOnlyWhenSourceSupportsIt(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
instance, _, _ := newTestAgent(t, fakeContainerSource{fakeSource: &fakeSource{}}, writer, time.Second)
|
||||
if len(instance.hello().Capabilities) != 3 {
|
||||
t.Fatalf("capabilities = %+v, want host/processes/containers", instance.hello().Capabilities)
|
||||
}
|
||||
instance.collectOnce(context.Background())
|
||||
if writer.countFor(agentstore.CapabilityContainers) != 1 {
|
||||
t.Fatalf("container snapshot was not published: %+v", writer.recorded())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInventoryCapabilitiesArePublishedOnlyForConfiguredUnraidSource(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
instance, _, _ := newTestAgent(t, fakeInventorySource{fakeSource: &fakeSource{}}, writer, time.Second)
|
||||
if got := len(instance.hello().Capabilities); got != 7 {
|
||||
t.Fatalf("capabilities = %+v, want host/processes/containers/array/disks/pools/shares", instance.hello().Capabilities)
|
||||
}
|
||||
instance.collectOnce(context.Background())
|
||||
for _, capability := range []agentstore.Capability{agentstore.CapabilityContainers, agentstore.CapabilityArray, agentstore.CapabilityDisks, agentstore.CapabilityPools, agentstore.CapabilityShares} {
|
||||
if writer.countFor(capability) != 1 {
|
||||
t.Fatalf("capability %q was not published: %+v", capability, writer.recorded())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeatFailureIsNotFatal(t *testing.T) {
|
||||
writer := newFakeWriter()
|
||||
instance, _, _ := newTestAgent(t, &fakeSource{}, writer, time.Second)
|
||||
instance.heartbeatPath = filepath.Join(t.TempDir(), "missing-directory", "healthy")
|
||||
|
||||
instance.iterate(context.Background())
|
||||
|
||||
if len(writer.recorded()) == 0 {
|
||||
t.Fatal("a heartbeat write failure must not stop the collection loop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotWriterRefusesAMissingPool(t *testing.T) {
|
||||
// A nil pool would let every publish fail deep in the driver. The agent must refuse
|
||||
// to start instead, so an operator sees the misconfiguration rather than an agent
|
||||
// that appears healthy while storing nothing.
|
||||
if _, err := newSnapshotWriter(nil); !errors.Is(err, errStoreNotLinked) {
|
||||
t.Fatalf("error = %v, want errStoreNotLinked", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
// Command pulse-agent collects read-only host telemetry and publishes it as bounded
|
||||
// snapshots for pulse-api to read.
|
||||
//
|
||||
// It exposes no network port, holds no Docker socket, and performs no mutation: it
|
||||
// reads procfs/sysfs and writes one row per capability through agentstore.Writer. See
|
||||
// docs/architecture/SYSTEM_ARCHITECTURE.md ("pulse-agent"), ADR-0005 and
|
||||
// docs/operations/WORKER_AGENT_HEALTHCHECK_CONTRACT.md.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/itworx/pulse/internal/array"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/disk"
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
"github.com/itworx/pulse/internal/hostcollect"
|
||||
"github.com/itworx/pulse/internal/pool"
|
||||
"github.com/itworx/pulse/internal/process"
|
||||
"github.com/itworx/pulse/internal/runtimeconfig"
|
||||
"github.com/itworx/pulse/internal/share"
|
||||
"github.com/itworx/pulse/internal/unraid"
|
||||
)
|
||||
|
||||
type configuredSource struct {
|
||||
host *hostcollect.Collector
|
||||
}
|
||||
|
||||
func (s configuredSource) Host(ctx context.Context) (host.RawSnapshot, error) {
|
||||
return s.host.Host(ctx)
|
||||
}
|
||||
func (s configuredSource) Processes(ctx context.Context) (process.RawSnapshot, error) {
|
||||
return s.host.Processes(ctx)
|
||||
}
|
||||
|
||||
type configuredUnraidSource struct {
|
||||
configuredSource
|
||||
containers interface {
|
||||
Snapshot(context.Context) (container.RawSnapshot, error)
|
||||
}
|
||||
array interface {
|
||||
Snapshot(context.Context) (array.RawSnapshot, error)
|
||||
}
|
||||
disks interface {
|
||||
Snapshot(context.Context) (disk.RawSnapshot, error)
|
||||
}
|
||||
pools interface {
|
||||
Snapshot(context.Context) (pool.RawSnapshot, error)
|
||||
}
|
||||
shares interface {
|
||||
Snapshot(context.Context) (share.RawSnapshot, error)
|
||||
}
|
||||
}
|
||||
|
||||
func (s configuredUnraidSource) Containers(ctx context.Context) (container.RawSnapshot, error) {
|
||||
if s.containers == nil {
|
||||
return container.RawSnapshot{}, errUnraidSourceNotConfigured
|
||||
}
|
||||
return s.containers.Snapshot(ctx)
|
||||
}
|
||||
func (s configuredUnraidSource) Array(ctx context.Context) (array.RawSnapshot, error) {
|
||||
if s.array == nil {
|
||||
return array.RawSnapshot{}, errUnraidSourceNotConfigured
|
||||
}
|
||||
return s.array.Snapshot(ctx)
|
||||
}
|
||||
func (s configuredUnraidSource) Disks(ctx context.Context) (disk.RawSnapshot, error) {
|
||||
if s.disks == nil {
|
||||
return disk.RawSnapshot{}, errUnraidSourceNotConfigured
|
||||
}
|
||||
return s.disks.Snapshot(ctx)
|
||||
}
|
||||
func (s configuredUnraidSource) Pools(ctx context.Context) (pool.RawSnapshot, error) {
|
||||
if s.pools == nil {
|
||||
return pool.RawSnapshot{}, errUnraidSourceNotConfigured
|
||||
}
|
||||
return s.pools.Snapshot(ctx)
|
||||
}
|
||||
func (s configuredUnraidSource) Shares(ctx context.Context) (share.RawSnapshot, error) {
|
||||
if s.shares == nil {
|
||||
return share.RawSnapshot{}, errUnraidSourceNotConfigured
|
||||
}
|
||||
return s.shares.Snapshot(ctx)
|
||||
}
|
||||
|
||||
var errUnraidSourceNotConfigured = errors.New("Unraid source is not configured")
|
||||
|
||||
func main() {
|
||||
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
||||
if err := run(logger); err != nil {
|
||||
logger.Error("pulse agent failed", "error", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(logger *slog.Logger) error {
|
||||
config, err := runtimeconfig.LoadAgent()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
collector, err := hostcollect.New(hostcollect.Options{
|
||||
ProcRoot: config.ProcRoot,
|
||||
SysRoot: config.SysRoot,
|
||||
FilesystemRoot: config.FilesystemRoot,
|
||||
HostName: config.HostName,
|
||||
SourceID: "host",
|
||||
ProcessLimits: process.Limits{MaxRows: config.MaxProcesses},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
writer, closeStore, err := openStore(ctx, config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeStore()
|
||||
|
||||
logger.Info("pulse agent configuration loaded",
|
||||
"agent_id", config.AgentID,
|
||||
"proc_root", config.ProcRoot,
|
||||
"sys_root", config.SysRoot,
|
||||
"filesystem_root", config.FilesystemRoot,
|
||||
"collect_interval", config.CollectInterval.String(),
|
||||
"shutdown_timeout", config.Service.ShutdownAfter.String(),
|
||||
)
|
||||
var source snapshotSource = configuredSource{host: collector}
|
||||
if config.UnraidURL != "" {
|
||||
var client *unraid.Client
|
||||
if config.UnraidCAFile == "" {
|
||||
client, err = unraid.New(config.UnraidURL, config.UnraidAPIToken, nil)
|
||||
} else {
|
||||
info, statErr := os.Stat(config.UnraidCAFile)
|
||||
if statErr != nil || info.Size() <= 0 || info.Size() > 1<<20 {
|
||||
return errors.New("PULSE_UNRAID_CA_FILE must be a readable certificate no larger than 1 MiB")
|
||||
}
|
||||
caPEM, readErr := os.ReadFile(config.UnraidCAFile)
|
||||
if readErr != nil {
|
||||
return errors.New("read PULSE_UNRAID_CA_FILE")
|
||||
}
|
||||
client, err = unraid.NewWithCAPEM(config.UnraidURL, config.UnraidAPIToken, caPEM)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
source = configuredUnraidSource{configuredSource: configuredSource{host: collector}, containers: unraid.ContainerSource{Client: client}, array: unraid.ArraySource{Client: client}, disks: unraid.DiskSource{Client: client}, pools: unraid.PoolSource{Client: client}, shares: unraid.ShareSource{Client: client}}
|
||||
}
|
||||
return newAgent(config, source, writer, logger).run(ctx)
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
"github.com/itworx/pulse/internal/runtimeconfig"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const (
|
||||
// storeConnectTimeout bounds the start-up reachability check. The healthcheck
|
||||
// contract wants the first heartbeat written after the database is reachable, so
|
||||
// this must stay far below the 45 second staleness threshold.
|
||||
storeConnectTimeout = 5 * time.Second
|
||||
// agentPoolMaxConns keeps the agent's footprint on the shared database small: it
|
||||
// issues one small write per capability per interval and never reads.
|
||||
agentPoolMaxConns = 2
|
||||
)
|
||||
|
||||
// errStoreNotLinked is returned by every write while no snapshot store implementation
|
||||
// is compiled into this binary. It is deliberately an error on the write path rather
|
||||
// than a silent drop: the reader turns a missing snapshot into Unknown, and the agent
|
||||
// log states plainly why nothing arrives.
|
||||
var errStoreNotLinked = errors.New("no agent snapshot store is linked into this build")
|
||||
|
||||
// openStore connects to PostgreSQL and returns the narrow Writer the agent publishes
|
||||
// through, plus a close function.
|
||||
//
|
||||
// The agent depends on the agentstore.Writer interface only; the concrete PostgreSQL
|
||||
// store owns migration 0016 and the agent_snapshots table.
|
||||
func openStore(ctx context.Context, config runtimeconfig.AgentConfig) (agentstore.Writer, func(), error) {
|
||||
pool, err := database.NewPool(ctx, database.Config{
|
||||
URL: config.DatabaseURL,
|
||||
MaxConns: agentPoolMaxConns,
|
||||
MinConns: 1,
|
||||
})
|
||||
if err != nil {
|
||||
// Never wrap the URL itself into the error: it carries the database password.
|
||||
return nil, nil, errors.New("agent database pool could not be created")
|
||||
}
|
||||
reachable, cancel := context.WithTimeout(ctx, storeConnectTimeout)
|
||||
defer cancel()
|
||||
if err := database.Ping(reachable, pool); err != nil {
|
||||
pool.Close()
|
||||
return nil, nil, fmt.Errorf("agent database is unreachable: %w", err)
|
||||
}
|
||||
writer, err := newSnapshotWriter(pool)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
return writer, pool.Close, nil
|
||||
}
|
||||
|
||||
// newSnapshotWriter builds the store the agent writes through. A nil pool would make
|
||||
// every publish fail silently at the driver, so it is refused here instead: the agent
|
||||
// must either have a real store or fail to start.
|
||||
func newSnapshotWriter(pool *pgxpool.Pool) (agentstore.Writer, error) {
|
||||
if pool == nil {
|
||||
return nil, errStoreNotLinked
|
||||
}
|
||||
return agentstore.PostgresStore{Pool: pool}, nil
|
||||
}
|
||||
|
||||
var _ agentstore.Writer = agentstore.PostgresStore{}
|
||||
+569
@@ -0,0 +1,569 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentsource"
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/alertapi"
|
||||
"github.com/itworx/pulse/internal/alertcontrol"
|
||||
"github.com/itworx/pulse/internal/alertcontrolapi"
|
||||
"github.com/itworx/pulse/internal/alertdefaults"
|
||||
"github.com/itworx/pulse/internal/alertopsapi"
|
||||
"github.com/itworx/pulse/internal/applicationapi"
|
||||
"github.com/itworx/pulse/internal/array"
|
||||
"github.com/itworx/pulse/internal/arrayapi"
|
||||
"github.com/itworx/pulse/internal/audit"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/authapi"
|
||||
"github.com/itworx/pulse/internal/backup"
|
||||
"github.com/itworx/pulse/internal/backupapi"
|
||||
"github.com/itworx/pulse/internal/config"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/containerapi"
|
||||
"github.com/itworx/pulse/internal/correlation"
|
||||
"github.com/itworx/pulse/internal/dashboard"
|
||||
"github.com/itworx/pulse/internal/dashboardapi"
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
"github.com/itworx/pulse/internal/disk"
|
||||
"github.com/itworx/pulse/internal/diskapi"
|
||||
"github.com/itworx/pulse/internal/eventapi"
|
||||
forecastdomain "github.com/itworx/pulse/internal/forecast"
|
||||
"github.com/itworx/pulse/internal/forecastapi"
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
"github.com/itworx/pulse/internal/hostapi"
|
||||
"github.com/itworx/pulse/internal/incident"
|
||||
"github.com/itworx/pulse/internal/incidentapi"
|
||||
"github.com/itworx/pulse/internal/inventory"
|
||||
"github.com/itworx/pulse/internal/inventoryapi"
|
||||
"github.com/itworx/pulse/internal/live"
|
||||
"github.com/itworx/pulse/internal/livesampler"
|
||||
"github.com/itworx/pulse/internal/metriccatalog"
|
||||
"github.com/itworx/pulse/internal/metricquery"
|
||||
"github.com/itworx/pulse/internal/metricsapi"
|
||||
"github.com/itworx/pulse/internal/network"
|
||||
"github.com/itworx/pulse/internal/networkapi"
|
||||
"github.com/itworx/pulse/internal/observability"
|
||||
"github.com/itworx/pulse/internal/onboarding"
|
||||
"github.com/itworx/pulse/internal/onboardingapi"
|
||||
pooldomain "github.com/itworx/pulse/internal/pool"
|
||||
"github.com/itworx/pulse/internal/poolapi"
|
||||
"github.com/itworx/pulse/internal/problem"
|
||||
"github.com/itworx/pulse/internal/process"
|
||||
"github.com/itworx/pulse/internal/processapi"
|
||||
"github.com/itworx/pulse/internal/prometheus"
|
||||
"github.com/itworx/pulse/internal/queryplan"
|
||||
"github.com/itworx/pulse/internal/reverseproxy"
|
||||
"github.com/itworx/pulse/internal/reverseproxyapi"
|
||||
"github.com/itworx/pulse/internal/runtimeconfig"
|
||||
"github.com/itworx/pulse/internal/service"
|
||||
"github.com/itworx/pulse/internal/serviceapi"
|
||||
sharedomain "github.com/itworx/pulse/internal/share"
|
||||
"github.com/itworx/pulse/internal/shareapi"
|
||||
"github.com/itworx/pulse/internal/systemstatus"
|
||||
"github.com/itworx/pulse/internal/systemstatusapi"
|
||||
"github.com/itworx/pulse/internal/widget"
|
||||
"github.com/itworx/pulse/internal/widgetapi"
|
||||
"github.com/itworx/pulse/internal/workerruntime"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
||||
if err := run(logger); err != nil {
|
||||
logger.Error("pulse api failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(logger *slog.Logger) error {
|
||||
runtime, err := runtimeconfig.Load("api")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
application, err := config.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
registry, err := metriccatalog.DefaultRegistry()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx := context.Background()
|
||||
internalMetrics := observability.NewRegistry(time.Now().UTC())
|
||||
var pool *pgxpool.Pool
|
||||
var inventoryRepo *inventory.Repository
|
||||
var dashboardRepo dashboard.Repository
|
||||
var alertRepo alert.Store
|
||||
var alertControlStore alertcontrol.Store
|
||||
var alertOperationsStore alertopsapi.Store
|
||||
var incidentStore incident.Store
|
||||
var serviceProvider service.Provider = service.UnknownProvider{Reason: "source_unavailable"}
|
||||
var reverseProxyProvider reverseproxy.Provider = reverseproxy.DisabledProvider{SourceID: "reverse-proxy", SourceType: "connector", Reason: "connector_disabled"}
|
||||
var dependencyRepo *service.DependencyRepository
|
||||
var onboardingService onboarding.Service
|
||||
// agentReader is the read half of the pulse-agent telemetry transport. It stays nil
|
||||
// without a database, which keeps every monitoring surface Unknown instead of
|
||||
// inventing state (ADR-0008).
|
||||
var agentReader agentstore.Reader
|
||||
databaseReady := false
|
||||
if application.DatabaseURL != "" {
|
||||
pool, err = database.NewPool(ctx, database.Config{URL: application.DatabaseURL})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Ping(ctx, pool); err != nil {
|
||||
return err
|
||||
}
|
||||
databaseReady = true
|
||||
agentReader = agentstore.PostgresStore{Pool: pool}
|
||||
dashboardRepo = dashboard.Repository{Pool: pool}
|
||||
alertRepo = alert.Repository{Pool: pool, Registry: registry}
|
||||
if report, seedErr := alertdefaults.Seed(ctx, alertRepo, registry, "system-defaults"); seedErr != nil {
|
||||
return fmt.Errorf("seed alert defaults: %w", seedErr)
|
||||
} else {
|
||||
logger.Info("alert defaults reconciled", "added", report.Added, "existing", report.Existing)
|
||||
}
|
||||
alertControlStore = alertcontrol.Repository{Pool: pool}
|
||||
alertOperationsStore = alert.StateRepository{Pool: pool}
|
||||
incidentStore, err = incident.NewRepository(pool)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
serviceProvider, err = service.NewPostgresProvider(pool, service.StatusPolicy{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dependencyRepo, err = service.NewDependencyRepository(pool, audit.PostgresStore{Pool: pool})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inventoryRepo, err = inventory.NewRepository(pool)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
onboardingService = onboarding.Service{State: onboarding.StateStore{Pool: pool}, Pool: pool, Dashboards: dashboardRepo, Alerts: alertRepo, AuthMode: application.AuthMode, OIDCIssuer: application.OIDCIssuer, OIDCClient: application.OIDCClientID, OIDCRedirect: application.OIDCRedirectURL, Prometheus: application.PrometheusURL != "", Unraid: application.UnraidURL != "" && application.UnraidAPIToken != ""}
|
||||
}
|
||||
|
||||
var queryService *metricquery.Service
|
||||
var liveSampler live.Sampler
|
||||
var promSource *prometheus.Client
|
||||
if application.PrometheusURL != "" {
|
||||
source, sourceErr := prometheus.New(application.PrometheusURL, nil, prometheus.Limits{Timeout: application.PrometheusTimeout})
|
||||
if sourceErr != nil {
|
||||
return sourceErr
|
||||
}
|
||||
promSource = source
|
||||
queryService = metricquery.NewService(queryplan.NewPlanner(registry, queryplan.Limits{}), source, nil)
|
||||
sampler, samplerErr := livesampler.New(registry, source, livesampler.Options{})
|
||||
if samplerErr != nil {
|
||||
return samplerErr
|
||||
}
|
||||
liveSampler = sampler
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// publishSourceMetrics refreshes adapter counters into the internal registry just
|
||||
// before it is read, so operators see current Prometheus latency and error counts
|
||||
// rather than the values captured at process start.
|
||||
publishSourceMetrics := func() {
|
||||
if promSource != nil {
|
||||
promSource.PublishMetrics(internalMetrics)
|
||||
}
|
||||
}
|
||||
sessions := auth.NewSlidingSessionManager("pulse_session", application.SessionIdleTTL, application.SessionAbsoluteTTL, application.Environment == config.Production)
|
||||
backupManager := &backup.Manager{Pool: pool, Directory: application.BackupDirectory, Retention: application.BackupRetention}
|
||||
var backupObservation struct {
|
||||
sync.Mutex
|
||||
checkedAt time.Time
|
||||
latest time.Time
|
||||
verified time.Time
|
||||
err error
|
||||
}
|
||||
readBackupObservation := func(ctx context.Context, now time.Time) (time.Time, time.Time, error) {
|
||||
backupObservation.Lock()
|
||||
defer backupObservation.Unlock()
|
||||
if !backupObservation.checkedAt.IsZero() && now.Sub(backupObservation.checkedAt) < 5*time.Minute {
|
||||
return backupObservation.latest, backupObservation.verified, backupObservation.err
|
||||
}
|
||||
results, err := backupManager.List(ctx)
|
||||
latest := time.Time{}
|
||||
if len(results) > 0 {
|
||||
latest = results[0].Created
|
||||
}
|
||||
verified := time.Time{}
|
||||
if err == nil {
|
||||
verified = now
|
||||
}
|
||||
backupObservation.checkedAt, backupObservation.latest, backupObservation.verified, backupObservation.err = now, latest, verified, err
|
||||
return latest, verified, err
|
||||
}
|
||||
invalidateBackupObservation := func() {
|
||||
backupObservation.Lock()
|
||||
defer backupObservation.Unlock()
|
||||
backupObservation.checkedAt = time.Time{}
|
||||
}
|
||||
mux := service.HealthMuxWithReadiness(func() bool { return databaseReady })
|
||||
mux.HandleFunc("/auth/test-login", func(response http.ResponseWriter, request *http.Request) {
|
||||
if application.Environment == config.Production || application.AuthMode != "mock" {
|
||||
problem.Write(response, request, http.StatusNotFound, "NOT_FOUND", "Not found", "The requested resource does not exist.", nil)
|
||||
return
|
||||
}
|
||||
principal := auth.Principal{Subject: "development-user", Role: auth.RoleAdministrator}
|
||||
if err := sessions.Issue(response, principal, time.Now().UTC()); err != nil {
|
||||
problem.Write(response, request, http.StatusInternalServerError, "SESSION_ERROR", "Session unavailable", "The session could not be created.", nil)
|
||||
return
|
||||
}
|
||||
if pool != nil {
|
||||
if err := audit.RecordSecurityAction(request.Context(), audit.PostgresStore{Pool: pool}, principal.Subject, "auth.test_login", "success", correlation.FromContext(request.Context())); err != nil {
|
||||
problem.Write(response, request, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "Authentication unavailable", "The authentication event could not be recorded.", nil)
|
||||
return
|
||||
}
|
||||
}
|
||||
response.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(response).Encode(map[string]string{"status": "authenticated", "mode": "mock-development"})
|
||||
})
|
||||
mux.HandleFunc("/session/logout", func(response http.ResponseWriter, request *http.Request) {
|
||||
sessions.Clear(response, request)
|
||||
response.WriteHeader(http.StatusNoContent)
|
||||
})
|
||||
if application.AuthMode == "oidc" && application.OIDCIssuer != "" {
|
||||
oidcLogin, loginErr := authapi.New(authapi.Options{
|
||||
OIDC: auth.OIDCConfig{
|
||||
Issuer: application.OIDCIssuer,
|
||||
ClientID: application.OIDCClientID,
|
||||
ClientSecret: application.OIDCClientSecret,
|
||||
RedirectURL: application.OIDCRedirectURL,
|
||||
},
|
||||
RoleMapping: roleMapping(application.OIDCRoleMapping),
|
||||
GroupsClaim: application.OIDCGroupsClaim,
|
||||
Sessions: sessions,
|
||||
Secure: application.Environment == config.Production,
|
||||
Logger: logger,
|
||||
// Failures land on the overview route, where the web app reads the reason
|
||||
// code from the query string, shows a localized notice and strips it from
|
||||
// the URL. There is deliberately no dedicated error page to maintain.
|
||||
ErrorPath: "/",
|
||||
Audit: func(ctx context.Context, actor, result string) error {
|
||||
if pool == nil {
|
||||
return nil
|
||||
}
|
||||
return audit.RecordSecurityAction(ctx, audit.PostgresStore{Pool: pool}, actor, "auth.login", result, correlation.FromContext(ctx))
|
||||
},
|
||||
})
|
||||
if loginErr != nil {
|
||||
return loginErr
|
||||
}
|
||||
mux.Handle("/auth/login", oidcLogin.LoginHandler())
|
||||
mux.Handle("/auth/callback", oidcLogin.CallbackHandler())
|
||||
logger.Info("oidc login enabled", "mapped_claims", len(application.OIDCRoleMapping))
|
||||
}
|
||||
// reportedJobs is the metadata-only view of the worker schedule. The API never runs
|
||||
// these jobs; it reads their recorded outcomes from job_runs so the status surface
|
||||
// reflects what the worker actually did instead of the hardcoded "not recorded"
|
||||
// placeholders it used before the worker runtime existed.
|
||||
reportedJobs := workerruntime.Schedule(workerruntime.ScheduleRuns{})
|
||||
snapshot := func(requestContext context.Context) (systemstatus.Snapshot, error) {
|
||||
publishSourceMetrics()
|
||||
var auditEvents *int64
|
||||
var statusOptions []systemstatus.Option
|
||||
now := time.Now().UTC()
|
||||
_, authenticated := auth.PrincipalFromContext(requestContext)
|
||||
statusOptions = append(statusOptions, systemstatus.WithAuthenticatedSession(authenticated))
|
||||
sourceHealth := make([]systemstatus.SourceHealth, 0, 3)
|
||||
if promSource != nil {
|
||||
sourceHealth = append(sourceHealth, systemstatus.FromDatasource("prometheus", promSource.Health(requestContext), now))
|
||||
}
|
||||
if agentReader != nil {
|
||||
unraidHealth, storageHealth, healthErr := readAgentSourceHealth(requestContext, agentReader, now)
|
||||
if healthErr != nil {
|
||||
return systemstatus.Snapshot{}, fmt.Errorf("read agent source health: %w", healthErr)
|
||||
}
|
||||
if unraidHealth.ReasonCode != agentsource.ReasonUnavailable || storageHealth.ReasonCode != agentsource.ReasonUnavailable {
|
||||
internalMetrics.SetGauge("pulse_unraid_configured", 1)
|
||||
}
|
||||
sourceHealth = append(sourceHealth, systemstatus.FromDatasource("unraid", unraidHealth, now), systemstatus.FromDatasource("storage", storageHealth, now))
|
||||
}
|
||||
if len(sourceHealth) > 0 {
|
||||
statusOptions = append(statusOptions, systemstatus.WithSources(sourceHealth...))
|
||||
}
|
||||
if application.BackupDirectory != "" {
|
||||
latestBackup, verifiedAt, backupErr := readBackupObservation(requestContext, now)
|
||||
if backupErr != nil {
|
||||
statusOptions = append(statusOptions, systemstatus.WithBackupObservation(time.Time{}, time.Time{}, backupErr))
|
||||
} else if !latestBackup.IsZero() {
|
||||
statusOptions = append(statusOptions, systemstatus.WithBackupObservation(latestBackup, verifiedAt, nil))
|
||||
}
|
||||
}
|
||||
if pool != nil && databaseReady {
|
||||
var count int64
|
||||
if err := pool.QueryRow(requestContext, `SELECT count(*) FROM audit_events`).Scan(&count); err != nil {
|
||||
return systemstatus.Snapshot{}, fmt.Errorf("read audit event count: %w", err)
|
||||
}
|
||||
auditEvents = &count
|
||||
var migrationVersion string
|
||||
if err := pool.QueryRow(requestContext, `SELECT id FROM schema_migrations ORDER BY applied_at DESC, id DESC LIMIT 1`).Scan(&migrationVersion); err != nil {
|
||||
return systemstatus.Snapshot{}, fmt.Errorf("read migration version: %w", err)
|
||||
}
|
||||
statusOptions = append(statusOptions, systemstatus.WithMigrationVersion(migrationVersion))
|
||||
jobs, jobsErr := workerruntime.ReadJobHealth(requestContext, pool, reportedJobs)
|
||||
if jobsErr != nil {
|
||||
// A failed read must not be reported as healthy. Omitting the option
|
||||
// leaves every job component Unknown, which is the honest answer when
|
||||
// the worker's recorded state cannot be established (ADR-0008).
|
||||
logger.Error("read worker job health", "error", jobsErr)
|
||||
} else {
|
||||
statusOptions = append(statusOptions, systemstatus.WithJobs(0, jobs...))
|
||||
}
|
||||
}
|
||||
return systemstatus.Build(application, databaseReady, now, auditEvents, statusOptions...), nil
|
||||
}
|
||||
internalMetrics.SetGauge("pulse_database_ready", boolMetric(databaseReady))
|
||||
internalMetrics.SetGauge("pulse_prometheus_configured", boolMetric(application.PrometheusURL != ""))
|
||||
internalMetrics.SetGauge("pulse_unraid_configured", boolMetric(application.UnraidURL != "" && application.UnraidAPIToken != ""))
|
||||
if onboardingService.State.Pool != nil {
|
||||
onboardingService.RuntimeCapabilities = func(requestContext context.Context) ([]onboarding.Capability, error) {
|
||||
status, statusErr := snapshot(requestContext)
|
||||
if statusErr != nil {
|
||||
return nil, statusErr
|
||||
}
|
||||
capabilities := make([]onboarding.Capability, 0, 2)
|
||||
for _, component := range status.Components {
|
||||
if component.ID != "prometheus" && component.ID != "unraid" {
|
||||
continue
|
||||
}
|
||||
state, detail := "unknown", "Geen actuele runtimewaarneming beschikbaar."
|
||||
switch component.State {
|
||||
case systemstatus.StateHealthy:
|
||||
state, detail = "ready", "Actuele telemetrie wordt ontvangen via de veilige runtimebron."
|
||||
case systemstatus.StateDegraded:
|
||||
state, detail = "incomplete", "De runtimebron vraagt aandacht."
|
||||
case systemstatus.StateDisabled:
|
||||
state, detail = "not-ready", "De runtimebron is niet geconfigureerd."
|
||||
}
|
||||
capabilities = append(capabilities, onboarding.Capability{ID: component.ID, State: state, Detail: detail})
|
||||
}
|
||||
return capabilities, nil
|
||||
}
|
||||
}
|
||||
statusHandler := systemstatusapi.Handler{
|
||||
Snapshot: snapshot,
|
||||
Diagnostics: func(requestContext context.Context) (systemstatusapi.Diagnostics, error) {
|
||||
status, err := snapshot(requestContext)
|
||||
if err != nil {
|
||||
return systemstatusapi.Diagnostics{}, err
|
||||
}
|
||||
return systemstatusapi.Diagnostics{
|
||||
Status: status,
|
||||
Config: systemstatusapi.ConfigSummary{
|
||||
Environment: string(application.Environment), Timezone: application.Timezone, Locale: application.DefaultLocale, AuthMode: application.AuthMode,
|
||||
PublicURLConfigured: application.PublicURL != "", DatabaseConfigured: application.DatabaseURL != "", PrometheusConfigured: application.PrometheusURL != "",
|
||||
UnraidConfigured: application.UnraidURL != "" && application.UnraidAPIToken != "", OIDCConfigured: application.OIDCIssuer != "" && application.OIDCClientID != "",
|
||||
},
|
||||
Runtime: systemstatusapi.Runtime(), Metrics: internalMetrics.Exposition(time.Now().UTC()),
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
protectedStatus := withSession(sessions, auth.Require(auth.PermissionView, statusHandler))
|
||||
protectedDiagnostics := withSession(sessions, auth.Require(auth.PermissionOperate, statusHandler))
|
||||
mux.Handle("/api/v1/system/status", protectedStatus)
|
||||
mux.Handle("/api/v1/system/diagnostics", protectedDiagnostics)
|
||||
metricsExposition := internalMetrics.Handler()
|
||||
mux.Handle("/api/v1/system/metrics", withSession(sessions, auth.Require(auth.PermissionOperate, http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
publishSourceMetrics()
|
||||
metricsExposition.ServeHTTP(response, request)
|
||||
}))))
|
||||
mux.Handle("/api/v1/system/backups", withSession(sessions, auth.Require(auth.PermissionAdmin, backupapi.Handler{Manager: backupManager, OnCreated: invalidateBackupObservation, Audit: func(ctx context.Context, actor, result string) error {
|
||||
return audit.RecordSecurityAction(ctx, audit.PostgresStore{Pool: pool}, actor, "backup.create", result, correlation.FromContext(ctx))
|
||||
}})))
|
||||
|
||||
onboardingHandler := onboardingapi.Handler{Service: onboardingService, Audit: audit.PostgresStore{Pool: pool}}
|
||||
mux.Handle("/api/v1/onboarding", withSession(sessions, auth.Require(auth.PermissionView, onboardingHandler)))
|
||||
metricsHandler := metricsapi.Handler{Registry: registry}
|
||||
widgetRegistry, err := widget.NewRegistry(widget.DefaultDefinitions())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
widgetHandler := widgetapi.Handler{Registry: widgetRegistry}
|
||||
mux.Handle("/api/v1/widgets/catalog", withSession(sessions, auth.Require(auth.PermissionView, widgetHandler)))
|
||||
mux.Handle("/api/v1/widgets/preview", withSession(sessions, auth.Require(auth.PermissionEdit, widgetHandler)))
|
||||
mux.Handle("/api/v1/metrics/catalog", withSession(sessions, auth.Require(auth.PermissionView, metricsHandler)))
|
||||
queryHandler := metricquery.Handler{Service: queryService}
|
||||
mux.Handle("/api/v1/metrics/query", withSession(sessions, auth.Require(auth.PermissionView, queryHandler)))
|
||||
mux.Handle("/api/v1/metrics/query-range", withSession(sessions, auth.Require(auth.PermissionView, queryHandler)))
|
||||
mux.Handle("/api/v1/metrics/inspect", withSession(sessions, auth.Require(auth.PermissionOperate, queryHandler)))
|
||||
livePlanner := queryplan.NewPlanner(registry, queryplan.Limits{})
|
||||
liveRegistry := live.NewRegistry(liveSampler, live.RegistryOptions{})
|
||||
liveHandler := live.Handler{Planner: &livePlanner, Registry: liveRegistry}
|
||||
mux.Handle("/api/v1/live", withSession(sessions, auth.Require(auth.PermissionView, liveHandler)))
|
||||
if alertRepo != nil {
|
||||
alertHandler := alertapi.Handler{Repository: alertRepo, Registry: registry, Audit: audit.PostgresStore{Pool: pool}}
|
||||
mux.Handle("/api/v1/alert-rules", withSession(sessions, alertHandler))
|
||||
mux.Handle("/api/v1/alert-rules/", withSession(sessions, alertHandler))
|
||||
operationsHandler := alertopsapi.Handler{Store: alertOperationsStore, Audit: audit.PostgresStore{Pool: pool}}
|
||||
protectedOperations := withSession(sessions, auth.Require(auth.PermissionView, operationsHandler))
|
||||
mux.Handle("/api/v1/alerts", protectedOperations)
|
||||
mux.Handle("/api/v1/alerts/", protectedOperations)
|
||||
controlHandler := alertcontrolapi.Handler{Store: alertControlStore, Audit: audit.PostgresStore{Pool: pool}}
|
||||
protectedControls := withSession(sessions, auth.Require(auth.PermissionView, controlHandler))
|
||||
mux.Handle("/api/v1/alert-silences", protectedControls)
|
||||
mux.Handle("/api/v1/alert-silences/", protectedControls)
|
||||
mux.Handle("/api/v1/maintenance-windows", protectedControls)
|
||||
mux.Handle("/api/v1/maintenance-windows/", protectedControls)
|
||||
}
|
||||
if incidentStore != nil {
|
||||
incidentHandler := incidentapi.Handler{Store: incidentStore, Audit: audit.PostgresStore{Pool: pool}}
|
||||
protectedIncidents := withSession(sessions, auth.Require(auth.PermissionView, incidentHandler))
|
||||
mux.Handle("/api/v1/incidents", protectedIncidents)
|
||||
mux.Handle("/api/v1/incidents/", protectedIncidents)
|
||||
}
|
||||
if dashboardRepo.Pool != nil {
|
||||
dashboardHandler := dashboardapi.Handler{Repository: dashboardRepo, Audit: audit.PostgresStore{Pool: pool}}
|
||||
mux.Handle("/api/v1/dashboards", withSession(sessions, dashboardHandler))
|
||||
mux.Handle("/api/v1/dashboards/", withSession(sessions, dashboardHandler))
|
||||
}
|
||||
if inventoryRepo != nil {
|
||||
inventoryHandler := inventoryapi.Handler{Repository: inventoryRepo}
|
||||
protectedInventory := withSession(sessions, auth.Require(auth.PermissionView, inventoryHandler))
|
||||
mux.Handle("/api/v1/entities", protectedInventory)
|
||||
mux.Handle("/api/v1/entities/", protectedInventory)
|
||||
}
|
||||
if pool != nil {
|
||||
eventsHandler := eventapi.Handler{Store: eventapi.PostgresStore{Pool: pool}}
|
||||
mux.Handle("/api/v1/events", withSession(sessions, auth.Require(auth.PermissionView, eventsHandler)))
|
||||
}
|
||||
// Monitoring surfaces are served from the bounded snapshots pulse-agent writes into
|
||||
// PostgreSQL. Without a database there is no transport at all, so each surface keeps
|
||||
// the empty adapter it had before, which resolves to Unknown rather than Healthy.
|
||||
agentWindows := agentsource.Windows{}
|
||||
if err := agentWindows.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
var hostProvider host.Provider = host.UnknownProvider{SourceID: "host", SourceType: "agent", Reason: "source_unavailable"}
|
||||
var processProvider interface {
|
||||
Snapshot(context.Context) (process.Snapshot, error)
|
||||
} = process.Adapter{}
|
||||
var containerProvider container.Provider = container.Adapter{}
|
||||
var arrayProvider array.Provider = array.Adapter{}
|
||||
var diskProvider disk.Provider = disk.Adapter{}
|
||||
var poolProvider pooldomain.Provider = pooldomain.Adapter{}
|
||||
var shareProvider sharedomain.Provider = sharedomain.Adapter{}
|
||||
if agentReader != nil {
|
||||
hostProvider = agentsource.HostProvider{Reader: agentReader, Windows: agentWindows}
|
||||
processProvider = agentsource.ProcessProvider{Reader: agentReader, Windows: agentWindows}
|
||||
containerProvider = agentsource.ContainerProvider{Reader: agentReader, Windows: agentWindows}
|
||||
arrayProvider = agentsource.ArrayProvider{Reader: agentReader, Windows: agentWindows}
|
||||
diskProvider = agentsource.DiskProvider{Reader: agentReader, Windows: agentWindows}
|
||||
poolProvider = agentsource.PoolProvider{Reader: agentReader, Windows: agentWindows}
|
||||
shareProvider = agentsource.ShareProvider{Reader: agentReader, Windows: agentWindows}
|
||||
}
|
||||
// Applications have no capability of their own: they aggregate the container
|
||||
// inventory with the service probe results, and report Unknown when either input is
|
||||
// missing or stale.
|
||||
applicationProvider := agentsource.ApplicationProvider{Containers: containerProvider, Services: serviceProvider}
|
||||
|
||||
hostHandler := hostapi.Handler{Provider: hostProvider}
|
||||
mux.Handle("/api/v1/host", withSession(sessions, auth.Require(auth.PermissionView, hostHandler)))
|
||||
processHandler := processapi.Handler{Provider: processProvider}
|
||||
mux.Handle("/api/v1/processes", withSession(sessions, auth.Require(auth.PermissionView, processHandler)))
|
||||
containerHandler := containerapi.Handler{Provider: containerProvider}
|
||||
mux.Handle("/api/v1/containers", withSession(sessions, auth.Require(auth.PermissionView, containerHandler)))
|
||||
mux.Handle("/api/v1/containers/", withSession(sessions, auth.Require(auth.PermissionView, containerHandler)))
|
||||
applicationHandler := applicationapi.Handler{Provider: applicationProvider}
|
||||
mux.Handle("/api/v1/applications", withSession(sessions, auth.Require(auth.PermissionView, applicationHandler)))
|
||||
mux.Handle("/api/v1/applications/", withSession(sessions, auth.Require(auth.PermissionView, applicationHandler)))
|
||||
arrayHandler := arrayapi.Handler{Provider: arrayProvider}
|
||||
mux.Handle("/api/v1/array", withSession(sessions, auth.Require(auth.PermissionView, arrayHandler)))
|
||||
diskHandler := diskapi.Handler{Provider: diskProvider}
|
||||
mux.Handle("/api/v1/disks", withSession(sessions, auth.Require(auth.PermissionView, diskHandler)))
|
||||
mux.Handle("/api/v1/disks/", withSession(sessions, auth.Require(auth.PermissionView, diskHandler)))
|
||||
poolHandler := poolapi.Handler{Provider: poolProvider}
|
||||
mux.Handle("/api/v1/pools", withSession(sessions, auth.Require(auth.PermissionView, poolHandler)))
|
||||
mux.Handle("/api/v1/pools/", withSession(sessions, auth.Require(auth.PermissionView, poolHandler)))
|
||||
shareHandler := shareapi.Handler{Provider: shareProvider}
|
||||
mux.Handle("/api/v1/shares", withSession(sessions, auth.Require(auth.PermissionView, shareHandler)))
|
||||
mux.Handle("/api/v1/shares/", withSession(sessions, auth.Require(auth.PermissionView, shareHandler)))
|
||||
|
||||
forecastHandler := forecastapi.Handler{Provider: forecastdomain.StorageProvider{Shares: shareProvider, Pools: poolProvider, History: forecastdomain.PostgresHistory{Pool: pool}, Policy: forecastdomain.Policy{Enabled: true}}}
|
||||
mux.Handle("/api/v1/forecasts", withSession(sessions, auth.Require(auth.PermissionView, forecastHandler)))
|
||||
|
||||
serviceHandler := serviceapi.Handler{Provider: serviceProvider, Dependencies: dependencyRepo, ReverseProxy: reverseProxyProvider}
|
||||
networkHandler := networkapi.Handler{Provider: network.Aggregator{Host: hostProvider, Services: serviceProvider}}
|
||||
mux.Handle("/api/v1/services", withSession(sessions, auth.Require(auth.PermissionView, serviceHandler)))
|
||||
mux.Handle("/api/v1/services/", withSession(sessions, auth.Require(auth.PermissionView, serviceHandler)))
|
||||
mux.Handle("/api/v1/topology", withSession(sessions, auth.Require(auth.PermissionView, serviceHandler)))
|
||||
mux.Handle("/api/v1/network", withSession(sessions, auth.Require(auth.PermissionView, networkHandler)))
|
||||
reverseProxyHandler := reverseproxyapi.Handler{Provider: reverseProxyProvider}
|
||||
mux.Handle("/api/v1/reverse-proxy", withSession(sessions, auth.Require(auth.PermissionView, reverseProxyHandler)))
|
||||
|
||||
server := &http.Server{Addr: runtime.ListenAddress, Handler: observability.Middleware(internalMetrics, correlation.Middleware(mux)), ReadHeaderTimeout: 5 * time.Second}
|
||||
go func() {
|
||||
logger.Info("pulse api listening", "addr", runtime.ListenAddress, "environment", application.Environment)
|
||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
logger.Error("pulse api stopped", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
stopContext, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
go alertcontrol.RunExpiryLoop(stopContext, alertControlStore, time.Minute, logger)
|
||||
service.WaitForStop(stopContext, nil)
|
||||
shutdownContext, cancel := context.WithTimeout(context.Background(), runtime.ShutdownAfter)
|
||||
defer cancel()
|
||||
if err := server.Shutdown(shutdownContext); err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Info("pulse api stopped")
|
||||
return nil
|
||||
}
|
||||
|
||||
func boolMetric(value bool) float64 {
|
||||
if value {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// roleMapping converts the validated configuration mapping of identity provider
|
||||
// group claims onto the typed roles used by the authorization layer. Configuration
|
||||
// already rejects unknown role names, so no further validation is needed here.
|
||||
func roleMapping(configured map[string]string) map[string]auth.Role {
|
||||
if len(configured) == 0 {
|
||||
return nil
|
||||
}
|
||||
mapping := make(map[string]auth.Role, len(configured))
|
||||
for claim, role := range configured {
|
||||
mapping[claim] = auth.Role(role)
|
||||
}
|
||||
return mapping
|
||||
}
|
||||
|
||||
func withSession(manager *auth.SessionManager, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
authentication, ok := manager.AuthenticateSession(response, request, time.Now().UTC())
|
||||
if !ok {
|
||||
next.ServeHTTP(response, request)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithCancel(request.Context())
|
||||
stop := context.AfterFunc(authentication.Context, cancel)
|
||||
defer func() {
|
||||
stop()
|
||||
cancel()
|
||||
}()
|
||||
next.ServeHTTP(response, request.WithContext(auth.WithPrincipal(ctx, authentication.Principal)))
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/live"
|
||||
)
|
||||
|
||||
func TestClearingSessionClosesAuthenticatedLiveConnection(t *testing.T) {
|
||||
manager := auth.NewSlidingSessionManager("pulse_test_session", time.Minute, time.Hour, false)
|
||||
issued := httptest.NewRecorder()
|
||||
if err := manager.Issue(issued, auth.Principal{Subject: "viewer", Role: auth.RoleViewer}, time.Now().UTC()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cookie := issued.Result().Cookies()[0]
|
||||
server := httptest.NewServer(withSession(manager, auth.Require(auth.PermissionView, live.Handler{})))
|
||||
defer server.Close()
|
||||
options := &websocket.DialOptions{HTTPHeader: http.Header{"Cookie": []string{cookie.String()}}}
|
||||
conn, _, err := websocket.Dial(context.Background(), "ws"+server.URL[4:]+"/api/v1/live", options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
clearRequest := httptest.NewRequest(http.MethodPost, "/auth/logout", nil)
|
||||
clearRequest.AddCookie(cookie)
|
||||
manager.Clear(httptest.NewRecorder(), clearRequest)
|
||||
readCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if _, _, err := conn.Read(readCtx); err == nil {
|
||||
t.Fatal("live connection survived session revocation")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentsource"
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/datasource"
|
||||
)
|
||||
|
||||
var requiredUnraidCapabilities = []agentstore.Capability{
|
||||
agentstore.CapabilityHost,
|
||||
agentstore.CapabilityProcesses,
|
||||
agentstore.CapabilityContainers,
|
||||
agentstore.CapabilityArray,
|
||||
agentstore.CapabilityDisks,
|
||||
agentstore.CapabilityPools,
|
||||
agentstore.CapabilityShares,
|
||||
}
|
||||
|
||||
var requiredStorageCapabilities = []agentstore.Capability{
|
||||
agentstore.CapabilityArray,
|
||||
agentstore.CapabilityDisks,
|
||||
agentstore.CapabilityPools,
|
||||
agentstore.CapabilityShares,
|
||||
}
|
||||
|
||||
func readAgentSourceHealth(ctx context.Context, reader agentstore.Reader, now time.Time) (datasource.SourceHealth, datasource.SourceHealth, error) {
|
||||
unraidHealth, err := agentsource.Health(ctx, reader, requiredUnraidCapabilities, agentsource.Windows{}, now)
|
||||
if err != nil {
|
||||
return datasource.SourceHealth{}, datasource.SourceHealth{}, fmt.Errorf("summarize Unraid capabilities: %w", err)
|
||||
}
|
||||
storageHealth, err := agentsource.Health(ctx, reader, requiredStorageCapabilities, agentsource.Windows{}, now)
|
||||
if err != nil {
|
||||
return datasource.SourceHealth{}, datasource.SourceHealth{}, fmt.Errorf("summarize storage capabilities: %w", err)
|
||||
}
|
||||
return unraidHealth, storageHealth, nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentsource"
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/datasource"
|
||||
)
|
||||
|
||||
type sourceHealthReader map[agentstore.Capability]agentstore.Snapshot
|
||||
|
||||
func (reader sourceHealthReader) Latest(_ context.Context, capability agentstore.Capability) (agentstore.Snapshot, error) {
|
||||
snapshot, ok := reader[capability]
|
||||
if !ok {
|
||||
return agentstore.Snapshot{}, agentstore.ErrNoSnapshot
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func TestReadAgentSourceHealthRequiresContainersForHolisticUnraidHealth(t *testing.T) {
|
||||
now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC)
|
||||
reader := sourceHealthReader{}
|
||||
for _, capability := range requiredUnraidCapabilities {
|
||||
reader[capability] = agentstore.Snapshot{Capability: capability, ObservedAt: now.Add(-time.Second), ReceivedAt: now}
|
||||
}
|
||||
stale := reader[agentstore.CapabilityContainers]
|
||||
stale.ObservedAt = now.Add(-time.Hour)
|
||||
reader[agentstore.CapabilityContainers] = stale
|
||||
|
||||
unraidHealth, storageHealth, err := readAgentSourceHealth(context.Background(), reader, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if unraidHealth.State != datasource.HealthUnknown || unraidHealth.ReasonCode != agentsource.ReasonStale {
|
||||
t.Fatalf("partially stale Unraid health = %#v", unraidHealth)
|
||||
}
|
||||
if storageHealth.State != datasource.HealthHealthy {
|
||||
t.Fatalf("fresh storage health = %#v", storageHealth)
|
||||
}
|
||||
|
||||
fresh := reader[agentstore.CapabilityContainers]
|
||||
fresh.ObservedAt = now.Add(-time.Second)
|
||||
reader[agentstore.CapabilityContainers] = fresh
|
||||
unraidHealth, storageHealth, err = readAgentSourceHealth(context.Background(), reader, now)
|
||||
if err != nil || unraidHealth.State != datasource.HealthHealthy || storageHealth.State != datasource.HealthHealthy {
|
||||
t.Fatalf("fully fresh health unraid=%#v storage=%#v err=%v", unraidHealth, storageHealth, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadAgentSourceHealthPropagatesCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, _, err := readAgentSourceHealth(ctx, sourceHealthReader{}, time.Now().UTC())
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancellation error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
databaseURL := os.Getenv("PULSE_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
logger.Error("migration failed", "error", "PULSE_DATABASE_URL is required")
|
||||
os.Exit(1)
|
||||
}
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: databaseURL})
|
||||
if err == nil {
|
||||
err = database.Ping(ctx, pool)
|
||||
}
|
||||
if err == nil {
|
||||
err = database.Migrate(ctx, pool)
|
||||
}
|
||||
if pool != nil {
|
||||
pool.Close()
|
||||
}
|
||||
if err != nil {
|
||||
logger.Error("migration failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
logger.Info("database migrations complete")
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
// Command worker is the ITWorx Pulse background runtime.
|
||||
//
|
||||
// It runs discovery/reconciliation, alert evaluation, service probes and the
|
||||
// notification outbox drain on independent schedules, coordinated with any
|
||||
// other worker through database leases. It is strictly observational
|
||||
// (ADR-0001): it reads sources and writes Pulse's own state, and never mutates
|
||||
// Unraid, Docker, the array or volumes.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"os"
|
||||
"os/signal"
|
||||
"reflect"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentsource"
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/alertworker"
|
||||
"github.com/itworx/pulse/internal/config"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
"github.com/itworx/pulse/internal/discovery"
|
||||
"github.com/itworx/pulse/internal/inventory"
|
||||
"github.com/itworx/pulse/internal/metriccatalog"
|
||||
"github.com/itworx/pulse/internal/metricquery"
|
||||
"github.com/itworx/pulse/internal/notification"
|
||||
"github.com/itworx/pulse/internal/observability"
|
||||
"github.com/itworx/pulse/internal/probe"
|
||||
"github.com/itworx/pulse/internal/prometheus"
|
||||
"github.com/itworx/pulse/internal/queryplan"
|
||||
"github.com/itworx/pulse/internal/runtimeconfig"
|
||||
"github.com/itworx/pulse/internal/servicedefaults"
|
||||
"github.com/itworx/pulse/internal/workerruntime"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// startupTimeout bounds every blocking call made before the scheduling loop
|
||||
// starts, so a slow database cannot hold the process before its first
|
||||
// heartbeat.
|
||||
const startupTimeout = 15 * time.Second
|
||||
|
||||
func main() {
|
||||
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
||||
if err := run(logger); err != nil {
|
||||
logger.Error("pulse worker failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(logger *slog.Logger) error {
|
||||
runtimeConfig, err := runtimeconfig.Load("worker")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
application, err := config.LoadWorker()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(application.DatabaseURL) == "" {
|
||||
return errors.New("PULSE_DATABASE_URL is required: every worker job is database-coordinated")
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
startupCtx, cancelStartup := context.WithTimeout(ctx, startupTimeout)
|
||||
pool, err := database.NewPool(startupCtx, database.Config{URL: application.DatabaseURL, MaxConns: 8, MinConns: 1})
|
||||
if err != nil {
|
||||
cancelStartup()
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Ping(startupCtx, pool); err != nil {
|
||||
cancelStartup()
|
||||
return err
|
||||
}
|
||||
cancelStartup()
|
||||
|
||||
owner := workerOwner()
|
||||
metrics := observability.NewRegistry(time.Now().UTC())
|
||||
jobs, probeJob, err := buildJobs(application, pool, owner, logger)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
runtime, err := workerruntime.New(workerruntime.Config{
|
||||
Owner: owner,
|
||||
Tick: workerruntime.DefaultTick,
|
||||
HeartbeatFile: runtimeConfig.HeartbeatFile,
|
||||
DrainTimeout: runtimeConfig.ShutdownAfter,
|
||||
Leases: workerruntime.PostgresLeaseStore{Pool: pool},
|
||||
Logger: logger,
|
||||
Metrics: metrics,
|
||||
}, jobs...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Info("pulse worker started",
|
||||
"owner", owner, "environment", application.Environment, "jobs", jobNames(jobs),
|
||||
"heartbeat_file", runtimeConfig.HeartbeatFile, "shutdown_timeout", runtimeConfig.ShutdownAfter.String(),
|
||||
"config", application.String())
|
||||
|
||||
runErr := runtime.Run(ctx)
|
||||
|
||||
// Probe execution owns goroutines of its own; give them the same bounded
|
||||
// grace as the scheduler before the process exits.
|
||||
shutdownCtx, cancelShutdown := context.WithTimeout(context.WithoutCancel(ctx), runtimeConfig.ShutdownAfter)
|
||||
defer cancelShutdown()
|
||||
if probeJob != nil {
|
||||
if err := probeJob.Shutdown(shutdownCtx); err != nil {
|
||||
logger.Warn("probe shutdown incomplete", "error", err.Error())
|
||||
}
|
||||
}
|
||||
for _, status := range runtime.Status() {
|
||||
logger.Info("worker job final state", "job", status.Name, "component", status.Component,
|
||||
"last_status", status.LastStatus, "runs", status.Runs, "failures", status.Failures, "skips", status.Skips)
|
||||
}
|
||||
logger.Info("pulse worker stopped")
|
||||
return runErr
|
||||
}
|
||||
|
||||
// buildJobs wires the repositories each job needs. A capability without its
|
||||
// dependencies is scheduled anyway and reports Disabled with a reason, so an
|
||||
// unconfigured feature is visible in system status instead of missing.
|
||||
func buildJobs(application config.Config, pool *pgxpool.Pool, owner string, logger *slog.Logger) ([]workerruntime.Job, *workerruntime.ProbeJob, error) {
|
||||
registry, err := metriccatalog.DefaultRegistry()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
inventoryRepo, err := inventory.NewRepository(pool)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
discoveryStore, err := discovery.NewPostgresStore(pool, owner)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
notificationRepo, err := notification.NewRepository(pool)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
configureCtx, cancelConfigure := context.WithTimeout(context.Background(), startupTimeout)
|
||||
defer cancelConfigure()
|
||||
notificationFactories, err := configureWebhookChannel(configureCtx, application, notificationRepo)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
discoveryJob := workerruntime.DiscoveryJob{
|
||||
SourceID: application.ContainerSourceID,
|
||||
// Reuse the same bounded agent snapshot transport as the public API. A
|
||||
// missing or stale snapshot resolves to Unknown and is skipped without
|
||||
// tombstoning inventory; a fresh snapshot drives idempotent reconciliation.
|
||||
Provider: containerDiscoveryProvider(pool),
|
||||
Aliases: workerruntime.PostgresContainerAliasStore{Pool: pool},
|
||||
Inventory: inventoryRepo,
|
||||
Runner: discovery.Runner{Store: discoveryStore, MaxAttempts: 2, BaseRetry: 250 * time.Millisecond},
|
||||
}
|
||||
|
||||
evaluator := &workerruntime.AlertEvaluator{
|
||||
States: alert.StateRepository{Pool: pool},
|
||||
Prior: workerruntime.PostgresAlertStateReader{Pool: pool},
|
||||
Versions: alert.Repository{Pool: pool, Registry: registry},
|
||||
Notifications: notificationRepo,
|
||||
Logger: logger,
|
||||
}
|
||||
alertJob := workerruntime.AlertEvaluationJob{Reason: "metric_source_not_configured"}
|
||||
if application.PrometheusURL != "" {
|
||||
source, sourceErr := prometheus.New(application.PrometheusURL, nil, prometheus.Limits{Timeout: application.PrometheusTimeout})
|
||||
if sourceErr != nil {
|
||||
return nil, nil, sourceErr
|
||||
}
|
||||
planner := queryplan.NewPlanner(registry, queryplan.Limits{})
|
||||
evaluator.Metrics = workerruntime.PrometheusMetricSource{Service: metricquery.NewService(planner, source, nil)}
|
||||
worker, workerErr := alertworker.New(alert.Repository{Pool: pool, Registry: registry}, alertworker.PostgresLeaseStore{Pool: pool}, evaluator, alertworker.Config{
|
||||
MaxConcurrent: 8, MaxBatch: workerruntime.MaxAlertRules, AttemptTimeout: 15 * time.Second, LeaseTTL: 2 * time.Minute, Owner: owner, Now: time.Now,
|
||||
})
|
||||
if workerErr != nil {
|
||||
return nil, nil, workerErr
|
||||
}
|
||||
alertJob = workerruntime.AlertEvaluationJob{Worker: &worker, Enabled: true}
|
||||
}
|
||||
|
||||
policy, err := probePolicy(application)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
serviceSummary, err := servicedefaults.Seed(configureCtx, pool, servicedefaults.Options{
|
||||
PublicURL: application.PublicURL, OIDCIssuer: application.OIDCIssuer,
|
||||
}, policy, probe.NetResolver{})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("configure system service monitoring: %w", err)
|
||||
}
|
||||
if serviceSummary.Services > 0 {
|
||||
logger.Info("system service monitoring configured", "services", serviceSummary.Services,
|
||||
"endpoints", serviceSummary.Endpoints, "probes", serviceSummary.Probes, "dependencies", serviceSummary.Dependencies)
|
||||
}
|
||||
probeJob, err := workerruntime.NewProbeJob(workerruntime.PostgresProbeStore{Pool: pool}, policy, logger)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
notificationJob := workerruntime.NotificationDrainJob{
|
||||
Store: notificationRepo,
|
||||
Channels: notificationRepo,
|
||||
Senders: map[string]notification.ChannelSender{},
|
||||
Factories: notificationFactories,
|
||||
Logger: logger,
|
||||
}
|
||||
|
||||
jobs := workerruntime.Schedule(workerruntime.ScheduleRuns{
|
||||
Discovery: discoveryJob.Run,
|
||||
AlertEvaluation: alertJob.Run,
|
||||
ProbeExecution: probeJob.Run,
|
||||
NotificationDrain: notificationJob.Run,
|
||||
})
|
||||
return jobs, probeJob, nil
|
||||
}
|
||||
|
||||
func containerDiscoveryProvider(pool *pgxpool.Pool) container.Provider {
|
||||
return agentsource.ContainerProvider{
|
||||
Reader: agentstore.PostgresStore{Pool: pool},
|
||||
Windows: agentsource.Windows{},
|
||||
}
|
||||
}
|
||||
|
||||
func configureWebhookChannel(ctx context.Context, application config.Config, repository notification.ChannelStore) (map[string]notification.ChannelSenderFactory, error) {
|
||||
if repository == nil {
|
||||
return nil, notification.ErrUnavailable
|
||||
}
|
||||
current, getErr := repository.GetChannel(ctx, notification.DefaultWebhookChannelID)
|
||||
if application.NotificationWebhookURL == "" {
|
||||
if errors.Is(getErr, notification.ErrNotFound) {
|
||||
return map[string]notification.ChannelSenderFactory{}, nil
|
||||
}
|
||||
if getErr != nil {
|
||||
return nil, fmt.Errorf("read system webhook channel: %w", getErr)
|
||||
}
|
||||
if current.Enabled {
|
||||
current.Enabled = false
|
||||
if _, err := repository.UpdateChannel(ctx, current, current.Revision); err != nil {
|
||||
return nil, fmt.Errorf("disable system webhook channel: %w", err)
|
||||
}
|
||||
}
|
||||
return map[string]notification.ChannelSenderFactory{}, nil
|
||||
}
|
||||
desired := notification.Channel{
|
||||
ID: notification.DefaultWebhookChannelID, Name: "Pulse webhook", Type: "webhook", Enabled: true,
|
||||
SecretRef: notification.SecretRef{ID: notification.WebhookSecretReference},
|
||||
Configuration: map[string]any{"url": application.NotificationWebhookURL, "timeoutSeconds": application.NotificationWebhookTimeout.Seconds()},
|
||||
Revision: 1,
|
||||
}
|
||||
if errors.Is(getErr, notification.ErrNotFound) {
|
||||
if _, err := repository.CreateChannel(ctx, desired); err != nil {
|
||||
return nil, fmt.Errorf("create system webhook channel: %w", err)
|
||||
}
|
||||
} else if getErr != nil {
|
||||
return nil, fmt.Errorf("read system webhook channel: %w", getErr)
|
||||
} else if current.Name != desired.Name || current.Type != desired.Type || !current.Enabled || current.SecretRef != desired.SecretRef || !reflect.DeepEqual(current.Configuration, desired.Configuration) {
|
||||
desired.Revision = current.Revision
|
||||
if _, err := repository.UpdateChannel(ctx, desired, current.Revision); err != nil {
|
||||
return nil, fmt.Errorf("update system webhook channel: %w", err)
|
||||
}
|
||||
}
|
||||
resolver := notification.SecretResolverFunc(func(_ context.Context, ref notification.SecretRef) (string, error) {
|
||||
if ref.ID != notification.WebhookSecretReference {
|
||||
return "", notification.ErrNotFound
|
||||
}
|
||||
return application.NotificationWebhookToken, nil
|
||||
})
|
||||
return map[string]notification.ChannelSenderFactory{
|
||||
"webhook": notification.WebhookFactory{
|
||||
Secrets: resolver,
|
||||
AllowHTTP: application.Environment != config.Production,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// probePolicy builds the probe network policy. Only administrator-configured
|
||||
// private ranges are added to the allowlist; every other protection in
|
||||
// internal/probe/policy.go keeps its default, so link-local, multicast, cloud
|
||||
// metadata and unlisted private addresses stay blocked.
|
||||
func probePolicy(application config.Config) (probe.NetworkPolicy, error) {
|
||||
policy := probe.NetworkPolicy{}
|
||||
for _, entry := range application.ProbeAllowedNetworks {
|
||||
prefix, err := netip.ParsePrefix(entry)
|
||||
if err != nil {
|
||||
return probe.NetworkPolicy{}, fmt.Errorf("probe allowlist entry %q is invalid", entry)
|
||||
}
|
||||
policy.AllowedNetworks = append(policy.AllowedNetworks, prefix)
|
||||
}
|
||||
if err := policy.Validate(); err != nil {
|
||||
return probe.NetworkPolicy{}, err
|
||||
}
|
||||
return policy, nil
|
||||
}
|
||||
|
||||
// workerOwner identifies this process in job_runs.lease_owner. It contains no
|
||||
// secret and stays stable for the lifetime of the process.
|
||||
func workerOwner() string {
|
||||
host, err := os.Hostname()
|
||||
if err != nil || strings.TrimSpace(host) == "" {
|
||||
host = "worker"
|
||||
}
|
||||
owner := fmt.Sprintf("%s/%d", host, os.Getpid())
|
||||
if len(owner) > 120 {
|
||||
owner = owner[:120]
|
||||
}
|
||||
return owner
|
||||
}
|
||||
|
||||
func jobNames(jobs []workerruntime.Job) []string {
|
||||
names := make([]string, 0, len(jobs))
|
||||
for _, job := range jobs {
|
||||
names = append(names, job.Name+"@"+job.Interval.String())
|
||||
}
|
||||
return names
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentsource"
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/config"
|
||||
"github.com/itworx/pulse/internal/notification"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type channelStoreStub struct {
|
||||
channels map[string]notification.Channel
|
||||
creates int
|
||||
updates int
|
||||
}
|
||||
|
||||
func TestContainerDiscoveryProviderUsesAgentSnapshotStore(t *testing.T) {
|
||||
pool := &pgxpool.Pool{}
|
||||
provider, ok := containerDiscoveryProvider(pool).(agentsource.ContainerProvider)
|
||||
if !ok {
|
||||
t.Fatalf("container discovery provider = %T, want agentsource.ContainerProvider", containerDiscoveryProvider(pool))
|
||||
}
|
||||
store, ok := provider.Reader.(agentstore.PostgresStore)
|
||||
if !ok || store.Pool != pool {
|
||||
t.Fatalf("container discovery reader = %#v, want PostgresStore with worker pool", provider.Reader)
|
||||
}
|
||||
}
|
||||
|
||||
func (store *channelStoreStub) CreateChannel(_ context.Context, channel notification.Channel) (notification.Channel, error) {
|
||||
if _, exists := store.channels[channel.ID]; exists {
|
||||
return notification.Channel{}, notification.ErrConflict
|
||||
}
|
||||
store.creates++
|
||||
channel.Revision = 1
|
||||
store.channels[channel.ID] = channel
|
||||
return channel, nil
|
||||
}
|
||||
func (store *channelStoreStub) GetChannel(_ context.Context, id string) (notification.Channel, error) {
|
||||
channel, exists := store.channels[id]
|
||||
if !exists {
|
||||
return notification.Channel{}, notification.ErrNotFound
|
||||
}
|
||||
return channel, nil
|
||||
}
|
||||
func (store *channelStoreStub) ListChannels(context.Context, int) ([]notification.Channel, error) {
|
||||
channels := make([]notification.Channel, 0, len(store.channels))
|
||||
for _, channel := range store.channels {
|
||||
channels = append(channels, channel)
|
||||
}
|
||||
return channels, nil
|
||||
}
|
||||
func (store *channelStoreStub) UpdateChannel(_ context.Context, channel notification.Channel, expected int64) (notification.Channel, error) {
|
||||
current, exists := store.channels[channel.ID]
|
||||
if !exists {
|
||||
return notification.Channel{}, notification.ErrNotFound
|
||||
}
|
||||
if current.Revision != expected {
|
||||
return notification.Channel{}, notification.ErrConflict
|
||||
}
|
||||
store.updates++
|
||||
channel.Revision = expected + 1
|
||||
store.channels[channel.ID] = channel
|
||||
return channel, nil
|
||||
}
|
||||
func (store *channelStoreStub) DeleteChannel(_ context.Context, id string) error {
|
||||
if _, exists := store.channels[id]; !exists {
|
||||
return notification.ErrNotFound
|
||||
}
|
||||
delete(store.channels, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestConfigureWebhookChannelReconcilesWithoutRevisionChurn(t *testing.T) {
|
||||
store := &channelStoreStub{channels: map[string]notification.Channel{}}
|
||||
application := config.Config{
|
||||
Environment: config.Development, NotificationWebhookURL: "http://127.0.0.1:18080/pulse",
|
||||
NotificationWebhookToken: "runtime-only", NotificationWebhookTimeout: 3 * time.Second,
|
||||
}
|
||||
factories, err := configureWebhookChannel(context.Background(), application, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
channel := store.channels[notification.DefaultWebhookChannelID]
|
||||
if store.creates != 1 || store.updates != 0 || !channel.Enabled || channel.SecretRef.ID != notification.WebhookSecretReference {
|
||||
t.Fatalf("channel=%+v creates=%d updates=%d", channel, store.creates, store.updates)
|
||||
}
|
||||
if _, exists := channel.Configuration["token"]; exists {
|
||||
t.Fatal("runtime credential entered persistent configuration")
|
||||
}
|
||||
if _, err := factories["webhook"].Sender(context.Background(), channel); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := configureWebhookChannel(context.Background(), application, store); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if store.creates != 1 || store.updates != 0 {
|
||||
t.Fatalf("idempotent reconciliation created=%d updated=%d", store.creates, store.updates)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureWebhookChannelDisablesRemovedRuntimeConfiguration(t *testing.T) {
|
||||
store := &channelStoreStub{channels: map[string]notification.Channel{
|
||||
notification.DefaultWebhookChannelID: {
|
||||
ID: notification.DefaultWebhookChannelID, Name: "Pulse webhook", Type: "webhook", Enabled: true,
|
||||
SecretRef: notification.SecretRef{ID: notification.WebhookSecretReference}, Revision: 4,
|
||||
},
|
||||
}}
|
||||
factories, err := configureWebhookChannel(context.Background(), config.Config{}, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(factories) != 0 || store.updates != 1 || store.channels[notification.DefaultWebhookChannelID].Enabled {
|
||||
t.Fatalf("factories=%v updates=%d channel=%+v", factories, store.updates, store.channels[notification.DefaultWebhookChannelID])
|
||||
}
|
||||
if _, err := configureWebhookChannel(context.Background(), config.Config{}, nil); !errors.Is(err, notification.ErrUnavailable) {
|
||||
t.Fatalf("nil repository error=%v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user