This commit is contained in:
@@ -0,0 +1,289 @@
|
||||
// Package agentsource turns the bounded snapshots pulse-agent writes through
|
||||
// internal/agentstore into the normalized domain snapshots pulse-api serves.
|
||||
//
|
||||
// Every provider in this package follows the same three steps: read the newest snapshot
|
||||
// for one capability, decide whether it is usable, and normalize it through the domain's
|
||||
// own Adapter. Only the middle step is interesting, and it is the ADR-0008 enforcement
|
||||
// point for the whole read path: telemetry that is missing, stale, or undecodable
|
||||
// resolves to that domain's Unknown snapshot and never to Healthy.
|
||||
//
|
||||
// The freshness decision itself is delegated to internal/freshness so this package does
|
||||
// not become another private copy of the rule. freshness.Evaluate is called with the
|
||||
// snapshot presented as a required, currently healthy datasource.SourceHealth; if it
|
||||
// answers with anything other than Healthy the snapshot is discarded as stale. That keeps
|
||||
// one tested implementation of "missing telemetry never becomes Healthy" in the codebase.
|
||||
//
|
||||
// # Reason codes
|
||||
//
|
||||
// A provider that cannot serve real telemetry reports exactly one machine-readable reason
|
||||
// from this closed set, which the web app maps onto localized copy:
|
||||
//
|
||||
// source_unavailable — nothing has been recorded for the capability yet, or the store
|
||||
// could not be read at all.
|
||||
// source_stale — the newest snapshot is older than the capability's freshness
|
||||
// window, or its timestamps are not usable.
|
||||
// source_invalid — the payload could not be decoded into the domain's RawSnapshot,
|
||||
// or the domain rejected it during normalization.
|
||||
//
|
||||
// # Error handling
|
||||
//
|
||||
// Context cancellation and deadlines propagate to the caller. Any other store failure is
|
||||
// reported as an Unknown snapshot with source_unavailable rather than an error: a
|
||||
// monitoring surface that says explicitly "this source is unavailable" is more useful to
|
||||
// an operator than a bare 503, and it keeps a single database hiccup from taking every
|
||||
// monitoring page down at once.
|
||||
package agentsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/datasource"
|
||||
"github.com/itworx/pulse/internal/freshness"
|
||||
)
|
||||
|
||||
// Reason codes reported when a provider cannot serve real telemetry. The set is closed;
|
||||
// see the package documentation for the meaning of each code.
|
||||
const (
|
||||
ReasonUnavailable = "source_unavailable"
|
||||
ReasonStale = "source_stale"
|
||||
ReasonInvalid = "source_invalid"
|
||||
)
|
||||
|
||||
// Default freshness windows per capability. They mirror the freshness policy defaults of
|
||||
// the domain packages, so a snapshot that passes this gate is never marked stale again
|
||||
// further down the pipeline, and they follow the agent's sampling cadence: the host is
|
||||
// sampled every few seconds, processes and containers roughly every half minute, and
|
||||
// share usage is an expensive scan that runs far less often.
|
||||
const (
|
||||
DefaultHostWindow = 30 * time.Second
|
||||
DefaultProcessesWindow = 60 * time.Second
|
||||
DefaultContainersWindow = 60 * time.Second
|
||||
DefaultArrayWindow = 60 * time.Second
|
||||
DefaultDisksWindow = 60 * time.Second
|
||||
DefaultPoolsWindow = 60 * time.Second
|
||||
DefaultSharesWindow = 2 * time.Minute
|
||||
)
|
||||
|
||||
// Windows configures how old a snapshot may be before its capability resolves to Unknown.
|
||||
// A zero field takes the documented default for that capability.
|
||||
type Windows struct {
|
||||
Host time.Duration
|
||||
Processes time.Duration
|
||||
Containers time.Duration
|
||||
Array time.Duration
|
||||
Disks time.Duration
|
||||
Pools time.Duration
|
||||
Shares time.Duration
|
||||
}
|
||||
|
||||
// WithDefaults fills every unset window with its documented default.
|
||||
func (w Windows) WithDefaults() Windows {
|
||||
if w.Host == 0 {
|
||||
w.Host = DefaultHostWindow
|
||||
}
|
||||
if w.Processes == 0 {
|
||||
w.Processes = DefaultProcessesWindow
|
||||
}
|
||||
if w.Containers == 0 {
|
||||
w.Containers = DefaultContainersWindow
|
||||
}
|
||||
if w.Array == 0 {
|
||||
w.Array = DefaultArrayWindow
|
||||
}
|
||||
if w.Disks == 0 {
|
||||
w.Disks = DefaultDisksWindow
|
||||
}
|
||||
if w.Pools == 0 {
|
||||
w.Pools = DefaultPoolsWindow
|
||||
}
|
||||
if w.Shares == 0 {
|
||||
w.Shares = DefaultSharesWindow
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// Validate reports whether every configured window is inside safe bounds. Windows are
|
||||
// bounded above at 24 hours because datasource.FreshnessPolicy refuses anything longer:
|
||||
// a source nobody has heard from for a day is not fresh under any reading.
|
||||
func (w Windows) Validate() error {
|
||||
for capability, window := range w.WithDefaults().byCapability() {
|
||||
if window <= 0 || window > 24*time.Hour {
|
||||
return fmt.Errorf("freshness window for %q is outside safe bounds", capability)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// For returns the freshness window for one capability, applying defaults.
|
||||
func (w Windows) For(capability agentstore.Capability) time.Duration {
|
||||
if window, ok := w.WithDefaults().byCapability()[capability]; ok {
|
||||
return window
|
||||
}
|
||||
// An unrecognized capability cannot be served at all; the tightest window keeps a
|
||||
// caller that ignores that from treating anything as fresh.
|
||||
return DefaultHostWindow
|
||||
}
|
||||
|
||||
func (w Windows) byCapability() map[agentstore.Capability]time.Duration {
|
||||
return map[agentstore.Capability]time.Duration{
|
||||
agentstore.CapabilityHost: w.Host,
|
||||
agentstore.CapabilityProcesses: w.Processes,
|
||||
agentstore.CapabilityContainers: w.Containers,
|
||||
agentstore.CapabilityArray: w.Array,
|
||||
agentstore.CapabilityDisks: w.Disks,
|
||||
agentstore.CapabilityPools: w.Pools,
|
||||
agentstore.CapabilityShares: w.Shares,
|
||||
}
|
||||
}
|
||||
|
||||
// staticRaw presents an already decoded RawSnapshot as the raw source interface each
|
||||
// domain Adapter expects, so normalization keeps running through the domain's own code.
|
||||
type staticRaw[R any] struct{ raw R }
|
||||
|
||||
func (s staticRaw[R]) Snapshot(context.Context) (R, error) { return s.raw, nil }
|
||||
|
||||
// resolve implements the shared read path. It is generic over the domain's RawSnapshot
|
||||
// and normalized Snapshot so every capability enforces the same rules in the same order.
|
||||
func resolve[R any, S any](
|
||||
ctx context.Context,
|
||||
reader agentstore.Reader,
|
||||
capability agentstore.Capability,
|
||||
window time.Duration,
|
||||
now time.Time,
|
||||
unknown func(time.Time, string) S,
|
||||
normalize func(R, time.Time) (S, error),
|
||||
) (S, error) {
|
||||
var zero S
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, err
|
||||
}
|
||||
if reader == nil {
|
||||
return unknown(now, ReasonUnavailable), nil
|
||||
}
|
||||
stored, err := reader.Latest(ctx, capability)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return zero, err
|
||||
}
|
||||
return unknown(now, ReasonUnavailable), nil
|
||||
}
|
||||
if reason := usableReason(string(capability), stored, window, now); reason != "" {
|
||||
return unknown(now, reason), nil
|
||||
}
|
||||
var raw R
|
||||
if err := json.Unmarshal(stored.Payload, &raw); err != nil {
|
||||
return unknown(now, ReasonInvalid), nil
|
||||
}
|
||||
snapshot, err := normalize(raw, now)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return zero, err
|
||||
}
|
||||
return unknown(now, ReasonInvalid), nil
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
// usableReason returns an empty string when the snapshot may be normalized, or the reason
|
||||
// code that must be reported instead. The freshness judgement is made by
|
||||
// internal/freshness rather than re-derived here.
|
||||
func usableReason(sourceID string, stored agentstore.Snapshot, window time.Duration, now time.Time) string {
|
||||
result, err := freshness.Evaluate(freshness.Input{
|
||||
SourceID: sourceID,
|
||||
Required: true,
|
||||
Now: now,
|
||||
Health: datasource.SourceHealth{
|
||||
State: datasource.HealthHealthy,
|
||||
ObservedAt: stored.ObservedAt,
|
||||
ReceivedAt: stored.ReceivedAt,
|
||||
LastSuccess: stored.ObservedAt,
|
||||
Policy: datasource.FreshnessPolicy{MaxAge: window},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
// Unusable timestamps (missing, or materially in the future) are indistinguishable
|
||||
// from staleness for a consumer: in both cases the observation cannot be trusted
|
||||
// to describe the present.
|
||||
return ReasonStale
|
||||
}
|
||||
if result.State != datasource.HealthHealthy {
|
||||
return ReasonStale
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// clockNow resolves the injected clock, defaulting to the wall clock in UTC.
|
||||
func clockNow(clock func() time.Time) time.Time {
|
||||
if clock == nil {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
return clock().UTC()
|
||||
}
|
||||
|
||||
func fixedClock(now time.Time) func() time.Time { return func() time.Time { return now } }
|
||||
|
||||
// Health summarizes a bounded set of agent capabilities into one datasource health
|
||||
// observation. Every requested capability is required: a source is only Healthy when
|
||||
// each capability has a recent snapshot. This is used by self-observability so it reads
|
||||
// the same persisted transport as the domain providers instead of inferring agent
|
||||
// configuration from API process environment variables.
|
||||
func Health(ctx context.Context, reader agentstore.Reader, capabilities []agentstore.Capability, windows Windows, now time.Time) (datasource.SourceHealth, error) {
|
||||
now = now.UTC()
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
health := datasource.SourceHealth{
|
||||
State: datasource.HealthUnknown,
|
||||
ReceivedAt: now,
|
||||
Policy: datasource.FreshnessPolicy{MaxAge: DefaultHostWindow},
|
||||
ReasonCode: ReasonUnavailable,
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return datasource.SourceHealth{}, err
|
||||
}
|
||||
if reader == nil || len(capabilities) == 0 {
|
||||
return health, nil
|
||||
}
|
||||
for _, capability := range capabilities {
|
||||
if window := windows.For(capability); window > health.Policy.MaxAge {
|
||||
health.Policy.MaxAge = window
|
||||
}
|
||||
}
|
||||
oldestObserved := now
|
||||
oldestReceived := now
|
||||
for _, capability := range capabilities {
|
||||
if !capability.Valid() {
|
||||
return datasource.SourceHealth{}, fmt.Errorf("summarize unknown agent capability %q", capability)
|
||||
}
|
||||
stored, err := reader.Latest(ctx, capability)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return datasource.SourceHealth{}, err
|
||||
}
|
||||
return health, nil
|
||||
}
|
||||
if reason := usableReason(string(capability), stored, windows.For(capability), now); reason != "" {
|
||||
health.ObservedAt = stored.ObservedAt.UTC()
|
||||
health.ReceivedAt = stored.ReceivedAt.UTC()
|
||||
health.ReasonCode = reason
|
||||
return health, nil
|
||||
}
|
||||
if stored.ObservedAt.Before(oldestObserved) {
|
||||
oldestObserved = stored.ObservedAt.UTC()
|
||||
}
|
||||
if stored.ReceivedAt.Before(oldestReceived) {
|
||||
oldestReceived = stored.ReceivedAt.UTC()
|
||||
}
|
||||
}
|
||||
health.State = datasource.HealthHealthy
|
||||
health.ObservedAt = oldestObserved
|
||||
health.ReceivedAt = oldestReceived
|
||||
health.LastSuccess = oldestObserved
|
||||
health.ReasonCode = "source_sampled"
|
||||
return health, nil
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package agentsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/application"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/service"
|
||||
)
|
||||
|
||||
// ApplicationProvider derives application health from two existing surfaces rather than
|
||||
// from a capability of its own: the container inventory the agent records, and the
|
||||
// service probe results the API already stores. Containers are grouped into applications
|
||||
// by their Compose project (falling back to the container name for a standalone
|
||||
// container), and a probe result for a service with the same name refines the component
|
||||
// status of the matching container.
|
||||
//
|
||||
// Fresh container evidence is required. Service probes refine matching
|
||||
// components when configured; their absence does not erase container-runtime
|
||||
// availability, because those are separate claims and surfaces.
|
||||
type ApplicationProvider struct {
|
||||
Containers container.Provider
|
||||
Services service.Provider
|
||||
// MaxApplications bounds the number of groups built before the payload is refused.
|
||||
// Zero takes DefaultMaxApplications.
|
||||
MaxApplications int
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ application.Provider = ApplicationProvider{}
|
||||
|
||||
// DefaultMaxApplications matches the bound application.BuildSnapshot enforces, so an
|
||||
// oversized inventory is reported as Unknown instead of failing the request.
|
||||
const DefaultMaxApplications = 150
|
||||
|
||||
func (p ApplicationProvider) Snapshot(ctx context.Context) (application.Snapshot, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return application.Snapshot{}, err
|
||||
}
|
||||
now := clockNow(p.Now)
|
||||
unknown := func(reason string) application.Snapshot {
|
||||
return application.UnknownSnapshot(now, applicationSources, agentSourceType, reason)
|
||||
}
|
||||
if p.Containers == nil {
|
||||
return unknown(ReasonUnavailable), nil
|
||||
}
|
||||
containers, err := p.Containers.Snapshot(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return application.Snapshot{}, err
|
||||
}
|
||||
return unknown(ReasonUnavailable), nil
|
||||
}
|
||||
if reason := unusableContainerReason(containers); reason != "" {
|
||||
return unknown(reason), nil
|
||||
}
|
||||
// Service probes refine application health when configured, but are not an
|
||||
// identity prerequisite. Fresh container state is complete evidence for
|
||||
// runtime availability; an empty/disabled probe module must not erase every
|
||||
// discovered application from the product.
|
||||
services := service.Snapshot{ContractVersion: service.ContractVersion, ObservedAt: containers.Source.ObservedAt}
|
||||
if p.Services != nil {
|
||||
observed, serviceErr := p.Services.Snapshot(ctx)
|
||||
if serviceErr != nil {
|
||||
if errors.Is(serviceErr, context.Canceled) || errors.Is(serviceErr, context.DeadlineExceeded) {
|
||||
return application.Snapshot{}, serviceErr
|
||||
}
|
||||
} else if strings.TrimSpace(observed.Reason) == "" {
|
||||
services = observed
|
||||
}
|
||||
}
|
||||
maximum := p.MaxApplications
|
||||
if maximum <= 0 {
|
||||
maximum = DefaultMaxApplications
|
||||
}
|
||||
discovered := groupApplications(containers, services, maximum)
|
||||
if discovered == nil {
|
||||
return unknown(ReasonInvalid), nil
|
||||
}
|
||||
source := application.Source{
|
||||
ID: applicationSources,
|
||||
Type: agentSourceType,
|
||||
ObservedAt: containers.Source.ObservedAt,
|
||||
ReceivedAt: containers.Source.ReceivedAt,
|
||||
}
|
||||
snapshot, err := application.BuildSnapshot(source, discovered, nil, now)
|
||||
if err != nil {
|
||||
return unknown(ReasonInvalid), nil
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
// unusableContainerReason maps the container source state onto an application reason code.
|
||||
// The container provider has already applied the freshness rule, so an Unknown container
|
||||
// source here means the same thing for applications.
|
||||
func unusableContainerReason(snapshot container.Snapshot) string {
|
||||
if snapshot.Source.State == "" || snapshot.Source.State == "unknown" {
|
||||
switch snapshot.Source.Reason {
|
||||
case ReasonStale, "stale_source":
|
||||
return ReasonStale
|
||||
case ReasonInvalid:
|
||||
return ReasonInvalid
|
||||
default:
|
||||
return ReasonUnavailable
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// groupApplications builds one discovered application per Compose project. It returns nil
|
||||
// when the inventory exceeds the configured bound.
|
||||
func groupApplications(containers container.Snapshot, services service.Snapshot, maximum int) []application.DiscoveredApplication {
|
||||
states := make(map[string]application.State, len(services.Services))
|
||||
for _, item := range services.Services {
|
||||
states[normalizeKey(item.Name)] = serviceState(item.State)
|
||||
}
|
||||
// application.evaluateComponent normalizes an empty service state to Unknown, so a
|
||||
// component with no probe would drag its application to Unknown even though the
|
||||
// container telemetry is fresh. There is no probe constraint on such a component, and
|
||||
// Healthy is the neutral element of the domain's aggregation: it leaves the component
|
||||
// status equal to the container status instead of inventing an unknown.
|
||||
stateFor := func(name string) application.State {
|
||||
if state, ok := states[normalizeKey(name)]; ok {
|
||||
return state
|
||||
}
|
||||
return application.StateHealthy
|
||||
}
|
||||
order := make([]string, 0, len(containers.Containers))
|
||||
groups := make(map[string]*application.DiscoveredApplication, len(containers.Containers))
|
||||
for _, item := range containers.Containers {
|
||||
key := strings.TrimSpace(item.Project)
|
||||
if key == "" {
|
||||
key = strings.TrimPrefix(strings.TrimSpace(item.Name), "/")
|
||||
}
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
group, seen := groups[key]
|
||||
if !seen {
|
||||
if len(order) >= maximum {
|
||||
return nil
|
||||
}
|
||||
group = &application.DiscoveredApplication{ID: application.StableApplicationID(application.SourceID, key), Name: key}
|
||||
groups[key] = group
|
||||
order = append(order, key)
|
||||
}
|
||||
group.Components = append(group.Components, application.ComponentInput{
|
||||
ID: item.ID,
|
||||
Name: strings.TrimPrefix(item.Name, "/"),
|
||||
Kind: "container",
|
||||
ContainerState: containerState(item),
|
||||
ServiceState: stateFor(item.Name),
|
||||
// Every discovered component is critical until an operator overrides it;
|
||||
// treating an unmapped component as optional would hide real failures.
|
||||
Critical: true,
|
||||
})
|
||||
}
|
||||
discovered := make([]application.DiscoveredApplication, 0, len(order))
|
||||
for _, key := range order {
|
||||
discovered = append(discovered, *groups[key])
|
||||
}
|
||||
return discovered
|
||||
}
|
||||
|
||||
// containerState maps a normalized container onto the application state vocabulary. A
|
||||
// container the operator stopped on purpose is not a failure; anything the collector
|
||||
// could not classify is Unknown rather than Healthy.
|
||||
func containerState(item container.Container) application.State {
|
||||
switch strings.ToLower(strings.TrimSpace(item.State)) {
|
||||
case "running":
|
||||
switch strings.ToLower(strings.TrimSpace(item.Health)) {
|
||||
case "healthy":
|
||||
return application.StateHealthy
|
||||
case "unhealthy":
|
||||
return application.StateDegraded
|
||||
case "starting", "unknown", "":
|
||||
return application.StateUnknown
|
||||
default:
|
||||
return application.StateUnknown
|
||||
}
|
||||
case "restarting", "paused", "removing":
|
||||
return application.StateDegraded
|
||||
case "exited", "dead", "stopped":
|
||||
if item.IntentionalStop {
|
||||
return application.StateUnknown
|
||||
}
|
||||
return application.StateDown
|
||||
default:
|
||||
return application.StateUnknown
|
||||
}
|
||||
}
|
||||
|
||||
// serviceState maps a probe verdict onto the application state vocabulary.
|
||||
func serviceState(state string) application.State {
|
||||
switch strings.ToLower(strings.TrimSpace(state)) {
|
||||
case service.StateUp:
|
||||
return application.StateHealthy
|
||||
case service.StateDegraded:
|
||||
return application.StateDegraded
|
||||
case service.StateDown:
|
||||
return application.StateDown
|
||||
default:
|
||||
return application.StateUnknown
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeKey(value string) string {
|
||||
return strings.ToLower(strings.TrimPrefix(strings.TrimSpace(value), "/"))
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package agentsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/application"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/service"
|
||||
)
|
||||
|
||||
type stubContainers struct {
|
||||
snapshot container.Snapshot
|
||||
err error
|
||||
}
|
||||
|
||||
func (s stubContainers) Snapshot(context.Context) (container.Snapshot, error) {
|
||||
return s.snapshot, s.err
|
||||
}
|
||||
|
||||
type stubServices struct {
|
||||
snapshot service.Snapshot
|
||||
err error
|
||||
}
|
||||
|
||||
func (s stubServices) Snapshot(context.Context) (service.Snapshot, error) { return s.snapshot, s.err }
|
||||
|
||||
func containerSnapshot(t *testing.T, now time.Time, containers ...container.RawContainer) container.Snapshot {
|
||||
t.Helper()
|
||||
snapshot, err := container.Normalize(container.RawSnapshot{
|
||||
Source: container.Source{ID: "container", Type: "agent"},
|
||||
Containers: containers,
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}, now, container.Limits{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func TestApplicationProviderRequiresFreshContainers(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
services := service.Snapshot{ContractVersion: service.ContractVersion, ObservedAt: now, Services: []service.ServiceStatus{}}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
containers container.Provider
|
||||
services service.Provider
|
||||
reason string
|
||||
}{
|
||||
{"no container provider", nil, stubServices{snapshot: services}, ReasonUnavailable},
|
||||
{"container read fails", stubContainers{err: errors.New("boom")}, stubServices{snapshot: services}, ReasonUnavailable},
|
||||
{"containers unavailable", stubContainers{snapshot: container.UnknownSnapshot(now, "container", "agent", ReasonUnavailable)}, stubServices{snapshot: services}, ReasonUnavailable},
|
||||
{"containers invalid", stubContainers{snapshot: container.UnknownSnapshot(now, "container", "agent", ReasonInvalid)}, stubServices{snapshot: services}, ReasonInvalid},
|
||||
{"containers stale", stubContainers{snapshot: container.UnknownSnapshot(now, "container", "agent", ReasonStale)}, stubServices{snapshot: services}, ReasonStale},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
provider := ApplicationProvider{Containers: testCase.containers, Services: testCase.services, Now: fixedClock(now)}
|
||||
snapshot, err := provider.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unusable input must not fail the request: %v", err)
|
||||
}
|
||||
if snapshot.Source.State != "unknown" || snapshot.Source.Reason != testCase.reason {
|
||||
t.Fatalf("source = %+v, want unknown/%s", snapshot.Source, testCase.reason)
|
||||
}
|
||||
for _, item := range snapshot.Applications {
|
||||
if item.Status == application.StateHealthy {
|
||||
t.Fatal("an application became healthy without complete evidence, violating ADR-0008")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationProviderUsesContainersWhenProbesAreDisabled(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
containers := containerSnapshot(t, now, container.RawContainer{ID: "pulse-1", Name: "pulse-api", State: "running", Health: "healthy"})
|
||||
for name, services := range map[string]service.Provider{
|
||||
"not configured": nil,
|
||||
"unavailable": stubServices{snapshot: service.UnknownSnapshot(now, "source_unavailable")},
|
||||
"read failure": stubServices{err: errors.New("probe store unavailable")},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
snapshot, err := (ApplicationProvider{Containers: stubContainers{snapshot: containers}, Services: services, Now: fixedClock(now)}).Snapshot(context.Background())
|
||||
if err != nil || snapshot.Total != 1 || snapshot.Applications[0].Status != application.StateHealthy {
|
||||
t.Fatalf("snapshot=%+v err=%v", snapshot, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationProviderGroupsContainersAndAppliesProbes(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
containers := containerSnapshot(t, now,
|
||||
container.RawContainer{ID: "web-1", Name: "media-web", State: "running", Health: "healthy", Project: "media"},
|
||||
container.RawContainer{ID: "db-1", Name: "media-db", State: "running", Health: "healthy", Project: "media"},
|
||||
container.RawContainer{ID: "solo-1", Name: "standalone", State: "running", Health: "healthy"},
|
||||
container.RawContainer{ID: "off-1", Name: "archived", State: "exited", IntentionalStop: true},
|
||||
container.RawContainer{ID: "bad-1", Name: "broken", State: "exited"},
|
||||
)
|
||||
services := service.Snapshot{ContractVersion: service.ContractVersion, ObservedAt: now, Services: []service.ServiceStatus{
|
||||
{ID: "svc-web", Name: "media-web", State: service.StateUp},
|
||||
{ID: "svc-db", Name: "media-db", State: service.StateDown},
|
||||
}}
|
||||
provider := ApplicationProvider{Containers: stubContainers{snapshot: containers}, Services: stubServices{snapshot: services}, Now: fixedClock(now)}
|
||||
snapshot, err := provider.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Source.State != "healthy" || snapshot.Source.Reason != "" {
|
||||
t.Fatalf("unexpected source %+v", snapshot.Source)
|
||||
}
|
||||
byName := make(map[string]application.Application, len(snapshot.Applications))
|
||||
for _, item := range snapshot.Applications {
|
||||
byName[item.Name] = item
|
||||
}
|
||||
if len(byName) != 4 {
|
||||
t.Fatalf("expected one application per compose project or standalone container, got %d: %+v", len(byName), snapshot.Applications)
|
||||
}
|
||||
media, ok := byName["media"]
|
||||
if !ok || len(media.Components) != 2 {
|
||||
t.Fatalf("media project was not grouped: %+v", byName)
|
||||
}
|
||||
if media.Status != application.StateDegraded {
|
||||
t.Fatalf("a down probe on a critical component must degrade the application, got %s", media.Status)
|
||||
}
|
||||
if standalone := byName["standalone"]; standalone.Status != application.StateHealthy {
|
||||
t.Fatalf("a running container without a probe must stay healthy, got %s", standalone.Status)
|
||||
}
|
||||
if archived := byName["archived"]; archived.Status != application.StateUnknown {
|
||||
t.Fatalf("an intentionally stopped critical component must remain explicit unknown, got %s", archived.Status)
|
||||
}
|
||||
if broken := byName["broken"]; broken.Status != application.StateDegraded {
|
||||
t.Fatalf("an unexpectedly stopped container must not read as healthy, got %s", broken.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationProviderBoundsTheInventory(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
raw := make([]container.RawContainer, 0, 4)
|
||||
for _, name := range []string{"a", "b", "c", "d"} {
|
||||
raw = append(raw, container.RawContainer{ID: name, Name: name, State: "running", Health: "healthy"})
|
||||
}
|
||||
provider := ApplicationProvider{
|
||||
Containers: stubContainers{snapshot: containerSnapshot(t, now, raw...)},
|
||||
Services: stubServices{snapshot: service.Snapshot{ContractVersion: service.ContractVersion, ObservedAt: now}},
|
||||
MaxApplications: 2,
|
||||
Now: fixedClock(now),
|
||||
}
|
||||
snapshot, err := provider.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Source.State != "unknown" || snapshot.Source.Reason != ReasonInvalid {
|
||||
t.Fatalf("an oversized inventory must resolve to unknown, got %+v", snapshot.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationProviderPropagatesCancellation(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
provider := ApplicationProvider{
|
||||
Containers: stubContainers{err: context.Canceled},
|
||||
Services: stubServices{},
|
||||
Now: fixedClock(now),
|
||||
}
|
||||
if _, err := provider.Snapshot(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected cancellation, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerStateMappingNeverInventsHealth(t *testing.T) {
|
||||
cases := map[string]application.State{
|
||||
"running": application.StateHealthy,
|
||||
"restarting": application.StateDegraded,
|
||||
"paused": application.StateDegraded,
|
||||
"exited": application.StateDown,
|
||||
"dead": application.StateDown,
|
||||
"created": application.StateUnknown,
|
||||
"": application.StateUnknown,
|
||||
"weird": application.StateUnknown,
|
||||
}
|
||||
for state, want := range cases {
|
||||
if got := containerState(container.Container{State: state, Health: "healthy"}); got != want {
|
||||
t.Fatalf("container state %q mapped to %s, want %s", state, got, want)
|
||||
}
|
||||
}
|
||||
if got := containerState(container.Container{State: "running", Health: "unhealthy"}); got != application.StateDegraded {
|
||||
t.Fatalf("an unhealthy running container mapped to %s", got)
|
||||
}
|
||||
if got := containerState(container.Container{State: "running", Health: "starting"}); got != application.StateUnknown {
|
||||
t.Fatalf("a starting container mapped to %s", got)
|
||||
}
|
||||
if got := containerState(container.Container{State: "RUNNING", Health: "HEALTHY"}); got != application.StateHealthy {
|
||||
t.Fatalf("uppercase runtime state mapped to %s", got)
|
||||
}
|
||||
if got := containerState(container.Container{State: "running", Health: "unknown"}); got != application.StateUnknown {
|
||||
t.Fatalf("missing health became %s", got)
|
||||
}
|
||||
if got := containerState(container.Container{State: "exited", Health: "unknown", IntentionalStop: true}); got != application.StateUnknown {
|
||||
t.Fatalf("intentional stop became %s", got)
|
||||
}
|
||||
if got := serviceState("nonsense"); got != application.StateUnknown {
|
||||
t.Fatalf("an unrecognized probe verdict mapped to %s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package agentsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/datasource"
|
||||
)
|
||||
|
||||
type healthReader map[agentstore.Capability]agentstore.Snapshot
|
||||
|
||||
func (r healthReader) Latest(_ context.Context, capability agentstore.Capability) (agentstore.Snapshot, error) {
|
||||
snapshot, ok := r[capability]
|
||||
if !ok {
|
||||
return agentstore.Snapshot{}, agentstore.ErrNoSnapshot
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func TestHealthRequiresEveryCapabilityToBeFresh(t *testing.T) {
|
||||
now := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
|
||||
reader := healthReader{
|
||||
agentstore.CapabilityHost: {
|
||||
Capability: agentstore.CapabilityHost, ObservedAt: now.Add(-time.Second), ReceivedAt: now.Add(-time.Second),
|
||||
},
|
||||
agentstore.CapabilityProcesses: {
|
||||
Capability: agentstore.CapabilityProcesses, ObservedAt: now.Add(-2 * time.Second), ReceivedAt: now.Add(-2 * time.Second),
|
||||
},
|
||||
}
|
||||
health, err := Health(context.Background(), reader, []agentstore.Capability{agentstore.CapabilityHost, agentstore.CapabilityProcesses}, Windows{}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if health.State != datasource.HealthHealthy || health.LastSuccess != now.Add(-2*time.Second) {
|
||||
t.Fatalf("health = %#v", health)
|
||||
}
|
||||
|
||||
missing, err := Health(context.Background(), reader, []agentstore.Capability{agentstore.CapabilityHost, agentstore.CapabilityContainers}, Windows{}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if missing.State != datasource.HealthUnknown || missing.ReasonCode != ReasonUnavailable {
|
||||
t.Fatalf("missing health = %#v", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthRejectsStaleCapabilityAndPropagatesCancellation(t *testing.T) {
|
||||
now := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
|
||||
reader := healthReader{agentstore.CapabilityHost: {
|
||||
Capability: agentstore.CapabilityHost, ObservedAt: now.Add(-time.Hour), ReceivedAt: now.Add(-time.Hour),
|
||||
}}
|
||||
health, err := Health(context.Background(), reader, []agentstore.Capability{agentstore.CapabilityHost}, Windows{}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if health.State != datasource.HealthUnknown || health.ReasonCode != ReasonStale {
|
||||
t.Fatalf("stale health = %#v", health)
|
||||
}
|
||||
|
||||
cancelled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := Health(cancelled, reader, []agentstore.Capability{agentstore.CapabilityHost}, Windows{}, now); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancellation error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package agentsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/application"
|
||||
"github.com/itworx/pulse/internal/array"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/disk"
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
"github.com/itworx/pulse/internal/pool"
|
||||
"github.com/itworx/pulse/internal/process"
|
||||
"github.com/itworx/pulse/internal/share"
|
||||
)
|
||||
|
||||
// Source identifiers and types reported to the API. They match the defaults each domain
|
||||
// Adapter already applies, so a snapshot served from the agent store is indistinguishable
|
||||
// in shape from one served by any other adapter. The type records where the observation
|
||||
// originates: the host capabilities are read by the agent itself, the storage
|
||||
// capabilities are read by the agent from Unraid.
|
||||
const (
|
||||
hostSourceID = "host"
|
||||
processSourceID = "process"
|
||||
containerSourceID = "container"
|
||||
arraySourceID = "array"
|
||||
diskSourceID = "disks"
|
||||
poolSourceID = "pools"
|
||||
shareSourceID = "shares"
|
||||
agentSourceType = "agent"
|
||||
unraidSourceType = "unraid"
|
||||
applicationSources = application.SourceID
|
||||
)
|
||||
|
||||
// HostProvider serves host telemetry recorded by the agent.
|
||||
type HostProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits host.Limits
|
||||
Policy host.Policy
|
||||
// Now overrides the clock in tests; production leaves it nil.
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ host.Provider = HostProvider{}
|
||||
|
||||
func (p HostProvider) Snapshot(ctx context.Context) (host.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityHost, p.Windows.For(agentstore.CapabilityHost), now,
|
||||
func(at time.Time, reason string) host.Snapshot {
|
||||
return host.UnknownSnapshot(at, hostSourceID, agentSourceType, reason)
|
||||
},
|
||||
func(raw host.RawSnapshot, at time.Time) (host.Snapshot, error) {
|
||||
return host.Adapter{Source: staticRaw[host.RawSnapshot]{raw}, Limits: p.Limits, Policy: p.Policy, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// ProcessProvider serves the process inventory recorded by the agent.
|
||||
type ProcessProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits process.Limits
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (p ProcessProvider) Snapshot(ctx context.Context) (process.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityProcesses, p.Windows.For(agentstore.CapabilityProcesses), now,
|
||||
func(at time.Time, reason string) process.Snapshot {
|
||||
return process.UnknownSnapshot(at, processSourceID, agentSourceType, reason)
|
||||
},
|
||||
func(raw process.RawSnapshot, at time.Time) (process.Snapshot, error) {
|
||||
return process.Adapter{Source: staticRaw[process.RawSnapshot]{raw}, Limits: p.Limits, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// ContainerProvider serves the container inventory recorded by the agent.
|
||||
type ContainerProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits container.Limits
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ container.Provider = ContainerProvider{}
|
||||
|
||||
func (p ContainerProvider) Snapshot(ctx context.Context) (container.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityContainers, p.Windows.For(agentstore.CapabilityContainers), now,
|
||||
func(at time.Time, reason string) container.Snapshot {
|
||||
return container.UnknownSnapshot(at, containerSourceID, agentSourceType, reason)
|
||||
},
|
||||
func(raw container.RawSnapshot, at time.Time) (container.Snapshot, error) {
|
||||
return container.Adapter{Source: staticRaw[container.RawSnapshot]{raw}, Limits: p.Limits, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// ArrayProvider serves Unraid array state recorded by the agent.
|
||||
type ArrayProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits array.Limits
|
||||
Policy array.Policy
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ array.Provider = ArrayProvider{}
|
||||
|
||||
func (p ArrayProvider) Snapshot(ctx context.Context) (array.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityArray, p.Windows.For(agentstore.CapabilityArray), now,
|
||||
func(at time.Time, reason string) array.Snapshot {
|
||||
return array.UnknownSnapshot(at, arraySourceID, unraidSourceType, reason)
|
||||
},
|
||||
func(raw array.RawSnapshot, at time.Time) (array.Snapshot, error) {
|
||||
return array.Adapter{Source: staticRaw[array.RawSnapshot]{raw}, Limits: p.Limits, Policy: p.Policy, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// DiskProvider serves disk inventory, SMART and performance data recorded by the agent.
|
||||
type DiskProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits disk.Limits
|
||||
Policy disk.Policy
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ disk.Provider = DiskProvider{}
|
||||
|
||||
func (p DiskProvider) Snapshot(ctx context.Context) (disk.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityDisks, p.Windows.For(agentstore.CapabilityDisks), now,
|
||||
func(at time.Time, reason string) disk.Snapshot {
|
||||
return disk.UnknownSnapshot(at, diskSourceID, unraidSourceType, reason)
|
||||
},
|
||||
func(raw disk.RawSnapshot, at time.Time) (disk.Snapshot, error) {
|
||||
return disk.Adapter{Source: staticRaw[disk.RawSnapshot]{raw}, Limits: p.Limits, Policy: p.Policy, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// PoolProvider serves cache and named pool state recorded by the agent.
|
||||
type PoolProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits pool.Limits
|
||||
Policy pool.Policy
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ pool.Provider = PoolProvider{}
|
||||
|
||||
func (p PoolProvider) Snapshot(ctx context.Context) (pool.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityPools, p.Windows.For(agentstore.CapabilityPools), now,
|
||||
func(at time.Time, reason string) pool.Snapshot {
|
||||
return pool.UnknownSnapshot(at, poolSourceID, unraidSourceType, reason)
|
||||
},
|
||||
func(raw pool.RawSnapshot, at time.Time) (pool.Snapshot, error) {
|
||||
return pool.Adapter{Source: staticRaw[pool.RawSnapshot]{raw}, Limits: p.Limits, Policy: p.Policy, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// ShareProvider serves user share usage recorded by the agent.
|
||||
type ShareProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits share.Limits
|
||||
Policy share.Policy
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ share.Provider = ShareProvider{}
|
||||
|
||||
func (p ShareProvider) Snapshot(ctx context.Context) (share.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityShares, p.Windows.For(agentstore.CapabilityShares), now,
|
||||
func(at time.Time, reason string) share.Snapshot {
|
||||
return share.UnknownSnapshot(at, shareSourceID, unraidSourceType, reason)
|
||||
},
|
||||
func(raw share.RawSnapshot, at time.Time) (share.Snapshot, error) {
|
||||
return share.Adapter{Source: staticRaw[share.RawSnapshot]{raw}, Limits: p.Limits, Policy: p.Policy, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
package agentsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/array"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/disk"
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
"github.com/itworx/pulse/internal/pool"
|
||||
"github.com/itworx/pulse/internal/process"
|
||||
"github.com/itworx/pulse/internal/share"
|
||||
)
|
||||
|
||||
// stubReader stands in for the PostgreSQL store so the read path can be exercised without
|
||||
// a database.
|
||||
type stubReader struct {
|
||||
snapshot agentstore.Snapshot
|
||||
err error
|
||||
asked agentstore.Capability
|
||||
}
|
||||
|
||||
func (r *stubReader) Latest(_ context.Context, capability agentstore.Capability) (agentstore.Snapshot, error) {
|
||||
r.asked = capability
|
||||
if r.err != nil {
|
||||
return agentstore.Snapshot{}, r.err
|
||||
}
|
||||
return r.snapshot, nil
|
||||
}
|
||||
|
||||
// observation is the part of every domain snapshot this package is responsible for.
|
||||
type observation struct {
|
||||
state string
|
||||
reason string
|
||||
healthy bool
|
||||
}
|
||||
|
||||
// domainUnderTest describes one capability generically so every domain runs the same
|
||||
// table: no snapshot, store failure, stale snapshot, corrupt payload, payload the domain
|
||||
// rejects, and a good payload.
|
||||
type domainUnderTest struct {
|
||||
name string
|
||||
capability agentstore.Capability
|
||||
window time.Duration
|
||||
good func(now time.Time) any
|
||||
rejected func(now time.Time) any
|
||||
observe func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error)
|
||||
}
|
||||
|
||||
func floatPtr(value float64) *float64 { return &value }
|
||||
|
||||
func domains() []domainUnderTest {
|
||||
return []domainUnderTest{
|
||||
{
|
||||
name: "host",
|
||||
capability: agentstore.CapabilityHost,
|
||||
window: DefaultHostWindow,
|
||||
good: func(now time.Time) any {
|
||||
used := uint64(6 * 1024 * 1024 * 1024)
|
||||
return host.RawSnapshot{
|
||||
Source: host.Source{ID: "agent-1", Type: "agent"},
|
||||
Identity: host.HostIdentity{Name: "pulse-host", Version: "7.2.2", Arch: "amd64"},
|
||||
UptimeSeconds: 3600,
|
||||
CPU: host.RawCPU{TotalPercent: floatPtr(42.5), PerCore: []float64{40, 45}, IOWaitPercent: floatPtr(2)},
|
||||
Load: host.RawLoad{One: 0.2, Five: 0.1, Fifteen: 0.1},
|
||||
Memory: host.RawMemory{TotalBytes: 8 * 1024 * 1024 * 1024, AvailableBytes: 2 * 1024 * 1024 * 1024, UsedBytes: &used},
|
||||
Filesystems: []host.RawFilesystem{{Mount: "/", Filesystem: "xfs", CapacityBytes: 100, UsedBytes: 25}},
|
||||
Network: []host.RawNetworkInterface{{Name: "eth0", State: "up"}},
|
||||
Time: host.RawTime{Synchronized: true, OffsetSeconds: 0.002, Stratum: 2},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return host.RawSnapshot{Identity: host.HostIdentity{Name: ""}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := HostProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{
|
||||
state: snapshot.Source.State,
|
||||
reason: snapshot.Source.Reason,
|
||||
healthy: snapshot.Source.Freshness == host.Fresh && snapshot.Status.State == host.StatusHealthy,
|
||||
}, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "processes",
|
||||
capability: agentstore.CapabilityProcesses,
|
||||
window: DefaultProcessesWindow,
|
||||
good: func(now time.Time) any {
|
||||
return process.RawSnapshot{
|
||||
Source: process.Source{ID: "agent-1", Type: "agent"},
|
||||
Processes: []process.RawProcess{{PID: 1, Name: "init", State: "sleeping", RuntimeSeconds: 100, CPUPercent: 1, MemoryBytes: 500}},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return process.RawSnapshot{Processes: []process.RawProcess{{PID: 0, Name: ""}}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := ProcessProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{state: snapshot.Source.State, reason: snapshot.Source.Reason, healthy: snapshot.Source.State == "healthy"}, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "containers",
|
||||
capability: agentstore.CapabilityContainers,
|
||||
window: DefaultContainersWindow,
|
||||
good: func(now time.Time) any {
|
||||
return container.RawSnapshot{
|
||||
Source: container.Source{ID: "agent-1", Type: "agent"},
|
||||
Containers: []container.RawContainer{{ID: "a", Name: "alpha", State: "running", Health: "healthy"}},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return container.RawSnapshot{Containers: []container.RawContainer{{ID: "", Name: ""}}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := ContainerProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{
|
||||
state: snapshot.Source.State,
|
||||
reason: snapshot.Source.Reason,
|
||||
healthy: snapshot.Source.Freshness == "fresh" && snapshot.Source.State == "healthy",
|
||||
}, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "array",
|
||||
capability: agentstore.CapabilityArray,
|
||||
window: DefaultArrayWindow,
|
||||
good: func(now time.Time) any {
|
||||
return array.RawSnapshot{
|
||||
Source: array.Source{ID: "agent-1", Type: "unraid"},
|
||||
State: array.StateOperational,
|
||||
Parity: array.RawParity{Present: true, State: "idle"},
|
||||
Members: []array.RawMember{
|
||||
{ID: "disk1", Name: "Disk 1", Role: "data", State: "online", CapacityBytes: 100},
|
||||
{ID: "parity", Name: "Parity", Role: "parity", State: "online", CapacityBytes: 100},
|
||||
},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return array.RawSnapshot{State: array.StateOperational, Members: []array.RawMember{{ID: "", Name: ""}}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := ArrayProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{
|
||||
state: snapshot.Source.State,
|
||||
reason: snapshot.Source.Reason,
|
||||
healthy: snapshot.Source.Freshness == array.Fresh && snapshot.State == array.StateOperational,
|
||||
}, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "disks",
|
||||
capability: agentstore.CapabilityDisks,
|
||||
window: DefaultDisksWindow,
|
||||
good: func(now time.Time) any {
|
||||
return disk.RawSnapshot{
|
||||
Source: disk.Source{ID: "agent-1", Type: "unraid"},
|
||||
Disks: []disk.RawDisk{{ID: "disk1", Name: "Disk 1", Role: "data", State: disk.StateOnline, SizeBytes: 100}},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return disk.RawSnapshot{Disks: []disk.RawDisk{{ID: "", Name: ""}}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := DiskProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{
|
||||
state: snapshot.Source.State,
|
||||
reason: snapshot.Source.Reason,
|
||||
healthy: snapshot.Source.Freshness == disk.Fresh && snapshot.Source.State != disk.StateUnknown,
|
||||
}, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "pools",
|
||||
capability: agentstore.CapabilityPools,
|
||||
window: DefaultPoolsWindow,
|
||||
good: func(now time.Time) any {
|
||||
return pool.RawSnapshot{
|
||||
Source: pool.Source{ID: "agent-1", Type: "unraid"},
|
||||
Pools: []pool.RawPool{{ID: "cache", Name: "Cache", Filesystem: "btrfs", State: pool.StateHealthy, UsableBytes: 1000, UsedBytes: 100}},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return pool.RawSnapshot{Pools: []pool.RawPool{{ID: "", Name: ""}}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := PoolProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{
|
||||
state: snapshot.Source.State,
|
||||
reason: snapshot.Source.Reason,
|
||||
healthy: snapshot.Source.Freshness == pool.Fresh && snapshot.Source.State != pool.StateUnknown,
|
||||
}, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "shares",
|
||||
capability: agentstore.CapabilityShares,
|
||||
window: DefaultSharesWindow,
|
||||
good: func(now time.Time) any {
|
||||
return share.RawSnapshot{
|
||||
Source: share.Source{ID: "agent-1", Type: "unraid"},
|
||||
Shares: []share.RawShare{{ID: "share-media", Name: "Media", UsedBytes: 700, SizeObservedAt: now, SizeState: share.SizeCached}},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return share.RawSnapshot{Shares: []share.RawShare{{ID: "", Name: ""}}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := ShareProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{
|
||||
state: snapshot.Source.State,
|
||||
reason: snapshot.Source.Reason,
|
||||
healthy: snapshot.Source.Freshness == share.Fresh && snapshot.Source.State != share.StateUnknown,
|
||||
}, err
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func encode(t *testing.T, value any) json.RawMessage {
|
||||
t.Helper()
|
||||
payload, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
func TestProvidersMapUnusableTelemetryToUnknown(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
for _, domain := range domains() {
|
||||
t.Run(domain.name, func(t *testing.T) {
|
||||
fresh := agentstore.Snapshot{AgentID: "agent-1", Capability: domain.capability, ObservedAt: now, ReceivedAt: now}
|
||||
cases := []struct {
|
||||
name string
|
||||
reader *stubReader
|
||||
reason string
|
||||
}{
|
||||
{"no snapshot", &stubReader{err: agentstore.ErrNoSnapshot}, ReasonUnavailable},
|
||||
{"store unreachable", &stubReader{err: errors.New("connection refused")}, ReasonUnavailable},
|
||||
{"stale snapshot", &stubReader{snapshot: func() agentstore.Snapshot {
|
||||
stale := fresh
|
||||
stale.ObservedAt = now.Add(-domain.window - time.Second)
|
||||
stale.Payload = encode(t, domain.good(stale.ObservedAt))
|
||||
return stale
|
||||
}()}, ReasonStale},
|
||||
{"missing observation time", &stubReader{snapshot: func() agentstore.Snapshot {
|
||||
broken := fresh
|
||||
broken.ObservedAt = time.Time{}
|
||||
broken.Payload = encode(t, domain.good(now))
|
||||
return broken
|
||||
}()}, ReasonStale},
|
||||
{"corrupt payload", &stubReader{snapshot: func() agentstore.Snapshot {
|
||||
corrupt := fresh
|
||||
corrupt.Payload = json.RawMessage(`{"source":"not-an-object"}`)
|
||||
return corrupt
|
||||
}()}, ReasonInvalid},
|
||||
{"payload the domain rejects", &stubReader{snapshot: func() agentstore.Snapshot {
|
||||
invalid := fresh
|
||||
invalid.Payload = encode(t, domain.rejected(now))
|
||||
return invalid
|
||||
}()}, ReasonInvalid},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
got, err := domain.observe(context.Background(), testCase.reader, now)
|
||||
if err != nil {
|
||||
t.Fatalf("unusable telemetry must not fail the request: %v", err)
|
||||
}
|
||||
if got.healthy {
|
||||
t.Fatal("unusable telemetry became healthy, violating ADR-0008")
|
||||
}
|
||||
if got.state != "unknown" {
|
||||
t.Fatalf("source state = %q, want unknown", got.state)
|
||||
}
|
||||
if got.reason != testCase.reason {
|
||||
t.Fatalf("reason = %q, want %q", got.reason, testCase.reason)
|
||||
}
|
||||
if testCase.reader.asked != domain.capability {
|
||||
t.Fatalf("read capability %q, want %q", testCase.reader.asked, domain.capability)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvidersNormalizeGoodTelemetry(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
for _, domain := range domains() {
|
||||
t.Run(domain.name, func(t *testing.T) {
|
||||
observed := now.Add(-time.Second)
|
||||
reader := &stubReader{snapshot: agentstore.Snapshot{
|
||||
AgentID: "agent-1", Capability: domain.capability, ObservedAt: observed, ReceivedAt: observed,
|
||||
Payload: encode(t, domain.good(observed)),
|
||||
}}
|
||||
got, err := domain.observe(context.Background(), reader, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got.healthy {
|
||||
t.Fatalf("fresh telemetry did not normalize to a healthy source: %+v", got)
|
||||
}
|
||||
if got.reason != "" {
|
||||
t.Fatalf("healthy source carries reason %q", got.reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvidersTreatTheWindowBoundaryAsFresh(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
for _, domain := range domains() {
|
||||
t.Run(domain.name, func(t *testing.T) {
|
||||
observed := now.Add(-domain.window)
|
||||
reader := &stubReader{snapshot: agentstore.Snapshot{
|
||||
AgentID: "agent-1", Capability: domain.capability, ObservedAt: observed, ReceivedAt: observed,
|
||||
Payload: encode(t, domain.good(observed)),
|
||||
}}
|
||||
got, err := domain.observe(context.Background(), reader, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.state == "unknown" {
|
||||
t.Fatalf("a snapshot exactly at the window edge must still be usable: %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvidersWithoutAReaderReportUnavailable(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
for _, domain := range domains() {
|
||||
t.Run(domain.name, func(t *testing.T) {
|
||||
got, err := domain.observe(context.Background(), nil, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.healthy || got.state != "unknown" || got.reason != ReasonUnavailable {
|
||||
t.Fatalf("unexpected observation %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvidersPropagateContextCancellation(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for _, domain := range domains() {
|
||||
t.Run(domain.name, func(t *testing.T) {
|
||||
reader := &stubReader{err: context.Canceled}
|
||||
if _, err := domain.observe(ctx, reader, now); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected cancellation, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWindowsApplyDocumentedDefaults(t *testing.T) {
|
||||
windows := Windows{}
|
||||
expected := map[agentstore.Capability]time.Duration{
|
||||
agentstore.CapabilityHost: DefaultHostWindow,
|
||||
agentstore.CapabilityProcesses: DefaultProcessesWindow,
|
||||
agentstore.CapabilityContainers: DefaultContainersWindow,
|
||||
agentstore.CapabilityArray: DefaultArrayWindow,
|
||||
agentstore.CapabilityDisks: DefaultDisksWindow,
|
||||
agentstore.CapabilityPools: DefaultPoolsWindow,
|
||||
agentstore.CapabilityShares: DefaultSharesWindow,
|
||||
}
|
||||
for _, capability := range agentstore.Capabilities() {
|
||||
if got := windows.For(capability); got != expected[capability] {
|
||||
t.Fatalf("window for %q = %s, want %s", capability, got, expected[capability])
|
||||
}
|
||||
}
|
||||
if windows.For(agentstore.CapabilityHost) >= windows.For(agentstore.CapabilityShares) {
|
||||
t.Fatal("the host is sampled far more often than shares and must have the tighter window")
|
||||
}
|
||||
if err := windows.Validate(); err != nil {
|
||||
t.Fatalf("defaults must validate: %v", err)
|
||||
}
|
||||
if err := (Windows{Host: 48 * time.Hour}).Validate(); err == nil {
|
||||
t.Fatal("a window beyond a day must be rejected")
|
||||
}
|
||||
if err := (Windows{Shares: -time.Second}).Validate(); err == nil {
|
||||
t.Fatal("a negative window must be rejected")
|
||||
}
|
||||
custom := Windows{Host: 5 * time.Second}.WithDefaults()
|
||||
if custom.Host != 5*time.Second || custom.Shares != DefaultSharesWindow {
|
||||
t.Fatalf("configuration was not preserved: %+v", custom)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredWindowOverridesTheDefault(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
observed := now.Add(-10 * time.Second)
|
||||
payload := encode(t, process.RawSnapshot{
|
||||
Source: process.Source{ID: "agent-1", Type: "agent"},
|
||||
Processes: []process.RawProcess{{PID: 1, Name: "init", State: "sleeping"}},
|
||||
ObservedAt: observed, ReceivedAt: observed,
|
||||
})
|
||||
reader := &stubReader{snapshot: agentstore.Snapshot{AgentID: "agent-1", Capability: agentstore.CapabilityProcesses, ObservedAt: observed, ReceivedAt: observed, Payload: payload}}
|
||||
provider := ProcessProvider{Reader: reader, Windows: Windows{Processes: 5 * time.Second}, Now: fixedClock(now)}
|
||||
snapshot, err := provider.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Source.State != "unknown" || snapshot.Source.Reason != ReasonStale {
|
||||
t.Fatalf("a tightened window must make the snapshot stale: %+v", snapshot.Source)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user