This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
package agentprotocol
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const Version = "v1"
|
||||
|
||||
type Capability struct {
|
||||
ID string `json:"id"`
|
||||
Version string `json:"version"`
|
||||
ReadOnly bool `json:"readOnly"`
|
||||
}
|
||||
type Hello struct {
|
||||
Protocol string `json:"protocol"`
|
||||
AgentID string `json:"agentId"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
Capabilities []Capability `json:"capabilities"`
|
||||
}
|
||||
|
||||
func (h Hello) Validate(now time.Time) error {
|
||||
if h.Protocol != Version {
|
||||
return errors.New("unsupported agent protocol")
|
||||
}
|
||||
if strings.TrimSpace(h.AgentID) == "" || len(h.AgentID) > 120 {
|
||||
return errors.New("invalid agent id")
|
||||
}
|
||||
if h.ObservedAt.IsZero() || h.ObservedAt.After(now.Add(time.Minute)) {
|
||||
return errors.New("invalid agent observation time")
|
||||
}
|
||||
if len(h.Capabilities) > 50 {
|
||||
return errors.New("too many agent capabilities")
|
||||
}
|
||||
for _, capability := range h.Capabilities {
|
||||
if strings.TrimSpace(capability.ID) == "" || capability.Version == "" || !capability.ReadOnly {
|
||||
return errors.New("agent capability must be explicit and read-only")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package agentprotocol
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHelloRejectsMutationCapability(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
hello := Hello{Protocol: Version, AgentID: "agent-1", ObservedAt: now, Capabilities: []Capability{{ID: "containers.read", Version: Version, ReadOnly: false}}}
|
||||
if err := hello.Validate(now); err == nil {
|
||||
t.Fatal("expected mutation capability rejection")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
// Package agentsource turns the bounded snapshots pulse-agent writes through
|
||||
// internal/agentstore into the normalized domain snapshots pulse-api serves.
|
||||
//
|
||||
// Every provider in this package follows the same three steps: read the newest snapshot
|
||||
// for one capability, decide whether it is usable, and normalize it through the domain's
|
||||
// own Adapter. Only the middle step is interesting, and it is the ADR-0008 enforcement
|
||||
// point for the whole read path: telemetry that is missing, stale, or undecodable
|
||||
// resolves to that domain's Unknown snapshot and never to Healthy.
|
||||
//
|
||||
// The freshness decision itself is delegated to internal/freshness so this package does
|
||||
// not become another private copy of the rule. freshness.Evaluate is called with the
|
||||
// snapshot presented as a required, currently healthy datasource.SourceHealth; if it
|
||||
// answers with anything other than Healthy the snapshot is discarded as stale. That keeps
|
||||
// one tested implementation of "missing telemetry never becomes Healthy" in the codebase.
|
||||
//
|
||||
// # Reason codes
|
||||
//
|
||||
// A provider that cannot serve real telemetry reports exactly one machine-readable reason
|
||||
// from this closed set, which the web app maps onto localized copy:
|
||||
//
|
||||
// source_unavailable — nothing has been recorded for the capability yet, or the store
|
||||
// could not be read at all.
|
||||
// source_stale — the newest snapshot is older than the capability's freshness
|
||||
// window, or its timestamps are not usable.
|
||||
// source_invalid — the payload could not be decoded into the domain's RawSnapshot,
|
||||
// or the domain rejected it during normalization.
|
||||
//
|
||||
// # Error handling
|
||||
//
|
||||
// Context cancellation and deadlines propagate to the caller. Any other store failure is
|
||||
// reported as an Unknown snapshot with source_unavailable rather than an error: a
|
||||
// monitoring surface that says explicitly "this source is unavailable" is more useful to
|
||||
// an operator than a bare 503, and it keeps a single database hiccup from taking every
|
||||
// monitoring page down at once.
|
||||
package agentsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/datasource"
|
||||
"github.com/itworx/pulse/internal/freshness"
|
||||
)
|
||||
|
||||
// Reason codes reported when a provider cannot serve real telemetry. The set is closed;
|
||||
// see the package documentation for the meaning of each code.
|
||||
const (
|
||||
ReasonUnavailable = "source_unavailable"
|
||||
ReasonStale = "source_stale"
|
||||
ReasonInvalid = "source_invalid"
|
||||
)
|
||||
|
||||
// Default freshness windows per capability. They mirror the freshness policy defaults of
|
||||
// the domain packages, so a snapshot that passes this gate is never marked stale again
|
||||
// further down the pipeline, and they follow the agent's sampling cadence: the host is
|
||||
// sampled every few seconds, processes and containers roughly every half minute, and
|
||||
// share usage is an expensive scan that runs far less often.
|
||||
const (
|
||||
DefaultHostWindow = 30 * time.Second
|
||||
DefaultProcessesWindow = 60 * time.Second
|
||||
DefaultContainersWindow = 60 * time.Second
|
||||
DefaultArrayWindow = 60 * time.Second
|
||||
DefaultDisksWindow = 60 * time.Second
|
||||
DefaultPoolsWindow = 60 * time.Second
|
||||
DefaultSharesWindow = 2 * time.Minute
|
||||
)
|
||||
|
||||
// Windows configures how old a snapshot may be before its capability resolves to Unknown.
|
||||
// A zero field takes the documented default for that capability.
|
||||
type Windows struct {
|
||||
Host time.Duration
|
||||
Processes time.Duration
|
||||
Containers time.Duration
|
||||
Array time.Duration
|
||||
Disks time.Duration
|
||||
Pools time.Duration
|
||||
Shares time.Duration
|
||||
}
|
||||
|
||||
// WithDefaults fills every unset window with its documented default.
|
||||
func (w Windows) WithDefaults() Windows {
|
||||
if w.Host == 0 {
|
||||
w.Host = DefaultHostWindow
|
||||
}
|
||||
if w.Processes == 0 {
|
||||
w.Processes = DefaultProcessesWindow
|
||||
}
|
||||
if w.Containers == 0 {
|
||||
w.Containers = DefaultContainersWindow
|
||||
}
|
||||
if w.Array == 0 {
|
||||
w.Array = DefaultArrayWindow
|
||||
}
|
||||
if w.Disks == 0 {
|
||||
w.Disks = DefaultDisksWindow
|
||||
}
|
||||
if w.Pools == 0 {
|
||||
w.Pools = DefaultPoolsWindow
|
||||
}
|
||||
if w.Shares == 0 {
|
||||
w.Shares = DefaultSharesWindow
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// Validate reports whether every configured window is inside safe bounds. Windows are
|
||||
// bounded above at 24 hours because datasource.FreshnessPolicy refuses anything longer:
|
||||
// a source nobody has heard from for a day is not fresh under any reading.
|
||||
func (w Windows) Validate() error {
|
||||
for capability, window := range w.WithDefaults().byCapability() {
|
||||
if window <= 0 || window > 24*time.Hour {
|
||||
return fmt.Errorf("freshness window for %q is outside safe bounds", capability)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// For returns the freshness window for one capability, applying defaults.
|
||||
func (w Windows) For(capability agentstore.Capability) time.Duration {
|
||||
if window, ok := w.WithDefaults().byCapability()[capability]; ok {
|
||||
return window
|
||||
}
|
||||
// An unrecognized capability cannot be served at all; the tightest window keeps a
|
||||
// caller that ignores that from treating anything as fresh.
|
||||
return DefaultHostWindow
|
||||
}
|
||||
|
||||
func (w Windows) byCapability() map[agentstore.Capability]time.Duration {
|
||||
return map[agentstore.Capability]time.Duration{
|
||||
agentstore.CapabilityHost: w.Host,
|
||||
agentstore.CapabilityProcesses: w.Processes,
|
||||
agentstore.CapabilityContainers: w.Containers,
|
||||
agentstore.CapabilityArray: w.Array,
|
||||
agentstore.CapabilityDisks: w.Disks,
|
||||
agentstore.CapabilityPools: w.Pools,
|
||||
agentstore.CapabilityShares: w.Shares,
|
||||
}
|
||||
}
|
||||
|
||||
// staticRaw presents an already decoded RawSnapshot as the raw source interface each
|
||||
// domain Adapter expects, so normalization keeps running through the domain's own code.
|
||||
type staticRaw[R any] struct{ raw R }
|
||||
|
||||
func (s staticRaw[R]) Snapshot(context.Context) (R, error) { return s.raw, nil }
|
||||
|
||||
// resolve implements the shared read path. It is generic over the domain's RawSnapshot
|
||||
// and normalized Snapshot so every capability enforces the same rules in the same order.
|
||||
func resolve[R any, S any](
|
||||
ctx context.Context,
|
||||
reader agentstore.Reader,
|
||||
capability agentstore.Capability,
|
||||
window time.Duration,
|
||||
now time.Time,
|
||||
unknown func(time.Time, string) S,
|
||||
normalize func(R, time.Time) (S, error),
|
||||
) (S, error) {
|
||||
var zero S
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, err
|
||||
}
|
||||
if reader == nil {
|
||||
return unknown(now, ReasonUnavailable), nil
|
||||
}
|
||||
stored, err := reader.Latest(ctx, capability)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return zero, err
|
||||
}
|
||||
return unknown(now, ReasonUnavailable), nil
|
||||
}
|
||||
if reason := usableReason(string(capability), stored, window, now); reason != "" {
|
||||
return unknown(now, reason), nil
|
||||
}
|
||||
var raw R
|
||||
if err := json.Unmarshal(stored.Payload, &raw); err != nil {
|
||||
return unknown(now, ReasonInvalid), nil
|
||||
}
|
||||
snapshot, err := normalize(raw, now)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return zero, err
|
||||
}
|
||||
return unknown(now, ReasonInvalid), nil
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
// usableReason returns an empty string when the snapshot may be normalized, or the reason
|
||||
// code that must be reported instead. The freshness judgement is made by
|
||||
// internal/freshness rather than re-derived here.
|
||||
func usableReason(sourceID string, stored agentstore.Snapshot, window time.Duration, now time.Time) string {
|
||||
result, err := freshness.Evaluate(freshness.Input{
|
||||
SourceID: sourceID,
|
||||
Required: true,
|
||||
Now: now,
|
||||
Health: datasource.SourceHealth{
|
||||
State: datasource.HealthHealthy,
|
||||
ObservedAt: stored.ObservedAt,
|
||||
ReceivedAt: stored.ReceivedAt,
|
||||
LastSuccess: stored.ObservedAt,
|
||||
Policy: datasource.FreshnessPolicy{MaxAge: window},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
// Unusable timestamps (missing, or materially in the future) are indistinguishable
|
||||
// from staleness for a consumer: in both cases the observation cannot be trusted
|
||||
// to describe the present.
|
||||
return ReasonStale
|
||||
}
|
||||
if result.State != datasource.HealthHealthy {
|
||||
return ReasonStale
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// clockNow resolves the injected clock, defaulting to the wall clock in UTC.
|
||||
func clockNow(clock func() time.Time) time.Time {
|
||||
if clock == nil {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
return clock().UTC()
|
||||
}
|
||||
|
||||
func fixedClock(now time.Time) func() time.Time { return func() time.Time { return now } }
|
||||
|
||||
// Health summarizes a bounded set of agent capabilities into one datasource health
|
||||
// observation. Every requested capability is required: a source is only Healthy when
|
||||
// each capability has a recent snapshot. This is used by self-observability so it reads
|
||||
// the same persisted transport as the domain providers instead of inferring agent
|
||||
// configuration from API process environment variables.
|
||||
func Health(ctx context.Context, reader agentstore.Reader, capabilities []agentstore.Capability, windows Windows, now time.Time) (datasource.SourceHealth, error) {
|
||||
now = now.UTC()
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
health := datasource.SourceHealth{
|
||||
State: datasource.HealthUnknown,
|
||||
ReceivedAt: now,
|
||||
Policy: datasource.FreshnessPolicy{MaxAge: DefaultHostWindow},
|
||||
ReasonCode: ReasonUnavailable,
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return datasource.SourceHealth{}, err
|
||||
}
|
||||
if reader == nil || len(capabilities) == 0 {
|
||||
return health, nil
|
||||
}
|
||||
for _, capability := range capabilities {
|
||||
if window := windows.For(capability); window > health.Policy.MaxAge {
|
||||
health.Policy.MaxAge = window
|
||||
}
|
||||
}
|
||||
oldestObserved := now
|
||||
oldestReceived := now
|
||||
for _, capability := range capabilities {
|
||||
if !capability.Valid() {
|
||||
return datasource.SourceHealth{}, fmt.Errorf("summarize unknown agent capability %q", capability)
|
||||
}
|
||||
stored, err := reader.Latest(ctx, capability)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return datasource.SourceHealth{}, err
|
||||
}
|
||||
return health, nil
|
||||
}
|
||||
if reason := usableReason(string(capability), stored, windows.For(capability), now); reason != "" {
|
||||
health.ObservedAt = stored.ObservedAt.UTC()
|
||||
health.ReceivedAt = stored.ReceivedAt.UTC()
|
||||
health.ReasonCode = reason
|
||||
return health, nil
|
||||
}
|
||||
if stored.ObservedAt.Before(oldestObserved) {
|
||||
oldestObserved = stored.ObservedAt.UTC()
|
||||
}
|
||||
if stored.ReceivedAt.Before(oldestReceived) {
|
||||
oldestReceived = stored.ReceivedAt.UTC()
|
||||
}
|
||||
}
|
||||
health.State = datasource.HealthHealthy
|
||||
health.ObservedAt = oldestObserved
|
||||
health.ReceivedAt = oldestReceived
|
||||
health.LastSuccess = oldestObserved
|
||||
health.ReasonCode = "source_sampled"
|
||||
return health, nil
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package agentsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/application"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/service"
|
||||
)
|
||||
|
||||
// ApplicationProvider derives application health from two existing surfaces rather than
|
||||
// from a capability of its own: the container inventory the agent records, and the
|
||||
// service probe results the API already stores. Containers are grouped into applications
|
||||
// by their Compose project (falling back to the container name for a standalone
|
||||
// container), and a probe result for a service with the same name refines the component
|
||||
// status of the matching container.
|
||||
//
|
||||
// Fresh container evidence is required. Service probes refine matching
|
||||
// components when configured; their absence does not erase container-runtime
|
||||
// availability, because those are separate claims and surfaces.
|
||||
type ApplicationProvider struct {
|
||||
Containers container.Provider
|
||||
Services service.Provider
|
||||
// MaxApplications bounds the number of groups built before the payload is refused.
|
||||
// Zero takes DefaultMaxApplications.
|
||||
MaxApplications int
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ application.Provider = ApplicationProvider{}
|
||||
|
||||
// DefaultMaxApplications matches the bound application.BuildSnapshot enforces, so an
|
||||
// oversized inventory is reported as Unknown instead of failing the request.
|
||||
const DefaultMaxApplications = 150
|
||||
|
||||
func (p ApplicationProvider) Snapshot(ctx context.Context) (application.Snapshot, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return application.Snapshot{}, err
|
||||
}
|
||||
now := clockNow(p.Now)
|
||||
unknown := func(reason string) application.Snapshot {
|
||||
return application.UnknownSnapshot(now, applicationSources, agentSourceType, reason)
|
||||
}
|
||||
if p.Containers == nil {
|
||||
return unknown(ReasonUnavailable), nil
|
||||
}
|
||||
containers, err := p.Containers.Snapshot(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return application.Snapshot{}, err
|
||||
}
|
||||
return unknown(ReasonUnavailable), nil
|
||||
}
|
||||
if reason := unusableContainerReason(containers); reason != "" {
|
||||
return unknown(reason), nil
|
||||
}
|
||||
// Service probes refine application health when configured, but are not an
|
||||
// identity prerequisite. Fresh container state is complete evidence for
|
||||
// runtime availability; an empty/disabled probe module must not erase every
|
||||
// discovered application from the product.
|
||||
services := service.Snapshot{ContractVersion: service.ContractVersion, ObservedAt: containers.Source.ObservedAt}
|
||||
if p.Services != nil {
|
||||
observed, serviceErr := p.Services.Snapshot(ctx)
|
||||
if serviceErr != nil {
|
||||
if errors.Is(serviceErr, context.Canceled) || errors.Is(serviceErr, context.DeadlineExceeded) {
|
||||
return application.Snapshot{}, serviceErr
|
||||
}
|
||||
} else if strings.TrimSpace(observed.Reason) == "" {
|
||||
services = observed
|
||||
}
|
||||
}
|
||||
maximum := p.MaxApplications
|
||||
if maximum <= 0 {
|
||||
maximum = DefaultMaxApplications
|
||||
}
|
||||
discovered := groupApplications(containers, services, maximum)
|
||||
if discovered == nil {
|
||||
return unknown(ReasonInvalid), nil
|
||||
}
|
||||
source := application.Source{
|
||||
ID: applicationSources,
|
||||
Type: agentSourceType,
|
||||
ObservedAt: containers.Source.ObservedAt,
|
||||
ReceivedAt: containers.Source.ReceivedAt,
|
||||
}
|
||||
snapshot, err := application.BuildSnapshot(source, discovered, nil, now)
|
||||
if err != nil {
|
||||
return unknown(ReasonInvalid), nil
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
// unusableContainerReason maps the container source state onto an application reason code.
|
||||
// The container provider has already applied the freshness rule, so an Unknown container
|
||||
// source here means the same thing for applications.
|
||||
func unusableContainerReason(snapshot container.Snapshot) string {
|
||||
if snapshot.Source.State == "" || snapshot.Source.State == "unknown" {
|
||||
switch snapshot.Source.Reason {
|
||||
case ReasonStale, "stale_source":
|
||||
return ReasonStale
|
||||
case ReasonInvalid:
|
||||
return ReasonInvalid
|
||||
default:
|
||||
return ReasonUnavailable
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// groupApplications builds one discovered application per Compose project. It returns nil
|
||||
// when the inventory exceeds the configured bound.
|
||||
func groupApplications(containers container.Snapshot, services service.Snapshot, maximum int) []application.DiscoveredApplication {
|
||||
states := make(map[string]application.State, len(services.Services))
|
||||
for _, item := range services.Services {
|
||||
states[normalizeKey(item.Name)] = serviceState(item.State)
|
||||
}
|
||||
// application.evaluateComponent normalizes an empty service state to Unknown, so a
|
||||
// component with no probe would drag its application to Unknown even though the
|
||||
// container telemetry is fresh. There is no probe constraint on such a component, and
|
||||
// Healthy is the neutral element of the domain's aggregation: it leaves the component
|
||||
// status equal to the container status instead of inventing an unknown.
|
||||
stateFor := func(name string) application.State {
|
||||
if state, ok := states[normalizeKey(name)]; ok {
|
||||
return state
|
||||
}
|
||||
return application.StateHealthy
|
||||
}
|
||||
order := make([]string, 0, len(containers.Containers))
|
||||
groups := make(map[string]*application.DiscoveredApplication, len(containers.Containers))
|
||||
for _, item := range containers.Containers {
|
||||
key := strings.TrimSpace(item.Project)
|
||||
if key == "" {
|
||||
key = strings.TrimPrefix(strings.TrimSpace(item.Name), "/")
|
||||
}
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
group, seen := groups[key]
|
||||
if !seen {
|
||||
if len(order) >= maximum {
|
||||
return nil
|
||||
}
|
||||
group = &application.DiscoveredApplication{ID: application.StableApplicationID(application.SourceID, key), Name: key}
|
||||
groups[key] = group
|
||||
order = append(order, key)
|
||||
}
|
||||
group.Components = append(group.Components, application.ComponentInput{
|
||||
ID: item.ID,
|
||||
Name: strings.TrimPrefix(item.Name, "/"),
|
||||
Kind: "container",
|
||||
ContainerState: containerState(item),
|
||||
ServiceState: stateFor(item.Name),
|
||||
// Every discovered component is critical until an operator overrides it;
|
||||
// treating an unmapped component as optional would hide real failures.
|
||||
Critical: true,
|
||||
})
|
||||
}
|
||||
discovered := make([]application.DiscoveredApplication, 0, len(order))
|
||||
for _, key := range order {
|
||||
discovered = append(discovered, *groups[key])
|
||||
}
|
||||
return discovered
|
||||
}
|
||||
|
||||
// containerState maps a normalized container onto the application state vocabulary. A
|
||||
// container the operator stopped on purpose is not a failure; anything the collector
|
||||
// could not classify is Unknown rather than Healthy.
|
||||
func containerState(item container.Container) application.State {
|
||||
switch strings.ToLower(strings.TrimSpace(item.State)) {
|
||||
case "running":
|
||||
switch strings.ToLower(strings.TrimSpace(item.Health)) {
|
||||
case "healthy":
|
||||
return application.StateHealthy
|
||||
case "unhealthy":
|
||||
return application.StateDegraded
|
||||
case "starting", "unknown", "":
|
||||
return application.StateUnknown
|
||||
default:
|
||||
return application.StateUnknown
|
||||
}
|
||||
case "restarting", "paused", "removing":
|
||||
return application.StateDegraded
|
||||
case "exited", "dead", "stopped":
|
||||
if item.IntentionalStop {
|
||||
return application.StateUnknown
|
||||
}
|
||||
return application.StateDown
|
||||
default:
|
||||
return application.StateUnknown
|
||||
}
|
||||
}
|
||||
|
||||
// serviceState maps a probe verdict onto the application state vocabulary.
|
||||
func serviceState(state string) application.State {
|
||||
switch strings.ToLower(strings.TrimSpace(state)) {
|
||||
case service.StateUp:
|
||||
return application.StateHealthy
|
||||
case service.StateDegraded:
|
||||
return application.StateDegraded
|
||||
case service.StateDown:
|
||||
return application.StateDown
|
||||
default:
|
||||
return application.StateUnknown
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeKey(value string) string {
|
||||
return strings.ToLower(strings.TrimPrefix(strings.TrimSpace(value), "/"))
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package agentsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/application"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/service"
|
||||
)
|
||||
|
||||
type stubContainers struct {
|
||||
snapshot container.Snapshot
|
||||
err error
|
||||
}
|
||||
|
||||
func (s stubContainers) Snapshot(context.Context) (container.Snapshot, error) {
|
||||
return s.snapshot, s.err
|
||||
}
|
||||
|
||||
type stubServices struct {
|
||||
snapshot service.Snapshot
|
||||
err error
|
||||
}
|
||||
|
||||
func (s stubServices) Snapshot(context.Context) (service.Snapshot, error) { return s.snapshot, s.err }
|
||||
|
||||
func containerSnapshot(t *testing.T, now time.Time, containers ...container.RawContainer) container.Snapshot {
|
||||
t.Helper()
|
||||
snapshot, err := container.Normalize(container.RawSnapshot{
|
||||
Source: container.Source{ID: "container", Type: "agent"},
|
||||
Containers: containers,
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}, now, container.Limits{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func TestApplicationProviderRequiresFreshContainers(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
services := service.Snapshot{ContractVersion: service.ContractVersion, ObservedAt: now, Services: []service.ServiceStatus{}}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
containers container.Provider
|
||||
services service.Provider
|
||||
reason string
|
||||
}{
|
||||
{"no container provider", nil, stubServices{snapshot: services}, ReasonUnavailable},
|
||||
{"container read fails", stubContainers{err: errors.New("boom")}, stubServices{snapshot: services}, ReasonUnavailable},
|
||||
{"containers unavailable", stubContainers{snapshot: container.UnknownSnapshot(now, "container", "agent", ReasonUnavailable)}, stubServices{snapshot: services}, ReasonUnavailable},
|
||||
{"containers invalid", stubContainers{snapshot: container.UnknownSnapshot(now, "container", "agent", ReasonInvalid)}, stubServices{snapshot: services}, ReasonInvalid},
|
||||
{"containers stale", stubContainers{snapshot: container.UnknownSnapshot(now, "container", "agent", ReasonStale)}, stubServices{snapshot: services}, ReasonStale},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
provider := ApplicationProvider{Containers: testCase.containers, Services: testCase.services, Now: fixedClock(now)}
|
||||
snapshot, err := provider.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unusable input must not fail the request: %v", err)
|
||||
}
|
||||
if snapshot.Source.State != "unknown" || snapshot.Source.Reason != testCase.reason {
|
||||
t.Fatalf("source = %+v, want unknown/%s", snapshot.Source, testCase.reason)
|
||||
}
|
||||
for _, item := range snapshot.Applications {
|
||||
if item.Status == application.StateHealthy {
|
||||
t.Fatal("an application became healthy without complete evidence, violating ADR-0008")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationProviderUsesContainersWhenProbesAreDisabled(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
containers := containerSnapshot(t, now, container.RawContainer{ID: "pulse-1", Name: "pulse-api", State: "running", Health: "healthy"})
|
||||
for name, services := range map[string]service.Provider{
|
||||
"not configured": nil,
|
||||
"unavailable": stubServices{snapshot: service.UnknownSnapshot(now, "source_unavailable")},
|
||||
"read failure": stubServices{err: errors.New("probe store unavailable")},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
snapshot, err := (ApplicationProvider{Containers: stubContainers{snapshot: containers}, Services: services, Now: fixedClock(now)}).Snapshot(context.Background())
|
||||
if err != nil || snapshot.Total != 1 || snapshot.Applications[0].Status != application.StateHealthy {
|
||||
t.Fatalf("snapshot=%+v err=%v", snapshot, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationProviderGroupsContainersAndAppliesProbes(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
containers := containerSnapshot(t, now,
|
||||
container.RawContainer{ID: "web-1", Name: "media-web", State: "running", Health: "healthy", Project: "media"},
|
||||
container.RawContainer{ID: "db-1", Name: "media-db", State: "running", Health: "healthy", Project: "media"},
|
||||
container.RawContainer{ID: "solo-1", Name: "standalone", State: "running", Health: "healthy"},
|
||||
container.RawContainer{ID: "off-1", Name: "archived", State: "exited", IntentionalStop: true},
|
||||
container.RawContainer{ID: "bad-1", Name: "broken", State: "exited"},
|
||||
)
|
||||
services := service.Snapshot{ContractVersion: service.ContractVersion, ObservedAt: now, Services: []service.ServiceStatus{
|
||||
{ID: "svc-web", Name: "media-web", State: service.StateUp},
|
||||
{ID: "svc-db", Name: "media-db", State: service.StateDown},
|
||||
}}
|
||||
provider := ApplicationProvider{Containers: stubContainers{snapshot: containers}, Services: stubServices{snapshot: services}, Now: fixedClock(now)}
|
||||
snapshot, err := provider.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Source.State != "healthy" || snapshot.Source.Reason != "" {
|
||||
t.Fatalf("unexpected source %+v", snapshot.Source)
|
||||
}
|
||||
byName := make(map[string]application.Application, len(snapshot.Applications))
|
||||
for _, item := range snapshot.Applications {
|
||||
byName[item.Name] = item
|
||||
}
|
||||
if len(byName) != 4 {
|
||||
t.Fatalf("expected one application per compose project or standalone container, got %d: %+v", len(byName), snapshot.Applications)
|
||||
}
|
||||
media, ok := byName["media"]
|
||||
if !ok || len(media.Components) != 2 {
|
||||
t.Fatalf("media project was not grouped: %+v", byName)
|
||||
}
|
||||
if media.Status != application.StateDegraded {
|
||||
t.Fatalf("a down probe on a critical component must degrade the application, got %s", media.Status)
|
||||
}
|
||||
if standalone := byName["standalone"]; standalone.Status != application.StateHealthy {
|
||||
t.Fatalf("a running container without a probe must stay healthy, got %s", standalone.Status)
|
||||
}
|
||||
if archived := byName["archived"]; archived.Status != application.StateUnknown {
|
||||
t.Fatalf("an intentionally stopped critical component must remain explicit unknown, got %s", archived.Status)
|
||||
}
|
||||
if broken := byName["broken"]; broken.Status != application.StateDegraded {
|
||||
t.Fatalf("an unexpectedly stopped container must not read as healthy, got %s", broken.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationProviderBoundsTheInventory(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
raw := make([]container.RawContainer, 0, 4)
|
||||
for _, name := range []string{"a", "b", "c", "d"} {
|
||||
raw = append(raw, container.RawContainer{ID: name, Name: name, State: "running", Health: "healthy"})
|
||||
}
|
||||
provider := ApplicationProvider{
|
||||
Containers: stubContainers{snapshot: containerSnapshot(t, now, raw...)},
|
||||
Services: stubServices{snapshot: service.Snapshot{ContractVersion: service.ContractVersion, ObservedAt: now}},
|
||||
MaxApplications: 2,
|
||||
Now: fixedClock(now),
|
||||
}
|
||||
snapshot, err := provider.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Source.State != "unknown" || snapshot.Source.Reason != ReasonInvalid {
|
||||
t.Fatalf("an oversized inventory must resolve to unknown, got %+v", snapshot.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationProviderPropagatesCancellation(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
provider := ApplicationProvider{
|
||||
Containers: stubContainers{err: context.Canceled},
|
||||
Services: stubServices{},
|
||||
Now: fixedClock(now),
|
||||
}
|
||||
if _, err := provider.Snapshot(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected cancellation, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerStateMappingNeverInventsHealth(t *testing.T) {
|
||||
cases := map[string]application.State{
|
||||
"running": application.StateHealthy,
|
||||
"restarting": application.StateDegraded,
|
||||
"paused": application.StateDegraded,
|
||||
"exited": application.StateDown,
|
||||
"dead": application.StateDown,
|
||||
"created": application.StateUnknown,
|
||||
"": application.StateUnknown,
|
||||
"weird": application.StateUnknown,
|
||||
}
|
||||
for state, want := range cases {
|
||||
if got := containerState(container.Container{State: state, Health: "healthy"}); got != want {
|
||||
t.Fatalf("container state %q mapped to %s, want %s", state, got, want)
|
||||
}
|
||||
}
|
||||
if got := containerState(container.Container{State: "running", Health: "unhealthy"}); got != application.StateDegraded {
|
||||
t.Fatalf("an unhealthy running container mapped to %s", got)
|
||||
}
|
||||
if got := containerState(container.Container{State: "running", Health: "starting"}); got != application.StateUnknown {
|
||||
t.Fatalf("a starting container mapped to %s", got)
|
||||
}
|
||||
if got := containerState(container.Container{State: "RUNNING", Health: "HEALTHY"}); got != application.StateHealthy {
|
||||
t.Fatalf("uppercase runtime state mapped to %s", got)
|
||||
}
|
||||
if got := containerState(container.Container{State: "running", Health: "unknown"}); got != application.StateUnknown {
|
||||
t.Fatalf("missing health became %s", got)
|
||||
}
|
||||
if got := containerState(container.Container{State: "exited", Health: "unknown", IntentionalStop: true}); got != application.StateUnknown {
|
||||
t.Fatalf("intentional stop became %s", got)
|
||||
}
|
||||
if got := serviceState("nonsense"); got != application.StateUnknown {
|
||||
t.Fatalf("an unrecognized probe verdict mapped to %s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package agentsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/datasource"
|
||||
)
|
||||
|
||||
type healthReader map[agentstore.Capability]agentstore.Snapshot
|
||||
|
||||
func (r healthReader) Latest(_ context.Context, capability agentstore.Capability) (agentstore.Snapshot, error) {
|
||||
snapshot, ok := r[capability]
|
||||
if !ok {
|
||||
return agentstore.Snapshot{}, agentstore.ErrNoSnapshot
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func TestHealthRequiresEveryCapabilityToBeFresh(t *testing.T) {
|
||||
now := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
|
||||
reader := healthReader{
|
||||
agentstore.CapabilityHost: {
|
||||
Capability: agentstore.CapabilityHost, ObservedAt: now.Add(-time.Second), ReceivedAt: now.Add(-time.Second),
|
||||
},
|
||||
agentstore.CapabilityProcesses: {
|
||||
Capability: agentstore.CapabilityProcesses, ObservedAt: now.Add(-2 * time.Second), ReceivedAt: now.Add(-2 * time.Second),
|
||||
},
|
||||
}
|
||||
health, err := Health(context.Background(), reader, []agentstore.Capability{agentstore.CapabilityHost, agentstore.CapabilityProcesses}, Windows{}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if health.State != datasource.HealthHealthy || health.LastSuccess != now.Add(-2*time.Second) {
|
||||
t.Fatalf("health = %#v", health)
|
||||
}
|
||||
|
||||
missing, err := Health(context.Background(), reader, []agentstore.Capability{agentstore.CapabilityHost, agentstore.CapabilityContainers}, Windows{}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if missing.State != datasource.HealthUnknown || missing.ReasonCode != ReasonUnavailable {
|
||||
t.Fatalf("missing health = %#v", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthRejectsStaleCapabilityAndPropagatesCancellation(t *testing.T) {
|
||||
now := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
|
||||
reader := healthReader{agentstore.CapabilityHost: {
|
||||
Capability: agentstore.CapabilityHost, ObservedAt: now.Add(-time.Hour), ReceivedAt: now.Add(-time.Hour),
|
||||
}}
|
||||
health, err := Health(context.Background(), reader, []agentstore.Capability{agentstore.CapabilityHost}, Windows{}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if health.State != datasource.HealthUnknown || health.ReasonCode != ReasonStale {
|
||||
t.Fatalf("stale health = %#v", health)
|
||||
}
|
||||
|
||||
cancelled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := Health(cancelled, reader, []agentstore.Capability{agentstore.CapabilityHost}, Windows{}, now); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancellation error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package agentsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/application"
|
||||
"github.com/itworx/pulse/internal/array"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/disk"
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
"github.com/itworx/pulse/internal/pool"
|
||||
"github.com/itworx/pulse/internal/process"
|
||||
"github.com/itworx/pulse/internal/share"
|
||||
)
|
||||
|
||||
// Source identifiers and types reported to the API. They match the defaults each domain
|
||||
// Adapter already applies, so a snapshot served from the agent store is indistinguishable
|
||||
// in shape from one served by any other adapter. The type records where the observation
|
||||
// originates: the host capabilities are read by the agent itself, the storage
|
||||
// capabilities are read by the agent from Unraid.
|
||||
const (
|
||||
hostSourceID = "host"
|
||||
processSourceID = "process"
|
||||
containerSourceID = "container"
|
||||
arraySourceID = "array"
|
||||
diskSourceID = "disks"
|
||||
poolSourceID = "pools"
|
||||
shareSourceID = "shares"
|
||||
agentSourceType = "agent"
|
||||
unraidSourceType = "unraid"
|
||||
applicationSources = application.SourceID
|
||||
)
|
||||
|
||||
// HostProvider serves host telemetry recorded by the agent.
|
||||
type HostProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits host.Limits
|
||||
Policy host.Policy
|
||||
// Now overrides the clock in tests; production leaves it nil.
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ host.Provider = HostProvider{}
|
||||
|
||||
func (p HostProvider) Snapshot(ctx context.Context) (host.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityHost, p.Windows.For(agentstore.CapabilityHost), now,
|
||||
func(at time.Time, reason string) host.Snapshot {
|
||||
return host.UnknownSnapshot(at, hostSourceID, agentSourceType, reason)
|
||||
},
|
||||
func(raw host.RawSnapshot, at time.Time) (host.Snapshot, error) {
|
||||
return host.Adapter{Source: staticRaw[host.RawSnapshot]{raw}, Limits: p.Limits, Policy: p.Policy, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// ProcessProvider serves the process inventory recorded by the agent.
|
||||
type ProcessProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits process.Limits
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (p ProcessProvider) Snapshot(ctx context.Context) (process.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityProcesses, p.Windows.For(agentstore.CapabilityProcesses), now,
|
||||
func(at time.Time, reason string) process.Snapshot {
|
||||
return process.UnknownSnapshot(at, processSourceID, agentSourceType, reason)
|
||||
},
|
||||
func(raw process.RawSnapshot, at time.Time) (process.Snapshot, error) {
|
||||
return process.Adapter{Source: staticRaw[process.RawSnapshot]{raw}, Limits: p.Limits, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// ContainerProvider serves the container inventory recorded by the agent.
|
||||
type ContainerProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits container.Limits
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ container.Provider = ContainerProvider{}
|
||||
|
||||
func (p ContainerProvider) Snapshot(ctx context.Context) (container.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityContainers, p.Windows.For(agentstore.CapabilityContainers), now,
|
||||
func(at time.Time, reason string) container.Snapshot {
|
||||
return container.UnknownSnapshot(at, containerSourceID, agentSourceType, reason)
|
||||
},
|
||||
func(raw container.RawSnapshot, at time.Time) (container.Snapshot, error) {
|
||||
return container.Adapter{Source: staticRaw[container.RawSnapshot]{raw}, Limits: p.Limits, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// ArrayProvider serves Unraid array state recorded by the agent.
|
||||
type ArrayProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits array.Limits
|
||||
Policy array.Policy
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ array.Provider = ArrayProvider{}
|
||||
|
||||
func (p ArrayProvider) Snapshot(ctx context.Context) (array.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityArray, p.Windows.For(agentstore.CapabilityArray), now,
|
||||
func(at time.Time, reason string) array.Snapshot {
|
||||
return array.UnknownSnapshot(at, arraySourceID, unraidSourceType, reason)
|
||||
},
|
||||
func(raw array.RawSnapshot, at time.Time) (array.Snapshot, error) {
|
||||
return array.Adapter{Source: staticRaw[array.RawSnapshot]{raw}, Limits: p.Limits, Policy: p.Policy, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// DiskProvider serves disk inventory, SMART and performance data recorded by the agent.
|
||||
type DiskProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits disk.Limits
|
||||
Policy disk.Policy
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ disk.Provider = DiskProvider{}
|
||||
|
||||
func (p DiskProvider) Snapshot(ctx context.Context) (disk.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityDisks, p.Windows.For(agentstore.CapabilityDisks), now,
|
||||
func(at time.Time, reason string) disk.Snapshot {
|
||||
return disk.UnknownSnapshot(at, diskSourceID, unraidSourceType, reason)
|
||||
},
|
||||
func(raw disk.RawSnapshot, at time.Time) (disk.Snapshot, error) {
|
||||
return disk.Adapter{Source: staticRaw[disk.RawSnapshot]{raw}, Limits: p.Limits, Policy: p.Policy, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// PoolProvider serves cache and named pool state recorded by the agent.
|
||||
type PoolProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits pool.Limits
|
||||
Policy pool.Policy
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ pool.Provider = PoolProvider{}
|
||||
|
||||
func (p PoolProvider) Snapshot(ctx context.Context) (pool.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityPools, p.Windows.For(agentstore.CapabilityPools), now,
|
||||
func(at time.Time, reason string) pool.Snapshot {
|
||||
return pool.UnknownSnapshot(at, poolSourceID, unraidSourceType, reason)
|
||||
},
|
||||
func(raw pool.RawSnapshot, at time.Time) (pool.Snapshot, error) {
|
||||
return pool.Adapter{Source: staticRaw[pool.RawSnapshot]{raw}, Limits: p.Limits, Policy: p.Policy, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// ShareProvider serves user share usage recorded by the agent.
|
||||
type ShareProvider struct {
|
||||
Reader agentstore.Reader
|
||||
Windows Windows
|
||||
Limits share.Limits
|
||||
Policy share.Policy
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
var _ share.Provider = ShareProvider{}
|
||||
|
||||
func (p ShareProvider) Snapshot(ctx context.Context) (share.Snapshot, error) {
|
||||
now := clockNow(p.Now)
|
||||
return resolve(ctx, p.Reader, agentstore.CapabilityShares, p.Windows.For(agentstore.CapabilityShares), now,
|
||||
func(at time.Time, reason string) share.Snapshot {
|
||||
return share.UnknownSnapshot(at, shareSourceID, unraidSourceType, reason)
|
||||
},
|
||||
func(raw share.RawSnapshot, at time.Time) (share.Snapshot, error) {
|
||||
return share.Adapter{Source: staticRaw[share.RawSnapshot]{raw}, Limits: p.Limits, Policy: p.Policy, Now: fixedClock(at)}.Snapshot(ctx)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
package agentsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/agentstore"
|
||||
"github.com/itworx/pulse/internal/array"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/disk"
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
"github.com/itworx/pulse/internal/pool"
|
||||
"github.com/itworx/pulse/internal/process"
|
||||
"github.com/itworx/pulse/internal/share"
|
||||
)
|
||||
|
||||
// stubReader stands in for the PostgreSQL store so the read path can be exercised without
|
||||
// a database.
|
||||
type stubReader struct {
|
||||
snapshot agentstore.Snapshot
|
||||
err error
|
||||
asked agentstore.Capability
|
||||
}
|
||||
|
||||
func (r *stubReader) Latest(_ context.Context, capability agentstore.Capability) (agentstore.Snapshot, error) {
|
||||
r.asked = capability
|
||||
if r.err != nil {
|
||||
return agentstore.Snapshot{}, r.err
|
||||
}
|
||||
return r.snapshot, nil
|
||||
}
|
||||
|
||||
// observation is the part of every domain snapshot this package is responsible for.
|
||||
type observation struct {
|
||||
state string
|
||||
reason string
|
||||
healthy bool
|
||||
}
|
||||
|
||||
// domainUnderTest describes one capability generically so every domain runs the same
|
||||
// table: no snapshot, store failure, stale snapshot, corrupt payload, payload the domain
|
||||
// rejects, and a good payload.
|
||||
type domainUnderTest struct {
|
||||
name string
|
||||
capability agentstore.Capability
|
||||
window time.Duration
|
||||
good func(now time.Time) any
|
||||
rejected func(now time.Time) any
|
||||
observe func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error)
|
||||
}
|
||||
|
||||
func floatPtr(value float64) *float64 { return &value }
|
||||
|
||||
func domains() []domainUnderTest {
|
||||
return []domainUnderTest{
|
||||
{
|
||||
name: "host",
|
||||
capability: agentstore.CapabilityHost,
|
||||
window: DefaultHostWindow,
|
||||
good: func(now time.Time) any {
|
||||
used := uint64(6 * 1024 * 1024 * 1024)
|
||||
return host.RawSnapshot{
|
||||
Source: host.Source{ID: "agent-1", Type: "agent"},
|
||||
Identity: host.HostIdentity{Name: "pulse-host", Version: "7.2.2", Arch: "amd64"},
|
||||
UptimeSeconds: 3600,
|
||||
CPU: host.RawCPU{TotalPercent: floatPtr(42.5), PerCore: []float64{40, 45}, IOWaitPercent: floatPtr(2)},
|
||||
Load: host.RawLoad{One: 0.2, Five: 0.1, Fifteen: 0.1},
|
||||
Memory: host.RawMemory{TotalBytes: 8 * 1024 * 1024 * 1024, AvailableBytes: 2 * 1024 * 1024 * 1024, UsedBytes: &used},
|
||||
Filesystems: []host.RawFilesystem{{Mount: "/", Filesystem: "xfs", CapacityBytes: 100, UsedBytes: 25}},
|
||||
Network: []host.RawNetworkInterface{{Name: "eth0", State: "up"}},
|
||||
Time: host.RawTime{Synchronized: true, OffsetSeconds: 0.002, Stratum: 2},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return host.RawSnapshot{Identity: host.HostIdentity{Name: ""}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := HostProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{
|
||||
state: snapshot.Source.State,
|
||||
reason: snapshot.Source.Reason,
|
||||
healthy: snapshot.Source.Freshness == host.Fresh && snapshot.Status.State == host.StatusHealthy,
|
||||
}, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "processes",
|
||||
capability: agentstore.CapabilityProcesses,
|
||||
window: DefaultProcessesWindow,
|
||||
good: func(now time.Time) any {
|
||||
return process.RawSnapshot{
|
||||
Source: process.Source{ID: "agent-1", Type: "agent"},
|
||||
Processes: []process.RawProcess{{PID: 1, Name: "init", State: "sleeping", RuntimeSeconds: 100, CPUPercent: 1, MemoryBytes: 500}},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return process.RawSnapshot{Processes: []process.RawProcess{{PID: 0, Name: ""}}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := ProcessProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{state: snapshot.Source.State, reason: snapshot.Source.Reason, healthy: snapshot.Source.State == "healthy"}, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "containers",
|
||||
capability: agentstore.CapabilityContainers,
|
||||
window: DefaultContainersWindow,
|
||||
good: func(now time.Time) any {
|
||||
return container.RawSnapshot{
|
||||
Source: container.Source{ID: "agent-1", Type: "agent"},
|
||||
Containers: []container.RawContainer{{ID: "a", Name: "alpha", State: "running", Health: "healthy"}},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return container.RawSnapshot{Containers: []container.RawContainer{{ID: "", Name: ""}}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := ContainerProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{
|
||||
state: snapshot.Source.State,
|
||||
reason: snapshot.Source.Reason,
|
||||
healthy: snapshot.Source.Freshness == "fresh" && snapshot.Source.State == "healthy",
|
||||
}, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "array",
|
||||
capability: agentstore.CapabilityArray,
|
||||
window: DefaultArrayWindow,
|
||||
good: func(now time.Time) any {
|
||||
return array.RawSnapshot{
|
||||
Source: array.Source{ID: "agent-1", Type: "unraid"},
|
||||
State: array.StateOperational,
|
||||
Parity: array.RawParity{Present: true, State: "idle"},
|
||||
Members: []array.RawMember{
|
||||
{ID: "disk1", Name: "Disk 1", Role: "data", State: "online", CapacityBytes: 100},
|
||||
{ID: "parity", Name: "Parity", Role: "parity", State: "online", CapacityBytes: 100},
|
||||
},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return array.RawSnapshot{State: array.StateOperational, Members: []array.RawMember{{ID: "", Name: ""}}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := ArrayProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{
|
||||
state: snapshot.Source.State,
|
||||
reason: snapshot.Source.Reason,
|
||||
healthy: snapshot.Source.Freshness == array.Fresh && snapshot.State == array.StateOperational,
|
||||
}, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "disks",
|
||||
capability: agentstore.CapabilityDisks,
|
||||
window: DefaultDisksWindow,
|
||||
good: func(now time.Time) any {
|
||||
return disk.RawSnapshot{
|
||||
Source: disk.Source{ID: "agent-1", Type: "unraid"},
|
||||
Disks: []disk.RawDisk{{ID: "disk1", Name: "Disk 1", Role: "data", State: disk.StateOnline, SizeBytes: 100}},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return disk.RawSnapshot{Disks: []disk.RawDisk{{ID: "", Name: ""}}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := DiskProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{
|
||||
state: snapshot.Source.State,
|
||||
reason: snapshot.Source.Reason,
|
||||
healthy: snapshot.Source.Freshness == disk.Fresh && snapshot.Source.State != disk.StateUnknown,
|
||||
}, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "pools",
|
||||
capability: agentstore.CapabilityPools,
|
||||
window: DefaultPoolsWindow,
|
||||
good: func(now time.Time) any {
|
||||
return pool.RawSnapshot{
|
||||
Source: pool.Source{ID: "agent-1", Type: "unraid"},
|
||||
Pools: []pool.RawPool{{ID: "cache", Name: "Cache", Filesystem: "btrfs", State: pool.StateHealthy, UsableBytes: 1000, UsedBytes: 100}},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return pool.RawSnapshot{Pools: []pool.RawPool{{ID: "", Name: ""}}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := PoolProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{
|
||||
state: snapshot.Source.State,
|
||||
reason: snapshot.Source.Reason,
|
||||
healthy: snapshot.Source.Freshness == pool.Fresh && snapshot.Source.State != pool.StateUnknown,
|
||||
}, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "shares",
|
||||
capability: agentstore.CapabilityShares,
|
||||
window: DefaultSharesWindow,
|
||||
good: func(now time.Time) any {
|
||||
return share.RawSnapshot{
|
||||
Source: share.Source{ID: "agent-1", Type: "unraid"},
|
||||
Shares: []share.RawShare{{ID: "share-media", Name: "Media", UsedBytes: 700, SizeObservedAt: now, SizeState: share.SizeCached}},
|
||||
ObservedAt: now, ReceivedAt: now,
|
||||
}
|
||||
},
|
||||
rejected: func(now time.Time) any {
|
||||
return share.RawSnapshot{Shares: []share.RawShare{{ID: "", Name: ""}}, ObservedAt: now, ReceivedAt: now}
|
||||
},
|
||||
observe: func(ctx context.Context, reader agentstore.Reader, now time.Time) (observation, error) {
|
||||
snapshot, err := ShareProvider{Reader: reader, Now: fixedClock(now)}.Snapshot(ctx)
|
||||
return observation{
|
||||
state: snapshot.Source.State,
|
||||
reason: snapshot.Source.Reason,
|
||||
healthy: snapshot.Source.Freshness == share.Fresh && snapshot.Source.State != share.StateUnknown,
|
||||
}, err
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func encode(t *testing.T, value any) json.RawMessage {
|
||||
t.Helper()
|
||||
payload, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
func TestProvidersMapUnusableTelemetryToUnknown(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
for _, domain := range domains() {
|
||||
t.Run(domain.name, func(t *testing.T) {
|
||||
fresh := agentstore.Snapshot{AgentID: "agent-1", Capability: domain.capability, ObservedAt: now, ReceivedAt: now}
|
||||
cases := []struct {
|
||||
name string
|
||||
reader *stubReader
|
||||
reason string
|
||||
}{
|
||||
{"no snapshot", &stubReader{err: agentstore.ErrNoSnapshot}, ReasonUnavailable},
|
||||
{"store unreachable", &stubReader{err: errors.New("connection refused")}, ReasonUnavailable},
|
||||
{"stale snapshot", &stubReader{snapshot: func() agentstore.Snapshot {
|
||||
stale := fresh
|
||||
stale.ObservedAt = now.Add(-domain.window - time.Second)
|
||||
stale.Payload = encode(t, domain.good(stale.ObservedAt))
|
||||
return stale
|
||||
}()}, ReasonStale},
|
||||
{"missing observation time", &stubReader{snapshot: func() agentstore.Snapshot {
|
||||
broken := fresh
|
||||
broken.ObservedAt = time.Time{}
|
||||
broken.Payload = encode(t, domain.good(now))
|
||||
return broken
|
||||
}()}, ReasonStale},
|
||||
{"corrupt payload", &stubReader{snapshot: func() agentstore.Snapshot {
|
||||
corrupt := fresh
|
||||
corrupt.Payload = json.RawMessage(`{"source":"not-an-object"}`)
|
||||
return corrupt
|
||||
}()}, ReasonInvalid},
|
||||
{"payload the domain rejects", &stubReader{snapshot: func() agentstore.Snapshot {
|
||||
invalid := fresh
|
||||
invalid.Payload = encode(t, domain.rejected(now))
|
||||
return invalid
|
||||
}()}, ReasonInvalid},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
got, err := domain.observe(context.Background(), testCase.reader, now)
|
||||
if err != nil {
|
||||
t.Fatalf("unusable telemetry must not fail the request: %v", err)
|
||||
}
|
||||
if got.healthy {
|
||||
t.Fatal("unusable telemetry became healthy, violating ADR-0008")
|
||||
}
|
||||
if got.state != "unknown" {
|
||||
t.Fatalf("source state = %q, want unknown", got.state)
|
||||
}
|
||||
if got.reason != testCase.reason {
|
||||
t.Fatalf("reason = %q, want %q", got.reason, testCase.reason)
|
||||
}
|
||||
if testCase.reader.asked != domain.capability {
|
||||
t.Fatalf("read capability %q, want %q", testCase.reader.asked, domain.capability)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvidersNormalizeGoodTelemetry(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
for _, domain := range domains() {
|
||||
t.Run(domain.name, func(t *testing.T) {
|
||||
observed := now.Add(-time.Second)
|
||||
reader := &stubReader{snapshot: agentstore.Snapshot{
|
||||
AgentID: "agent-1", Capability: domain.capability, ObservedAt: observed, ReceivedAt: observed,
|
||||
Payload: encode(t, domain.good(observed)),
|
||||
}}
|
||||
got, err := domain.observe(context.Background(), reader, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got.healthy {
|
||||
t.Fatalf("fresh telemetry did not normalize to a healthy source: %+v", got)
|
||||
}
|
||||
if got.reason != "" {
|
||||
t.Fatalf("healthy source carries reason %q", got.reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvidersTreatTheWindowBoundaryAsFresh(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
for _, domain := range domains() {
|
||||
t.Run(domain.name, func(t *testing.T) {
|
||||
observed := now.Add(-domain.window)
|
||||
reader := &stubReader{snapshot: agentstore.Snapshot{
|
||||
AgentID: "agent-1", Capability: domain.capability, ObservedAt: observed, ReceivedAt: observed,
|
||||
Payload: encode(t, domain.good(observed)),
|
||||
}}
|
||||
got, err := domain.observe(context.Background(), reader, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.state == "unknown" {
|
||||
t.Fatalf("a snapshot exactly at the window edge must still be usable: %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvidersWithoutAReaderReportUnavailable(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
for _, domain := range domains() {
|
||||
t.Run(domain.name, func(t *testing.T) {
|
||||
got, err := domain.observe(context.Background(), nil, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.healthy || got.state != "unknown" || got.reason != ReasonUnavailable {
|
||||
t.Fatalf("unexpected observation %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvidersPropagateContextCancellation(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for _, domain := range domains() {
|
||||
t.Run(domain.name, func(t *testing.T) {
|
||||
reader := &stubReader{err: context.Canceled}
|
||||
if _, err := domain.observe(ctx, reader, now); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected cancellation, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWindowsApplyDocumentedDefaults(t *testing.T) {
|
||||
windows := Windows{}
|
||||
expected := map[agentstore.Capability]time.Duration{
|
||||
agentstore.CapabilityHost: DefaultHostWindow,
|
||||
agentstore.CapabilityProcesses: DefaultProcessesWindow,
|
||||
agentstore.CapabilityContainers: DefaultContainersWindow,
|
||||
agentstore.CapabilityArray: DefaultArrayWindow,
|
||||
agentstore.CapabilityDisks: DefaultDisksWindow,
|
||||
agentstore.CapabilityPools: DefaultPoolsWindow,
|
||||
agentstore.CapabilityShares: DefaultSharesWindow,
|
||||
}
|
||||
for _, capability := range agentstore.Capabilities() {
|
||||
if got := windows.For(capability); got != expected[capability] {
|
||||
t.Fatalf("window for %q = %s, want %s", capability, got, expected[capability])
|
||||
}
|
||||
}
|
||||
if windows.For(agentstore.CapabilityHost) >= windows.For(agentstore.CapabilityShares) {
|
||||
t.Fatal("the host is sampled far more often than shares and must have the tighter window")
|
||||
}
|
||||
if err := windows.Validate(); err != nil {
|
||||
t.Fatalf("defaults must validate: %v", err)
|
||||
}
|
||||
if err := (Windows{Host: 48 * time.Hour}).Validate(); err == nil {
|
||||
t.Fatal("a window beyond a day must be rejected")
|
||||
}
|
||||
if err := (Windows{Shares: -time.Second}).Validate(); err == nil {
|
||||
t.Fatal("a negative window must be rejected")
|
||||
}
|
||||
custom := Windows{Host: 5 * time.Second}.WithDefaults()
|
||||
if custom.Host != 5*time.Second || custom.Shares != DefaultSharesWindow {
|
||||
t.Fatalf("configuration was not preserved: %+v", custom)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredWindowOverridesTheDefault(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
observed := now.Add(-10 * time.Second)
|
||||
payload := encode(t, process.RawSnapshot{
|
||||
Source: process.Source{ID: "agent-1", Type: "agent"},
|
||||
Processes: []process.RawProcess{{PID: 1, Name: "init", State: "sleeping"}},
|
||||
ObservedAt: observed, ReceivedAt: observed,
|
||||
})
|
||||
reader := &stubReader{snapshot: agentstore.Snapshot{AgentID: "agent-1", Capability: agentstore.CapabilityProcesses, ObservedAt: observed, ReceivedAt: observed, Payload: payload}}
|
||||
provider := ProcessProvider{Reader: reader, Windows: Windows{Processes: 5 * time.Second}, Now: fixedClock(now)}
|
||||
snapshot, err := provider.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Source.State != "unknown" || snapshot.Source.Reason != ReasonStale {
|
||||
t.Fatalf("a tightened window must make the snapshot stale: %+v", snapshot.Source)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
// Package agentstore is the transport boundary between pulse-agent and pulse-api.
|
||||
//
|
||||
// The agent runs with the narrow read-only host access it needs and never exposes a
|
||||
// network endpoint; the API never reaches out to the host. Both processes already share
|
||||
// PostgreSQL on the internal-only Compose network (see deploy/compose.yaml and ADR-0010),
|
||||
// so the agent writes bounded, normalized snapshots into the database and the API reads
|
||||
// the most recent one per capability. This keeps the privilege separation required by
|
||||
// SYSTEM_ARCHITECTURE section "pulse-agent" without adding an inbound port to the agent.
|
||||
//
|
||||
// Freshness is deliberately the reader's problem, not the writer's: a snapshot carries
|
||||
// the time the agent observed it, and the reader decides whether that is still usable.
|
||||
// A capability with no snapshot, or one older than its freshness window, resolves to
|
||||
// Unknown and never to Healthy (ADR-0008).
|
||||
package agentstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Capability identifies one bounded telemetry surface an agent can report. The set is
|
||||
// closed: a reader must never accept a capability it does not recognize, because an
|
||||
// unknown capability cannot be normalized or bounded.
|
||||
type Capability string
|
||||
|
||||
const (
|
||||
CapabilityHost Capability = "host"
|
||||
CapabilityProcesses Capability = "processes"
|
||||
CapabilityContainers Capability = "containers"
|
||||
CapabilityArray Capability = "array"
|
||||
CapabilityDisks Capability = "disks"
|
||||
CapabilityPools Capability = "pools"
|
||||
CapabilityShares Capability = "shares"
|
||||
)
|
||||
|
||||
// Capabilities lists every capability the platform recognizes, in a stable order.
|
||||
func Capabilities() []Capability {
|
||||
return []Capability{
|
||||
CapabilityHost, CapabilityProcesses, CapabilityContainers,
|
||||
CapabilityArray, CapabilityDisks, CapabilityPools, CapabilityShares,
|
||||
}
|
||||
}
|
||||
|
||||
// Valid reports whether the capability is one this platform recognizes.
|
||||
func (c Capability) Valid() bool {
|
||||
for _, known := range Capabilities() {
|
||||
if c == known {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// MaxPayloadBytes bounds a single snapshot. The largest realistic payload is the process
|
||||
// inventory at the documented scale target; this leaves generous headroom while keeping a
|
||||
// misbehaving or compromised agent from filling the database.
|
||||
const MaxPayloadBytes = 2 << 20
|
||||
|
||||
// ErrNoSnapshot is returned by Reader.Latest when the capability has never been reported.
|
||||
// It is an expected condition on a fresh install, not a failure: callers translate it into
|
||||
// an Unknown status with an explicit reason.
|
||||
var ErrNoSnapshot = errors.New("no agent snapshot recorded")
|
||||
|
||||
// Snapshot is one bounded observation of a single capability.
|
||||
type Snapshot struct {
|
||||
// AgentID identifies the reporting agent.
|
||||
AgentID string
|
||||
// Capability is the telemetry surface this payload describes.
|
||||
Capability Capability
|
||||
// ObservedAt is when the agent read the underlying source, in UTC.
|
||||
ObservedAt time.Time
|
||||
// ReceivedAt is when the store accepted the snapshot, in UTC. It is set by the
|
||||
// writer implementation, never by the agent, so a skewed agent clock cannot make
|
||||
// stale data look fresh.
|
||||
ReceivedAt time.Time
|
||||
// Payload is the domain RawSnapshot for this capability, JSON encoded.
|
||||
Payload json.RawMessage
|
||||
}
|
||||
|
||||
// Age reports how long ago the agent observed this snapshot.
|
||||
func (s Snapshot) Age(now time.Time) time.Duration { return now.Sub(s.ObservedAt) }
|
||||
|
||||
// Writer is the narrow interface pulse-agent depends on. The agent must not be able to
|
||||
// read other agents' data or mutate anything else.
|
||||
type Writer interface {
|
||||
// Put records the newest snapshot for one capability, replacing any previous one.
|
||||
// Implementations reject an unknown capability, an oversized payload, a zero or
|
||||
// future ObservedAt, and payloads that are not valid JSON objects.
|
||||
Put(ctx context.Context, snapshot Snapshot) error
|
||||
}
|
||||
|
||||
// Reader is the narrow interface pulse-api depends on.
|
||||
type Reader interface {
|
||||
// Latest returns the most recent snapshot for the capability, or ErrNoSnapshot.
|
||||
Latest(ctx context.Context, capability Capability) (Snapshot, error)
|
||||
}
|
||||
|
||||
// Store is the combined boundary. Only the migration-owning implementation satisfies it.
|
||||
type Store interface {
|
||||
Writer
|
||||
Reader
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
package agentstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// MaxAgentIDBytes bounds the reporting agent identifier. It matches the column check in
|
||||
// migration 0016 so a rejection surfaces as a terse store error rather than a constraint
|
||||
// violation from PostgreSQL.
|
||||
const MaxAgentIDBytes = 128
|
||||
|
||||
// MaxClockSkew is the only tolerance granted to an agent clock that runs ahead of the
|
||||
// database. The agent and the database share one host in the supported topology, so real
|
||||
// skew is sub-millisecond; one second absorbs scheduling jitter while staying far below
|
||||
// the tightest freshness window, so a skewed clock cannot mask meaningful staleness.
|
||||
const MaxClockSkew = time.Second
|
||||
|
||||
// ErrUnavailable is returned when the store has no database pool. Callers translate it
|
||||
// into an Unknown status exactly as they do ErrNoSnapshot.
|
||||
var ErrUnavailable = errors.New("agent snapshot store is unavailable")
|
||||
|
||||
// PostgresStore is the only implementation of Store. It owns the agent_snapshots table
|
||||
// created by migration 0016 and keeps exactly one row per (agent, capability).
|
||||
//
|
||||
// Transaction boundary: Put commits the latest snapshot and any derived capacity samples
|
||||
// atomically. Latest remains a single read statement.
|
||||
type PostgresStore struct {
|
||||
// Pool is the shared pgx pool. A nil pool makes every call return ErrUnavailable.
|
||||
Pool *pgxpool.Pool
|
||||
// Clock supplies ReceivedAt and the future-observation check. It exists for tests;
|
||||
// production leaves it nil and the store uses the wall clock in UTC.
|
||||
Clock func() time.Time
|
||||
}
|
||||
|
||||
var _ Store = PostgresStore{}
|
||||
|
||||
func (s PostgresStore) now() time.Time {
|
||||
if s.Clock == nil {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
return s.Clock().UTC()
|
||||
}
|
||||
|
||||
// Put records the newest snapshot for one capability, replacing any previous one for the
|
||||
// same agent. It rejects an unknown capability, a missing or oversized payload, a payload
|
||||
// that is not a JSON object, and a zero or future ObservedAt.
|
||||
//
|
||||
// ReceivedAt is always taken from the store clock: the caller's value is ignored so a
|
||||
// skewed or hostile agent clock cannot make stale data look freshly received.
|
||||
//
|
||||
// A snapshot that is older than the row already stored is accepted but does not overwrite
|
||||
// it, so a delayed retry cannot resurrect superseded telemetry.
|
||||
func (s PostgresStore) Put(ctx context.Context, snapshot Snapshot) error {
|
||||
row, err := prepare(snapshot, s.now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s.Pool == nil {
|
||||
return ErrUnavailable
|
||||
}
|
||||
tx, err := s.Pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin agent snapshot write: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rollbackCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = tx.Rollback(rollbackCtx)
|
||||
}()
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO agent_snapshots (agent_id, capability, observed_at, received_at, payload)
|
||||
VALUES ($1, $2, $3, $4, $5::jsonb)
|
||||
ON CONFLICT (agent_id, capability) DO UPDATE
|
||||
SET observed_at = EXCLUDED.observed_at, received_at = EXCLUDED.received_at, payload = EXCLUDED.payload
|
||||
WHERE agent_snapshots.observed_at <= EXCLUDED.observed_at`,
|
||||
row.AgentID, string(row.Capability), row.ObservedAt, row.ReceivedAt, []byte(row.Payload)); err != nil {
|
||||
return fmt.Errorf("write agent snapshot: %w", err)
|
||||
}
|
||||
if err := persistCapacitySamples(ctx, tx, row); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit agent snapshot write: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type capacitySample struct {
|
||||
Kind string `json:"kind"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
UsedBytes uint64 `json:"used_bytes"`
|
||||
CapacityBytes uint64 `json:"capacity_bytes"`
|
||||
}
|
||||
|
||||
func persistCapacitySamples(ctx context.Context, tx pgx.Tx, snapshot Snapshot) error {
|
||||
samples, err := capacitySamples(snapshot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(samples) == 0 {
|
||||
return nil
|
||||
}
|
||||
payload, err := json.Marshal(samples)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode capacity samples: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `INSERT INTO capacity_samples (entity_kind,entity_id,entity_name,source_id,sampled_at,observed_at,used_bytes,capacity_bytes)
|
||||
SELECT sample.kind,sample.id,sample.name,$1,
|
||||
date_trunc('day',sample.observed_at) + floor(extract(hour FROM sample.observed_at)/6)*interval '6 hours',
|
||||
sample.observed_at,sample.used_bytes,sample.capacity_bytes
|
||||
FROM jsonb_to_recordset($2::jsonb) AS sample(kind text,id text,name text,observed_at timestamptz,used_bytes bigint,capacity_bytes bigint)
|
||||
ON CONFLICT (entity_kind,entity_id,source_id,sampled_at) DO UPDATE
|
||||
SET entity_name=EXCLUDED.entity_name,observed_at=EXCLUDED.observed_at,used_bytes=EXCLUDED.used_bytes,capacity_bytes=EXCLUDED.capacity_bytes
|
||||
WHERE capacity_samples.observed_at <= EXCLUDED.observed_at`, snapshot.AgentID, payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("persist capacity samples: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func capacitySamples(snapshot Snapshot) ([]capacitySample, error) {
|
||||
kind := ""
|
||||
collection := ""
|
||||
capacityField := ""
|
||||
switch snapshot.Capability {
|
||||
case CapabilityShares:
|
||||
kind, collection = "share", "shares"
|
||||
case CapabilityPools:
|
||||
kind, collection, capacityField = "pool", "pools", "usableBytes"
|
||||
case CapabilityDisks:
|
||||
kind, collection, capacityField = "disk", "disks", "sizeBytes"
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
var document map[string]json.RawMessage
|
||||
if err := json.Unmarshal(snapshot.Payload, &document); err != nil {
|
||||
return nil, errors.New("decode capacity snapshot payload")
|
||||
}
|
||||
var items []map[string]json.RawMessage
|
||||
if err := json.Unmarshal(document[collection], &items); err != nil {
|
||||
return nil, fmt.Errorf("decode %s capacity collection", kind)
|
||||
}
|
||||
samples := make([]capacitySample, 0, len(items))
|
||||
for _, item := range items {
|
||||
var id, name string
|
||||
var used, capacity uint64
|
||||
if json.Unmarshal(item["id"], &id) != nil || json.Unmarshal(item["name"], &name) != nil || json.Unmarshal(item["usedBytes"], &used) != nil {
|
||||
continue
|
||||
}
|
||||
if capacityField != "" {
|
||||
if json.Unmarshal(item[capacityField], &capacity) != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
observed := snapshot.ObservedAt
|
||||
if kind == "share" {
|
||||
var sizeObserved time.Time
|
||||
if json.Unmarshal(item["sizeObservedAt"], &sizeObserved) == nil && !sizeObserved.IsZero() {
|
||||
observed = sizeObserved.UTC()
|
||||
}
|
||||
}
|
||||
id, name = strings.TrimSpace(id), strings.TrimSpace(name)
|
||||
if id == "" || name == "" || len(id) > 128 || len(name) > 255 || used > math.MaxInt64 || capacity > math.MaxInt64 || observed.IsZero() || observed.After(snapshot.ReceivedAt.Add(MaxClockSkew)) {
|
||||
continue
|
||||
}
|
||||
samples = append(samples, capacitySample{Kind: kind, ID: id, Name: name, ObservedAt: observed, UsedBytes: used, CapacityBytes: capacity})
|
||||
}
|
||||
return samples, nil
|
||||
}
|
||||
|
||||
// Latest returns the most recent snapshot for the capability across every reporting
|
||||
// agent, or ErrNoSnapshot when none has been recorded.
|
||||
func (s PostgresStore) Latest(ctx context.Context, capability Capability) (Snapshot, error) {
|
||||
if !capability.Valid() {
|
||||
return Snapshot{}, fmt.Errorf("unknown agent capability %q", capability)
|
||||
}
|
||||
if s.Pool == nil {
|
||||
return Snapshot{}, ErrUnavailable
|
||||
}
|
||||
return scanSnapshot(s.Pool.QueryRow(ctx, `SELECT agent_id, capability, observed_at, received_at, payload
|
||||
FROM agent_snapshots WHERE capability = $1 ORDER BY observed_at DESC, received_at DESC LIMIT 1`,
|
||||
string(capability)))
|
||||
}
|
||||
|
||||
// scanSnapshot reads one row and maps the absence of a row onto ErrNoSnapshot, which is an
|
||||
// expected condition on a fresh install rather than a failure.
|
||||
func scanSnapshot(row pgx.Row) (Snapshot, error) {
|
||||
var (
|
||||
snapshot Snapshot
|
||||
name string
|
||||
payload []byte
|
||||
)
|
||||
err := row.Scan(&snapshot.AgentID, &name, &snapshot.ObservedAt, &snapshot.ReceivedAt, &payload)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Snapshot{}, ErrNoSnapshot
|
||||
}
|
||||
if err != nil {
|
||||
return Snapshot{}, fmt.Errorf("read agent snapshot: %w", err)
|
||||
}
|
||||
snapshot.Capability = Capability(name)
|
||||
snapshot.ObservedAt = snapshot.ObservedAt.UTC()
|
||||
snapshot.ReceivedAt = snapshot.ReceivedAt.UTC()
|
||||
snapshot.Payload = json.RawMessage(payload)
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
// prepare enforces every bound the Writer contract documents and returns the row the
|
||||
// store persists. It is the single place where an inbound snapshot is trusted, and it
|
||||
// overwrites ReceivedAt with the store clock so the caller cannot influence freshness.
|
||||
func prepare(snapshot Snapshot, now time.Time) (Snapshot, error) {
|
||||
if snapshot.AgentID == "" || len(snapshot.AgentID) > MaxAgentIDBytes {
|
||||
return Snapshot{}, errors.New("agent id is required and bounded")
|
||||
}
|
||||
if !snapshot.Capability.Valid() {
|
||||
return Snapshot{}, fmt.Errorf("unknown agent capability %q", snapshot.Capability)
|
||||
}
|
||||
if len(snapshot.Payload) == 0 {
|
||||
return Snapshot{}, errors.New("agent snapshot payload is required")
|
||||
}
|
||||
if len(snapshot.Payload) > MaxPayloadBytes {
|
||||
return Snapshot{}, fmt.Errorf("agent snapshot payload exceeds %d bytes", MaxPayloadBytes)
|
||||
}
|
||||
if snapshot.ObservedAt.IsZero() {
|
||||
return Snapshot{}, errors.New("agent snapshot observed time is required")
|
||||
}
|
||||
if snapshot.ObservedAt.After(now.Add(MaxClockSkew)) {
|
||||
return Snapshot{}, errors.New("agent snapshot observed time is in the future")
|
||||
}
|
||||
var object map[string]json.RawMessage
|
||||
if err := json.Unmarshal(snapshot.Payload, &object); err != nil || object == nil {
|
||||
return Snapshot{}, errors.New("agent snapshot payload must be a JSON object")
|
||||
}
|
||||
snapshot.ObservedAt = snapshot.ObservedAt.UTC()
|
||||
snapshot.ReceivedAt = now
|
||||
return snapshot, nil
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package agentstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
)
|
||||
|
||||
// TestPostgreSQLSnapshotRoundTrip exercises the real table created by migration 0016. It
|
||||
// skips unless PULSE_TEST_DATABASE_URL points at a disposable database.
|
||||
func TestPostgreSQLSnapshotRoundTrip(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `DELETE FROM agent_snapshots WHERE agent_id = 'integration-agent'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `DELETE FROM capacity_samples WHERE source_id = 'integration-agent'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
cleanup, cancelCleanup := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancelCleanup()
|
||||
_, _ = pool.Exec(cleanup, `DELETE FROM agent_snapshots WHERE agent_id = 'integration-agent'`)
|
||||
_, _ = pool.Exec(cleanup, `DELETE FROM capacity_samples WHERE source_id = 'integration-agent'`)
|
||||
})
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
store := PostgresStore{Pool: pool, Clock: func() time.Time { return now }}
|
||||
readOwn := func() Snapshot {
|
||||
t.Helper()
|
||||
stored := Snapshot{AgentID: "integration-agent", Capability: CapabilityPools}
|
||||
var payload []byte
|
||||
if err := pool.QueryRow(ctx, `SELECT observed_at,received_at,payload FROM agent_snapshots WHERE agent_id=$1 AND capability=$2`, stored.AgentID, stored.Capability).Scan(&stored.ObservedAt, &stored.ReceivedAt, &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored.Payload = payload
|
||||
return stored
|
||||
}
|
||||
if _, err := store.Latest(ctx, CapabilityPools); err != nil && !errors.Is(err, ErrNoSnapshot) {
|
||||
t.Fatalf("unexpected error reading an empty capability: %v", err)
|
||||
}
|
||||
|
||||
first := Snapshot{AgentID: "integration-agent", Capability: CapabilityPools, ObservedAt: now.Add(-30 * time.Second), ReceivedAt: now.Add(72 * time.Hour), Payload: json.RawMessage(`{"pools":[{"id":"cache","name":"Cache","usedBytes":100,"usableBytes":1000}],"generation":1}`)}
|
||||
if err := store.Put(ctx, first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored := readOwn()
|
||||
if !stored.ReceivedAt.Equal(now) {
|
||||
t.Fatalf("received at = %s, want the store clock %s", stored.ReceivedAt, now)
|
||||
}
|
||||
if !stored.ObservedAt.Equal(first.ObservedAt) {
|
||||
t.Fatalf("observed at = %s, want %s", stored.ObservedAt, first.ObservedAt)
|
||||
}
|
||||
|
||||
newer := first
|
||||
newer.ObservedAt = now.Add(-5 * time.Second)
|
||||
newer.Payload = json.RawMessage(`{"pools":[{"id":"cache","name":"Cache","usedBytes":200,"usableBytes":1000}],"generation":2}`)
|
||||
if err := store.Put(ctx, newer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
older := first
|
||||
older.ObservedAt = now.Add(-120 * time.Second)
|
||||
older.Payload = json.RawMessage(`{"pools":[{"id":"cache","name":"Cache","usedBytes":50,"usableBytes":1000}],"generation":3}`)
|
||||
if err := store.Put(ctx, older); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored = readOwn()
|
||||
var decoded struct {
|
||||
Generation int `json:"generation"`
|
||||
}
|
||||
if err := json.Unmarshal(stored.Payload, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decoded.Generation != 2 {
|
||||
t.Fatalf("a delayed retry must not resurrect superseded telemetry, got generation %d", decoded.Generation)
|
||||
}
|
||||
|
||||
var rows int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM agent_snapshots WHERE agent_id = 'integration-agent'`).Scan(&rows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Fatalf("agent snapshot rows = %d, want exactly one per (agent, capability)", rows)
|
||||
}
|
||||
var sampleRows int
|
||||
var usedBytes int64
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*),max(used_bytes) FROM capacity_samples WHERE source_id='integration-agent' AND entity_kind='pool' AND entity_id='cache'`).Scan(&sampleRows, &usedBytes); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sampleRows != 1 || usedBytes != 200 {
|
||||
t.Fatalf("six-hour bucket was not idempotent or accepted an older retry: rows=%d used=%d", sampleRows, usedBytes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
package agentstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func fixedClock(now time.Time) func() time.Time { return func() time.Time { return now } }
|
||||
|
||||
func TestPutRejectsEveryDocumentedBound(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
valid := Snapshot{AgentID: "agent-1", Capability: CapabilityHost, ObservedAt: now.Add(-5 * time.Second), Payload: json.RawMessage(`{"identity":{"name":"tower"}}`)}
|
||||
oversized := make([]byte, MaxPayloadBytes+1)
|
||||
oversized[0] = '{'
|
||||
for i := 1; i < len(oversized)-1; i++ {
|
||||
oversized[i] = ' '
|
||||
}
|
||||
oversized[len(oversized)-1] = '}'
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
mutate func(Snapshot) Snapshot
|
||||
contains string
|
||||
}{
|
||||
{"missing agent", func(s Snapshot) Snapshot { s.AgentID = ""; return s }, "agent id"},
|
||||
{"oversized agent", func(s Snapshot) Snapshot { s.AgentID = strings.Repeat("a", MaxAgentIDBytes+1); return s }, "agent id"},
|
||||
{"unknown capability", func(s Snapshot) Snapshot { s.Capability = "gpu"; return s }, "unknown agent capability"},
|
||||
{"empty capability", func(s Snapshot) Snapshot { s.Capability = ""; return s }, "unknown agent capability"},
|
||||
{"missing payload", func(s Snapshot) Snapshot { s.Payload = nil; return s }, "payload is required"},
|
||||
{"oversized payload", func(s Snapshot) Snapshot { s.Payload = oversized; return s }, "exceeds"},
|
||||
{"zero observed at", func(s Snapshot) Snapshot { s.ObservedAt = time.Time{}; return s }, "observed time is required"},
|
||||
{"future observed at", func(s Snapshot) Snapshot { s.ObservedAt = now.Add(time.Hour); return s }, "in the future"},
|
||||
{"array payload", func(s Snapshot) Snapshot { s.Payload = json.RawMessage(`[]`); return s }, "JSON object"},
|
||||
{"scalar payload", func(s Snapshot) Snapshot { s.Payload = json.RawMessage(`42`); return s }, "JSON object"},
|
||||
{"null payload", func(s Snapshot) Snapshot { s.Payload = json.RawMessage(`null`); return s }, "JSON object"},
|
||||
{"corrupt payload", func(s Snapshot) Snapshot { s.Payload = json.RawMessage(`{"a":`); return s }, "JSON object"},
|
||||
}
|
||||
store := PostgresStore{Clock: fixedClock(now)}
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
err := store.Put(context.Background(), testCase.mutate(valid))
|
||||
if err == nil {
|
||||
t.Fatal("expected rejection")
|
||||
}
|
||||
if errors.Is(err, ErrUnavailable) {
|
||||
t.Fatalf("bounds must be enforced before availability: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), testCase.contains) {
|
||||
t.Fatalf("error %q does not mention %q", err, testCase.contains)
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := store.Put(context.Background(), valid); !errors.Is(err, ErrUnavailable) {
|
||||
t.Fatalf("a valid snapshot with no pool must report unavailability, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutAcceptsEveryKnownCapability(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
for _, capability := range Capabilities() {
|
||||
snapshot := Snapshot{AgentID: "agent-1", Capability: capability, ObservedAt: now, Payload: json.RawMessage(`{}`)}
|
||||
if _, err := prepare(snapshot, now); err != nil {
|
||||
t.Fatalf("capability %q rejected: %v", capability, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapacitySamplesAreBoundedAndCapabilityAware(t *testing.T) {
|
||||
now := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
|
||||
snapshot := Snapshot{AgentID: "agent-1", Capability: CapabilityShares, ObservedAt: now, ReceivedAt: now, Payload: json.RawMessage(`{"shares":[{"id":"media","name":"Media","usedBytes":123,"sizeObservedAt":"2026-08-12T01:00:00Z"},{"id":"","name":"invalid","usedBytes":1}]}`)}
|
||||
samples, err := capacitySamples(snapshot)
|
||||
if err != nil || len(samples) != 1 || samples[0].Kind != "share" || samples[0].ID != "media" || samples[0].UsedBytes != 123 {
|
||||
t.Fatalf("share capacity extraction failed: %+v, %v", samples, err)
|
||||
}
|
||||
snapshot.Capability = CapabilityHost
|
||||
samples, err = capacitySamples(snapshot)
|
||||
if err != nil || len(samples) != 0 {
|
||||
t.Fatalf("non-capacity capability emitted samples: %+v, %v", samples, err)
|
||||
}
|
||||
snapshot.Capability = CapabilityPools
|
||||
snapshot.Payload = json.RawMessage(`{"pools":[{"id":"cache","name":"Cache","usedBytes":"invalid","usableBytes":1000}]}`)
|
||||
samples, err = capacitySamples(snapshot)
|
||||
if err != nil || len(samples) != 0 {
|
||||
t.Fatalf("malformed capacity values must fail closed: %+v, %v", samples, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareTakesReceivedAtFromStoreClock(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
// A hostile agent claims it was received in the future and observed just now.
|
||||
row, err := prepare(Snapshot{
|
||||
AgentID: "agent-1",
|
||||
Capability: CapabilityProcesses,
|
||||
ObservedAt: now.Add(-90 * time.Second),
|
||||
ReceivedAt: now.Add(48 * time.Hour),
|
||||
Payload: json.RawMessage(`{"processes":[]}`),
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !row.ReceivedAt.Equal(now) {
|
||||
t.Fatalf("received at = %s, want the store clock %s", row.ReceivedAt, now)
|
||||
}
|
||||
if age := row.Age(now); age != 90*time.Second {
|
||||
t.Fatalf("age = %s, want 90s", age)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareToleratesOnlyBenignClockSkew(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
base := Snapshot{AgentID: "agent-1", Capability: CapabilityHost, Payload: json.RawMessage(`{}`)}
|
||||
base.ObservedAt = now.Add(MaxClockSkew)
|
||||
if _, err := prepare(base, now); err != nil {
|
||||
t.Fatalf("skew within tolerance must be accepted: %v", err)
|
||||
}
|
||||
base.ObservedAt = now.Add(MaxClockSkew + time.Millisecond)
|
||||
if _, err := prepare(base, now); err == nil {
|
||||
t.Fatal("skew beyond tolerance must be rejected")
|
||||
}
|
||||
if MaxClockSkew >= 30*time.Second {
|
||||
t.Fatal("clock skew tolerance must stay well below the tightest freshness window")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareNormalizesObservedAtToUTC(t *testing.T) {
|
||||
zone := time.FixedZone("CEST", 2*60*60)
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
row, err := prepare(Snapshot{AgentID: "agent-1", Capability: CapabilityShares, ObservedAt: now.Add(-time.Minute).In(zone), Payload: json.RawMessage(`{}`)}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if row.ObservedAt.Location() != time.UTC {
|
||||
t.Fatalf("observed at location = %s, want UTC", row.ObservedAt.Location())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLatestRejectsUnknownCapabilityBeforeTouchingTheDatabase(t *testing.T) {
|
||||
store := PostgresStore{}
|
||||
if _, err := store.Latest(context.Background(), "gpu"); err == nil || !strings.Contains(err.Error(), "unknown agent capability") {
|
||||
t.Fatalf("unexpected error %v", err)
|
||||
}
|
||||
if _, err := store.Latest(context.Background(), CapabilityHost); !errors.Is(err, ErrUnavailable) {
|
||||
t.Fatalf("unexpected error %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// stubRow stands in for one PostgreSQL row so the ErrNoSnapshot and decoding paths of
|
||||
// Latest can be exercised without a live database.
|
||||
type stubRow struct {
|
||||
err error
|
||||
values []any
|
||||
}
|
||||
|
||||
func (r stubRow) Scan(dest ...any) error {
|
||||
if r.err != nil {
|
||||
return r.err
|
||||
}
|
||||
for index, target := range dest {
|
||||
switch typed := target.(type) {
|
||||
case *string:
|
||||
*typed = r.values[index].(string)
|
||||
case *time.Time:
|
||||
*typed = r.values[index].(time.Time)
|
||||
case *[]byte:
|
||||
*typed = r.values[index].([]byte)
|
||||
default:
|
||||
return errors.New("unsupported destination")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestLatestReportsErrNoSnapshotWhenNothingWasRecorded(t *testing.T) {
|
||||
if _, err := scanSnapshot(stubRow{err: pgx.ErrNoRows}); !errors.Is(err, ErrNoSnapshot) {
|
||||
t.Fatalf("missing row must map to ErrNoSnapshot, got %v", err)
|
||||
}
|
||||
failure := errors.New("connection reset")
|
||||
_, err := scanSnapshot(stubRow{err: failure})
|
||||
if err == nil || errors.Is(err, ErrNoSnapshot) || !errors.Is(err, failure) {
|
||||
t.Fatalf("a read failure must not look like an absent snapshot, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLatestDecodesRowIntoUTCSnapshot(t *testing.T) {
|
||||
zone := time.FixedZone("CEST", 2*60*60)
|
||||
observed := time.Date(2026, 8, 4, 12, 0, 0, 0, zone)
|
||||
received := observed.Add(time.Second)
|
||||
snapshot, err := scanSnapshot(stubRow{values: []any{"agent-1", string(CapabilityDisks), observed, received, []byte(`{"disks":[]}`)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.AgentID != "agent-1" || snapshot.Capability != CapabilityDisks {
|
||||
t.Fatalf("unexpected identity %+v", snapshot)
|
||||
}
|
||||
if snapshot.ObservedAt.Location() != time.UTC || snapshot.ReceivedAt.Location() != time.UTC {
|
||||
t.Fatalf("timestamps must be UTC: %+v", snapshot)
|
||||
}
|
||||
if string(snapshot.Payload) != `{"disks":[]}` {
|
||||
t.Fatalf("unexpected payload %s", snapshot.Payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreClockDefaultsToWallClockInUTC(t *testing.T) {
|
||||
store := PostgresStore{}
|
||||
if location := store.now().Location(); location != time.UTC {
|
||||
t.Fatalf("store clock location = %s, want UTC", location)
|
||||
}
|
||||
fixed := time.Date(2026, 8, 4, 12, 0, 0, 0, time.FixedZone("CEST", 2*60*60))
|
||||
if got := (PostgresStore{Clock: fixedClock(fixed)}).now(); got.Location() != time.UTC || !got.Equal(fixed) {
|
||||
t.Fatalf("store clock = %s, want the injected instant in UTC", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
type Alert struct {
|
||||
Instance
|
||||
RuleName string `json:"ruleName"`
|
||||
Severity string `json:"severity"`
|
||||
EntityType string `json:"entityType,omitempty"`
|
||||
EntityName string `json:"entityName,omitempty"`
|
||||
Occurrences []Occurrence `json:"occurrences,omitempty"`
|
||||
}
|
||||
|
||||
type AlertReader interface {
|
||||
ListAlerts(context.Context, int, string) ([]Alert, error)
|
||||
GetAlert(context.Context, string, int) (Alert, error)
|
||||
}
|
||||
|
||||
func (r StateRepository) ListAlerts(ctx context.Context, limit int, state string) ([]Alert, error) {
|
||||
if r.Pool == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if limit < 1 || limit > 100 {
|
||||
return nil, errors.New("alert limit is invalid")
|
||||
}
|
||||
if state != "" && !validState(State(state)) {
|
||||
return nil, errors.New("alert state filter is invalid")
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, `SELECT i.id,i.rule_id,i.rule_version_id,i.fingerprint,COALESCE(i.entity_id::text,''),i.current_state,i.retained_state,i.active_since,i.recovery_since,i.cooldown_until,i.last_evaluated_at,i.last_known_at,i.last_value,i.reason,i.source_health,COALESCE(i.acknowledged_by,''),i.acknowledged_at,i.revision,i.created_at,i.updated_at,r.name,r.severity,COALESCE(e.entity_type,''),COALESCE(e.display_name,'') FROM alert_instances i JOIN alert_rules r ON r.id=i.rule_id LEFT JOIN entities e ON e.id=i.entity_id WHERE (($2='' AND i.current_state <> 'inactive') OR ($2<>'' AND i.current_state=$2)) ORDER BY CASE i.current_state WHEN 'firing' THEN 1 WHEN 'acknowledged' THEN 2 WHEN 'pending' THEN 3 WHEN 'unknown' THEN 4 WHEN 'resolved' THEN 5 ELSE 6 END,i.updated_at DESC,i.id ASC LIMIT $1`, limit, state)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list alerts: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]Alert, 0, limit)
|
||||
for rows.Next() {
|
||||
item, err := scanAlert(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate alerts: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (r StateRepository) GetAlert(ctx context.Context, id string, occurrenceLimit int) (Alert, error) {
|
||||
if r.Pool == nil {
|
||||
return Alert{}, ErrUnavailable
|
||||
}
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return Alert{}, ErrInstanceNotFound
|
||||
}
|
||||
if occurrenceLimit < 1 || occurrenceLimit > 500 {
|
||||
return Alert{}, errors.New("alert occurrence limit is invalid")
|
||||
}
|
||||
row := r.Pool.QueryRow(ctx, `SELECT i.id,i.rule_id,i.rule_version_id,i.fingerprint,COALESCE(i.entity_id::text,''),i.current_state,i.retained_state,i.active_since,i.recovery_since,i.cooldown_until,i.last_evaluated_at,i.last_known_at,i.last_value,i.reason,i.source_health,COALESCE(i.acknowledged_by,''),i.acknowledged_at,i.revision,i.created_at,i.updated_at,r.name,r.severity,COALESCE(e.entity_type,''),COALESCE(e.display_name,'') FROM alert_instances i JOIN alert_rules r ON r.id=i.rule_id LEFT JOIN entities e ON e.id=i.entity_id WHERE i.id=$1`, id)
|
||||
item, err := scanAlert(row)
|
||||
if errors.Is(err, ErrInstanceNotFound) || errors.Is(err, pgx.ErrNoRows) {
|
||||
return Alert{}, ErrInstanceNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Alert{}, fmt.Errorf("get alert: %w", err)
|
||||
}
|
||||
occurrences, err := r.ListOccurrences(ctx, id, occurrenceLimit)
|
||||
if err != nil {
|
||||
return Alert{}, err
|
||||
}
|
||||
item.Occurrences = occurrences
|
||||
return item, nil
|
||||
}
|
||||
|
||||
type alertRow interface{ Scan(...any) error }
|
||||
|
||||
func scanAlert(row alertRow) (Alert, error) {
|
||||
var item Alert
|
||||
var valueJSON, healthJSON []byte
|
||||
var state, retained State
|
||||
err := row.Scan(&item.ID, &item.RuleID, &item.RuleVersionID, &item.Fingerprint, &item.EntityID, &state, &retained, &item.ActiveSince, &item.RecoverySince, &item.CooldownUntil, &item.LastEvaluatedAt, &item.LastKnownAt, &valueJSON, &item.Reason, &healthJSON, &item.AcknowledgedBy, &item.AcknowledgedAt, &item.Revision, &item.CreatedAt, &item.UpdatedAt, &item.RuleName, &item.Severity, &item.EntityType, &item.EntityName)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Alert{}, ErrInstanceNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Alert{}, fmt.Errorf("scan alert: %w", err)
|
||||
}
|
||||
item.State, item.RetainedState = state, retained
|
||||
item.LastValue, err = decodeJSON(valueJSON)
|
||||
if err != nil {
|
||||
return Alert{}, fmt.Errorf("decode alert value: %w", err)
|
||||
}
|
||||
item.SourceHealth, err = decodeMap(healthJSON)
|
||||
if err != nil {
|
||||
return Alert{}, fmt.Errorf("decode alert source health: %w", err)
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxFingerprintLabels = 10
|
||||
MaxAlertGroups = 1000
|
||||
MaxSignalsPerGroup = 500
|
||||
)
|
||||
|
||||
var ErrInvalidAlertIdentity = errors.New("invalid alert identity")
|
||||
var alertLabelPattern = regexp.MustCompile("^[A-Za-z0-9_.:/-]+$")
|
||||
|
||||
type Signal struct {
|
||||
InstanceID string
|
||||
RuleID string
|
||||
RuleVersionID string
|
||||
EntityID string
|
||||
Severity string
|
||||
State State
|
||||
Fingerprint string
|
||||
EvaluationKey string
|
||||
ObservedAt time.Time
|
||||
Labels map[string]string
|
||||
GroupBy []string
|
||||
SuppressWhen []string
|
||||
}
|
||||
|
||||
type Group struct {
|
||||
Key string
|
||||
Severity string
|
||||
Labels map[string]string
|
||||
Signals []Signal
|
||||
}
|
||||
|
||||
type Cause struct {
|
||||
Key string
|
||||
State State
|
||||
Confirmed bool
|
||||
Confidence float64
|
||||
ObservedAt time.Time
|
||||
}
|
||||
|
||||
type SuppressionDecision struct {
|
||||
Suppressed bool `json:"suppressed"`
|
||||
CauseKey string `json:"causeKey,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
func BuildFingerprint(ruleID, ruleVersionID, entityID string, labels map[string]string) (string, error) {
|
||||
if strings.TrimSpace(ruleID) == "" || strings.TrimSpace(ruleVersionID) == "" || strings.TrimSpace(entityID) == "" || validateLabel(ruleID, 160) != nil || validateLabel(ruleVersionID, 160) != nil || validateLabel(entityID, 160) != nil {
|
||||
return "", ErrInvalidAlertIdentity
|
||||
}
|
||||
canonical, err := canonicalLabels(labels, MaxFingerprintLabels)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
value := "rule=" + ruleID + "\x00version=" + ruleVersionID + "\x00entity=" + entityID + "\x00" + canonical
|
||||
digest := sha256.Sum256([]byte(value))
|
||||
return hex.EncodeToString(digest[:]), nil
|
||||
}
|
||||
|
||||
func GroupSignals(signals []Signal) ([]Group, error) {
|
||||
groups := make(map[string]*Group)
|
||||
for _, signal := range signals {
|
||||
if err := validateSignal(signal); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, labels, err := groupKey(signal)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
group := groups[key]
|
||||
if group == nil {
|
||||
if len(groups) >= MaxAlertGroups {
|
||||
return nil, fmt.Errorf("%w: too many alert groups", ErrInvalidAlertIdentity)
|
||||
}
|
||||
group = &Group{Key: key, Severity: signal.Severity, Labels: labels}
|
||||
groups[key] = group
|
||||
}
|
||||
if len(group.Signals) >= MaxSignalsPerGroup {
|
||||
return nil, fmt.Errorf("%w: too many signals in group", ErrInvalidAlertIdentity)
|
||||
}
|
||||
group.Signals = append(group.Signals, signal)
|
||||
}
|
||||
result := make([]Group, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
sort.SliceStable(group.Signals, func(i, j int) bool { return signalSortKey(group.Signals[i]) < signalSortKey(group.Signals[j]) })
|
||||
result = append(result, *group)
|
||||
}
|
||||
sort.SliceStable(result, func(i, j int) bool { return result[i].Key < result[j].Key })
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func DeduplicateSignals(signals []Signal) ([]Signal, error) {
|
||||
byKey := make(map[string]Signal, len(signals))
|
||||
for _, signal := range signals {
|
||||
if err := validateSignal(signal); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := signal.InstanceID + "\x00" + signal.EvaluationKey
|
||||
if previous, exists := byKey[key]; !exists || signalSortKey(signal) > signalSortKey(previous) {
|
||||
byKey[key] = signal
|
||||
}
|
||||
}
|
||||
result := make([]Signal, 0, len(byKey))
|
||||
for _, signal := range byKey {
|
||||
result = append(result, signal)
|
||||
}
|
||||
sort.SliceStable(result, func(i, j int) bool { return signalSortKey(result[i]) < signalSortKey(result[j]) })
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func EvaluateSuppression(signal Signal, causes []Cause) (SuppressionDecision, error) {
|
||||
if err := validateSignal(signal); err != nil {
|
||||
return SuppressionDecision{}, err
|
||||
}
|
||||
if signal.State != StatePending && signal.State != StateFiring && signal.State != StateAcknowledged && signal.State != StateUnknown {
|
||||
return SuppressionDecision{Reason: "alert_not_active"}, nil
|
||||
}
|
||||
wanted := make(map[string]struct{}, len(signal.SuppressWhen))
|
||||
for _, key := range signal.SuppressWhen {
|
||||
if err := validateLabel(key, 160); err != nil {
|
||||
return SuppressionDecision{}, err
|
||||
}
|
||||
wanted[key] = struct{}{}
|
||||
}
|
||||
ordered := append([]Cause(nil), causes...)
|
||||
sort.SliceStable(ordered, func(i, j int) bool { return causeSortKey(ordered[i]) < causeSortKey(ordered[j]) })
|
||||
for _, cause := range ordered {
|
||||
if _, ok := wanted[cause.Key]; !ok || !causeActive(cause) {
|
||||
continue
|
||||
}
|
||||
if !cause.Confirmed && cause.Confidence < .75 {
|
||||
continue
|
||||
}
|
||||
reason := "dependency_failure"
|
||||
if strings.HasPrefix(cause.Key, "source.") {
|
||||
reason = "source_outage"
|
||||
}
|
||||
return SuppressionDecision{Suppressed: true, CauseKey: cause.Key, Reason: reason}, nil
|
||||
}
|
||||
return SuppressionDecision{Reason: "no_active_suppression_cause"}, nil
|
||||
}
|
||||
|
||||
func groupKey(signal Signal) (string, map[string]string, error) {
|
||||
labels := make(map[string]string, len(signal.GroupBy))
|
||||
for _, key := range signal.GroupBy {
|
||||
if err := validateLabel(key, 80); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if value, ok := signal.Labels[key]; ok {
|
||||
if err := validateLabel(value, 160); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
labels[key] = value
|
||||
}
|
||||
}
|
||||
canonical, err := canonicalLabels(labels, MaxFingerprintLabels)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return signal.RuleID + "|" + signal.Severity + "|" + canonical, labels, nil
|
||||
}
|
||||
|
||||
func validateSignal(signal Signal) error {
|
||||
if signal.InstanceID == "" || signal.RuleID == "" || signal.RuleVersionID == "" || signal.EvaluationKey == "" || signal.Severity == "" || !validState(signal.State) || validateLabel(signal.InstanceID, 160) != nil || validateLabel(signal.RuleID, 160) != nil || validateLabel(signal.RuleVersionID, 160) != nil || validateLabel(signal.EvaluationKey, 160) != nil || validateLabel(signal.Severity, 40) != nil {
|
||||
return ErrInvalidAlertIdentity
|
||||
}
|
||||
if len(signal.GroupBy) > MaxFingerprintLabels || len(signal.Labels) > MaxFingerprintLabels {
|
||||
return fmt.Errorf("%w: label cardinality exceeds limit", ErrInvalidAlertIdentity)
|
||||
}
|
||||
for key, value := range signal.Labels {
|
||||
if err := validateLabel(key, 80); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateLabel(value, 160); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func canonicalLabels(labels map[string]string, max int) (string, error) {
|
||||
if len(labels) > max {
|
||||
return "", fmt.Errorf("%w: too many labels", ErrInvalidAlertIdentity)
|
||||
}
|
||||
keys := make([]string, 0, len(labels))
|
||||
for key, value := range labels {
|
||||
if err := validateLabel(key, 80); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := validateLabel(value, 160); err != nil {
|
||||
return "", err
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
parts = append(parts, key+"="+labels[key])
|
||||
}
|
||||
return strings.Join(parts, "\x00"), nil
|
||||
}
|
||||
|
||||
func validateLabel(value string, max int) error {
|
||||
if value == "" || len(value) > max || strings.ContainsAny(value, "\r\n\x00") || !alertLabelPattern.MatchString(value) {
|
||||
return ErrInvalidAlertIdentity
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func signalSortKey(signal Signal) string {
|
||||
return signal.InstanceID + "|" + signal.EvaluationKey + "|" + signal.ObservedAt.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
func causeSortKey(cause Cause) string {
|
||||
return cause.Key + "|" + string(cause.State) + "|" + cause.ObservedAt.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
func causeActive(cause Cause) bool {
|
||||
return cause.State == StateFiring || cause.State == StateAcknowledged || (cause.State == StateUnknown && cause.Confirmed)
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func signal(id, rule, evaluation string, state State, labels map[string]string) Signal {
|
||||
return Signal{InstanceID: id, RuleID: rule, RuleVersionID: "version-1", Severity: SeverityDegraded, State: state, EvaluationKey: evaluation, ObservedAt: time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC), Labels: labels, GroupBy: []string{"host", "application"}, SuppressWhen: []string{"host.unreachable", "dns.failure", "source.unavailable"}}
|
||||
}
|
||||
|
||||
func TestBuildFingerprintIsStableAndIncludesRuleBehavior(t *testing.T) {
|
||||
first, err := BuildFingerprint("rule-1", "version-1", "entity-1", map[string]string{"application": "media", "host": "pulse"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := BuildFingerprint("rule-1", "version-1", "entity-1", map[string]string{"host": "pulse", "application": "media"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first != second || len(first) != 64 {
|
||||
t.Fatalf("fingerprint instability: %q %q", first, second)
|
||||
}
|
||||
changedVersion, err := BuildFingerprint("rule-1", "version-2", "entity-1", map[string]string{"host": "pulse", "application": "media"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if changedVersion == first {
|
||||
t.Fatal("rule version did not affect fingerprint")
|
||||
}
|
||||
tooMany := make(map[string]string, MaxFingerprintLabels+1)
|
||||
for i := 0; i <= MaxFingerprintLabels; i++ {
|
||||
tooMany["label"+string(rune('a'+i))] = "value"
|
||||
}
|
||||
if _, err := BuildFingerprint("rule-1", "version-1", "entity-1", tooMany); !errors.Is(err, ErrInvalidAlertIdentity) {
|
||||
t.Fatalf("too many labels error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupAndDeduplicateSignalsAreDeterministic(t *testing.T) {
|
||||
inputs := []Signal{
|
||||
signal("instance-b", "rule-1", "slot-1", StateFiring, map[string]string{"host": "pulse", "application": "media"}),
|
||||
signal("instance-a", "rule-1", "slot-1", StateFiring, map[string]string{"host": "pulse", "application": "media"}),
|
||||
signal("instance-a", "rule-1", "slot-1", StateFiring, map[string]string{"host": "pulse", "application": "media"}),
|
||||
signal("instance-c", "rule-1", "slot-1", StateFiring, map[string]string{"host": "other", "application": "media"}),
|
||||
}
|
||||
deduplicated, err := DeduplicateSignals(inputs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(deduplicated) != 3 {
|
||||
t.Fatalf("deduplicated signals = %d, want 3", len(deduplicated))
|
||||
}
|
||||
groups, err := GroupSignals(deduplicated)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(groups) != 2 || len(groups[0].Signals) != 1 || len(groups[1].Signals) != 2 {
|
||||
t.Fatalf("unexpected groups: %#v", groups)
|
||||
}
|
||||
if groups[0].Key > groups[1].Key {
|
||||
t.Fatal("groups are not sorted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuppressionScenariosRemainInspectablyBounded(t *testing.T) {
|
||||
base := signal("instance-1", "rule-service", "slot-1", StateFiring, map[string]string{"host": "pulse", "application": "web"})
|
||||
tests := []struct {
|
||||
name string
|
||||
cause Cause
|
||||
want bool
|
||||
reason string
|
||||
}{
|
||||
{name: "host outage", cause: Cause{Key: "host.unreachable", State: StateFiring, Confirmed: true}, want: true, reason: "dependency_failure"},
|
||||
{name: "dns outage", cause: Cause{Key: "dns.failure", State: StateAcknowledged, Confidence: .9}, want: true, reason: "dependency_failure"},
|
||||
{name: "source outage", cause: Cause{Key: "source.unavailable", State: StateUnknown, Confirmed: true}, want: true, reason: "source_outage"},
|
||||
{name: "low confidence", cause: Cause{Key: "host.unreachable", State: StateFiring, Confidence: .5}, want: false, reason: "no_active_suppression_cause"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
decision, err := EvaluateSuppression(base, []Cause{test.cause})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decision.Suppressed != test.want || decision.Reason != test.reason {
|
||||
t.Fatalf("decision = %#v, want suppressed=%v reason=%s", decision, test.want, test.reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
resolved := base
|
||||
resolved.State = StateResolved
|
||||
decision, err := EvaluateSuppression(resolved, []Cause{{Key: "host.unreachable", State: StateFiring, Confirmed: true}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decision.Suppressed {
|
||||
t.Fatal("resolved alert was suppressed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func Unacknowledge(current Snapshot, actor string, at time.Time) (TransitionResult, error) {
|
||||
current = current.normalized()
|
||||
if actor == "" || len(actor) > 160 || at.IsZero() {
|
||||
return TransitionResult{}, ErrInvalidObservation
|
||||
}
|
||||
if current.State != StateAcknowledged {
|
||||
return TransitionResult{}, ErrStateConflict
|
||||
}
|
||||
result := current
|
||||
result.State = StateFiring
|
||||
result.RetainedState = StateFiring
|
||||
result.AcknowledgedBy = ""
|
||||
result.AcknowledgedAt = nil
|
||||
result.Reason = "unacknowledged"
|
||||
return finish(current, result, StateFiring, "unacknowledge"), nil
|
||||
}
|
||||
|
||||
func (r StateRepository) AcknowledgeRevision(ctx context.Context, instanceID, actor, evaluationKey string, at time.Time, expectedRevision int64) (Instance, Occurrence, bool, error) {
|
||||
return r.applyOperation(ctx, instanceID, actor, evaluationKey, at, expectedRevision, true)
|
||||
}
|
||||
|
||||
func (r StateRepository) Unacknowledge(ctx context.Context, instanceID, actor, evaluationKey string, at time.Time, expectedRevision int64) (Instance, Occurrence, bool, error) {
|
||||
return r.applyOperation(ctx, instanceID, actor, evaluationKey, at, expectedRevision, false)
|
||||
}
|
||||
|
||||
func (r StateRepository) applyOperation(ctx context.Context, instanceID, actor, evaluationKey string, at time.Time, expectedRevision int64, acknowledge bool) (Instance, Occurrence, bool, error) {
|
||||
if r.Pool == nil {
|
||||
return Instance{}, Occurrence{}, false, ErrUnavailable
|
||||
}
|
||||
if instanceID == "" || actor == "" || len(actor) > 160 || evaluationKey == "" || len(evaluationKey) > 160 || at.IsZero() || expectedRevision < 1 {
|
||||
return Instance{}, Occurrence{}, false, ErrInvalidObservation
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, fmt.Errorf("begin alert operation: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var current Instance
|
||||
if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1 FOR UPDATE`, instanceID), ¤t); errors.Is(err, ErrInstanceNotFound) {
|
||||
return Instance{}, Occurrence{}, false, ErrInstanceNotFound
|
||||
} else if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
if occurrence, err := scanOccurrence(tx.QueryRow(ctx, `SELECT id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at FROM alert_occurrences WHERE instance_id=$1 AND evaluation_key=$2`, instanceID, evaluationKey)); err == nil {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Instance{}, Occurrence{}, false, fmt.Errorf("commit idempotent alert operation: %w", err)
|
||||
}
|
||||
return current, occurrence, true, nil
|
||||
} else if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
if current.Revision != expectedRevision {
|
||||
return Instance{}, Occurrence{}, false, ErrRevisionConflict
|
||||
}
|
||||
var transition TransitionResult
|
||||
if acknowledge {
|
||||
transition, err = Acknowledge(snapshotFromInstance(current), actor, at.UTC())
|
||||
} else {
|
||||
transition, err = Unacknowledge(snapshotFromInstance(current), actor, at.UTC())
|
||||
}
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
valueJSON, err := boundedJSON(current.LastValue, 128<<10)
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
healthJSON, err := boundedJSON(nonNilMap(current.SourceHealth), 64<<10)
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
acknowledgedBy := nullableText(transition.Snapshot.AcknowledgedBy)
|
||||
acknowledgedAt := transition.Snapshot.AcknowledgedAt
|
||||
if _, err := tx.Exec(ctx, `UPDATE alert_instances SET current_state=$1,retained_state=$2,reason=$3,acknowledged_by=$4,acknowledged_at=$5,revision=revision+1,updated_at=now() WHERE id=$6 AND revision=$7`, transition.Snapshot.State, transition.Snapshot.RetainedState, transition.Snapshot.Reason, acknowledgedBy, acknowledgedAt, instanceID, expectedRevision); err != nil {
|
||||
return Instance{}, Occurrence{}, false, mapStateError(fmt.Errorf("update alert operation: %w", err))
|
||||
}
|
||||
occurrence, err := insertOccurrence(ctx, tx, instanceID, evaluationKey, transition, Observation{ObservedAt: at.UTC(), Reason: transition.Snapshot.Reason, Value: current.LastValue}, valueJSON, healthJSON)
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1`, instanceID), ¤t); err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Instance{}, Occurrence{}, false, fmt.Errorf("commit alert operation: %w", err)
|
||||
}
|
||||
return current, occurrence, false, nil
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
)
|
||||
|
||||
func TestPostgreSQLAlertOperationsAreRevisionSafeAndRestartable(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 8, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
document, registry := validDocument(t)
|
||||
document.Enabled = true
|
||||
rules := Repository{Pool: pool, Registry: registry}
|
||||
created, version, err := rules.Create(ctx, "operations-integration", document, "operations test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := StateRepository{Pool: pool}
|
||||
base := time.Date(2026, time.January, 4, 12, 0, 0, 0, time.UTC)
|
||||
policy := Policy{PendingSeconds: 0, ResolveSeconds: 0, UnknownBehavior: UnknownRetain}
|
||||
firing, _, _, err := store.ApplyObservation(ctx, StateInput{RuleID: created.ID, RuleVersionID: version.ID, Fingerprint: "operations:test", Policy: policy, Observation: observation(base, "evaluation-1", true)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if firing.State != StateFiring {
|
||||
t.Fatalf("state = %s", firing.State)
|
||||
}
|
||||
ack, occurrence, duplicate, err := store.AcknowledgeRevision(ctx, firing.ID, "operator", "ack-operation-1", base.Add(time.Second), firing.Revision)
|
||||
if err != nil || duplicate || ack.State != StateAcknowledged || occurrence.EventType != "acknowledge" {
|
||||
t.Fatalf("ack result=%#v occurrence=%#v duplicate=%v err=%v", ack, occurrence, duplicate, err)
|
||||
}
|
||||
retry, _, duplicate, err := store.AcknowledgeRevision(ctx, firing.ID, "operator", "ack-operation-1", base.Add(time.Second), firing.Revision)
|
||||
if err != nil || !duplicate || retry.Revision != ack.Revision {
|
||||
t.Fatalf("ack retry=%#v duplicate=%v err=%v", retry, duplicate, err)
|
||||
}
|
||||
restarted := StateRepository{Pool: pool}
|
||||
persisted, err := restarted.GetInstance(ctx, firing.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if persisted.State != StateAcknowledged || persisted.AcknowledgedBy != "operator" {
|
||||
t.Fatalf("ack did not survive restart: %#v", persisted)
|
||||
}
|
||||
unack, occurrence, duplicate, err := restarted.Unacknowledge(ctx, firing.ID, "operator", "unack-operation-1", base.Add(2*time.Second), ack.Revision)
|
||||
if err != nil || duplicate || unack.State != StateFiring || unack.AcknowledgedBy != "" || occurrence.EventType != "unacknowledge" {
|
||||
t.Fatalf("unack result=%#v occurrence=%#v duplicate=%v err=%v", unack, occurrence, duplicate, err)
|
||||
}
|
||||
resolved, _, _, err := restarted.ApplyObservation(ctx, StateInput{RuleID: created.ID, RuleVersionID: version.ID, Fingerprint: "operations:test", Policy: policy, Observation: observation(base.Add(3*time.Second), "evaluation-2", false)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resolved.State != StateResolved {
|
||||
t.Fatalf("resolved state was lost after unacknowledge: %#v", resolved)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestUnacknowledgeReturnsFiringAndClearsActor(t *testing.T) {
|
||||
at := time.Date(2026, 8, 2, 12, 0, 0, 0, time.UTC)
|
||||
result, err := Unacknowledge(Snapshot{State: StateAcknowledged, RetainedState: StateAcknowledged, AcknowledgedBy: "operator", AcknowledgedAt: &at, LastValue: 90}, "operator", at)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.To != StateFiring || result.Snapshot.AcknowledgedBy != "" || result.Snapshot.AcknowledgedAt != nil || result.EventType != "unacknowledge" {
|
||||
t.Fatalf("unexpected unacknowledge result: %#v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnacknowledgeRejectsResolvedAndInactive(t *testing.T) {
|
||||
at := time.Now().UTC()
|
||||
for _, state := range []State{StateInactive, StatePending, StateFiring, StateResolved, StateUnknown} {
|
||||
if _, err := Unacknowledge(Snapshot{State: state}, "operator", at); err != ErrStateConflict {
|
||||
t.Fatalf("state %s error = %v", state, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/itworx/pulse/internal/metriccatalog"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
type Store interface {
|
||||
Create(context.Context, string, Document, string) (Rule, Version, error)
|
||||
Get(context.Context, string) (Rule, error)
|
||||
List(context.Context, int) ([]Rule, error)
|
||||
Update(context.Context, string, string, int64, Document, string) (Rule, error)
|
||||
Versions(context.Context, string, int) ([]Version, error)
|
||||
SetEnabled(context.Context, string, int64, bool) (Rule, error)
|
||||
}
|
||||
|
||||
type Repository struct {
|
||||
Pool *pgxpool.Pool
|
||||
Registry metriccatalog.Registry
|
||||
}
|
||||
|
||||
func (r Repository) Create(ctx context.Context, actor string, document Document, changeSummary string) (Rule, Version, error) {
|
||||
if r.Pool == nil {
|
||||
return Rule{}, Version{}, ErrUnavailable
|
||||
}
|
||||
if err := document.Validate(r.Registry); err != nil {
|
||||
return Rule{}, Version{}, err
|
||||
}
|
||||
if changeSummary == "" {
|
||||
changeSummary = "initial version"
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Rule{}, Version{}, fmt.Errorf("begin alert rule create: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
docJSON, err := document.MarshalCanonical()
|
||||
if err != nil {
|
||||
return Rule{}, Version{}, fmt.Errorf("marshal alert rule: %w", err)
|
||||
}
|
||||
conditionJSON, _ := json.Marshal(document.Condition)
|
||||
scopeJSON, _ := json.Marshal(nonNilMap(document.Scope))
|
||||
groupJSON, _ := json.Marshal(nonNilStrings(document.GroupBy))
|
||||
suppressJSON, _ := json.Marshal(nonNilStrings(document.SuppressWhen))
|
||||
messageJSON, _ := json.Marshal(document.Message)
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO alert_rules (id,schema_version,name,enabled,severity,scope,condition,evaluation_interval_seconds,pending_seconds,resolve_seconds,cooldown_seconds,unknown_behavior,group_by,suppress_when,message,revision,created_by) VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7::jsonb,$8,$9,$10,$11,$12,$13::jsonb,$14::jsonb,$15::jsonb,1,(SELECT id FROM users WHERE external_subject=$16))`, document.ID, document.SchemaVersion, document.Name, document.Enabled, document.Severity, scopeJSON, conditionJSON, document.EvaluationIntervalSeconds, document.PendingSeconds, document.ResolveSeconds, document.CooldownSeconds, document.UnknownBehavior, groupJSON, suppressJSON, messageJSON, actor); err != nil {
|
||||
return Rule{}, Version{}, mapError(fmt.Errorf("create alert rule: %w", err))
|
||||
}
|
||||
versionID := NewID()
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO alert_rule_versions (id,rule_id,version_number,document,change_summary,created_by) VALUES ($1,$2,1,$3::jsonb,$4,(SELECT id FROM users WHERE external_subject=$5))`, versionID, document.ID, docJSON, changeSummary, actor); err != nil {
|
||||
return Rule{}, Version{}, mapError(fmt.Errorf("create alert rule version: %w", err))
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE alert_rules SET current_version_id=$1 WHERE id=$2`, versionID, document.ID); err != nil {
|
||||
return Rule{}, Version{}, fmt.Errorf("set current alert rule version: %w", err)
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return Rule{}, Version{}, fmt.Errorf("commit alert rule create: %w", err)
|
||||
}
|
||||
rule, err := r.Get(ctx, document.ID)
|
||||
if err != nil {
|
||||
return Rule{}, Version{}, err
|
||||
}
|
||||
versions, err := r.Versions(ctx, document.ID, 1)
|
||||
if err != nil || len(versions) == 0 {
|
||||
return Rule{}, Version{}, err
|
||||
}
|
||||
return rule, versions[0], nil
|
||||
}
|
||||
|
||||
func (r Repository) Get(ctx context.Context, id string) (Rule, error) {
|
||||
if r.Pool == nil {
|
||||
return Rule{}, ErrUnavailable
|
||||
}
|
||||
var rule Rule
|
||||
var scopeJSON, conditionJSON, groupJSON, suppressJSON, messageJSON []byte
|
||||
var createdBy *string
|
||||
err := r.Pool.QueryRow(ctx, `SELECT r.id,r.schema_version,r.name,r.enabled,r.severity,r.scope,r.condition,r.evaluation_interval_seconds,r.pending_seconds,r.resolve_seconds,r.cooldown_seconds,r.unknown_behavior,r.group_by,r.suppress_when,r.message,r.revision,v.version_number,COALESCE(u.external_subject,''),r.created_at,r.updated_at FROM alert_rules r JOIN alert_rule_versions v ON v.id=r.current_version_id LEFT JOIN users u ON u.id=r.created_by WHERE r.id=$1`, id).Scan(&rule.ID, &rule.SchemaVersion, &rule.Name, &rule.Enabled, &rule.Severity, &scopeJSON, &conditionJSON, &rule.EvaluationIntervalSeconds, &rule.PendingSeconds, &rule.ResolveSeconds, &rule.CooldownSeconds, &rule.UnknownBehavior, &groupJSON, &suppressJSON, &messageJSON, &rule.Revision, &rule.CurrentVersion, &createdBy, &rule.CreatedAt, &rule.UpdatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Rule{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Rule{}, fmt.Errorf("get alert rule: %w", err)
|
||||
}
|
||||
rule.CreatedBy = valueOrEmpty(createdBy)
|
||||
if err := decodeStored(&rule.Document, scopeJSON, conditionJSON, groupJSON, suppressJSON, messageJSON, rule.CooldownSeconds); err != nil {
|
||||
return Rule{}, err
|
||||
}
|
||||
return rule, nil
|
||||
}
|
||||
|
||||
func (r Repository) List(ctx context.Context, limit int) ([]Rule, error) {
|
||||
if r.Pool == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if limit < 1 || limit > 100 {
|
||||
return nil, errors.New("alert rule limit is invalid")
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, `SELECT r.id,r.schema_version,r.name,r.enabled,r.severity,r.scope,r.condition,r.evaluation_interval_seconds,r.pending_seconds,r.resolve_seconds,r.cooldown_seconds,r.unknown_behavior,r.group_by,r.suppress_when,r.message,r.revision,v.version_number,COALESCE(u.external_subject,''),r.created_at,r.updated_at FROM alert_rules r JOIN alert_rule_versions v ON v.id=r.current_version_id LEFT JOIN users u ON u.id=r.created_by ORDER BY r.name ASC,r.id ASC LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list alert rules: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
result := make([]Rule, 0, limit)
|
||||
for rows.Next() {
|
||||
var rule Rule
|
||||
var scopeJSON, conditionJSON, groupJSON, suppressJSON, messageJSON []byte
|
||||
if err := rows.Scan(&rule.ID, &rule.SchemaVersion, &rule.Name, &rule.Enabled, &rule.Severity, &scopeJSON, &conditionJSON, &rule.EvaluationIntervalSeconds, &rule.PendingSeconds, &rule.ResolveSeconds, &rule.CooldownSeconds, &rule.UnknownBehavior, &groupJSON, &suppressJSON, &messageJSON, &rule.Revision, &rule.CurrentVersion, &rule.CreatedBy, &rule.CreatedAt, &rule.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan alert rule: %w", err)
|
||||
}
|
||||
if err := decodeStored(&rule.Document, scopeJSON, conditionJSON, groupJSON, suppressJSON, messageJSON, rule.CooldownSeconds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, rule)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func (r Repository) Update(ctx context.Context, id, actor string, expected int64, document Document, changeSummary string) (Rule, error) {
|
||||
if r.Pool == nil {
|
||||
return Rule{}, ErrUnavailable
|
||||
}
|
||||
document.ID = id
|
||||
if err := document.Validate(r.Registry); err != nil {
|
||||
return Rule{}, err
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Rule{}, fmt.Errorf("begin alert rule update: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var currentRevision int64
|
||||
var currentVersionID string
|
||||
var currentJSON []byte
|
||||
err = tx.QueryRow(ctx, `SELECT revision,current_version_id FROM alert_rules WHERE id=$1 FOR UPDATE`, id).Scan(¤tRevision, ¤tVersionID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Rule{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Rule{}, fmt.Errorf("lock alert rule: %w", err)
|
||||
}
|
||||
if currentRevision != expected {
|
||||
return Rule{}, ErrConflict
|
||||
}
|
||||
if err = tx.QueryRow(ctx, `SELECT document FROM alert_rule_versions WHERE id=$1`, currentVersionID).Scan(¤tJSON); err != nil {
|
||||
return Rule{}, fmt.Errorf("read current alert rule version: %w", err)
|
||||
}
|
||||
nextJSON, err := document.MarshalCanonical()
|
||||
if err != nil {
|
||||
return Rule{}, err
|
||||
}
|
||||
if sameJSON(currentJSON, nextJSON) {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Rule{}, err
|
||||
}
|
||||
return r.Get(ctx, id)
|
||||
}
|
||||
var currentVersion int
|
||||
if err := tx.QueryRow(ctx, `SELECT version_number FROM alert_rule_versions WHERE id=$1`, currentVersionID).Scan(¤tVersion); err != nil {
|
||||
return Rule{}, err
|
||||
}
|
||||
if changeSummary == "" {
|
||||
changeSummary = "rule update"
|
||||
}
|
||||
versionID := NewID()
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO alert_rule_versions (id,rule_id,version_number,document,change_summary,created_by) VALUES ($1,$2,$3,$4::jsonb,$5,(SELECT id FROM users WHERE external_subject=$6))`, versionID, id, currentVersion+1, nextJSON, changeSummary, actor); err != nil {
|
||||
return Rule{}, mapError(err)
|
||||
}
|
||||
conditionJSON, _ := json.Marshal(document.Condition)
|
||||
scopeJSON, _ := json.Marshal(nonNilMap(document.Scope))
|
||||
groupJSON, _ := json.Marshal(nonNilStrings(document.GroupBy))
|
||||
suppressJSON, _ := json.Marshal(nonNilStrings(document.SuppressWhen))
|
||||
messageJSON, _ := json.Marshal(document.Message)
|
||||
tag, err := tx.Exec(ctx, `UPDATE alert_rules SET schema_version=$1,name=$2,enabled=$3,severity=$4,scope=$5::jsonb,condition=$6::jsonb,evaluation_interval_seconds=$7,pending_seconds=$8,resolve_seconds=$9,cooldown_seconds=$10,unknown_behavior=$11,group_by=$12::jsonb,suppress_when=$13::jsonb,message=$14::jsonb,current_version_id=$15,revision=revision+1,updated_at=now() WHERE id=$16 AND revision=$17`, document.SchemaVersion, document.Name, document.Enabled, document.Severity, scopeJSON, conditionJSON, document.EvaluationIntervalSeconds, document.PendingSeconds, document.ResolveSeconds, document.CooldownSeconds, document.UnknownBehavior, groupJSON, suppressJSON, messageJSON, versionID, id, expected)
|
||||
if err != nil {
|
||||
return Rule{}, mapError(err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return Rule{}, ErrConflict
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return Rule{}, fmt.Errorf("commit alert rule update: %w", err)
|
||||
}
|
||||
return r.Get(ctx, id)
|
||||
}
|
||||
|
||||
func (r Repository) Versions(ctx context.Context, id string, limit int) ([]Version, error) {
|
||||
if r.Pool == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if limit < 1 || limit > 100 {
|
||||
return nil, errors.New("version limit is invalid")
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, `SELECT v.id,v.rule_id,v.version_number,v.document,v.change_summary,COALESCE(u.external_subject,''),v.created_at FROM alert_rule_versions v LEFT JOIN users u ON u.id=v.created_by WHERE v.rule_id=$1 ORDER BY v.version_number DESC,v.id ASC LIMIT $2`, id, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list alert rule versions: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
result := make([]Version, 0, limit)
|
||||
for rows.Next() {
|
||||
var version Version
|
||||
var raw []byte
|
||||
if err := rows.Scan(&version.ID, &version.RuleID, &version.VersionNumber, &raw, &version.ChangeSummary, &version.CreatedBy, &version.CreatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan alert rule version: %w", err)
|
||||
}
|
||||
var document Document
|
||||
if _, err := DecodeDocument(raw, r.Registry); err != nil {
|
||||
return nil, fmt.Errorf("decode stored alert rule version: %w", err)
|
||||
} else {
|
||||
document = mustDecode(raw)
|
||||
}
|
||||
version.Document = document
|
||||
result = append(result, version)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(result) == 0 {
|
||||
var exists bool
|
||||
if err := r.Pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM alert_rules WHERE id=$1)`, id).Scan(&exists); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !exists {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r Repository) SetEnabled(ctx context.Context, id string, expected int64, enabled bool) (Rule, error) {
|
||||
if r.Pool == nil {
|
||||
return Rule{}, ErrUnavailable
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Rule{}, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var revision int64
|
||||
var current bool
|
||||
if err := tx.QueryRow(ctx, `SELECT revision,enabled FROM alert_rules WHERE id=$1 FOR UPDATE`, id).Scan(&revision, ¤t); errors.Is(err, pgx.ErrNoRows) {
|
||||
return Rule{}, ErrNotFound
|
||||
} else if err != nil {
|
||||
return Rule{}, err
|
||||
}
|
||||
if revision != expected {
|
||||
return Rule{}, ErrConflict
|
||||
}
|
||||
if current == enabled {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Rule{}, err
|
||||
}
|
||||
return r.Get(ctx, id)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE alert_rules SET enabled=$1,revision=revision+1,updated_at=now() WHERE id=$2 AND revision=$3`, enabled, id, expected); err != nil {
|
||||
return Rule{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Rule{}, err
|
||||
}
|
||||
return r.Get(ctx, id)
|
||||
}
|
||||
|
||||
func (r Repository) ListEnabled(ctx context.Context, limit int) ([]Rule, error) {
|
||||
if r.Pool == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if limit < 1 || limit > 100 {
|
||||
return nil, errors.New("enabled alert rule limit is invalid")
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, `SELECT r.id,r.schema_version,r.name,r.enabled,r.severity,r.scope,r.condition,r.evaluation_interval_seconds,r.pending_seconds,r.resolve_seconds,r.cooldown_seconds,r.unknown_behavior,r.group_by,r.suppress_when,r.message,r.revision,v.version_number,COALESCE(u.external_subject,''),r.created_at,r.updated_at FROM alert_rules r JOIN alert_rule_versions v ON v.id=r.current_version_id LEFT JOIN users u ON u.id=r.created_by WHERE r.enabled=true ORDER BY r.evaluation_interval_seconds ASC,r.name ASC,r.id ASC LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list enabled alert rules: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
result := make([]Rule, 0, limit)
|
||||
for rows.Next() {
|
||||
var rule Rule
|
||||
var scopeJSON, conditionJSON, groupJSON, suppressJSON, messageJSON []byte
|
||||
if err := rows.Scan(&rule.ID, &rule.SchemaVersion, &rule.Name, &rule.Enabled, &rule.Severity, &scopeJSON, &conditionJSON, &rule.EvaluationIntervalSeconds, &rule.PendingSeconds, &rule.ResolveSeconds, &rule.CooldownSeconds, &rule.UnknownBehavior, &groupJSON, &suppressJSON, &messageJSON, &rule.Revision, &rule.CurrentVersion, &rule.CreatedBy, &rule.CreatedAt, &rule.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan enabled alert rule: %w", err)
|
||||
}
|
||||
if err := decodeStored(&rule.Document, scopeJSON, conditionJSON, groupJSON, suppressJSON, messageJSON, rule.CooldownSeconds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, rule)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func decodeStored(document *Document, scopeJSON, conditionJSON, groupJSON, suppressJSON, messageJSON []byte, cooldownSeconds int) error {
|
||||
if err := json.Unmarshal(scopeJSON, &document.Scope); err != nil {
|
||||
return errors.New("invalid stored alert rule scope")
|
||||
}
|
||||
if err := json.Unmarshal(conditionJSON, &document.Condition); err != nil {
|
||||
return errors.New("invalid stored alert rule condition")
|
||||
}
|
||||
if err := json.Unmarshal(groupJSON, &document.GroupBy); err != nil {
|
||||
return errors.New("invalid stored alert rule groups")
|
||||
}
|
||||
if err := json.Unmarshal(suppressJSON, &document.SuppressWhen); err != nil {
|
||||
return errors.New("invalid stored alert rule suppression")
|
||||
}
|
||||
document.CooldownSeconds = cooldownSeconds
|
||||
if err := json.Unmarshal(messageJSON, &document.Message); err != nil {
|
||||
return errors.New("invalid stored alert rule message")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mustDecode(raw []byte) Document {
|
||||
var document Document
|
||||
_ = json.Unmarshal(raw, &document)
|
||||
return document
|
||||
}
|
||||
|
||||
func sameJSON(left, right []byte) bool {
|
||||
var a, b any
|
||||
if json.Unmarshal(left, &a) != nil || json.Unmarshal(right, &b) != nil {
|
||||
return bytes.Equal(bytes.TrimSpace(left), bytes.TrimSpace(right))
|
||||
}
|
||||
return reflect.DeepEqual(a, b)
|
||||
}
|
||||
|
||||
func nonNilMap(value map[string]any) map[string]any {
|
||||
if value == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
return value
|
||||
}
|
||||
func nonNilStrings(value []string) []string {
|
||||
if value == nil {
|
||||
return []string{}
|
||||
}
|
||||
return value
|
||||
}
|
||||
func valueOrEmpty(value *string) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
func mapError(err error) error {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return ErrConflict
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
)
|
||||
|
||||
func TestPostgreSQLRuleRepositoryLifecycle(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 4, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
document, registry := validDocument(t)
|
||||
repository := Repository{Pool: pool, Registry: registry}
|
||||
created, version, err := repository.Create(ctx, "integration-editor", document, "integration create")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if created.Revision != 1 || created.CurrentVersion != 1 || version.VersionNumber != 1 {
|
||||
t.Fatalf("unexpected create: %#v %#v", created, version)
|
||||
}
|
||||
|
||||
if _, _, err := repository.Create(ctx, "integration-editor", document, "duplicate"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("duplicate create error = %v, want conflict", err)
|
||||
}
|
||||
loaded, err := repository.Get(ctx, document.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loaded.Name != document.Name || loaded.Condition.Metric != document.Condition.Metric {
|
||||
t.Fatalf("loaded rule mismatch: %#v", loaded)
|
||||
}
|
||||
|
||||
same, err := repository.Update(ctx, document.ID, "integration-editor", 1, document, "idempotent")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if same.Revision != 1 || same.CurrentVersion != 1 {
|
||||
t.Fatalf("idempotent update changed revision: %#v", same)
|
||||
}
|
||||
|
||||
changed := document
|
||||
changed.Name = "CPU aandacht gewijzigd"
|
||||
updated, err := repository.Update(ctx, document.ID, "integration-editor", 1, changed, "change")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if updated.Revision != 2 || updated.CurrentVersion != 2 {
|
||||
t.Fatalf("unexpected update: %#v", updated)
|
||||
}
|
||||
if _, err := repository.Update(ctx, document.ID, "integration-editor", 1, changed, "stale"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("stale update error = %v, want conflict", err)
|
||||
}
|
||||
enabled, err := repository.SetEnabled(ctx, document.ID, 2, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !enabled.Enabled || enabled.Revision != 3 {
|
||||
t.Fatalf("unexpected enable: %#v", enabled)
|
||||
}
|
||||
versions, err := repository.Versions(ctx, document.ID, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(versions) != 2 || versions[0].VersionNumber != 2 || versions[1].VersionNumber != 1 {
|
||||
t.Fatalf("unexpected immutable versions: %#v", versions)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateInactive State = "inactive"
|
||||
StatePending State = "pending"
|
||||
StateFiring State = "firing"
|
||||
StateAcknowledged State = "acknowledged"
|
||||
StateResolved State = "resolved"
|
||||
StateUnknown State = "unknown"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidObservation = errors.New("invalid alert observation")
|
||||
ErrStaleObservation = errors.New("stale alert observation")
|
||||
ErrInstanceNotFound = errors.New("alert instance not found")
|
||||
ErrStateConflict = errors.New("alert instance state conflict")
|
||||
ErrRevisionConflict = errors.New("alert instance revision conflict")
|
||||
)
|
||||
|
||||
type Policy struct {
|
||||
PendingSeconds int
|
||||
ResolveSeconds int
|
||||
CooldownSeconds int
|
||||
UnknownBehavior string
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
State State
|
||||
RetainedState State
|
||||
ActiveSince *time.Time
|
||||
RecoverySince *time.Time
|
||||
CooldownUntil *time.Time
|
||||
LastEvaluatedAt time.Time
|
||||
LastKnownAt *time.Time
|
||||
LastValue any
|
||||
Reason string
|
||||
SourceHealth map[string]any
|
||||
AcknowledgedBy string
|
||||
AcknowledgedAt *time.Time
|
||||
}
|
||||
|
||||
type Notification string
|
||||
|
||||
const (
|
||||
NotificationNone Notification = ""
|
||||
NotificationFiring Notification = "firing"
|
||||
NotificationRecovery Notification = "recovery"
|
||||
NotificationUnknown Notification = "unknown"
|
||||
)
|
||||
|
||||
type TransitionResult struct {
|
||||
Snapshot Snapshot
|
||||
From State
|
||||
To State
|
||||
EventType string
|
||||
Notification Notification
|
||||
}
|
||||
type Observation struct {
|
||||
EvaluationKey string
|
||||
ObservedAt time.Time
|
||||
ConditionTrue bool
|
||||
Unknown bool
|
||||
Value any
|
||||
Reason string
|
||||
SourceHealth map[string]any
|
||||
}
|
||||
|
||||
func (s Snapshot) normalized() Snapshot {
|
||||
if s.State == "" {
|
||||
s.State = StateInactive
|
||||
}
|
||||
if s.RetainedState == "" {
|
||||
s.RetainedState = s.State
|
||||
}
|
||||
if s.SourceHealth == nil {
|
||||
s.SourceHealth = map[string]any{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func Transition(current Snapshot, policy Policy, observation Observation) (TransitionResult, error) {
|
||||
current = current.normalized()
|
||||
if observation.ObservedAt.IsZero() || observation.EvaluationKey == "" || len(observation.EvaluationKey) > 160 {
|
||||
return TransitionResult{}, ErrInvalidObservation
|
||||
}
|
||||
if !validState(current.State) || !validState(current.RetainedState) {
|
||||
return TransitionResult{}, fmt.Errorf("%w: invalid current state", ErrInvalidObservation)
|
||||
}
|
||||
if policy.PendingSeconds < 0 || policy.ResolveSeconds < 0 || policy.CooldownSeconds < 0 || policy.CooldownSeconds > 2592000 || policy.UnknownBehavior == "" {
|
||||
return TransitionResult{}, fmt.Errorf("%w: invalid policy", ErrInvalidObservation)
|
||||
}
|
||||
at := observation.ObservedAt.UTC()
|
||||
if !current.LastEvaluatedAt.IsZero() && at.Before(current.LastEvaluatedAt.UTC()) {
|
||||
return TransitionResult{}, ErrStaleObservation
|
||||
}
|
||||
result := current
|
||||
result.LastEvaluatedAt = at
|
||||
result.Reason = boundedReason(observation.Reason)
|
||||
result.SourceHealth = cloneMap(observation.SourceHealth)
|
||||
if observation.Unknown {
|
||||
if policy.UnknownBehavior == UnknownIgnoreGap {
|
||||
result.Reason = "unknown_input_ignored_short_gap"
|
||||
return finish(current, result, State(current.State), "evaluation"), nil
|
||||
}
|
||||
result.RetainedState = current.State
|
||||
if current.State == StateUnknown && current.RetainedState != StateUnknown {
|
||||
result.RetainedState = current.RetainedState
|
||||
}
|
||||
result.State = StateUnknown
|
||||
transition := finish(current, result, StateUnknown, "transition")
|
||||
transition.Notification = notificationFor(current, transition, at)
|
||||
return transition, nil
|
||||
}
|
||||
|
||||
base := current.State
|
||||
if base == StateUnknown {
|
||||
base = current.RetainedState
|
||||
if !validState(base) || base == StateUnknown {
|
||||
base = StateInactive
|
||||
}
|
||||
result.State = base
|
||||
}
|
||||
result.RetainedState = base
|
||||
result.LastKnownAt = timePtr(at)
|
||||
result.LastValue = observation.Value
|
||||
if observation.ConditionTrue {
|
||||
result.RecoverySince = nil
|
||||
switch base {
|
||||
case StateInactive, StateResolved:
|
||||
result.ActiveSince = timePtr(at)
|
||||
if policy.PendingSeconds == 0 {
|
||||
result.State = StateFiring
|
||||
} else {
|
||||
result.State = StatePending
|
||||
}
|
||||
case StatePending:
|
||||
if result.ActiveSince == nil {
|
||||
result.ActiveSince = timePtr(at)
|
||||
}
|
||||
if at.Sub(result.ActiveSince.UTC()) >= time.Duration(policy.PendingSeconds)*time.Second {
|
||||
result.State = StateFiring
|
||||
}
|
||||
case StateFiring, StateAcknowledged:
|
||||
result.State = base
|
||||
default:
|
||||
result.State = StateInactive
|
||||
}
|
||||
} else {
|
||||
result.ActiveSince = current.ActiveSince
|
||||
switch base {
|
||||
case StatePending:
|
||||
result.State = StateInactive
|
||||
result.ActiveSince = nil
|
||||
case StateFiring, StateAcknowledged:
|
||||
if result.RecoverySince == nil {
|
||||
result.RecoverySince = timePtr(at)
|
||||
}
|
||||
if at.Sub(result.RecoverySince.UTC()) >= time.Duration(policy.ResolveSeconds)*time.Second {
|
||||
result.State = StateResolved
|
||||
result.CooldownUntil = timePtr(at.Add(time.Duration(policy.CooldownSeconds) * time.Second))
|
||||
result.ActiveSince = nil
|
||||
result.RecoverySince = nil
|
||||
}
|
||||
default:
|
||||
result.State = StateInactive
|
||||
result.ActiveSince = nil
|
||||
result.RecoverySince = nil
|
||||
}
|
||||
}
|
||||
transition := finish(current, result, result.State, stateEvent(current.State, result.State))
|
||||
transition.Notification = notificationFor(current, transition, at)
|
||||
return transition, nil
|
||||
}
|
||||
|
||||
func Acknowledge(current Snapshot, actor string, at time.Time) (TransitionResult, error) {
|
||||
current = current.normalized()
|
||||
if actor == "" || len(actor) > 160 || at.IsZero() {
|
||||
return TransitionResult{}, ErrInvalidObservation
|
||||
}
|
||||
if current.State != StateFiring && current.State != StatePending {
|
||||
return TransitionResult{}, ErrStateConflict
|
||||
}
|
||||
result := current
|
||||
result.State = StateAcknowledged
|
||||
result.RetainedState = StateAcknowledged
|
||||
result.AcknowledgedBy = actor
|
||||
result.AcknowledgedAt = timePtr(at.UTC())
|
||||
result.Reason = "acknowledged"
|
||||
return finish(current, result, StateAcknowledged, "acknowledge"), nil
|
||||
}
|
||||
|
||||
func finish(current, result Snapshot, state State, eventType string) TransitionResult {
|
||||
result.State = state
|
||||
if result.SourceHealth == nil {
|
||||
result.SourceHealth = map[string]any{}
|
||||
}
|
||||
return TransitionResult{Snapshot: result, From: current.State, To: state, EventType: eventType}
|
||||
}
|
||||
|
||||
func notificationFor(current Snapshot, result TransitionResult, at time.Time) Notification {
|
||||
switch {
|
||||
case result.To == StateFiring && current.State != StateFiring && current.State != StateAcknowledged:
|
||||
if current.CooldownUntil != nil && at.Before(current.CooldownUntil.UTC()) {
|
||||
return NotificationNone
|
||||
}
|
||||
return NotificationFiring
|
||||
case result.To == StateResolved && (current.State == StateFiring || current.State == StateAcknowledged):
|
||||
return NotificationRecovery
|
||||
case result.To == StateUnknown && current.State != StateUnknown:
|
||||
return NotificationUnknown
|
||||
default:
|
||||
return NotificationNone
|
||||
}
|
||||
}
|
||||
|
||||
func stateEvent(from, to State) string {
|
||||
if from == to {
|
||||
return "evaluation"
|
||||
}
|
||||
return "transition"
|
||||
}
|
||||
|
||||
func validState(state State) bool {
|
||||
switch state {
|
||||
case StateInactive, StatePending, StateFiring, StateAcknowledged, StateResolved, StateUnknown:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func boundedReason(reason string) string {
|
||||
if len(reason) > 500 {
|
||||
return reason[:500]
|
||||
}
|
||||
return reason
|
||||
}
|
||||
|
||||
func cloneMap(source map[string]any) map[string]any {
|
||||
if source == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
copy := make(map[string]any, len(source))
|
||||
for key, value := range source {
|
||||
copy[key] = value
|
||||
}
|
||||
return copy
|
||||
}
|
||||
|
||||
func timePtr(value time.Time) *time.Time {
|
||||
value = value.UTC()
|
||||
return &value
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type StateInput struct {
|
||||
RuleID string
|
||||
RuleVersionID string
|
||||
Fingerprint string
|
||||
EntityID string
|
||||
Policy Policy
|
||||
Observation Observation
|
||||
}
|
||||
|
||||
type Instance struct {
|
||||
ID string `json:"id"`
|
||||
RuleID string `json:"ruleId"`
|
||||
RuleVersionID string `json:"ruleVersionId"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
EntityID string `json:"entityId,omitempty"`
|
||||
State State `json:"state"`
|
||||
RetainedState State `json:"retainedState"`
|
||||
ActiveSince *time.Time `json:"activeSince,omitempty"`
|
||||
RecoverySince *time.Time `json:"recoverySince,omitempty"`
|
||||
LastEvaluatedAt time.Time `json:"lastEvaluatedAt"`
|
||||
LastKnownAt *time.Time `json:"lastKnownAt,omitempty"`
|
||||
LastValue any `json:"lastValue,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
SourceHealth map[string]any `json:"sourceHealth"`
|
||||
AcknowledgedBy string `json:"acknowledgedBy,omitempty"`
|
||||
AcknowledgedAt *time.Time `json:"acknowledgedAt,omitempty"`
|
||||
CooldownUntil *time.Time `json:"cooldownUntil,omitempty"`
|
||||
Revision int64 `json:"revision"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type Occurrence struct {
|
||||
ID string `json:"id"`
|
||||
InstanceID string `json:"instanceId"`
|
||||
EvaluationKey string `json:"evaluationKey"`
|
||||
EventType string `json:"eventType"`
|
||||
From State `json:"from"`
|
||||
To State `json:"to"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
Value any `json:"value,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
SourceHealth map[string]any `json:"sourceHealth"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
type StateStore interface {
|
||||
ApplyObservation(context.Context, StateInput) (Instance, Occurrence, bool, error)
|
||||
GetInstance(context.Context, string) (Instance, error)
|
||||
ListOccurrences(context.Context, string, int) ([]Occurrence, error)
|
||||
Acknowledge(context.Context, string, string, string, time.Time) (Instance, Occurrence, bool, error)
|
||||
}
|
||||
|
||||
type StateRepository struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func (r StateRepository) ApplyObservation(ctx context.Context, input StateInput) (Instance, Occurrence, bool, error) {
|
||||
if r.Pool == nil {
|
||||
return Instance{}, Occurrence{}, false, ErrUnavailable
|
||||
}
|
||||
if err := validateStateInput(input); err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
healthJSON, err := boundedJSON(nonNilMap(input.Observation.SourceHealth), 64<<10)
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, fmt.Errorf("begin alert state transition: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
instanceID := NewID()
|
||||
if err := tx.QueryRow(ctx, `INSERT INTO alert_instances (id,rule_id,rule_version_id,fingerprint,entity_id,last_evaluated_at) VALUES ($1,$2,$3,$4,$5,$6) ON CONFLICT (rule_id,fingerprint) DO NOTHING RETURNING id`, instanceID, input.RuleID, input.RuleVersionID, input.Fingerprint, nullableID(input.EntityID), input.Observation.ObservedAt.UTC()).Scan(&instanceID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Instance{}, Occurrence{}, false, mapStateError(fmt.Errorf("create alert instance: %w", err))
|
||||
}
|
||||
var current Instance
|
||||
if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE rule_id=$1 AND fingerprint=$2 FOR UPDATE`, input.RuleID, input.Fingerprint), ¤t); err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
if current.EntityID != input.EntityID {
|
||||
return Instance{}, Occurrence{}, false, ErrStateConflict
|
||||
}
|
||||
if occurrence, err := scanOccurrence(tx.QueryRow(ctx, `SELECT id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at FROM alert_occurrences WHERE instance_id=$1 AND evaluation_key=$2`, current.ID, input.Observation.EvaluationKey)); err == nil {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Instance{}, Occurrence{}, false, fmt.Errorf("commit idempotent alert evaluation: %w", err)
|
||||
}
|
||||
return current, occurrence, true, nil
|
||||
} else if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
transition, err := Transition(snapshotFromInstance(current), input.Policy, input.Observation)
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
resultValue := transition.Snapshot.LastValue
|
||||
if input.Observation.Unknown || input.Policy.UnknownBehavior == UnknownIgnoreGap && transition.Snapshot.LastKnownAt == nil {
|
||||
resultValue = current.LastValue
|
||||
}
|
||||
resultValueJSON, err := boundedJSON(resultValue, 128<<10)
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE alert_instances SET rule_version_id=$1,current_state=$2,retained_state=$3,active_since=$4,recovery_since=$5,cooldown_until=$6,last_evaluated_at=$7,last_known_at=$8,last_value=$9::jsonb,reason=$10,source_health=$11::jsonb,acknowledged_by=$12,acknowledged_at=$13,revision=revision+1,updated_at=now() WHERE id=$14`, input.RuleVersionID, transition.Snapshot.State, transition.Snapshot.RetainedState, transition.Snapshot.ActiveSince, transition.Snapshot.RecoverySince, transition.Snapshot.CooldownUntil, transition.Snapshot.LastEvaluatedAt.UTC(), transition.Snapshot.LastKnownAt, resultValueJSON, transition.Snapshot.Reason, healthJSON, nullableText(transition.Snapshot.AcknowledgedBy), transition.Snapshot.AcknowledgedAt, current.ID); err != nil {
|
||||
return Instance{}, Occurrence{}, false, fmt.Errorf("update alert instance: %w", err)
|
||||
}
|
||||
occurrence, err := insertOccurrence(ctx, tx, current.ID, input.Observation.EvaluationKey, transition, input.Observation, resultValueJSON, healthJSON)
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1`, current.ID), ¤t); err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Instance{}, Occurrence{}, false, fmt.Errorf("commit alert state transition: %w", err)
|
||||
}
|
||||
return current, occurrence, false, nil
|
||||
}
|
||||
|
||||
func (r StateRepository) GetInstance(ctx context.Context, id string) (Instance, error) {
|
||||
if r.Pool == nil {
|
||||
return Instance{}, ErrUnavailable
|
||||
}
|
||||
var instance Instance
|
||||
err := scanInstance(r.Pool.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1`, id), &instance)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Instance{}, ErrInstanceNotFound
|
||||
}
|
||||
return instance, err
|
||||
}
|
||||
|
||||
func (r StateRepository) ListOccurrences(ctx context.Context, instanceID string, limit int) ([]Occurrence, error) {
|
||||
if r.Pool == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if limit < 1 || limit > 500 {
|
||||
return nil, errors.New("alert occurrence limit is invalid")
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, `SELECT id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at FROM alert_occurrences WHERE instance_id=$1 ORDER BY observed_at DESC,id ASC LIMIT $2`, instanceID, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list alert occurrences: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
result := make([]Occurrence, 0, limit)
|
||||
for rows.Next() {
|
||||
occurrence, err := scanOccurrence(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, occurrence)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(result) == 0 {
|
||||
if _, err := r.GetInstance(ctx, instanceID); errors.Is(err, ErrInstanceNotFound) {
|
||||
return nil, ErrInstanceNotFound
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r StateRepository) Acknowledge(ctx context.Context, instanceID, actor, evaluationKey string, at time.Time) (Instance, Occurrence, bool, error) {
|
||||
if r.Pool == nil {
|
||||
return Instance{}, Occurrence{}, false, ErrUnavailable
|
||||
}
|
||||
if instanceID == "" || actor == "" || len(actor) > 160 || evaluationKey == "" || len(evaluationKey) > 160 || at.IsZero() {
|
||||
return Instance{}, Occurrence{}, false, ErrInvalidObservation
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, fmt.Errorf("begin alert acknowledgement: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var current Instance
|
||||
if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1 FOR UPDATE`, instanceID), ¤t); errors.Is(err, pgx.ErrNoRows) {
|
||||
return Instance{}, Occurrence{}, false, ErrInstanceNotFound
|
||||
} else if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
if occurrence, err := scanOccurrence(tx.QueryRow(ctx, `SELECT id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at FROM alert_occurrences WHERE instance_id=$1 AND evaluation_key=$2`, instanceID, evaluationKey)); err == nil {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
return current, occurrence, true, nil
|
||||
} else if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
transition, err := Acknowledge(snapshotFromInstance(current), actor, at.UTC())
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
valueJSON, err := boundedJSON(current.LastValue, 128<<10)
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
healthJSON, err := boundedJSON(nonNilMap(current.SourceHealth), 64<<10)
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE alert_instances SET current_state=$1,retained_state=$2,reason=$3,acknowledged_by=$4,acknowledged_at=$5,revision=revision+1,updated_at=now() WHERE id=$6`, transition.Snapshot.State, transition.Snapshot.RetainedState, transition.Snapshot.Reason, actor, transition.Snapshot.AcknowledgedAt, instanceID); err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
occurrence, err := insertOccurrence(ctx, tx, instanceID, evaluationKey, transition, Observation{ObservedAt: at.UTC(), Reason: "acknowledged", Value: current.LastValue}, valueJSON, healthJSON)
|
||||
if err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1`, instanceID), ¤t); err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Instance{}, Occurrence{}, false, err
|
||||
}
|
||||
return current, occurrence, false, nil
|
||||
}
|
||||
|
||||
func scanInstance(row pgx.Row, instance *Instance) error {
|
||||
var valueJSON, healthJSON []byte
|
||||
err := row.Scan(&instance.ID, &instance.RuleID, &instance.RuleVersionID, &instance.Fingerprint, &instance.EntityID, &instance.State, &instance.RetainedState, &instance.ActiveSince, &instance.RecoverySince, &instance.CooldownUntil, &instance.LastEvaluatedAt, &instance.LastKnownAt, &valueJSON, &instance.Reason, &healthJSON, &instance.AcknowledgedBy, &instance.AcknowledgedAt, &instance.Revision, &instance.CreatedAt, &instance.UpdatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrInstanceNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("scan alert instance: %w", err)
|
||||
}
|
||||
instance.LastValue, err = decodeJSON(valueJSON)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decode alert instance value: %w", err)
|
||||
}
|
||||
instance.SourceHealth, err = decodeMap(healthJSON)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decode alert instance source health: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scanOccurrence(row interface{ Scan(...any) error }) (Occurrence, error) {
|
||||
var occurrence Occurrence
|
||||
var valueJSON, healthJSON []byte
|
||||
err := row.Scan(&occurrence.ID, &occurrence.InstanceID, &occurrence.EvaluationKey, &occurrence.EventType, &occurrence.From, &occurrence.To, &occurrence.ObservedAt, &valueJSON, &occurrence.Reason, &healthJSON, &occurrence.CreatedAt)
|
||||
if err != nil {
|
||||
return Occurrence{}, err
|
||||
}
|
||||
occurrence.Value, err = decodeJSON(valueJSON)
|
||||
if err != nil {
|
||||
return Occurrence{}, fmt.Errorf("decode alert occurrence value: %w", err)
|
||||
}
|
||||
occurrence.SourceHealth, err = decodeMap(healthJSON)
|
||||
if err != nil {
|
||||
return Occurrence{}, fmt.Errorf("decode alert occurrence source health: %w", err)
|
||||
}
|
||||
return occurrence, nil
|
||||
}
|
||||
|
||||
func insertOccurrence(ctx context.Context, tx pgx.Tx, instanceID, key string, transition TransitionResult, observation Observation, valueJSON, healthJSON []byte) (Occurrence, error) {
|
||||
var occurrence Occurrence
|
||||
err := tx.QueryRow(ctx, `INSERT INTO alert_occurrences (id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health) VALUES ($1,$2,$3,$4,$5,$6,$7,$8::jsonb,$9,$10::jsonb) RETURNING id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at`, NewID(), instanceID, key, transition.EventType, transition.From, transition.To, observation.ObservedAt.UTC(), valueJSON, transition.Snapshot.Reason, healthJSON).Scan(&occurrence.ID, &occurrence.InstanceID, &occurrence.EvaluationKey, &occurrence.EventType, &occurrence.From, &occurrence.To, &occurrence.ObservedAt, &valueJSON, &occurrence.Reason, &healthJSON, &occurrence.CreatedAt)
|
||||
if err != nil {
|
||||
return Occurrence{}, mapStateError(fmt.Errorf("insert alert occurrence: %w", err))
|
||||
}
|
||||
occurrence.Value, err = decodeJSON(valueJSON)
|
||||
if err != nil {
|
||||
return Occurrence{}, err
|
||||
}
|
||||
occurrence.SourceHealth, err = decodeMap(healthJSON)
|
||||
if err != nil {
|
||||
return Occurrence{}, err
|
||||
}
|
||||
return occurrence, nil
|
||||
}
|
||||
|
||||
func snapshotFromInstance(instance Instance) Snapshot {
|
||||
return Snapshot{State: instance.State, RetainedState: instance.RetainedState, ActiveSince: instance.ActiveSince, RecoverySince: instance.RecoverySince, CooldownUntil: instance.CooldownUntil, LastEvaluatedAt: instance.LastEvaluatedAt, LastKnownAt: instance.LastKnownAt, LastValue: instance.LastValue, Reason: instance.Reason, SourceHealth: instance.SourceHealth, AcknowledgedBy: instance.AcknowledgedBy, AcknowledgedAt: instance.AcknowledgedAt}
|
||||
}
|
||||
|
||||
func validateStateInput(input StateInput) error {
|
||||
if strings.TrimSpace(input.RuleID) == "" || strings.TrimSpace(input.RuleVersionID) == "" || input.Fingerprint == "" || len(input.Fingerprint) > 160 {
|
||||
return ErrInvalidObservation
|
||||
}
|
||||
if input.Policy.UnknownBehavior != UnknownRetain && input.Policy.UnknownBehavior != UnknownBecome && input.Policy.UnknownBehavior != UnknownIgnoreGap {
|
||||
return ErrInvalidObservation
|
||||
}
|
||||
if input.Policy.PendingSeconds < 0 || input.Policy.ResolveSeconds < 0 {
|
||||
return ErrInvalidObservation
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func boundedJSON(value any, max int) ([]byte, error) {
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode alert state JSON: %w", err)
|
||||
}
|
||||
if len(encoded) > max {
|
||||
return nil, ErrInvalidObservation
|
||||
}
|
||||
return encoded, nil
|
||||
}
|
||||
|
||||
func decodeJSON(raw []byte) (any, error) {
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return nil, nil
|
||||
}
|
||||
var value any
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func decodeMap(raw []byte) (map[string]any, error) {
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return map[string]any{}, nil
|
||||
}
|
||||
var value map[string]any
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if value == nil {
|
||||
return map[string]any{}, nil
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func nullableID(value string) any {
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func nullableText(value string) any {
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func mapStateError(err error) error {
|
||||
var pgErr interface{ SQLState() string }
|
||||
if errors.As(err, &pgErr) && pgErr.SQLState() == "23505" {
|
||||
return ErrStateConflict
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
)
|
||||
|
||||
func TestPostgreSQLAlertStateLifecycleAndIdempotence(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 8, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
document, registry := validDocument(t)
|
||||
document.Enabled = true
|
||||
rules := Repository{Pool: pool, Registry: registry}
|
||||
created, version, err := rules.Create(ctx, "state-integration", document, "state test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := StateRepository{Pool: pool}
|
||||
policy := Policy{PendingSeconds: document.PendingSeconds, ResolveSeconds: document.ResolveSeconds, UnknownBehavior: document.UnknownBehavior}
|
||||
base := time.Date(2026, time.January, 2, 12, 0, 0, 0, time.UTC)
|
||||
first, firstOccurrence, duplicate, err := store.ApplyObservation(ctx, StateInput{RuleID: created.ID, RuleVersionID: version.ID, Fingerprint: "host:test", Policy: policy, Observation: observation(base, "slot-1", true)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if duplicate || first.State != StatePending || firstOccurrence.To != StatePending {
|
||||
t.Fatalf("unexpected first state: %#v %#v", first, firstOccurrence)
|
||||
}
|
||||
replayed, replayOccurrence, duplicate, err := store.ApplyObservation(ctx, StateInput{RuleID: created.ID, RuleVersionID: version.ID, Fingerprint: "host:test", Policy: policy, Observation: observation(base, "slot-1", true)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !duplicate || replayed.Revision != first.Revision || replayOccurrence.ID != firstOccurrence.ID {
|
||||
t.Fatalf("replay was not idempotent: %#v %#v", replayed, replayOccurrence)
|
||||
}
|
||||
firing, _, _, err := store.ApplyObservation(ctx, StateInput{RuleID: created.ID, RuleVersionID: version.ID, Fingerprint: "host:test", Policy: policy, Observation: observation(base.Add(60*time.Second), "slot-2", true)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if firing.State != StateFiring {
|
||||
t.Fatalf("pending did not fire: %#v", firing)
|
||||
}
|
||||
acknowledged, _, _, err := store.Acknowledge(ctx, firing.ID, "operator", "ack-1", base.Add(61*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if acknowledged.State != StateAcknowledged {
|
||||
t.Fatalf("acknowledgement failed: %#v", acknowledged)
|
||||
}
|
||||
stillFiring, _, _, err := store.ApplyObservation(ctx, StateInput{RuleID: created.ID, RuleVersionID: version.ID, Fingerprint: "host:test", Policy: policy, Observation: observation(base.Add(62*time.Second), "slot-3", true)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stillFiring.State != StateAcknowledged {
|
||||
t.Fatalf("acknowledged firing alert changed state: %#v", stillFiring)
|
||||
}
|
||||
unknownObservation := observation(base.Add(90*time.Second), "slot-4", false)
|
||||
unknownObservation.Unknown = true
|
||||
unknown, _, _, err := store.ApplyObservation(ctx, StateInput{RuleID: created.ID, RuleVersionID: version.ID, Fingerprint: "host:test", Policy: policy, Observation: unknownObservation})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if unknown.State != StateUnknown || unknown.RetainedState != StateAcknowledged {
|
||||
t.Fatalf("unknown state lost acknowledgement context: %#v", unknown)
|
||||
}
|
||||
restarted := StateRepository{Pool: pool}
|
||||
loaded, err := restarted.GetInstance(ctx, unknown.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loaded.State != StateUnknown || loaded.LastKnownAt == nil || loaded.LastValue == nil {
|
||||
t.Fatalf("restart did not preserve state: %#v", loaded)
|
||||
}
|
||||
occurrences, err := restarted.ListOccurrences(ctx, unknown.ID, 20)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(occurrences) != 5 {
|
||||
t.Fatalf("occurrence count = %d, want 5", len(occurrences))
|
||||
}
|
||||
|
||||
missingVersion := StateInput{RuleID: created.ID, RuleVersionID: NewID(), Fingerprint: "rollback", Policy: policy, Observation: observation(base, "rollback", true)}
|
||||
if _, _, _, err := store.ApplyObservation(ctx, missingVersion); err == nil {
|
||||
t.Fatal("missing foreign key did not fail")
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `SELECT 1 FROM alert_instances WHERE rule_id=$1 AND fingerprint=$2`, created.ID, "rollback"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgreSQLAlertStateCoordinatesOverlappingWrites(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 8, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
document, registry := validDocument(t)
|
||||
rules := Repository{Pool: pool, Registry: registry}
|
||||
created, version, err := rules.Create(ctx, "state-concurrency", document, "state concurrency")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := StateRepository{Pool: pool}
|
||||
input := StateInput{RuleID: created.ID, RuleVersionID: version.ID, Fingerprint: "host:concurrent", Policy: Policy{PendingSeconds: 0, ResolveSeconds: 0, UnknownBehavior: UnknownRetain}, Observation: observation(time.Date(2026, time.January, 3, 12, 0, 0, 0, time.UTC), "same-slot", true)}
|
||||
const workers = 8
|
||||
results := make(chan bool, workers)
|
||||
errorsCh := make(chan error, workers)
|
||||
var group sync.WaitGroup
|
||||
for i := 0; i < workers; i++ {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
_, _, duplicate, err := store.ApplyObservation(ctx, input)
|
||||
if err != nil {
|
||||
errorsCh <- err
|
||||
return
|
||||
}
|
||||
results <- duplicate
|
||||
}()
|
||||
}
|
||||
group.Wait()
|
||||
close(results)
|
||||
close(errorsCh)
|
||||
for err := range errorsCh {
|
||||
t.Fatal(err)
|
||||
}
|
||||
createdCount := 0
|
||||
for duplicate := range results {
|
||||
if !duplicate {
|
||||
createdCount++
|
||||
}
|
||||
}
|
||||
if createdCount != 1 {
|
||||
t.Fatalf("non-idempotent concurrent writes = %d, want 1", createdCount)
|
||||
}
|
||||
var occurrenceCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM alert_occurrences WHERE instance_id=(SELECT id FROM alert_instances WHERE rule_id=$1 AND fingerprint=$2)`, created.ID, input.Fingerprint).Scan(&occurrenceCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if occurrenceCount != 1 {
|
||||
t.Fatalf("occurrences = %d, want 1", occurrenceCount)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testPolicy() Policy {
|
||||
return Policy{PendingSeconds: 60, ResolveSeconds: 30, UnknownBehavior: UnknownRetain}
|
||||
}
|
||||
|
||||
func observation(at time.Time, key string, fire bool) Observation {
|
||||
return Observation{EvaluationKey: key, ObservedAt: at, ConditionTrue: fire, Value: 90, Reason: "condition_evaluated", SourceHealth: map[string]any{"source": "test"}}
|
||||
}
|
||||
|
||||
func TestStateTransitionTable(t *testing.T) {
|
||||
start := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
||||
tests := []struct {
|
||||
name string
|
||||
state State
|
||||
at time.Time
|
||||
fire bool
|
||||
want State
|
||||
}{
|
||||
{name: "inactive enters pending", state: StateInactive, at: start, fire: true, want: StatePending},
|
||||
{name: "pending remains pending", state: StatePending, at: start.Add(30 * time.Second), fire: true, want: StatePending},
|
||||
{name: "pending fires after duration", state: StatePending, at: start.Add(60 * time.Second), fire: true, want: StateFiring},
|
||||
{name: "pending clears", state: StatePending, at: start.Add(30 * time.Second), fire: false, want: StateInactive},
|
||||
{name: "firing starts recovery", state: StateFiring, at: start, fire: false, want: StateFiring},
|
||||
{name: "firing resolves after duration", state: StateFiring, at: start.Add(30 * time.Second), fire: false, want: StateResolved},
|
||||
{name: "resolved reopens", state: StateResolved, at: start, fire: true, want: StatePending},
|
||||
{name: "acknowledged remains firing", state: StateAcknowledged, at: start, fire: true, want: StateAcknowledged},
|
||||
{name: "acknowledged starts recovery", state: StateAcknowledged, at: start, fire: false, want: StateAcknowledged},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
current := Snapshot{State: test.state, RetainedState: test.state}
|
||||
if test.state == StatePending {
|
||||
current.ActiveSince = timePtr(start)
|
||||
}
|
||||
if test.state == StateFiring || test.state == StateAcknowledged {
|
||||
current.RecoverySince = timePtr(start)
|
||||
}
|
||||
result, err := Transition(current, testPolicy(), observation(test.at, "slot-"+test.name, test.fire))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.To != test.want {
|
||||
t.Fatalf("state = %s, want %s", result.To, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcknowledgementIsExplicitAndPreservesFiringContext(t *testing.T) {
|
||||
at := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
||||
result, err := Acknowledge(Snapshot{State: StateFiring, RetainedState: StateFiring, LastValue: 92}, "operator", at)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.To != StateAcknowledged || result.Snapshot.LastValue != 92 || result.Snapshot.AcknowledgedBy != "operator" {
|
||||
t.Fatalf("unexpected acknowledgement: %#v", result)
|
||||
}
|
||||
later, err := Transition(result.Snapshot, testPolicy(), observation(at.Add(time.Second), "slot-ack", true))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if later.To != StateAcknowledged {
|
||||
t.Fatalf("acknowledged alert did not remain acknowledged while firing: %#v", later)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownDoesNotFalseResolveAndRecoveryUsesRetainedState(t *testing.T) {
|
||||
at := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
||||
unknown, err := Transition(Snapshot{State: StateFiring, RetainedState: StateFiring, LastKnownAt: timePtr(at), LastValue: 91}, testPolicy(), Observation{EvaluationKey: "unknown", ObservedAt: at.Add(time.Minute), Unknown: true, Reason: "source_stale", SourceHealth: map[string]any{"state": "unknown"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if unknown.To != StateUnknown || unknown.Snapshot.RetainedState != StateFiring || unknown.Snapshot.LastValue != 91 {
|
||||
t.Fatalf("unknown transition lost firing context: %#v", unknown)
|
||||
}
|
||||
recovered, err := Transition(unknown.Snapshot, testPolicy(), observation(at.Add(2*time.Minute), "recovery", false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if recovered.To != StateFiring {
|
||||
t.Fatalf("unknown recovery falsely resolved immediately: %#v", recovered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStateRejectsOutOfOrderObservation(t *testing.T) {
|
||||
at := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
||||
_, err := Transition(Snapshot{State: StateFiring, RetainedState: StateFiring, LastEvaluatedAt: at}, testPolicy(), observation(at.Add(-time.Second), "old", false))
|
||||
if !errors.Is(err, ErrStaleObservation) {
|
||||
t.Fatalf("error = %v, want ErrStaleObservation", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIgnoreUnknownGapPreservesState(t *testing.T) {
|
||||
at := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
||||
policy := testPolicy()
|
||||
policy.UnknownBehavior = UnknownIgnoreGap
|
||||
result, err := Transition(Snapshot{State: StateFiring, RetainedState: StateFiring, LastEvaluatedAt: at, LastValue: 80}, policy, Observation{EvaluationKey: "gap", ObservedAt: at.Add(time.Second), Unknown: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.To != StateFiring || result.Snapshot.LastValue != 80 {
|
||||
t.Fatalf("unknown gap changed state: %#v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskTemperatureScenarioUsesPendingRecoveryAndCooldownDeterministically(t *testing.T) {
|
||||
start := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
||||
policy := Policy{PendingSeconds: 300, ResolveSeconds: 300, UnknownBehavior: UnknownRetain}
|
||||
current := Snapshot{State: StateInactive, RetainedState: StateInactive}
|
||||
transition, err := Transition(current, policy, observation(start, "disk-1", true))
|
||||
if err != nil || transition.To != StatePending {
|
||||
t.Fatalf("initial transition = %#v, %v", transition, err)
|
||||
}
|
||||
current = transition.Snapshot
|
||||
transition, err = Transition(current, policy, observation(start.Add(299*time.Second), "disk-2", true))
|
||||
if err != nil || transition.To != StatePending {
|
||||
t.Fatalf("boundary pending transition = %#v, %v", transition, err)
|
||||
}
|
||||
current = transition.Snapshot
|
||||
transition, err = Transition(current, policy, observation(start.Add(300*time.Second), "disk-3", true))
|
||||
if err != nil || transition.To != StateFiring {
|
||||
t.Fatalf("fire transition = %#v, %v", transition, err)
|
||||
}
|
||||
current = transition.Snapshot
|
||||
transition, err = Transition(current, policy, observation(start.Add(330*time.Second), "disk-4", true))
|
||||
if err != nil || transition.To != StateFiring {
|
||||
t.Fatalf("temperature at recovery threshold changed state: %#v, %v", transition, err)
|
||||
}
|
||||
current = transition.Snapshot
|
||||
transition, err = Transition(current, policy, observation(start.Add(360*time.Second), "disk-5", false))
|
||||
if err != nil || transition.To != StateFiring {
|
||||
t.Fatalf("recovery start transition = %#v, %v", transition, err)
|
||||
}
|
||||
current = transition.Snapshot
|
||||
transition, err = Transition(current, policy, observation(start.Add(659*time.Second), "disk-6", false))
|
||||
if err != nil || transition.To != StateFiring {
|
||||
t.Fatalf("pre-recovery boundary transition = %#v, %v", transition, err)
|
||||
}
|
||||
transition, err = Transition(transition.Snapshot, policy, observation(start.Add(660*time.Second), "disk-7", false))
|
||||
if err != nil || transition.To != StateResolved {
|
||||
t.Fatalf("recovery boundary transition = %#v, %v", transition, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCooldownSuppressesRepeatedFiringNotification(t *testing.T) {
|
||||
start := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
||||
policy := Policy{PendingSeconds: 0, ResolveSeconds: 0, CooldownSeconds: 60, UnknownBehavior: UnknownRetain}
|
||||
first, err := Transition(Snapshot{State: StateInactive, RetainedState: StateInactive}, policy, observation(start, "cool-1", true))
|
||||
if err != nil || first.Notification != NotificationFiring {
|
||||
t.Fatalf("first notification = %#v, %v", first, err)
|
||||
}
|
||||
resolved, err := Transition(first.Snapshot, policy, observation(start.Add(time.Second), "cool-2", false))
|
||||
if err != nil || resolved.To != StateResolved || resolved.Notification != NotificationRecovery || resolved.Snapshot.CooldownUntil == nil {
|
||||
t.Fatalf("resolve notification = %#v, %v", resolved, err)
|
||||
}
|
||||
suppressed, err := Transition(resolved.Snapshot, policy, observation(start.Add(30*time.Second), "cool-3", true))
|
||||
if err != nil || suppressed.To != StateFiring || suppressed.Notification != NotificationNone {
|
||||
t.Fatalf("cooldown firing notification = %#v, %v", suppressed, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/metriccatalog"
|
||||
)
|
||||
|
||||
const SchemaVersion = 1
|
||||
|
||||
const (
|
||||
SeverityAttention = "attention"
|
||||
SeverityDegraded = "degraded"
|
||||
SeverityCritical = "critical"
|
||||
UnknownRetain = "retain-firing-as-unknown"
|
||||
UnknownBecome = "become-unknown"
|
||||
UnknownIgnoreGap = "ignore-short-gap"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidRule = errors.New("invalid alert rule")
|
||||
ErrConflict = errors.New("alert rule revision conflict")
|
||||
ErrNotFound = errors.New("alert rule not found")
|
||||
ErrUnavailable = errors.New("alert rule repository is unavailable")
|
||||
semanticKeyPattern = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9_.-]*$")
|
||||
uuidPattern = regexp.MustCompile("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$")
|
||||
)
|
||||
|
||||
type Condition struct {
|
||||
InputType string `json:"inputType"`
|
||||
Metric string `json:"metric,omitempty"`
|
||||
Operator string `json:"operator"`
|
||||
Threshold any `json:"threshold,omitempty"`
|
||||
RecoveryThreshold *float64 `json:"recoveryThreshold,omitempty"`
|
||||
Aggregation string `json:"aggregation,omitempty"`
|
||||
WindowSeconds int `json:"windowSeconds,omitempty"`
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
TitleKey string `json:"titleKey"`
|
||||
BodyKey string `json:"bodyKey"`
|
||||
}
|
||||
|
||||
type Document struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Severity string `json:"severity"`
|
||||
Scope map[string]any `json:"scope"`
|
||||
Condition Condition `json:"condition"`
|
||||
EvaluationIntervalSeconds int `json:"evaluationIntervalSeconds"`
|
||||
PendingSeconds int `json:"pendingSeconds"`
|
||||
ResolveSeconds int `json:"resolveSeconds"`
|
||||
CooldownSeconds int `json:"cooldownSeconds,omitempty"`
|
||||
UnknownBehavior string `json:"unknownBehavior"`
|
||||
GroupBy []string `json:"groupBy,omitempty"`
|
||||
SuppressWhen []string `json:"suppressWhen,omitempty"`
|
||||
Message Message `json:"message"`
|
||||
}
|
||||
|
||||
type Rule struct {
|
||||
Document
|
||||
Revision int64 `json:"revision"`
|
||||
CurrentVersion int `json:"currentVersion"`
|
||||
CreatedBy string `json:"createdBy,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type Version struct {
|
||||
ID string `json:"id"`
|
||||
RuleID string `json:"ruleId"`
|
||||
VersionNumber int `json:"versionNumber"`
|
||||
Document Document `json:"document"`
|
||||
ChangeSummary string `json:"changeSummary"`
|
||||
CreatedBy string `json:"createdBy,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
type PreviewRequest struct {
|
||||
Value any `json:"value,omitempty"`
|
||||
Unknown bool `json:"unknown,omitempty"`
|
||||
CurrentState string `json:"currentState,omitempty"`
|
||||
}
|
||||
|
||||
type PreviewResult struct {
|
||||
WouldFire bool `json:"wouldFire"`
|
||||
State string `json:"state"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
func (d Document) Validate(registry metriccatalog.Registry) error {
|
||||
if d.SchemaVersion != SchemaVersion {
|
||||
return invalid("schemaVersion", "must be 1")
|
||||
}
|
||||
if !uuidPattern.MatchString(d.ID) {
|
||||
return invalid("id", "must be a UUID")
|
||||
}
|
||||
if strings.TrimSpace(d.Name) != d.Name || d.Name == "" || len(d.Name) > 160 || strings.ContainsAny(d.Name, "\r\n") {
|
||||
return invalid("name", "must be 1-160 characters without line breaks")
|
||||
}
|
||||
if !oneOf(d.Severity, SeverityAttention, SeverityDegraded, SeverityCritical) {
|
||||
return invalid("severity", "is unsupported")
|
||||
}
|
||||
if len(d.Scope) > 20 {
|
||||
return invalid("scope", "has too many keys")
|
||||
}
|
||||
for key, value := range d.Scope {
|
||||
if len(key) == 0 || len(key) > 80 || !semanticKeyPattern.MatchString(key) {
|
||||
return invalid("scope", "contains an invalid key")
|
||||
}
|
||||
if err := validateValue(value, 160); err != nil {
|
||||
return invalid("scope", err.Error())
|
||||
}
|
||||
}
|
||||
if d.EvaluationIntervalSeconds < 5 || d.EvaluationIntervalSeconds > 3600 {
|
||||
return invalid("evaluationIntervalSeconds", "must be between 5 and 3600")
|
||||
}
|
||||
if d.PendingSeconds < 0 || d.PendingSeconds > 2592000 || d.ResolveSeconds < 0 || d.ResolveSeconds > 2592000 {
|
||||
return invalid("pendingSeconds", "is out of range")
|
||||
}
|
||||
if d.CooldownSeconds < 0 || d.CooldownSeconds > 2592000 {
|
||||
return invalid("cooldownSeconds", "is out of range")
|
||||
}
|
||||
|
||||
if !oneOf(d.UnknownBehavior, UnknownRetain, UnknownBecome, UnknownIgnoreGap) {
|
||||
return invalid("unknownBehavior", "is unsupported")
|
||||
}
|
||||
if len(d.GroupBy) > 10 || uniqueStrings(d.GroupBy, 80) != nil {
|
||||
return invalid("groupBy", "must contain at most 10 unique bounded keys")
|
||||
}
|
||||
if len(d.SuppressWhen) > 20 || uniqueStrings(d.SuppressWhen, 160) != nil {
|
||||
return invalid("suppressWhen", "must contain at most 20 unique bounded rules")
|
||||
}
|
||||
if len(d.Message.TitleKey) == 0 || len(d.Message.TitleKey) > 160 || len(d.Message.BodyKey) == 0 || len(d.Message.BodyKey) > 160 {
|
||||
return invalid("message", "titleKey and bodyKey are required and bounded")
|
||||
}
|
||||
if err := d.Condition.validate(registry); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c Condition) validate(registry metriccatalog.Registry) error {
|
||||
if !oneOf(c.InputType, "metric", "entity-status", "event", "datasource-health") {
|
||||
return invalid("condition.inputType", "is unsupported")
|
||||
}
|
||||
if !oneOf(c.Operator, ">", ">=", "<", "<=", "==", "!=", "matches", "absent") {
|
||||
return invalid("condition.operator", "is unsupported")
|
||||
}
|
||||
if c.InputType == "metric" {
|
||||
if c.Metric == "" || strings.ContainsAny(c.Metric, "{};$()[]") || len(c.Metric) > 160 {
|
||||
return invalid("condition.metric", "must be a semantic metric name")
|
||||
}
|
||||
if _, ok := registry.Find(c.Metric); !ok {
|
||||
return invalid("condition.metric", "is not in the semantic metric catalog")
|
||||
}
|
||||
} else if c.Metric != "" {
|
||||
return invalid("condition.metric", "is only valid for metric input")
|
||||
}
|
||||
if c.Aggregation != "" && !oneOf(c.Aggregation, "none", "avg", "sum", "min", "max", "rate", "increase", "count", "p50", "p95", "p99") {
|
||||
return invalid("condition.aggregation", "is unsupported")
|
||||
}
|
||||
if c.Aggregation == "count" && c.InputType != "event" {
|
||||
return invalid("condition.aggregation", "count is only valid for event input")
|
||||
}
|
||||
if c.WindowSeconds < 0 || c.WindowSeconds > 2592000 {
|
||||
return invalid("condition.windowSeconds", "is out of range")
|
||||
}
|
||||
if c.Operator == "matches" {
|
||||
value, ok := c.Threshold.(string)
|
||||
if !ok || len(value) == 0 || len(value) > 160 {
|
||||
return invalid("condition.threshold", "matches requires a bounded pattern")
|
||||
}
|
||||
if _, err := regexp.Compile(value); err != nil {
|
||||
return invalid("condition.threshold", "contains an invalid pattern")
|
||||
}
|
||||
} else if c.Operator == "absent" {
|
||||
if c.Threshold != nil {
|
||||
return invalid("condition.threshold", "absent does not accept a threshold")
|
||||
}
|
||||
} else if !isNumber(c.Threshold) && !(c.Operator == "==" || c.Operator == "!=" && isComparableString(c.Threshold)) {
|
||||
return invalid("condition.threshold", "a numeric threshold is required")
|
||||
}
|
||||
if c.RecoveryThreshold != nil {
|
||||
if math.IsNaN(*c.RecoveryThreshold) || math.IsInf(*c.RecoveryThreshold, 0) {
|
||||
return invalid("condition.recoveryThreshold", "must be finite")
|
||||
}
|
||||
if !oneOf(c.Operator, ">", ">=", "<", "<=") || !isNumber(c.Threshold) {
|
||||
return invalid("condition.recoveryThreshold", "requires a numeric ordered condition")
|
||||
}
|
||||
threshold, _ := number(c.Threshold)
|
||||
if (c.Operator == ">" || c.Operator == ">=") && *c.RecoveryThreshold >= threshold {
|
||||
return invalid("condition.recoveryThreshold", "must be below the firing threshold")
|
||||
}
|
||||
if (c.Operator == "<" || c.Operator == "<=") && *c.RecoveryThreshold <= threshold {
|
||||
return invalid("condition.recoveryThreshold", "must be above the firing threshold")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d Document) MarshalCanonical() ([]byte, error) {
|
||||
if d.Scope == nil {
|
||||
d.Scope = map[string]any{}
|
||||
}
|
||||
if d.GroupBy == nil {
|
||||
d.GroupBy = []string{}
|
||||
}
|
||||
if d.SuppressWhen == nil {
|
||||
d.SuppressWhen = []string{}
|
||||
}
|
||||
return json.Marshal(d)
|
||||
}
|
||||
|
||||
func DecodeDocument(data []byte, registry metriccatalog.Registry) (Document, error) {
|
||||
if len(data) == 0 || len(data) > 2<<20 {
|
||||
return Document{}, invalid("document", "is empty or too large")
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
var document Document
|
||||
if err := decoder.Decode(&document); err != nil {
|
||||
return Document{}, fmt.Errorf("%w: document: %v", ErrInvalidRule, err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return Document{}, invalid("document", "contains multiple JSON values")
|
||||
}
|
||||
if err := document.Validate(registry); err != nil {
|
||||
return Document{}, err
|
||||
}
|
||||
return document, nil
|
||||
}
|
||||
|
||||
func Preview(document Document, request PreviewRequest, registry metriccatalog.Registry) (PreviewResult, error) {
|
||||
if err := document.Validate(registry); err != nil {
|
||||
return PreviewResult{}, err
|
||||
}
|
||||
if request.Unknown {
|
||||
if document.UnknownBehavior == UnknownRetain {
|
||||
return PreviewResult{State: "unknown", Reason: "unknown_input_retains_last_state"}, nil
|
||||
}
|
||||
return PreviewResult{State: "unknown", Reason: "unknown_input"}, nil
|
||||
}
|
||||
active := request.CurrentState == string(StateFiring) || request.CurrentState == string(StateAcknowledged) || request.CurrentState == string(StateUnknown)
|
||||
fire, err := document.Condition.Evaluate(request.Value, active)
|
||||
if err != nil {
|
||||
return PreviewResult{}, err
|
||||
}
|
||||
state := "inactive"
|
||||
if fire {
|
||||
state = "firing"
|
||||
}
|
||||
return PreviewResult{WouldFire: fire, State: state, Reason: "condition_evaluated"}, nil
|
||||
}
|
||||
|
||||
func (c Condition) Evaluate(value any, active bool) (bool, error) {
|
||||
threshold := c.Threshold
|
||||
operator := c.Operator
|
||||
if active && c.RecoveryThreshold != nil {
|
||||
threshold = *c.RecoveryThreshold
|
||||
if operator == ">" || operator == ">=" {
|
||||
operator = ">="
|
||||
}
|
||||
if operator == "<" || operator == "<=" {
|
||||
operator = "<="
|
||||
}
|
||||
}
|
||||
return compare(operator, value, threshold)
|
||||
}
|
||||
func compare(operator string, value, threshold any) (bool, error) {
|
||||
if operator == "absent" {
|
||||
return value == nil, nil
|
||||
}
|
||||
if operator == "matches" {
|
||||
left, ok := value.(string)
|
||||
right, ok2 := threshold.(string)
|
||||
if !ok || !ok2 {
|
||||
return false, invalid("preview", "matches requires string input")
|
||||
}
|
||||
matched, err := regexp.MatchString(right, left)
|
||||
return matched, err
|
||||
}
|
||||
if left, ok := number(value); ok {
|
||||
right, ok := number(threshold)
|
||||
if !ok {
|
||||
return false, invalid("preview", "numeric threshold is required")
|
||||
}
|
||||
switch operator {
|
||||
case ">":
|
||||
return left > right, nil
|
||||
case ">=":
|
||||
return left >= right, nil
|
||||
case "<":
|
||||
return left < right, nil
|
||||
case "<=":
|
||||
return left <= right, nil
|
||||
case "==":
|
||||
return left == right, nil
|
||||
case "!=":
|
||||
return left != right, nil
|
||||
}
|
||||
}
|
||||
if operator == "==" || operator == "!=" {
|
||||
equal := fmt.Sprint(value) == fmt.Sprint(threshold)
|
||||
if operator == "!=" {
|
||||
equal = !equal
|
||||
}
|
||||
return equal, nil
|
||||
}
|
||||
return false, invalid("preview", "value type does not support operator")
|
||||
}
|
||||
|
||||
func number(value any) (float64, bool) {
|
||||
switch value := value.(type) {
|
||||
case float64:
|
||||
return value, !math.IsNaN(value) && !math.IsInf(value, 0)
|
||||
case float32:
|
||||
return float64(value), true
|
||||
case int:
|
||||
return float64(value), true
|
||||
case int64:
|
||||
return float64(value), true
|
||||
case json.Number:
|
||||
parsed, err := value.Float64()
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func isNumber(value any) bool { _, ok := number(value); return ok }
|
||||
func isComparableString(value any) bool { _, ok := value.(string); return ok }
|
||||
|
||||
func validateValue(value any, maxString int) error {
|
||||
switch value := value.(type) {
|
||||
case nil, bool:
|
||||
return nil
|
||||
case string:
|
||||
if len(value) > maxString || strings.ContainsAny(value, "\r\n") {
|
||||
return errors.New("contains an unsafe string")
|
||||
}
|
||||
case float64:
|
||||
if math.IsNaN(value) || math.IsInf(value, 0) {
|
||||
return errors.New("contains a non-finite number")
|
||||
}
|
||||
case []any:
|
||||
if len(value) > 20 {
|
||||
return errors.New("contains too many values")
|
||||
}
|
||||
for _, item := range value {
|
||||
if err := validateValue(item, maxString); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case map[string]any:
|
||||
if len(value) > 20 {
|
||||
return errors.New("contains too many keys")
|
||||
}
|
||||
keys := make([]string, 0, len(value))
|
||||
for key, item := range value {
|
||||
keys = append(keys, key)
|
||||
if err := validateValue(item, maxString); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
default:
|
||||
return errors.New("contains an unsupported value")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func uniqueStrings(values []string, max int) error {
|
||||
seen := map[string]struct{}{}
|
||||
for _, value := range values {
|
||||
if len(value) == 0 || len(value) > max || strings.TrimSpace(value) != value || strings.ContainsAny(value, "\r\n") {
|
||||
return errors.New("contains an invalid value")
|
||||
}
|
||||
if _, ok := seen[value]; ok {
|
||||
return errors.New("contains duplicate values")
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func oneOf(value string, allowed ...string) bool {
|
||||
for _, item := range allowed {
|
||||
if value == item {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func invalid(field, detail string) error {
|
||||
return fmt.Errorf("%w: %s %s", ErrInvalidRule, field, detail)
|
||||
}
|
||||
|
||||
func NewID() string {
|
||||
var raw [16]byte
|
||||
if _, err := rand.Read(raw[:]); err != nil {
|
||||
return "00000000-0000-4000-8000-000000000000"
|
||||
}
|
||||
raw[6] = (raw[6] & 0x0f) | 0x40
|
||||
raw[8] = (raw[8] & 0x3f) | 0x80
|
||||
encoded := hex.EncodeToString(raw[:])
|
||||
return encoded[0:8] + "-" + encoded[8:12] + "-" + encoded[12:16] + "-" + encoded[16:20] + "-" + encoded[20:32]
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/itworx/pulse/internal/metriccatalog"
|
||||
)
|
||||
|
||||
func validDocument(t *testing.T) (Document, metriccatalog.Registry) {
|
||||
t.Helper()
|
||||
registry, err := metriccatalog.DefaultRegistry()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Document{
|
||||
SchemaVersion: 1, ID: NewID(), Name: "CPU aandacht", Enabled: false, Severity: SeverityAttention,
|
||||
Scope: map[string]any{"entityType": "host"},
|
||||
Condition: Condition{InputType: "metric", Metric: registry.Metrics()[0].SemanticName, Operator: ">", Threshold: float64(80), Aggregation: "avg", WindowSeconds: 60},
|
||||
EvaluationIntervalSeconds: 30, PendingSeconds: 60, ResolveSeconds: 120, CooldownSeconds: 60, UnknownBehavior: UnknownRetain,
|
||||
GroupBy: []string{"instance"}, SuppressWhen: []string{"maintenance"}, Message: Message{TitleKey: "alerts.cpu.title", BodyKey: "alerts.cpu.body"},
|
||||
}, registry
|
||||
}
|
||||
|
||||
func TestDocumentValidationRejectsRawPromQLAndUnknownMetric(t *testing.T) {
|
||||
document, registry := validDocument(t)
|
||||
document.Condition.Metric = "rate(node_cpu_seconds_total[5m])"
|
||||
if !errors.Is(document.Validate(registry), ErrInvalidRule) {
|
||||
t.Fatal("expected raw query rejection")
|
||||
}
|
||||
document.Condition.Metric = "pulse.not_in_catalog"
|
||||
if !errors.Is(document.Validate(registry), ErrInvalidRule) {
|
||||
t.Fatal("expected unknown metric rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewHasDeterministicNoSideEffectEvaluation(t *testing.T) {
|
||||
document, registry := validDocument(t)
|
||||
result, err := Preview(document, PreviewRequest{Value: float64(90)}, registry)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.WouldFire || result.State != "firing" || result.Reason != "condition_evaluated" {
|
||||
t.Fatalf("unexpected preview: %#v", result)
|
||||
}
|
||||
unknown, err := Preview(document, PreviewRequest{Unknown: true}, registry)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if unknown.WouldFire || unknown.State != "unknown" || unknown.Reason != "unknown_input_retains_last_state" {
|
||||
t.Fatalf("unexpected unknown preview: %#v", unknown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeDocumentRejectsUnknownFields(t *testing.T) {
|
||||
_, registry := validDocument(t)
|
||||
if _, err := DecodeDocument([]byte("{\"schemaVersion\":1,\"id\":\""+NewID()+"\",\"name\":\"x\",\"unsafeQuery\":\"rate(foo[5m])\"}"), registry); !errors.Is(err, ErrInvalidRule) {
|
||||
t.Fatal("expected unknown field rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConditionEvaluatorAppliesDiskTemperatureHysteresisAtBoundaries(t *testing.T) {
|
||||
document, registry := validDocument(t)
|
||||
document.Condition.Operator = ">="
|
||||
document.Condition.Threshold = float64(50)
|
||||
recovery := float64(46)
|
||||
document.Condition.RecoveryThreshold = &recovery
|
||||
if err := document.Validate(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if firing, err := document.Condition.Evaluate(float64(49), false); err != nil || firing {
|
||||
t.Fatalf("activation at 49 = %v, %v; want false", firing, err)
|
||||
}
|
||||
for _, value := range []float64{49, 46} {
|
||||
firing, err := document.Condition.Evaluate(value, true)
|
||||
if err != nil || !firing {
|
||||
t.Fatalf("active evaluation at %v = %v, %v; want true", value, firing, err)
|
||||
}
|
||||
}
|
||||
if firing, err := document.Condition.Evaluate(float64(45.999), true); err != nil || firing {
|
||||
t.Fatalf("recovery at 45.999 = %v, %v; want false", firing, err)
|
||||
}
|
||||
recovery = 50
|
||||
if err := document.Validate(registry); !errors.Is(err, ErrInvalidRule) {
|
||||
t.Fatalf("inverted recovery threshold error = %v, want ErrInvalidRule", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
package alertapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/audit"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/correlation"
|
||||
"github.com/itworx/pulse/internal/metriccatalog"
|
||||
"github.com/itworx/pulse/internal/problem"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
Repository alert.Store
|
||||
Registry metriccatalog.Registry
|
||||
Audit audit.Store
|
||||
}
|
||||
|
||||
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
principal, ok := auth.PrincipalFromContext(r.Context())
|
||||
if !ok {
|
||||
fail(w, r, http.StatusUnauthorized, "UNAUTHORIZED", "Authentication required.")
|
||||
return
|
||||
}
|
||||
path := strings.TrimPrefix(r.URL.Path, "/api/v1/alert-rules")
|
||||
if path == "" || path == "/" {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
h.list(w, r)
|
||||
case http.MethodPost:
|
||||
if !requireEdit(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.create(w, r, principal.Subject)
|
||||
default:
|
||||
fail(w, r, http.StatusMethodNotAllowed, "METHOD_NOT_ALLOWED", "This method is not supported.")
|
||||
}
|
||||
return
|
||||
}
|
||||
parts := strings.Split(strings.Trim(path, "/"), "/")
|
||||
if len(parts) < 1 || parts[0] == "" || len(parts) > 2 {
|
||||
fail(w, r, http.StatusNotFound, "NOT_FOUND", "Alert rule route not found.")
|
||||
return
|
||||
}
|
||||
id := parts[0]
|
||||
if len(parts) == 1 && r.Method == http.MethodGet {
|
||||
h.get(w, r, id)
|
||||
return
|
||||
}
|
||||
if len(parts) == 2 && parts[1] == "versions" && r.Method == http.MethodGet {
|
||||
h.versions(w, r, id)
|
||||
return
|
||||
}
|
||||
if len(parts) == 2 && parts[1] == "test" && r.Method == http.MethodPost {
|
||||
if !requireEdit(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.test(w, r, id)
|
||||
return
|
||||
}
|
||||
if len(parts) == 2 && (parts[1] == "enable" || parts[1] == "disable") && r.Method == http.MethodPost {
|
||||
if !requireEdit(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.setEnabled(w, r, id, principal.Subject, parts[1] == "enable")
|
||||
return
|
||||
}
|
||||
if len(parts) == 1 && r.Method == http.MethodPut {
|
||||
if !requireEdit(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.update(w, r, id, principal.Subject)
|
||||
return
|
||||
}
|
||||
fail(w, r, http.StatusNotFound, "NOT_FOUND", "Alert rule route not found.")
|
||||
}
|
||||
|
||||
func (h Handler) list(w http.ResponseWriter, r *http.Request) {
|
||||
limit := 100
|
||||
if value := r.URL.Query().Get("limit"); value != "" {
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed < 1 || parsed > 100 {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_LIMIT", "The alert-rule limit must be between 1 and 100.")
|
||||
return
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
items, err := h.Repository.List(r.Context(), limit)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Alert rules are unavailable.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"items": items})
|
||||
}
|
||||
|
||||
func (h Handler) get(w http.ResponseWriter, r *http.Request, id string) {
|
||||
item, err := h.Repository.Get(r.Context(), id)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Alert rule not found.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"rule": item})
|
||||
}
|
||||
|
||||
func (h Handler) versions(w http.ResponseWriter, r *http.Request, id string) {
|
||||
limit := 100
|
||||
if value := r.URL.Query().Get("limit"); value != "" {
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed < 1 || parsed > 100 {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_LIMIT", "The version limit must be between 1 and 100.")
|
||||
return
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
items, err := h.Repository.Versions(r.Context(), id, limit)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Alert-rule versions are unavailable.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"items": items})
|
||||
}
|
||||
|
||||
func (h Handler) create(w http.ResponseWriter, r *http.Request, actor string) {
|
||||
var document alert.Document
|
||||
if err := decode(r, &document); err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_RULE", "The alert-rule document is invalid.")
|
||||
return
|
||||
}
|
||||
if document.ID == "" {
|
||||
document.ID = alert.NewID()
|
||||
}
|
||||
rule, version, err := h.Repository.Create(r.Context(), actor, document, "initial version")
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Alert rule could not be created.")
|
||||
return
|
||||
}
|
||||
if err := h.record(r, actor, "alert_rule.create", rule.ID, nil, map[string]any{"revision": rule.Revision, "version": version.VersionNumber}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusCreated, map[string]any{"rule": rule, "version": version})
|
||||
}
|
||||
|
||||
func (h Handler) update(w http.ResponseWriter, r *http.Request, id, actor string) {
|
||||
expected, err := revision(r)
|
||||
if err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_REVISION", "A valid If-Match or revision value is required.")
|
||||
return
|
||||
}
|
||||
var document alert.Document
|
||||
if err := decode(r, &document); err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_RULE", "The alert-rule document is invalid.")
|
||||
return
|
||||
}
|
||||
updated, err := h.Repository.Update(r.Context(), id, actor, expected, document, "rule update")
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Alert rule update failed.")
|
||||
return
|
||||
}
|
||||
if err := h.record(r, actor, "alert_rule.update", id, map[string]any{"revision": expected}, map[string]any{"revision": updated.Revision, "version": updated.CurrentVersion}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"rule": updated})
|
||||
}
|
||||
|
||||
func (h Handler) setEnabled(w http.ResponseWriter, r *http.Request, id, actor string, enabled bool) {
|
||||
expected, err := revision(r)
|
||||
if err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_REVISION", "A valid If-Match or revision value is required.")
|
||||
return
|
||||
}
|
||||
updated, err := h.Repository.SetEnabled(r.Context(), id, expected, enabled)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Alert rule state update failed.")
|
||||
return
|
||||
}
|
||||
action := "alert_rule.disable"
|
||||
if enabled {
|
||||
action = "alert_rule.enable"
|
||||
}
|
||||
if err := h.record(r, actor, action, id, map[string]any{"enabled": !enabled, "revision": expected}, map[string]any{"enabled": enabled, "revision": updated.Revision}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"rule": updated})
|
||||
}
|
||||
|
||||
func (h Handler) test(w http.ResponseWriter, r *http.Request, id string) {
|
||||
var request struct {
|
||||
Rule *alert.Document `json:"rule"`
|
||||
Value any `json:"value"`
|
||||
Unknown bool `json:"unknown"`
|
||||
}
|
||||
if err := decode(r, &request); err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_PREVIEW", "The alert-rule preview request is invalid.")
|
||||
return
|
||||
}
|
||||
document := request.Rule
|
||||
if document == nil {
|
||||
current, err := h.Repository.Get(r.Context(), id)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Alert rule preview is unavailable.")
|
||||
return
|
||||
}
|
||||
document = ¤t.Document
|
||||
}
|
||||
result, err := alert.Preview(*document, alert.PreviewRequest{Value: request.Value, Unknown: request.Unknown}, h.Registry)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "The alert-rule preview is invalid.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"preview": result})
|
||||
}
|
||||
|
||||
func (h Handler) record(r *http.Request, actor, action, resourceID string, before, after map[string]any) error {
|
||||
if h.Audit == nil {
|
||||
return nil
|
||||
}
|
||||
return h.Audit.Append(r.Context(), audit.Event{Actor: actor, Action: action, ResourceType: "alert_rule", ResourceID: resourceID, Result: "success", CorrelationID: correlation.FromContext(r.Context()), Before: before, After: after})
|
||||
}
|
||||
|
||||
func (h Handler) repositoryFailure(w http.ResponseWriter, r *http.Request, err error, fallback string) {
|
||||
switch {
|
||||
case errors.Is(err, alert.ErrInvalidRule):
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_RULE", fallback)
|
||||
case errors.Is(err, alert.ErrConflict):
|
||||
fail(w, r, http.StatusConflict, "REVISION_CONFLICT", "The alert rule was changed by another request.")
|
||||
case errors.Is(err, alert.ErrNotFound):
|
||||
fail(w, r, http.StatusNotFound, "NOT_FOUND", fallback)
|
||||
case errors.Is(err, alert.ErrUnavailable):
|
||||
fail(w, r, http.StatusServiceUnavailable, "DATABASE_UNAVAILABLE", fallback)
|
||||
default:
|
||||
fail(w, r, http.StatusInternalServerError, "ALERT_RULE_REQUEST_FAILED", fallback)
|
||||
}
|
||||
}
|
||||
|
||||
func requireEdit(w http.ResponseWriter, r *http.Request, role auth.Role) bool {
|
||||
if auth.Allows(role, auth.PermissionEdit) {
|
||||
return true
|
||||
}
|
||||
fail(w, r, http.StatusForbidden, "FORBIDDEN", "Alert-rule editing is not allowed for this role.")
|
||||
return false
|
||||
}
|
||||
|
||||
func decode(r *http.Request, target any) error {
|
||||
contentType := strings.ToLower(strings.TrimSpace(strings.Split(r.Header.Get("Content-Type"), ";")[0]))
|
||||
if contentType != "" && contentType != "application/json" {
|
||||
return errors.New("unsupported content type")
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, 2<<20+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer r.Body.Close()
|
||||
if len(body) > 2<<20 {
|
||||
return errors.New("request too large")
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(string(body)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(target); err != nil {
|
||||
return err
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return errors.New("multiple JSON values")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func revision(r *http.Request) (int64, error) {
|
||||
value := r.Header.Get("If-Match")
|
||||
if value == "" {
|
||||
value = r.URL.Query().Get("revision")
|
||||
}
|
||||
return strconv.ParseInt(strings.Trim(value, "\""), 10, 64)
|
||||
}
|
||||
|
||||
func fail(w http.ResponseWriter, r *http.Request, status int, code, detail string) {
|
||||
problem.Write(w, r, status, code, http.StatusText(status), detail, nil)
|
||||
}
|
||||
|
||||
func write(w http.ResponseWriter, status int, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package alertapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/audit"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/metriccatalog"
|
||||
)
|
||||
|
||||
type fakeStore struct {
|
||||
rule alert.Rule
|
||||
getCalls, createCalls, updateCalls, toggleCalls int
|
||||
}
|
||||
|
||||
func (f *fakeStore) Create(_ context.Context, _ string, document alert.Document, _ string) (alert.Rule, alert.Version, error) {
|
||||
f.createCalls++
|
||||
f.rule = alert.Rule{Document: document, Revision: 1, CurrentVersion: 1}
|
||||
return f.rule, alert.Version{RuleID: document.ID, VersionNumber: 1, Document: document}, nil
|
||||
}
|
||||
func (f *fakeStore) Get(_ context.Context, _ string) (alert.Rule, error) {
|
||||
f.getCalls++
|
||||
return f.rule, nil
|
||||
}
|
||||
func (f *fakeStore) List(_ context.Context, _ int) ([]alert.Rule, error) {
|
||||
return []alert.Rule{f.rule}, nil
|
||||
}
|
||||
func (f *fakeStore) Update(_ context.Context, id, _ string, _ int64, document alert.Document, _ string) (alert.Rule, error) {
|
||||
f.updateCalls++
|
||||
document.ID = id
|
||||
f.rule.Document = document
|
||||
f.rule.Revision++
|
||||
return f.rule, nil
|
||||
}
|
||||
func (f *fakeStore) Versions(_ context.Context, _ string, _ int) ([]alert.Version, error) {
|
||||
return []alert.Version{{Document: f.rule.Document, VersionNumber: 1}}, nil
|
||||
}
|
||||
func (f *fakeStore) SetEnabled(_ context.Context, _ string, _ int64, enabled bool) (alert.Rule, error) {
|
||||
f.toggleCalls++
|
||||
f.rule.Enabled = enabled
|
||||
f.rule.Document.Enabled = enabled
|
||||
f.rule.Revision++
|
||||
return f.rule, nil
|
||||
}
|
||||
|
||||
func requestWithPrincipal(method, path string, body any, role auth.Role) *http.Request {
|
||||
data, _ := json.Marshal(body)
|
||||
request := httptest.NewRequest(method, path, strings.NewReader(string(data)))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
return request.WithContext(auth.WithPrincipal(request.Context(), auth.Principal{Subject: "editor-1", Role: role}))
|
||||
}
|
||||
|
||||
func TestViewerCannotCreateAlertRule(t *testing.T) {
|
||||
document, registry := validDocumentForHandler(t)
|
||||
store := &fakeStore{}
|
||||
handler := Handler{Repository: store, Registry: registry}
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, requestWithPrincipal(http.MethodPost, "/api/v1/alert-rules", document, auth.RoleViewer))
|
||||
if response.Code != http.StatusForbidden || store.createCalls != 0 {
|
||||
t.Fatalf("status=%d creates=%d", response.Code, store.createCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewDoesNotWriteOrAudit(t *testing.T) {
|
||||
document, registry := validDocumentForHandler(t)
|
||||
store := &fakeStore{}
|
||||
auditStore := &audit.MemoryStore{}
|
||||
handler := Handler{Repository: store, Registry: registry, Audit: auditStore}
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, requestWithPrincipal(http.MethodPost, "/api/v1/alert-rules/"+document.ID+"/test", map[string]any{"rule": document, "value": 90}, auth.RoleEditor))
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("preview status=%d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
if store.createCalls != 0 || store.updateCalls != 0 || store.toggleCalls != 0 || len(auditStore.Events) != 0 {
|
||||
t.Fatalf("preview had side effects: store=%#v audit=%d", store, len(auditStore.Events))
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body["preview"] == nil {
|
||||
t.Fatal("preview result missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnableIsAudited(t *testing.T) {
|
||||
document, registry := validDocumentForHandler(t)
|
||||
store := &fakeStore{rule: alert.Rule{Document: document, Revision: 1}}
|
||||
auditStore := &audit.MemoryStore{}
|
||||
handler := Handler{Repository: store, Registry: registry, Audit: auditStore}
|
||||
request := requestWithPrincipal(http.MethodPost, "/api/v1/alert-rules/"+document.ID+"/enable?revision=1", map[string]any{}, auth.RoleEditor)
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusOK || store.toggleCalls != 1 {
|
||||
t.Fatalf("status=%d toggles=%d", response.Code, store.toggleCalls)
|
||||
}
|
||||
if len(auditStore.Events) != 1 || auditStore.Events[0].Action != "alert_rule.enable" {
|
||||
t.Fatalf("audit=%#v", auditStore.Events)
|
||||
}
|
||||
}
|
||||
|
||||
func validDocumentForHandler(t *testing.T) (alert.Document, metriccatalog.Registry) {
|
||||
t.Helper()
|
||||
registry, err := metriccatalog.DefaultRegistry()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return alert.Document{
|
||||
SchemaVersion: 1, ID: alert.NewID(), Name: "CPU aandacht", Severity: alert.SeverityAttention,
|
||||
Scope: map[string]any{"entityType": "host"},
|
||||
Condition: alert.Condition{InputType: "metric", Metric: registry.Metrics()[0].SemanticName, Operator: ">", Threshold: float64(80)},
|
||||
EvaluationIntervalSeconds: 30, UnknownBehavior: alert.UnknownRetain,
|
||||
Message: alert.Message{TitleKey: "alerts.cpu.title", BodyKey: "alerts.cpu.body"},
|
||||
}, registry
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package alertcontrol
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ExpiryJob struct {
|
||||
Store Store
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (job ExpiryJob) Run(ctx context.Context) (ExpiryResult, error) {
|
||||
if job.Now == nil {
|
||||
job.Now = time.Now
|
||||
}
|
||||
return job.Store.Expire(ctx, job.Now().UTC())
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
package alertcontrol
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type Store interface {
|
||||
CreateSilence(context.Context, string, Silence) (Silence, error)
|
||||
ListSilences(context.Context, int, time.Time) ([]Silence, error)
|
||||
RevokeSilence(context.Context, string, string, int64, time.Time) (Silence, error)
|
||||
CreateMaintenance(context.Context, string, MaintenanceWindow) (MaintenanceWindow, error)
|
||||
ListMaintenance(context.Context, int, time.Time) ([]MaintenanceWindow, error)
|
||||
RevokeMaintenance(context.Context, string, string, int64, time.Time) (MaintenanceWindow, error)
|
||||
Expire(context.Context, time.Time) (ExpiryResult, error)
|
||||
}
|
||||
|
||||
type Repository struct{ Pool *pgxpool.Pool }
|
||||
|
||||
type ExpiryResult struct {
|
||||
Silences int `json:"silences"`
|
||||
MaintenanceWindows int `json:"maintenanceWindows"`
|
||||
}
|
||||
|
||||
func (r Repository) CreateSilence(ctx context.Context, actor string, silence Silence) (Silence, error) {
|
||||
if r.Pool == nil {
|
||||
return Silence{}, ErrUnavailable
|
||||
}
|
||||
if actor == "" {
|
||||
return Silence{}, fmt.Errorf("%w: creator is required", ErrInvalid)
|
||||
}
|
||||
if silence.ID == "" {
|
||||
silence.ID = NewID()
|
||||
}
|
||||
if silence.Owner == "" {
|
||||
silence.Owner = actor
|
||||
}
|
||||
if err := silence.Validate(time.Now().UTC()); err != nil {
|
||||
return Silence{}, err
|
||||
}
|
||||
matcher, err := json.Marshal(silence.Matchers)
|
||||
if err != nil {
|
||||
return Silence{}, fmt.Errorf("marshal silence matcher: %w", err)
|
||||
}
|
||||
if silence.Owner == "" {
|
||||
silence.Owner = actor
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Silence{}, fmt.Errorf("begin silence create: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
_, err = tx.Exec(ctx, `INSERT INTO alert_silences (id,name,reason,owner,matchers,starts_at,expires_at,created_by) VALUES ($1::uuid,$2,$3,$4,$5::jsonb,$6,$7,$8)`, silence.ID, silence.Name, silence.Reason, silence.Owner, matcher, silence.StartsAt.UTC(), silence.ExpiresAt.UTC(), actor)
|
||||
if err != nil {
|
||||
return Silence{}, mapError(fmt.Errorf("create silence: %w", err))
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Silence{}, fmt.Errorf("commit silence create: %w", err)
|
||||
}
|
||||
return r.getSilence(ctx, silence.ID, time.Now().UTC())
|
||||
}
|
||||
|
||||
func (r Repository) ListSilences(ctx context.Context, limit int, now time.Time) ([]Silence, error) {
|
||||
if r.Pool == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if limit < 1 || limit > 100 {
|
||||
return nil, fmt.Errorf("%w: invalid list limit", ErrInvalid)
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, `SELECT id,name,reason,owner,matchers,starts_at,expires_at,status,created_by,created_at,revoked_by,revoked_at,expired_at,revision FROM alert_silences ORDER BY starts_at DESC,id DESC LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list silences: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]Silence, 0)
|
||||
for rows.Next() {
|
||||
item, err := scanSilence(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.State = item.StateAt(now.UTC())
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate silences: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (r Repository) RevokeSilence(ctx context.Context, id, actor string, expected int64, now time.Time) (Silence, error) {
|
||||
if r.Pool == nil {
|
||||
return Silence{}, ErrUnavailable
|
||||
}
|
||||
if id == "" || actor == "" {
|
||||
return Silence{}, fmt.Errorf("%w: id and actor are required", ErrInvalid)
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Silence{}, fmt.Errorf("begin silence revoke: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var query string
|
||||
var args []any
|
||||
if expected > 0 {
|
||||
query = `UPDATE alert_silences SET status='revoked',revoked_by=$2,revoked_at=$3,revision=revision+1 WHERE id=$1 AND status='active' AND revision=$4`
|
||||
args = []any{id, actor, now.UTC(), expected}
|
||||
} else {
|
||||
query = `UPDATE alert_silences SET status='revoked',revoked_by=$2,revoked_at=$3,revision=revision+1 WHERE id=$1 AND status='active'`
|
||||
args = []any{id, actor, now.UTC()}
|
||||
}
|
||||
result, err := tx.Exec(ctx, query, args...)
|
||||
if err != nil {
|
||||
return Silence{}, mapError(fmt.Errorf("revoke silence: %w", err))
|
||||
}
|
||||
if result.RowsAffected() == 0 {
|
||||
return Silence{}, r.revokeFailure(ctx, tx, id, expected, true)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Silence{}, fmt.Errorf("commit silence revoke: %w", err)
|
||||
}
|
||||
return r.getSilence(ctx, id, now.UTC())
|
||||
}
|
||||
|
||||
func (r Repository) CreateMaintenance(ctx context.Context, actor string, window MaintenanceWindow) (MaintenanceWindow, error) {
|
||||
if r.Pool == nil {
|
||||
return MaintenanceWindow{}, ErrUnavailable
|
||||
}
|
||||
if actor == "" {
|
||||
return MaintenanceWindow{}, fmt.Errorf("%w: creator is required", ErrInvalid)
|
||||
}
|
||||
if window.ID == "" {
|
||||
window.ID = NewID()
|
||||
}
|
||||
if err := window.Validate(time.Now().UTC()); err != nil {
|
||||
return MaintenanceWindow{}, err
|
||||
}
|
||||
selector, err := json.Marshal(window.Selector)
|
||||
if err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("marshal maintenance selector: %w", err)
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("begin maintenance create: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
_, err = tx.Exec(ctx, `INSERT INTO maintenance_windows (id,name,reason,selector,starts_at,ends_at,created_by) VALUES ($1::uuid,$2,$3,$4::jsonb,$5,$6,$7)`, window.ID, window.Name, window.Reason, selector, window.StartsAt.UTC(), window.EndsAt.UTC(), actor)
|
||||
if err != nil {
|
||||
return MaintenanceWindow{}, mapError(fmt.Errorf("create maintenance window: %w", err))
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("commit maintenance create: %w", err)
|
||||
}
|
||||
return r.getMaintenance(ctx, window.ID, time.Now().UTC())
|
||||
}
|
||||
|
||||
func (r Repository) ListMaintenance(ctx context.Context, limit int, now time.Time) ([]MaintenanceWindow, error) {
|
||||
if r.Pool == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if limit < 1 || limit > 100 {
|
||||
return nil, fmt.Errorf("%w: invalid list limit", ErrInvalid)
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, `SELECT id,name,reason,selector,starts_at,ends_at,status,created_by,created_at,revoked_by,revoked_at,expired_at,revision FROM maintenance_windows ORDER BY starts_at DESC,id DESC LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list maintenance windows: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]MaintenanceWindow, 0)
|
||||
for rows.Next() {
|
||||
item, err := scanMaintenance(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.State = item.StateAt(now.UTC())
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate maintenance windows: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (r Repository) RevokeMaintenance(ctx context.Context, id, actor string, expected int64, now time.Time) (MaintenanceWindow, error) {
|
||||
if r.Pool == nil {
|
||||
return MaintenanceWindow{}, ErrUnavailable
|
||||
}
|
||||
if id == "" || actor == "" {
|
||||
return MaintenanceWindow{}, fmt.Errorf("%w: id and actor are required", ErrInvalid)
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("begin maintenance revoke: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var query string
|
||||
var args []any
|
||||
if expected > 0 {
|
||||
query = `UPDATE maintenance_windows SET status='revoked',revoked_by=$2,revoked_at=$3,revision=revision+1 WHERE id=$1 AND status='active' AND revision=$4`
|
||||
args = []any{id, actor, now.UTC(), expected}
|
||||
} else {
|
||||
query = `UPDATE maintenance_windows SET status='revoked',revoked_by=$2,revoked_at=$3,revision=revision+1 WHERE id=$1 AND status='active'`
|
||||
args = []any{id, actor, now.UTC()}
|
||||
}
|
||||
result, err := tx.Exec(ctx, query, args...)
|
||||
if err != nil {
|
||||
return MaintenanceWindow{}, mapError(fmt.Errorf("revoke maintenance window: %w", err))
|
||||
}
|
||||
if result.RowsAffected() == 0 {
|
||||
return MaintenanceWindow{}, r.revokeFailure(ctx, tx, id, expected, false)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("commit maintenance revoke: %w", err)
|
||||
}
|
||||
return r.getMaintenance(ctx, id, now.UTC())
|
||||
}
|
||||
|
||||
func (r Repository) Expire(ctx context.Context, now time.Time) (ExpiryResult, error) {
|
||||
if r.Pool == nil {
|
||||
return ExpiryResult{}, ErrUnavailable
|
||||
}
|
||||
now = now.UTC()
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return ExpiryResult{}, fmt.Errorf("begin control expiry: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var result ExpiryResult
|
||||
if err := tx.QueryRow(ctx, `WITH expired AS (UPDATE alert_silences SET status='expired',expired_at=$1,revision=revision+1 WHERE status='active' AND expires_at <= $1 RETURNING id) SELECT count(*) FROM expired`, now).Scan(&result.Silences); err != nil {
|
||||
return ExpiryResult{}, fmt.Errorf("expire silences: %w", err)
|
||||
}
|
||||
if err := tx.QueryRow(ctx, `WITH expired AS (UPDATE maintenance_windows SET status='expired',expired_at=$1,revision=revision+1 WHERE status='active' AND ends_at <= $1 RETURNING id) SELECT count(*) FROM expired`, now).Scan(&result.MaintenanceWindows); err != nil {
|
||||
return ExpiryResult{}, fmt.Errorf("expire maintenance windows: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ExpiryResult{}, fmt.Errorf("commit control expiry: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r Repository) getSilence(ctx context.Context, id string, now time.Time) (Silence, error) {
|
||||
var item Silence
|
||||
row := r.Pool.QueryRow(ctx, `SELECT id,name,reason,owner,matchers,starts_at,expires_at,status,created_by,created_at,revoked_by,revoked_at,expired_at,revision FROM alert_silences WHERE id=$1`, id)
|
||||
scanned, err := scanSilence(row)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Silence{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Silence{}, fmt.Errorf("get silence: %w", err)
|
||||
}
|
||||
item = scanned
|
||||
item.State = item.StateAt(now.UTC())
|
||||
return item, nil
|
||||
}
|
||||
func (r Repository) getMaintenance(ctx context.Context, id string, now time.Time) (MaintenanceWindow, error) {
|
||||
row := r.Pool.QueryRow(ctx, `SELECT id,name,reason,selector,starts_at,ends_at,status,created_by,created_at,revoked_by,revoked_at,expired_at,revision FROM maintenance_windows WHERE id=$1`, id)
|
||||
item, err := scanMaintenance(row)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return MaintenanceWindow{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("get maintenance window: %w", err)
|
||||
}
|
||||
item.State = item.StateAt(now.UTC())
|
||||
return item, nil
|
||||
}
|
||||
|
||||
type rowScanner interface{ Scan(...any) error }
|
||||
|
||||
func scanSilence(row rowScanner) (Silence, error) {
|
||||
var item Silence
|
||||
var matcher []byte
|
||||
var status string
|
||||
var revokedBy, createdBy *string
|
||||
if err := row.Scan(&item.ID, &item.Name, &item.Reason, &item.Owner, &matcher, &item.StartsAt, &item.ExpiresAt, &status, &createdBy, &item.CreatedAt, &revokedBy, &item.RevokedAt, &item.ExpiredAt, &item.Revision); err != nil {
|
||||
return Silence{}, err
|
||||
}
|
||||
if err := json.Unmarshal(matcher, &item.Matchers); err != nil {
|
||||
return Silence{}, fmt.Errorf("decode silence matcher: %w", err)
|
||||
}
|
||||
item.CreatedBy = valueOrEmpty(createdBy)
|
||||
item.RevokedBy = valueOrEmpty(revokedBy)
|
||||
return item, nil
|
||||
}
|
||||
func scanMaintenance(row rowScanner) (MaintenanceWindow, error) {
|
||||
var item MaintenanceWindow
|
||||
var selector []byte
|
||||
var status string
|
||||
var revokedBy, createdBy *string
|
||||
if err := row.Scan(&item.ID, &item.Name, &item.Reason, &selector, &item.StartsAt, &item.EndsAt, &status, &createdBy, &item.CreatedAt, &revokedBy, &item.RevokedAt, &item.ExpiredAt, &item.Revision); err != nil {
|
||||
return MaintenanceWindow{}, err
|
||||
}
|
||||
if err := json.Unmarshal(selector, &item.Selector); err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("decode maintenance selector: %w", err)
|
||||
}
|
||||
item.CreatedBy = valueOrEmpty(createdBy)
|
||||
item.RevokedBy = valueOrEmpty(revokedBy)
|
||||
return item, nil
|
||||
}
|
||||
func valueOrEmpty(value *string) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
return *value
|
||||
}
|
||||
func (r Repository) revokeFailure(ctx context.Context, tx pgx.Tx, id string, expected int64, silence bool) error {
|
||||
var revision int64
|
||||
var status string
|
||||
table := "maintenance_windows"
|
||||
if silence {
|
||||
table = "alert_silences"
|
||||
}
|
||||
err := tx.QueryRow(ctx, "SELECT revision,status FROM "+table+" WHERE id=$1 FOR UPDATE", id).Scan(&revision, &status)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect control revoke: %w", err)
|
||||
}
|
||||
if expected > 0 && revision != expected {
|
||||
return ErrConflict
|
||||
}
|
||||
return fmt.Errorf("%w: control is %s", ErrConflict, status)
|
||||
}
|
||||
func mapError(err error) error {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
switch pgErr.Code {
|
||||
case "23505":
|
||||
return fmt.Errorf("%w: duplicate control", ErrConflict)
|
||||
case "23514", "22P02":
|
||||
return fmt.Errorf("%w: database constraint", ErrInvalid)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package alertcontrol
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
)
|
||||
|
||||
func TestPostgreSQLControlsAreExpiringAuditedAndIdempotent(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 8, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repo := Repository{Pool: pool}
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
silenceID, maintenanceID, expiringID := NewID(), NewID(), NewID()
|
||||
t.Cleanup(func() {
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cleanupCancel()
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM alert_silences WHERE id = ANY($1::uuid[])`, []string{silenceID, expiringID})
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM maintenance_windows WHERE id = $1::uuid`, maintenanceID)
|
||||
})
|
||||
|
||||
silence, err := repo.CreateSilence(ctx, "operator-1", Silence{ID: silenceID, Name: "planned deploy", Reason: "change window", Owner: "operator-1", Matchers: Matcher{Severities: []string{"critical"}}, StartsAt: now.Add(-time.Minute), ExpiresAt: now.Add(time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if silence.State != StateActive || silence.Revision != 1 {
|
||||
t.Fatalf("unexpected silence: %#v", silence)
|
||||
}
|
||||
window, err := repo.CreateMaintenance(ctx, "operator-1", MaintenanceWindow{ID: maintenanceID, Name: "maintenance", Reason: "firmware", Selector: Matcher{EntityTypes: []string{"host"}}, StartsAt: now.Add(-time.Minute), EndsAt: now.Add(time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if window.State != StateActive {
|
||||
t.Fatalf("maintenance state = %s", window.State)
|
||||
}
|
||||
items, err := repo.ListSilences(ctx, 100, now)
|
||||
if err != nil || len(items) != 1 {
|
||||
t.Fatalf("list silences: %d, %v", len(items), err)
|
||||
}
|
||||
|
||||
expiring, err := repo.CreateSilence(ctx, "operator-1", Silence{ID: expiringID, Name: "short", Reason: "test expiry", Owner: "operator-1", Matchers: Matcher{}, StartsAt: now.Add(-time.Minute), ExpiresAt: now.Add(-time.Second)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if expiring.State != StateExpired {
|
||||
t.Fatalf("expired control before job = %s", expiring.State)
|
||||
}
|
||||
first, err := repo.Expire(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := repo.Expire(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.Silences != 1 || first.MaintenanceWindows != 0 || second != (ExpiryResult{}) {
|
||||
t.Fatalf("expiry not idempotent: first=%#v second=%#v", first, second)
|
||||
}
|
||||
|
||||
concurrentID := NewID()
|
||||
defer func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM alert_silences WHERE id=$1::uuid`, concurrentID)
|
||||
}()
|
||||
if _, err := repo.CreateSilence(ctx, "operator-1", Silence{ID: concurrentID, Name: "concurrent", Reason: "test", Owner: "operator-1", Matchers: Matcher{}, StartsAt: now.Add(-time.Minute), ExpiresAt: now.Add(-time.Second)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
results := make(chan ExpiryResult, 2)
|
||||
errorsCh := make(chan error, 2)
|
||||
for i := 0; i < 2; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
result, err := repo.Expire(ctx, now.Add(time.Second))
|
||||
if err != nil {
|
||||
errorsCh <- err
|
||||
return
|
||||
}
|
||||
results <- result
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
close(errorsCh)
|
||||
total := 0
|
||||
for result := range results {
|
||||
total += result.Silences + result.MaintenanceWindows
|
||||
}
|
||||
for err := range errorsCh {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if total != 1 {
|
||||
t.Fatalf("concurrent expiry count = %d, want 1", total)
|
||||
}
|
||||
|
||||
if _, err := repo.CreateSilence(ctx, "operator-1", Silence{ID: silenceID, Name: "duplicate", Reason: "duplicate", Owner: "operator-1", Matchers: Matcher{}, StartsAt: now, ExpiresAt: now.Add(time.Hour)}); err == nil {
|
||||
t.Fatal("duplicate silence should fail")
|
||||
}
|
||||
if _, err := repo.RevokeSilence(ctx, silenceID, "operator-1", silence.Revision, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := repo.RevokeSilence(ctx, silenceID, "operator-1", silence.Revision+1, now); err == nil {
|
||||
t.Fatal("repeated revoke should conflict")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package alertcontrol
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
func RunExpiryLoop(ctx context.Context, store Store, interval time.Duration, logger *slog.Logger) {
|
||||
if store == nil || interval <= 0 {
|
||||
return
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
job := ExpiryJob{Store: store}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if _, err := job.Run(ctx); err != nil && logger != nil {
|
||||
logger.Warn("alert control expiry failed", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package alertcontrol
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxName = 160
|
||||
MaxReason = 500
|
||||
MaxOwner = 255
|
||||
MaxMatcherKeys = 20
|
||||
MaxMatcherValues = 50
|
||||
MaxDuration = 365 * 24 * time.Hour
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalid = errors.New("invalid alert control")
|
||||
ErrNotFound = errors.New("alert control not found")
|
||||
ErrUnavailable = errors.New("alert control repository is unavailable")
|
||||
ErrConflict = errors.New("alert control has already changed")
|
||||
keyPattern = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_.:/-]{0,63}$`)
|
||||
)
|
||||
|
||||
type Matcher struct {
|
||||
RuleIDs []string `json:"ruleIds,omitempty"`
|
||||
EntityIDs []string `json:"entityIds,omitempty"`
|
||||
EntityTypes []string `json:"entityTypes,omitempty"`
|
||||
Severities []string `json:"severities,omitempty"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
}
|
||||
|
||||
type Signal struct {
|
||||
InstanceID string `json:"instanceId"`
|
||||
RuleID string `json:"ruleId"`
|
||||
EntityID string `json:"entityId,omitempty"`
|
||||
Severity string `json:"severity"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
}
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateScheduled State = "scheduled"
|
||||
StateActive State = "active"
|
||||
StateExpired State = "expired"
|
||||
StateRevoked State = "revoked"
|
||||
)
|
||||
|
||||
type Silence struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Reason string `json:"reason"`
|
||||
Owner string `json:"owner"`
|
||||
Matchers Matcher `json:"matchers"`
|
||||
StartsAt time.Time `json:"startsAt"`
|
||||
ExpiresAt time.Time `json:"expiresAt"`
|
||||
State State `json:"state"`
|
||||
CreatedBy string `json:"createdBy"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
RevokedBy string `json:"revokedBy,omitempty"`
|
||||
RevokedAt *time.Time `json:"revokedAt,omitempty"`
|
||||
ExpiredAt *time.Time `json:"expiredAt,omitempty"`
|
||||
Revision int64 `json:"revision"`
|
||||
}
|
||||
|
||||
type MaintenanceWindow struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Reason string `json:"reason"`
|
||||
Selector Matcher `json:"selector"`
|
||||
StartsAt time.Time `json:"startsAt"`
|
||||
EndsAt time.Time `json:"endsAt"`
|
||||
State State `json:"state"`
|
||||
CreatedBy string `json:"createdBy"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
RevokedBy string `json:"revokedBy,omitempty"`
|
||||
RevokedAt *time.Time `json:"revokedAt,omitempty"`
|
||||
ExpiredAt *time.Time `json:"expiredAt,omitempty"`
|
||||
Revision int64 `json:"revision"`
|
||||
}
|
||||
|
||||
type Preview struct {
|
||||
Matched bool `json:"matched"`
|
||||
MatchedCount int `json:"matchedCount"`
|
||||
InstanceIDs []string `json:"instanceIds"`
|
||||
}
|
||||
|
||||
func NewID() string {
|
||||
b := make([]byte, 16)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "00000000-0000-4000-8000-000000000000"
|
||||
}
|
||||
b[6] = (b[6] & 0x0f) | 0x40
|
||||
b[8] = (b[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%s-%s-%s-%s-%s", hex.EncodeToString(b[0:4]), hex.EncodeToString(b[4:6]), hex.EncodeToString(b[6:8]), hex.EncodeToString(b[8:10]), hex.EncodeToString(b[10:16]))
|
||||
}
|
||||
|
||||
func (m Matcher) Validate() error {
|
||||
count := len(m.RuleIDs) + len(m.EntityIDs) + len(m.EntityTypes) + len(m.Severities) + len(m.Labels)
|
||||
if count > MaxMatcherKeys {
|
||||
return fmt.Errorf("%w: too many matcher keys", ErrInvalid)
|
||||
}
|
||||
for _, values := range [][]string{m.RuleIDs, m.EntityIDs, m.EntityTypes, m.Severities} {
|
||||
if len(values) > MaxMatcherValues {
|
||||
return fmt.Errorf("%w: too many matcher values", ErrInvalid)
|
||||
}
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
if err := validateValue(value); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, ok := seen[value]; ok {
|
||||
return fmt.Errorf("%w: duplicate matcher value", ErrInvalid)
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(m.Labels) > MaxMatcherValues {
|
||||
return fmt.Errorf("%w: too many label matchers", ErrInvalid)
|
||||
}
|
||||
for key, value := range m.Labels {
|
||||
if !keyPattern.MatchString(key) {
|
||||
return fmt.Errorf("%w: invalid label key", ErrInvalid)
|
||||
}
|
||||
if err := validateValue(value); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateValue(value string) error {
|
||||
if value == "" || len(value) > 160 || strings.ContainsAny(value, "\r\n\x00") {
|
||||
return fmt.Errorf("%w: invalid matcher value", ErrInvalid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m Matcher) Matches(signal Signal) bool {
|
||||
return containsOrWildcard(m.RuleIDs, signal.RuleID) && containsOrWildcard(m.EntityIDs, signal.EntityID) && containsOrWildcard(m.Severities, signal.Severity) && containsOrWildcard(m.EntityTypes, signal.Labels["entity.type"]) && labelsMatch(m.Labels, signal.Labels)
|
||||
}
|
||||
|
||||
func containsOrWildcard(values []string, value string) bool {
|
||||
if len(values) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, item := range values {
|
||||
if item == value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func labelsMatch(expected, actual map[string]string) bool {
|
||||
for key, value := range expected {
|
||||
if actual[key] != value {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s Silence) Validate(now time.Time) error {
|
||||
if err := validateCommon(s.Name, s.Reason, s.StartsAt, s.ExpiresAt, s.Matchers); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.Owner == "" || len(s.Owner) > MaxOwner {
|
||||
return fmt.Errorf("%w: invalid owner", ErrInvalid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w MaintenanceWindow) Validate(now time.Time) error {
|
||||
return validateCommon(w.Name, w.Reason, w.StartsAt, w.EndsAt, w.Selector)
|
||||
}
|
||||
|
||||
func validateCommon(name, reason string, starts, ends time.Time, matcher Matcher) error {
|
||||
if strings.TrimSpace(name) == "" || len(name) > MaxName || strings.ContainsAny(name, "\r\n\x00") {
|
||||
return fmt.Errorf("%w: invalid name", ErrInvalid)
|
||||
}
|
||||
if strings.TrimSpace(reason) == "" || len(reason) > MaxReason || strings.ContainsAny(reason, "\r\n\x00") {
|
||||
return fmt.Errorf("%w: invalid reason", ErrInvalid)
|
||||
}
|
||||
if starts.IsZero() || ends.IsZero() || !ends.After(starts) || ends.Sub(starts) > MaxDuration {
|
||||
return fmt.Errorf("%w: invalid expiry window", ErrInvalid)
|
||||
}
|
||||
if err := matcher.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s Silence) StateAt(now time.Time) State {
|
||||
return temporalState(s.StartsAt, s.ExpiresAt, s.RevokedAt, s.ExpiredAt, now)
|
||||
}
|
||||
func (w MaintenanceWindow) StateAt(now time.Time) State {
|
||||
return temporalState(w.StartsAt, w.EndsAt, w.RevokedAt, w.ExpiredAt, now)
|
||||
}
|
||||
func temporalState(starts, ends time.Time, revoked, expired *time.Time, now time.Time) State {
|
||||
if revoked != nil {
|
||||
return StateRevoked
|
||||
}
|
||||
if expired != nil || !now.Before(ends) {
|
||||
return StateExpired
|
||||
}
|
||||
if now.Before(starts) {
|
||||
return StateScheduled
|
||||
}
|
||||
return StateActive
|
||||
}
|
||||
|
||||
func (s Silence) Matches(signal Signal, now time.Time) bool {
|
||||
return s.StateAt(now) == StateActive && s.Matchers.Matches(signal)
|
||||
}
|
||||
func (w MaintenanceWindow) Matches(signal Signal, now time.Time) bool {
|
||||
return w.StateAt(now) == StateActive && w.Selector.Matches(signal)
|
||||
}
|
||||
|
||||
func PreviewSignals(m Matcher, signals []Signal) (Preview, error) {
|
||||
if err := m.Validate(); err != nil {
|
||||
return Preview{}, err
|
||||
}
|
||||
ids := make([]string, 0, len(signals))
|
||||
seen := make(map[string]struct{}, len(signals))
|
||||
for _, signal := range signals {
|
||||
if signal.InstanceID == "" {
|
||||
return Preview{}, fmt.Errorf("%w: signal instance id is required", ErrInvalid)
|
||||
}
|
||||
if m.Matches(signal) {
|
||||
if _, ok := seen[signal.InstanceID]; !ok {
|
||||
ids = append(ids, signal.InstanceID)
|
||||
seen[signal.InstanceID] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return Preview{Matched: len(ids) > 0, MatchedCount: len(ids), InstanceIDs: ids}, nil
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package alertcontrol
|
||||
|
||||
import "testing"
|
||||
|
||||
func BenchmarkPreviewSignals1000(b *testing.B) {
|
||||
matcher := Matcher{Severities: []string{"critical"}, Labels: map[string]string{"source.type": "prometheus"}}
|
||||
signals := make([]Signal, 1000)
|
||||
for index := range signals {
|
||||
signals[index] = Signal{InstanceID: NewID(), Severity: "critical", Labels: map[string]string{"source.type": "prometheus"}}
|
||||
}
|
||||
b.ResetTimer()
|
||||
for index := 0; index < b.N; index++ {
|
||||
if _, err := PreviewSignals(matcher, signals); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package alertcontrol
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMatcherIsBoundedAndDeterministic(t *testing.T) {
|
||||
m := Matcher{RuleIDs: []string{"rule-a"}, EntityTypes: []string{"host"}, Labels: map[string]string{"source.type": "prometheus"}}
|
||||
if err := m.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
signal := Signal{InstanceID: "instance-1", RuleID: "rule-a", Severity: "critical", Labels: map[string]string{"entity.type": "host", "source.type": "prometheus"}}
|
||||
if !m.Matches(signal) {
|
||||
t.Fatal("matcher should match signal")
|
||||
}
|
||||
signal.Labels["source.type"] = "agent"
|
||||
if m.Matches(signal) {
|
||||
t.Fatal("matcher should reject different labels")
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlStateAndPreview(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
|
||||
control := Silence{ID: "silence-1", Name: "Deploy", Reason: "planned change", Owner: "operator", Matchers: Matcher{Severities: []string{"critical"}}, StartsAt: now, ExpiresAt: now.Add(time.Hour)}
|
||||
if got := control.StateAt(now); got != StateActive {
|
||||
t.Fatalf("state at start = %s", got)
|
||||
}
|
||||
if got := control.StateAt(now.Add(time.Hour)); got != StateExpired {
|
||||
t.Fatalf("state at expiry = %s", got)
|
||||
}
|
||||
if err := control.Validate(now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
preview, err := PreviewSignals(control.Matchers, []Signal{{InstanceID: "b", Severity: "critical"}, {InstanceID: "a", Severity: "critical"}, {InstanceID: "x", Severity: "attention"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if preview.MatchedCount != 2 || preview.InstanceIDs[0] != "a" || preview.InstanceIDs[1] != "b" {
|
||||
t.Fatalf("unexpected preview: %#v", preview)
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlValidationRejectsUnboundedExpiryAndMatchers(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
window := MaintenanceWindow{Name: "window", Reason: "reason", Selector: Matcher{Labels: map[string]string{"bad key": "value"}}, StartsAt: now, EndsAt: now.Add(2 * MaxDuration)}
|
||||
if err := window.Validate(now); err == nil {
|
||||
t.Fatal("expected validation error")
|
||||
}
|
||||
matcher := Matcher{RuleIDs: make([]string, MaxMatcherValues+1)}
|
||||
if err := matcher.Validate(); err == nil {
|
||||
t.Fatal("expected matcher bound error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package alertcontrolapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alertcontrol"
|
||||
"github.com/itworx/pulse/internal/audit"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/correlation"
|
||||
"github.com/itworx/pulse/internal/problem"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
Store alertcontrol.Store
|
||||
Audit audit.Store
|
||||
}
|
||||
|
||||
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
principal, ok := auth.PrincipalFromContext(r.Context())
|
||||
if !ok {
|
||||
fail(w, r, http.StatusUnauthorized, "UNAUTHORIZED", "Authentication required.")
|
||||
return
|
||||
}
|
||||
base := "/api/v1/alert-silences"
|
||||
maintenance := strings.HasPrefix(r.URL.Path, "/api/v1/maintenance-windows")
|
||||
if maintenance {
|
||||
base = "/api/v1/maintenance-windows"
|
||||
}
|
||||
path := strings.Trim(strings.TrimPrefix(r.URL.Path, base), "/")
|
||||
if path == "" {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
h.list(w, r, maintenance)
|
||||
case http.MethodPost:
|
||||
if !requireOperate(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.create(w, r, principal.Subject, maintenance)
|
||||
default:
|
||||
fail(w, r, http.StatusMethodNotAllowed, "METHOD_NOT_ALLOWED", "This method is not supported.")
|
||||
}
|
||||
return
|
||||
}
|
||||
if path == "preview" && r.Method == http.MethodPost {
|
||||
h.preview(w, r, maintenance)
|
||||
return
|
||||
}
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) == 2 && parts[1] == "revoke" && r.Method == http.MethodPost {
|
||||
if !requireOperate(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.revoke(w, r, parts[0], principal.Subject, maintenance)
|
||||
return
|
||||
}
|
||||
fail(w, r, http.StatusNotFound, "NOT_FOUND", "Alert-control route not found.")
|
||||
}
|
||||
|
||||
func (h Handler) list(w http.ResponseWriter, r *http.Request, maintenance bool) {
|
||||
limit := 100
|
||||
if value := r.URL.Query().Get("limit"); value != "" {
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed < 1 || parsed > 100 {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_LIMIT", "The limit must be between 1 and 100.")
|
||||
return
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
if maintenance {
|
||||
items, err := h.Store.ListMaintenance(r.Context(), limit, time.Now().UTC())
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err)
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"items": items})
|
||||
return
|
||||
}
|
||||
items, err := h.Store.ListSilences(r.Context(), limit, time.Now().UTC())
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err)
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"items": items})
|
||||
}
|
||||
|
||||
func (h Handler) create(w http.ResponseWriter, r *http.Request, actor string, maintenance bool) {
|
||||
if maintenance {
|
||||
var item alertcontrol.MaintenanceWindow
|
||||
if err := decode(r, &item); err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_MAINTENANCE", "The maintenance-window document is invalid.")
|
||||
return
|
||||
}
|
||||
created, err := h.Store.CreateMaintenance(r.Context(), actor, item)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err)
|
||||
return
|
||||
}
|
||||
if err := h.record(r, actor, "maintenance_window.create", created.ID, "maintenance_window", nil, map[string]any{"state": created.State, "revision": created.Revision}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusCreated, map[string]any{"maintenance": created})
|
||||
return
|
||||
}
|
||||
var item alertcontrol.Silence
|
||||
if err := decode(r, &item); err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_SILENCE", "The silence document is invalid.")
|
||||
return
|
||||
}
|
||||
item.Owner = actor
|
||||
created, err := h.Store.CreateSilence(r.Context(), actor, item)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err)
|
||||
return
|
||||
}
|
||||
if err := h.record(r, actor, "alert_silence.create", created.ID, "alert_silence", nil, map[string]any{"state": created.State, "revision": created.Revision}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusCreated, map[string]any{"silence": created})
|
||||
}
|
||||
|
||||
func (h Handler) revoke(w http.ResponseWriter, r *http.Request, id, actor string, maintenance bool) {
|
||||
expected, err := revision(r)
|
||||
if err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_REVISION", "A valid If-Match or revision value is required.")
|
||||
return
|
||||
}
|
||||
if maintenance {
|
||||
item, err := h.Store.RevokeMaintenance(r.Context(), id, actor, expected, time.Now().UTC())
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err)
|
||||
return
|
||||
}
|
||||
if err := h.record(r, actor, "maintenance_window.revoke", id, "maintenance_window", map[string]any{"revision": expected}, map[string]any{"state": item.State, "revision": item.Revision}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"maintenance": item})
|
||||
return
|
||||
}
|
||||
item, err := h.Store.RevokeSilence(r.Context(), id, actor, expected, time.Now().UTC())
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err)
|
||||
return
|
||||
}
|
||||
if err := h.record(r, actor, "alert_silence.revoke", id, "alert_silence", map[string]any{"revision": expected}, map[string]any{"state": item.State, "revision": item.Revision}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"silence": item})
|
||||
}
|
||||
|
||||
func (h Handler) preview(w http.ResponseWriter, r *http.Request, maintenance bool) {
|
||||
var request struct {
|
||||
Matcher alertcontrol.Matcher `json:"matcher"`
|
||||
Selector alertcontrol.Matcher `json:"selector"`
|
||||
Signals []alertcontrol.Signal `json:"signals"`
|
||||
}
|
||||
if err := decode(r, &request); err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_PREVIEW", "The matcher preview request is invalid.")
|
||||
return
|
||||
}
|
||||
matcher := request.Matcher
|
||||
if maintenance {
|
||||
matcher = request.Selector
|
||||
}
|
||||
result, err := alertcontrol.PreviewSignals(matcher, request.Signals)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err)
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"preview": result})
|
||||
}
|
||||
|
||||
func (h Handler) record(r *http.Request, actor, action, id, resourceType string, before, after map[string]any) error {
|
||||
if h.Audit == nil {
|
||||
return nil
|
||||
}
|
||||
return h.Audit.Append(r.Context(), audit.Event{Actor: actor, Action: action, ResourceType: resourceType, ResourceID: id, Result: "success", CorrelationID: correlation.FromContext(r.Context()), Before: before, After: after})
|
||||
}
|
||||
|
||||
func (h Handler) repositoryFailure(w http.ResponseWriter, r *http.Request, err error) {
|
||||
switch {
|
||||
case errors.Is(err, alertcontrol.ErrInvalid):
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_ALERT_CONTROL", "The alert-control document is invalid.")
|
||||
case errors.Is(err, alertcontrol.ErrConflict):
|
||||
fail(w, r, http.StatusConflict, "REVISION_CONFLICT", "The alert control was changed or expired.")
|
||||
case errors.Is(err, alertcontrol.ErrNotFound):
|
||||
fail(w, r, http.StatusNotFound, "NOT_FOUND", "The alert control was not found.")
|
||||
case errors.Is(err, alertcontrol.ErrUnavailable):
|
||||
fail(w, r, http.StatusServiceUnavailable, "DATABASE_UNAVAILABLE", "Alert controls are unavailable.")
|
||||
default:
|
||||
fail(w, r, http.StatusInternalServerError, "ALERT_CONTROL_REQUEST_FAILED", "The alert-control request failed.")
|
||||
}
|
||||
}
|
||||
func requireOperate(w http.ResponseWriter, r *http.Request, role auth.Role) bool {
|
||||
if auth.Allows(role, auth.PermissionOperate) {
|
||||
return true
|
||||
}
|
||||
fail(w, r, http.StatusForbidden, "FORBIDDEN", "Alert-control editing is not allowed for this role.")
|
||||
return false
|
||||
}
|
||||
func revision(r *http.Request) (int64, error) {
|
||||
value := r.Header.Get("If-Match")
|
||||
if value == "" {
|
||||
value = r.URL.Query().Get("revision")
|
||||
}
|
||||
value = strings.Trim(value, "\"")
|
||||
if value == "" {
|
||||
return 0, errors.New("revision required")
|
||||
}
|
||||
return strconv.ParseInt(value, 10, 64)
|
||||
}
|
||||
func decode(r *http.Request, target any) error {
|
||||
contentType := strings.ToLower(strings.TrimSpace(strings.Split(r.Header.Get("Content-Type"), ";")[0]))
|
||||
if contentType != "" && contentType != "application/json" {
|
||||
return errors.New("unsupported content type")
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, 2<<20+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer r.Body.Close()
|
||||
if len(body) > 2<<20 {
|
||||
return errors.New("request too large")
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(string(body)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(target); err != nil {
|
||||
return err
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return errors.New("multiple JSON values")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func fail(w http.ResponseWriter, r *http.Request, status int, code, detail string) {
|
||||
problem.Write(w, r, status, code, http.StatusText(status), detail, nil)
|
||||
}
|
||||
func write(w http.ResponseWriter, status int, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package alertcontrolapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alertcontrol"
|
||||
"github.com/itworx/pulse/internal/audit"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
)
|
||||
|
||||
type memoryStore struct {
|
||||
mu sync.Mutex
|
||||
silences map[string]alertcontrol.Silence
|
||||
maintenance map[string]alertcontrol.MaintenanceWindow
|
||||
}
|
||||
|
||||
func newMemoryStore() *memoryStore {
|
||||
return &memoryStore{silences: map[string]alertcontrol.Silence{}, maintenance: map[string]alertcontrol.MaintenanceWindow{}}
|
||||
}
|
||||
func (s *memoryStore) CreateSilence(_ context.Context, actor string, item alertcontrol.Silence) (alertcontrol.Silence, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if item.ID == "" {
|
||||
item.ID = alertcontrol.NewID()
|
||||
}
|
||||
if item.Owner == "" {
|
||||
item.Owner = actor
|
||||
}
|
||||
if err := item.Validate(time.Now().UTC()); err != nil {
|
||||
return alertcontrol.Silence{}, err
|
||||
}
|
||||
if _, ok := s.silences[item.ID]; ok {
|
||||
return alertcontrol.Silence{}, alertcontrol.ErrConflict
|
||||
}
|
||||
item.CreatedBy, item.CreatedAt, item.Revision = actor, time.Now().UTC(), 1
|
||||
item.State = item.StateAt(time.Now().UTC())
|
||||
s.silences[item.ID] = item
|
||||
return item, nil
|
||||
}
|
||||
func (s *memoryStore) ListSilences(_ context.Context, _ int, now time.Time) ([]alertcontrol.Silence, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
items := make([]alertcontrol.Silence, 0, len(s.silences))
|
||||
for _, item := range s.silences {
|
||||
item.State = item.StateAt(now)
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
func (s *memoryStore) RevokeSilence(_ context.Context, id, actor string, expected int64, now time.Time) (alertcontrol.Silence, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
item, ok := s.silences[id]
|
||||
if !ok {
|
||||
return alertcontrol.Silence{}, alertcontrol.ErrNotFound
|
||||
}
|
||||
if item.Revision != expected || item.StateAt(now) != alertcontrol.StateActive {
|
||||
return alertcontrol.Silence{}, alertcontrol.ErrConflict
|
||||
}
|
||||
item.RevokedBy, item.RevokedAt, item.Revision, item.State = actor, &now, item.Revision+1, alertcontrol.StateRevoked
|
||||
s.silences[id] = item
|
||||
return item, nil
|
||||
}
|
||||
func (s *memoryStore) CreateMaintenance(_ context.Context, actor string, item alertcontrol.MaintenanceWindow) (alertcontrol.MaintenanceWindow, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if item.ID == "" {
|
||||
item.ID = alertcontrol.NewID()
|
||||
}
|
||||
if err := item.Validate(time.Now().UTC()); err != nil {
|
||||
return alertcontrol.MaintenanceWindow{}, err
|
||||
}
|
||||
item.CreatedBy, item.CreatedAt, item.Revision = actor, time.Now().UTC(), 1
|
||||
item.State = item.StateAt(time.Now().UTC())
|
||||
s.maintenance[item.ID] = item
|
||||
return item, nil
|
||||
}
|
||||
func (s *memoryStore) ListMaintenance(_ context.Context, _ int, now time.Time) ([]alertcontrol.MaintenanceWindow, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
items := make([]alertcontrol.MaintenanceWindow, 0, len(s.maintenance))
|
||||
for _, item := range s.maintenance {
|
||||
item.State = item.StateAt(now)
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
func (s *memoryStore) RevokeMaintenance(_ context.Context, id, actor string, expected int64, now time.Time) (alertcontrol.MaintenanceWindow, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
item, ok := s.maintenance[id]
|
||||
if !ok {
|
||||
return alertcontrol.MaintenanceWindow{}, alertcontrol.ErrNotFound
|
||||
}
|
||||
if item.Revision != expected {
|
||||
return alertcontrol.MaintenanceWindow{}, alertcontrol.ErrConflict
|
||||
}
|
||||
item.RevokedBy, item.RevokedAt, item.Revision, item.State = actor, &now, item.Revision+1, alertcontrol.StateRevoked
|
||||
s.maintenance[id] = item
|
||||
return item, nil
|
||||
}
|
||||
func (s *memoryStore) Expire(_ context.Context, now time.Time) (alertcontrol.ExpiryResult, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var result alertcontrol.ExpiryResult
|
||||
for id, item := range s.silences {
|
||||
if item.State == alertcontrol.StateActive && !now.Before(item.ExpiresAt) {
|
||||
item.ExpiredAt, item.Revision = &now, item.Revision+1
|
||||
s.silences[id] = item
|
||||
result.Silences++
|
||||
}
|
||||
}
|
||||
for id, item := range s.maintenance {
|
||||
if item.State == alertcontrol.StateActive && !now.Before(item.EndsAt) {
|
||||
item.ExpiredAt, item.Revision = &now, item.Revision+1
|
||||
s.maintenance[id] = item
|
||||
result.MaintenanceWindows++
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func TestHandlerEnforcesRBACAuditPreviewAndVisibleMaintenance(t *testing.T) {
|
||||
store := newMemoryStore()
|
||||
auditStore := &audit.MemoryStore{}
|
||||
handler := Handler{Store: store, Audit: auditStore}
|
||||
now := time.Now().UTC()
|
||||
silence := alertcontrol.Silence{Name: "deploy", Reason: "planned", Owner: "operator", Matchers: alertcontrol.Matcher{Severities: []string{"critical"}}, StartsAt: now.Add(-time.Minute), ExpiresAt: now.Add(time.Hour)}
|
||||
viewer := requestWithPrincipal(http.MethodPost, "/api/v1/alert-silences", silence, auth.RoleViewer)
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, viewer)
|
||||
if response.Code != http.StatusForbidden {
|
||||
t.Fatalf("viewer create status = %d", response.Code)
|
||||
}
|
||||
operator := requestWithPrincipal(http.MethodPost, "/api/v1/alert-silences", silence, auth.RoleOperator)
|
||||
response = httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, operator)
|
||||
if response.Code != http.StatusCreated {
|
||||
t.Fatalf("operator create status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
var created struct {
|
||||
Silence alertcontrol.Silence `json:"silence"`
|
||||
}
|
||||
if err := json.Unmarshal(response.Body.Bytes(), &created); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(auditStore.Events) != 1 || auditStore.Events[0].Action != "alert_silence.create" {
|
||||
t.Fatalf("audit events = %#v", auditStore.Events)
|
||||
}
|
||||
previewBody := map[string]any{"matcher": alertcontrol.Matcher{Severities: []string{"critical"}}, "signals": []alertcontrol.Signal{{InstanceID: "i-1", Severity: "critical"}}}
|
||||
preview := requestWithPrincipal(http.MethodPost, "/api/v1/alert-silences/preview", previewBody, auth.RoleViewer)
|
||||
response = httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, preview)
|
||||
if response.Code != http.StatusOK || !contains(response.Body.String(), `"matched":true`) {
|
||||
t.Fatalf("preview response = %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
window := alertcontrol.MaintenanceWindow{Name: "maintenance", Reason: "upgrade", Selector: alertcontrol.Matcher{EntityTypes: []string{"host"}}, StartsAt: now.Add(-time.Minute), EndsAt: now.Add(time.Hour)}
|
||||
request := requestWithPrincipal(http.MethodPost, "/api/v1/maintenance-windows", window, auth.RoleOperator)
|
||||
response = httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusCreated {
|
||||
t.Fatalf("maintenance create status = %d", response.Code)
|
||||
}
|
||||
request = requestWithPrincipal(http.MethodGet, "/api/v1/maintenance-windows", nil, auth.RoleViewer)
|
||||
response = httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusOK || !contains(response.Body.String(), `"state":"active"`) {
|
||||
t.Fatalf("maintenance list = %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
request = requestWithPrincipal(http.MethodPost, "/api/v1/alert-silences/"+created.Silence.ID+"/revoke?revision=999", nil, auth.RoleOperator)
|
||||
response = httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusConflict {
|
||||
t.Fatalf("stale revoke status = %d", response.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func requestWithPrincipal(method, path string, body any, role auth.Role) *http.Request {
|
||||
var reader *strings.Reader
|
||||
if body == nil {
|
||||
reader = strings.NewReader("")
|
||||
} else {
|
||||
encoded, _ := json.Marshal(body)
|
||||
reader = strings.NewReader(string(encoded))
|
||||
}
|
||||
request := httptest.NewRequest(method, path, reader).WithContext(auth.WithPrincipal(context.Background(), auth.Principal{Subject: "operator-1", Role: role}))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
return request
|
||||
}
|
||||
func contains(value, part string) bool { return strings.Contains(value, part) }
|
||||
@@ -0,0 +1,111 @@
|
||||
package alertdefaults
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/metriccatalog"
|
||||
)
|
||||
|
||||
//go:embed seed.json
|
||||
var seedFS embed.FS
|
||||
|
||||
type seedDocument struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
Rules []alert.Document `json:"rules"`
|
||||
}
|
||||
|
||||
type RuleStore interface {
|
||||
Create(context.Context, string, alert.Document, string) (alert.Rule, alert.Version, error)
|
||||
Get(context.Context, string) (alert.Rule, error)
|
||||
// Update is optional for the seed: stores that also implement it allow the
|
||||
// seed to refresh an implementation-owned default that has never been edited
|
||||
// (revision 1) when a newer seed corrects it. See Seed.
|
||||
}
|
||||
|
||||
// RuleUpdater is implemented by stores that support versioned updates. The seed
|
||||
// uses it only for rules that are still at revision 1 (created by the seed and
|
||||
// never changed by an operator), so operator edits are never overwritten.
|
||||
type RuleUpdater interface {
|
||||
Update(ctx context.Context, id, actor string, expected int64, document alert.Document, changeSummary string) (alert.Rule, error)
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Added int
|
||||
Existing int
|
||||
Refreshed int
|
||||
}
|
||||
|
||||
func Load(registry metriccatalog.Registry) ([]alert.Document, error) {
|
||||
raw, err := seedFS.ReadFile("seed.json")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read embedded alert defaults: %w", err)
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.DisallowUnknownFields()
|
||||
var bundle seedDocument
|
||||
if err := decoder.Decode(&bundle); err != nil {
|
||||
return nil, fmt.Errorf("decode embedded alert defaults: %w", err)
|
||||
}
|
||||
if bundle.SchemaVersion != 1 || len(bundle.Rules) == 0 {
|
||||
return nil, errors.New("embedded alert defaults have an invalid bundle")
|
||||
}
|
||||
seen := make(map[string]struct{}, len(bundle.Rules))
|
||||
for _, rule := range bundle.Rules {
|
||||
if _, ok := seen[rule.ID]; ok {
|
||||
return nil, fmt.Errorf("embedded alert defaults contain duplicate rule %q", rule.ID)
|
||||
}
|
||||
seen[rule.ID] = struct{}{}
|
||||
if err := rule.Validate(registry); err != nil {
|
||||
return nil, fmt.Errorf("validate embedded alert default %q: %w", rule.ID, err)
|
||||
}
|
||||
}
|
||||
return bundle.Rules, nil
|
||||
}
|
||||
|
||||
func Seed(ctx context.Context, store RuleStore, registry metriccatalog.Registry, actor string) (Report, error) {
|
||||
if store == nil {
|
||||
return Report{}, alert.ErrUnavailable
|
||||
}
|
||||
rules, err := Load(registry)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
report := Report{}
|
||||
for _, document := range rules {
|
||||
if _, _, err := store.Create(ctx, actor, document, "system default seed v1"); err == nil {
|
||||
report.Added++
|
||||
} else if errors.Is(err, alert.ErrConflict) {
|
||||
existing, getErr := store.Get(ctx, document.ID)
|
||||
if getErr != nil {
|
||||
return Report{}, fmt.Errorf("verify existing alert default %q: %w", document.ID, getErr)
|
||||
}
|
||||
if updater, ok := store.(RuleUpdater); ok && existing.Revision == 1 && !sameDocument(existing.Document, document) {
|
||||
// The stored default is untouched since the seed created it, but the
|
||||
// seed itself changed (for example a corrected metric binding). Refresh
|
||||
// it through the normal versioned update path so history is kept.
|
||||
if _, updErr := updater.Update(ctx, document.ID, actor, existing.Revision, document, "system default seed v1 refresh"); updErr != nil {
|
||||
return Report{}, fmt.Errorf("refresh alert default %q: %w", document.ID, updErr)
|
||||
}
|
||||
report.Refreshed++
|
||||
continue
|
||||
}
|
||||
report.Existing++
|
||||
} else {
|
||||
return Report{}, fmt.Errorf("seed alert default %q: %w", document.ID, err)
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// sameDocument compares two rule documents by their canonical JSON encoding.
|
||||
func sameDocument(a, b alert.Document) bool {
|
||||
left, errA := json.Marshal(a)
|
||||
right, errB := json.Marshal(b)
|
||||
return errA == nil && errB == nil && bytes.Equal(left, right)
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"rules": [
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"id": "71111111-1111-4111-8111-111111111111",
|
||||
"name": "Monitoringbron levert geen recente gegevens",
|
||||
"enabled": true,
|
||||
"severity": "degraded",
|
||||
"scope": {
|
||||
"entityType": "data-source",
|
||||
"required": true
|
||||
},
|
||||
"condition": {
|
||||
"inputType": "datasource-health",
|
||||
"operator": "==",
|
||||
"threshold": "stale",
|
||||
"windowSeconds": 120
|
||||
},
|
||||
"evaluationIntervalSeconds": 30,
|
||||
"pendingSeconds": 120,
|
||||
"resolveSeconds": 60,
|
||||
"unknownBehavior": "become-unknown",
|
||||
"groupBy": [
|
||||
"source"
|
||||
],
|
||||
"suppressWhen": [],
|
||||
"message": {
|
||||
"titleKey": "alerts.datasourceStale.title",
|
||||
"bodyKey": "alerts.datasourceStale.body"
|
||||
}
|
||||
},
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"id": "81111111-1111-4111-8111-111111111111",
|
||||
"name": "Container bevindt zich in een herstartlus",
|
||||
"enabled": true,
|
||||
"severity": "degraded",
|
||||
"scope": {
|
||||
"entityType": "container",
|
||||
"excludeIntentionalStopped": true
|
||||
},
|
||||
"condition": {
|
||||
"inputType": "event",
|
||||
"operator": ">=",
|
||||
"threshold": 3,
|
||||
"aggregation": "count",
|
||||
"windowSeconds": 900
|
||||
},
|
||||
"evaluationIntervalSeconds": 30,
|
||||
"pendingSeconds": 0,
|
||||
"resolveSeconds": 900,
|
||||
"unknownBehavior": "become-unknown",
|
||||
"groupBy": [
|
||||
"container",
|
||||
"application"
|
||||
],
|
||||
"suppressWhen": [
|
||||
"host.unreachable"
|
||||
],
|
||||
"message": {
|
||||
"titleKey": "alerts.containerRestartLoop.title",
|
||||
"bodyKey": "alerts.containerRestartLoop.body"
|
||||
}
|
||||
},
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"id": "91111111-1111-4111-8111-111111111111",
|
||||
"name": "Disktemperatuur te hoog",
|
||||
"enabled": true,
|
||||
"severity": "degraded",
|
||||
"scope": {
|
||||
"entityType": "disk"
|
||||
},
|
||||
"condition": {
|
||||
"inputType": "metric",
|
||||
"metric": "storage.disk.temperature.maximum",
|
||||
"operator": ">=",
|
||||
"threshold": 50,
|
||||
"recoveryThreshold": 46,
|
||||
"aggregation": "max",
|
||||
"windowSeconds": 300
|
||||
},
|
||||
"evaluationIntervalSeconds": 30,
|
||||
"pendingSeconds": 300,
|
||||
"resolveSeconds": 300,
|
||||
"unknownBehavior": "retain-firing-as-unknown",
|
||||
"groupBy": [
|
||||
"disk",
|
||||
"server"
|
||||
],
|
||||
"suppressWhen": [
|
||||
"host.unreachable",
|
||||
"storage.source.unavailable"
|
||||
],
|
||||
"message": {
|
||||
"titleKey": "alerts.diskTemperature.title",
|
||||
"bodyKey": "alerts.diskTemperature.body"
|
||||
}
|
||||
},
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"id": "a1111111-1111-4111-8111-111111111111",
|
||||
"name": "Service is niet bereikbaar",
|
||||
"enabled": true,
|
||||
"severity": "degraded",
|
||||
"scope": {
|
||||
"entityType": "service",
|
||||
"critical": true
|
||||
},
|
||||
"condition": {
|
||||
"inputType": "metric",
|
||||
"metric": "service.availability.minimum",
|
||||
"operator": "<",
|
||||
"threshold": 1,
|
||||
"aggregation": "min",
|
||||
"windowSeconds": 90
|
||||
},
|
||||
"evaluationIntervalSeconds": 30,
|
||||
"pendingSeconds": 90,
|
||||
"resolveSeconds": 60,
|
||||
"unknownBehavior": "become-unknown",
|
||||
"groupBy": [
|
||||
"service",
|
||||
"application"
|
||||
],
|
||||
"suppressWhen": [
|
||||
"host.unreachable",
|
||||
"network.gateway.unreachable",
|
||||
"dns.unavailable"
|
||||
],
|
||||
"message": {
|
||||
"titleKey": "alerts.serviceUnavailable.title",
|
||||
"bodyKey": "alerts.serviceUnavailable.body"
|
||||
}
|
||||
},
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"id": "b1111111-1111-4111-8111-111111111111",
|
||||
"name": "Opslagpool bijna vol",
|
||||
"enabled": true,
|
||||
"severity": "critical",
|
||||
"scope": {
|
||||
"entityType": "storage_pool"
|
||||
},
|
||||
"condition": {
|
||||
"inputType": "metric",
|
||||
"metric": "storage.pool.utilization.maximum",
|
||||
"operator": ">=",
|
||||
"threshold": 97,
|
||||
"recoveryThreshold": 90,
|
||||
"aggregation": "max",
|
||||
"windowSeconds": 300
|
||||
},
|
||||
"evaluationIntervalSeconds": 60,
|
||||
"pendingSeconds": 300,
|
||||
"resolveSeconds": 300,
|
||||
"unknownBehavior": "retain-firing-as-unknown",
|
||||
"groupBy": [
|
||||
"server"
|
||||
],
|
||||
"suppressWhen": [
|
||||
"host.unreachable",
|
||||
"storage.source.unavailable"
|
||||
],
|
||||
"message": {
|
||||
"titleKey": "alerts.storagePoolCritical.title",
|
||||
"bodyKey": "alerts.storagePoolCritical.body"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package alertdefaults
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
"github.com/itworx/pulse/internal/metriccatalog"
|
||||
)
|
||||
|
||||
func TestPostgreSQLDefaultSeedIsIdempotent(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 6, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry, err := metriccatalog.DefaultRegistry()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repository := alert.Repository{Pool: pool, Registry: registry}
|
||||
first, err := Seed(ctx, repository, registry, "system-defaults")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := Seed(ctx, repository, registry, "system-defaults")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.Added != 0 || second.Existing != first.Added+first.Existing {
|
||||
t.Fatalf("seed reports first=%+v second=%+v", first, second)
|
||||
}
|
||||
rules, err := Load(registry)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, document := range rules {
|
||||
loaded, err := repository.Get(ctx, document.ID)
|
||||
if err != nil || loaded.ID != document.ID {
|
||||
t.Fatalf("default %s was not readable after restart-safe seed: rule=%+v err=%v", document.ID, loaded, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package alertdefaults
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/metriccatalog"
|
||||
)
|
||||
|
||||
type fakeStore struct {
|
||||
rules map[string]alert.Rule
|
||||
adds int
|
||||
}
|
||||
|
||||
func (store *fakeStore) Create(_ context.Context, _ string, document alert.Document, _ string) (alert.Rule, alert.Version, error) {
|
||||
if _, exists := store.rules[document.ID]; exists {
|
||||
return alert.Rule{}, alert.Version{}, alert.ErrConflict
|
||||
}
|
||||
store.adds++
|
||||
rule := alert.Rule{Document: document}
|
||||
store.rules[document.ID] = rule
|
||||
return rule, alert.Version{RuleID: document.ID, VersionNumber: 1}, nil
|
||||
}
|
||||
|
||||
func (store *fakeStore) Update(_ context.Context, id, _ string, expected int64, document alert.Document, _ string) (alert.Rule, error) {
|
||||
rule, exists := store.rules[id]
|
||||
if !exists {
|
||||
return alert.Rule{}, alert.ErrNotFound
|
||||
}
|
||||
if rule.Revision != expected {
|
||||
return alert.Rule{}, alert.ErrConflict
|
||||
}
|
||||
rule.Document = document
|
||||
rule.Revision++
|
||||
store.rules[id] = rule
|
||||
return rule, nil
|
||||
}
|
||||
|
||||
func (store *fakeStore) Get(_ context.Context, id string) (alert.Rule, error) {
|
||||
rule, exists := store.rules[id]
|
||||
if !exists {
|
||||
return alert.Rule{}, alert.ErrNotFound
|
||||
}
|
||||
return rule, nil
|
||||
}
|
||||
|
||||
func TestLoadValidatesImplementationOwnedDefaults(t *testing.T) {
|
||||
registry, err := metriccatalog.DefaultRegistry()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rules, err := Load(registry)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rules) < 4 {
|
||||
t.Fatalf("default rule count=%d, want at least 4", len(rules))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedIsIdempotentAndDoesNotOverwriteExistingRule(t *testing.T) {
|
||||
registry, err := metriccatalog.DefaultRegistry()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rules, err := Load(registry)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
custom := rules[0]
|
||||
custom.Name = "Aangepaste naam"
|
||||
store := &fakeStore{rules: map[string]alert.Rule{custom.ID: {Document: custom}}}
|
||||
first, err := Seed(context.Background(), store, registry, "system-defaults")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.Added != len(rules)-1 || first.Existing != 1 || store.adds != len(rules)-1 {
|
||||
t.Fatalf("first seed report=%+v adds=%d", first, store.adds)
|
||||
}
|
||||
second, err := Seed(context.Background(), store, registry, "system-defaults")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.Added != 0 || second.Existing != len(rules) {
|
||||
t.Fatalf("second seed report=%+v", second)
|
||||
}
|
||||
loaded, err := store.Get(context.Background(), custom.ID)
|
||||
if err != nil || loaded.Name != "Aangepaste naam" {
|
||||
t.Fatalf("custom rule was overwritten: rule=%+v err=%v", loaded, err)
|
||||
}
|
||||
}
|
||||
func TestDefaultRulesReduceStormNoise(t *testing.T) {
|
||||
registry, err := metriccatalog.DefaultRegistry()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rules, err := Load(registry)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var serviceRule alert.Document
|
||||
for _, rule := range rules {
|
||||
if rule.Name == "Service is niet bereikbaar" {
|
||||
serviceRule = rule
|
||||
}
|
||||
}
|
||||
if serviceRule.ID == "" {
|
||||
t.Fatal("service default rule is missing")
|
||||
}
|
||||
at := time.Date(2026, time.January, 4, 12, 0, 0, 0, time.UTC)
|
||||
signals := make([]alert.Signal, 0, 5)
|
||||
for i := 0; i < 5; i++ {
|
||||
signals = append(signals, alert.Signal{InstanceID: "service-" + string(rune('a'+i)), RuleID: serviceRule.ID, RuleVersionID: "version-1", EntityID: "entity-" + string(rune('a'+i)), Severity: serviceRule.Severity, State: alert.StateFiring, Fingerprint: "fingerprint-" + string(rune('a'+i)), EvaluationKey: "evaluation-1", ObservedAt: at, Labels: map[string]string{"host": "pulse", "application": "media", "service": "svc-" + string(rune('a'+i))}, GroupBy: serviceRule.GroupBy, SuppressWhen: serviceRule.SuppressWhen})
|
||||
}
|
||||
deduplicated, err := alert.DeduplicateSignals(append(signals, signals[0]))
|
||||
if err != nil || len(deduplicated) != len(signals) {
|
||||
t.Fatalf("deduplicated=%d err=%v", len(deduplicated), err)
|
||||
}
|
||||
groups, err := alert.GroupSignals(deduplicated)
|
||||
if err != nil || len(groups) != len(signals) {
|
||||
t.Fatalf("groups=%+v err=%v", groups, err)
|
||||
}
|
||||
decision, err := alert.EvaluateSuppression(signals[0], []alert.Cause{{Key: "dns.unavailable", State: alert.StateFiring, Confirmed: true, ObservedAt: at}})
|
||||
if err != nil || !decision.Suppressed || decision.CauseKey != "dns.unavailable" {
|
||||
t.Fatalf("suppression=%+v err=%v", decision, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedRefreshesUntouchedDefaultButKeepsOperatorEdits(t *testing.T) {
|
||||
registry, err := metriccatalog.DefaultRegistry()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rules, err := Load(registry)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stale := rules[0]
|
||||
stale.PendingSeconds = stale.PendingSeconds + 60
|
||||
edited := rules[1]
|
||||
edited.Name = "Door operator aangepast"
|
||||
store := &fakeStore{rules: map[string]alert.Rule{
|
||||
stale.ID: {Document: stale, Revision: 1}, // seeded, never edited -> refreshed
|
||||
edited.ID: {Document: edited, Revision: 2}, // edited by an operator -> untouched
|
||||
}}
|
||||
report, err := Seed(context.Background(), store, registry, "system-defaults")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Refreshed != 1 || report.Existing != 1 || report.Added != len(rules)-2 {
|
||||
t.Fatalf("report=%+v", report)
|
||||
}
|
||||
refreshed, _ := store.Get(context.Background(), stale.ID)
|
||||
if refreshed.PendingSeconds != rules[0].PendingSeconds || refreshed.Revision != 2 {
|
||||
t.Fatalf("stale default not refreshed: %+v", refreshed)
|
||||
}
|
||||
kept, _ := store.Get(context.Background(), edited.ID)
|
||||
if kept.Name != "Door operator aangepast" || kept.Revision != 2 {
|
||||
t.Fatalf("operator edit overwritten: %+v", kept)
|
||||
}
|
||||
again, err := Seed(context.Background(), store, registry, "system-defaults")
|
||||
if err != nil || again.Refreshed != 0 || again.Added != 0 {
|
||||
t.Fatalf("second seed not idempotent: %+v err=%v", again, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package alertopsapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/audit"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/correlation"
|
||||
"github.com/itworx/pulse/internal/problem"
|
||||
)
|
||||
|
||||
type Store interface {
|
||||
alert.AlertReader
|
||||
AcknowledgeRevision(context.Context, string, string, string, time.Time, int64) (alert.Instance, alert.Occurrence, bool, error)
|
||||
Unacknowledge(context.Context, string, string, string, time.Time, int64) (alert.Instance, alert.Occurrence, bool, error)
|
||||
}
|
||||
|
||||
type Handler struct {
|
||||
Store Store
|
||||
Audit audit.Store
|
||||
}
|
||||
|
||||
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
principal, ok := auth.PrincipalFromContext(r.Context())
|
||||
if !ok {
|
||||
fail(w, r, http.StatusUnauthorized, "UNAUTHORIZED", "Authentication required.")
|
||||
return
|
||||
}
|
||||
path := strings.Trim(strings.TrimPrefix(r.URL.Path, "/api/v1/alerts"), "/")
|
||||
if path == "" && r.Method == http.MethodGet {
|
||||
h.list(w, r)
|
||||
return
|
||||
}
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) == 1 && parts[0] != "" && r.Method == http.MethodGet {
|
||||
h.get(w, r, parts[0])
|
||||
return
|
||||
}
|
||||
if len(parts) == 2 && (parts[1] == "acknowledge" || parts[1] == "unacknowledge") && r.Method == http.MethodPost {
|
||||
if !auth.Allows(principal.Role, auth.PermissionOperate) {
|
||||
fail(w, r, http.StatusForbidden, "FORBIDDEN", "Alert operations are not allowed for this role.")
|
||||
return
|
||||
}
|
||||
h.operate(w, r, parts[0], principal.Subject, parts[1] == "acknowledge")
|
||||
return
|
||||
}
|
||||
fail(w, r, http.StatusNotFound, "NOT_FOUND", "Alert route not found.")
|
||||
}
|
||||
|
||||
func (h Handler) list(w http.ResponseWriter, r *http.Request) {
|
||||
limit := 100
|
||||
if value := r.URL.Query().Get("limit"); value != "" {
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed < 1 || parsed > 100 {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_LIMIT", "The alert limit must be between 1 and 100.")
|
||||
return
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
items, err := h.Store.ListAlerts(r.Context(), limit, r.URL.Query().Get("state"))
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err)
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"items": items})
|
||||
}
|
||||
|
||||
func (h Handler) get(w http.ResponseWriter, r *http.Request, id string) {
|
||||
limit := 100
|
||||
if value := r.URL.Query().Get("occurrenceLimit"); value != "" {
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed < 1 || parsed > 500 {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_LIMIT", "The occurrence limit must be between 1 and 500.")
|
||||
return
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
item, err := h.Store.GetAlert(r.Context(), id, limit)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err)
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"alert": item})
|
||||
}
|
||||
|
||||
func (h Handler) operate(w http.ResponseWriter, r *http.Request, id, actor string, acknowledge bool) {
|
||||
expected, err := revision(r)
|
||||
if err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_REVISION", "A valid If-Match or revision value is required.")
|
||||
return
|
||||
}
|
||||
evaluationKey := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||
if evaluationKey == "" {
|
||||
var request struct {
|
||||
EvaluationKey string `json:"evaluationKey"`
|
||||
}
|
||||
if err := decode(r, &request); err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_OPERATION", "The alert operation request is invalid.")
|
||||
return
|
||||
}
|
||||
evaluationKey = strings.TrimSpace(request.EvaluationKey)
|
||||
}
|
||||
if evaluationKey == "" || len(evaluationKey) > 160 {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_OPERATION", "A bounded evaluation key or Idempotency-Key is required.")
|
||||
return
|
||||
}
|
||||
var instance alert.Instance
|
||||
var occurrence alert.Occurrence
|
||||
var duplicate bool
|
||||
if acknowledge {
|
||||
instance, occurrence, duplicate, err = h.Store.AcknowledgeRevision(r.Context(), id, actor, evaluationKey, time.Now().UTC(), expected)
|
||||
} else {
|
||||
instance, occurrence, duplicate, err = h.Store.Unacknowledge(r.Context(), id, actor, evaluationKey, time.Now().UTC(), expected)
|
||||
}
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err)
|
||||
return
|
||||
}
|
||||
action := "alert.unacknowledge"
|
||||
if acknowledge {
|
||||
action = "alert.acknowledge"
|
||||
}
|
||||
result := "success"
|
||||
if duplicate {
|
||||
result = "idempotent"
|
||||
}
|
||||
if h.Audit != nil {
|
||||
if err := h.Audit.Append(r.Context(), audit.Event{Actor: actor, Action: action, ResourceType: "alert_instance", ResourceID: id, Result: result, CorrelationID: correlation.FromContext(r.Context()), After: map[string]any{"state": instance.State, "revision": instance.Revision, "duplicate": duplicate}}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"instance": instance, "occurrence": occurrence, "duplicate": duplicate})
|
||||
}
|
||||
|
||||
func (h Handler) repositoryFailure(w http.ResponseWriter, r *http.Request, err error) {
|
||||
switch {
|
||||
case errors.Is(err, alert.ErrInvalidObservation):
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_ALERT_OPERATION", "The alert operation is invalid.")
|
||||
case errors.Is(err, alert.ErrRevisionConflict):
|
||||
fail(w, r, http.StatusConflict, "REVISION_CONFLICT", "The alert changed before this operation was applied.")
|
||||
case errors.Is(err, alert.ErrStateConflict):
|
||||
fail(w, r, http.StatusConflict, "STATE_CONFLICT", "The alert is not in a state that supports this operation.")
|
||||
case errors.Is(err, alert.ErrInstanceNotFound):
|
||||
fail(w, r, http.StatusNotFound, "NOT_FOUND", "The alert instance was not found.")
|
||||
case errors.Is(err, alert.ErrUnavailable):
|
||||
fail(w, r, http.StatusServiceUnavailable, "DATABASE_UNAVAILABLE", "Alerts are unavailable.")
|
||||
default:
|
||||
fail(w, r, http.StatusInternalServerError, "ALERT_REQUEST_FAILED", "The alert request failed.")
|
||||
}
|
||||
}
|
||||
func revision(r *http.Request) (int64, error) {
|
||||
value := r.Header.Get("If-Match")
|
||||
if value == "" {
|
||||
value = r.URL.Query().Get("revision")
|
||||
}
|
||||
value = strings.Trim(value, "\"")
|
||||
if value == "" {
|
||||
return 0, errors.New("revision required")
|
||||
}
|
||||
parsed, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || parsed < 1 {
|
||||
return 0, errors.New("invalid revision")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
func decode(r *http.Request, target any) error {
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, 2<<20+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer r.Body.Close()
|
||||
if len(body) > 2<<20 {
|
||||
return errors.New("request too large")
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(string(body)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(target); err != nil {
|
||||
return err
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return errors.New("multiple JSON values")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func fail(w http.ResponseWriter, r *http.Request, status int, code, detail string) {
|
||||
problem.Write(w, r, status, code, http.StatusText(status), detail, nil)
|
||||
}
|
||||
func write(w http.ResponseWriter, status int, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package alertopsapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/audit"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
)
|
||||
|
||||
type memoryStore struct {
|
||||
mu sync.Mutex
|
||||
items map[string]alert.Alert
|
||||
occurrences map[string]alert.Occurrence
|
||||
}
|
||||
|
||||
func newMemoryStore() *memoryStore {
|
||||
return &memoryStore{items: map[string]alert.Alert{}, occurrences: map[string]alert.Occurrence{}}
|
||||
}
|
||||
func (s *memoryStore) ListAlerts(_ context.Context, _ int, state string) ([]alert.Alert, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
result := make([]alert.Alert, 0)
|
||||
for _, item := range s.items {
|
||||
if state == "" || string(item.State) == state {
|
||||
result = append(result, item)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
func (s *memoryStore) GetAlert(_ context.Context, id string, _ int) (alert.Alert, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
item, ok := s.items[id]
|
||||
if !ok {
|
||||
return alert.Alert{}, alert.ErrInstanceNotFound
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
func (s *memoryStore) AcknowledgeRevision(_ context.Context, id, actor, key string, at time.Time, expected int64) (alert.Instance, alert.Occurrence, bool, error) {
|
||||
return s.mutate(id, actor, key, at, expected, true)
|
||||
}
|
||||
func (s *memoryStore) Unacknowledge(_ context.Context, id, actor, key string, at time.Time, expected int64) (alert.Instance, alert.Occurrence, bool, error) {
|
||||
return s.mutate(id, actor, key, at, expected, false)
|
||||
}
|
||||
func (s *memoryStore) mutate(id, actor, key string, at time.Time, expected int64, acknowledge bool) (alert.Instance, alert.Occurrence, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
item, ok := s.items[id]
|
||||
if !ok {
|
||||
return alert.Instance{}, alert.Occurrence{}, false, alert.ErrInstanceNotFound
|
||||
}
|
||||
if occurrence, ok := s.occurrences[key]; ok {
|
||||
return item.Instance, occurrence, true, nil
|
||||
}
|
||||
if item.Revision != expected {
|
||||
return alert.Instance{}, alert.Occurrence{}, false, alert.ErrRevisionConflict
|
||||
}
|
||||
if acknowledge {
|
||||
if item.State != alert.StateFiring && item.State != alert.StatePending {
|
||||
return alert.Instance{}, alert.Occurrence{}, false, alert.ErrStateConflict
|
||||
}
|
||||
item.State, item.RetainedState, item.AcknowledgedBy, item.AcknowledgedAt, item.Reason = alert.StateAcknowledged, alert.StateAcknowledged, actor, &at, "acknowledged"
|
||||
} else {
|
||||
if item.State != alert.StateAcknowledged {
|
||||
return alert.Instance{}, alert.Occurrence{}, false, alert.ErrStateConflict
|
||||
}
|
||||
item.State, item.RetainedState, item.AcknowledgedBy, item.AcknowledgedAt, item.Reason = alert.StateFiring, alert.StateFiring, "", nil, "unacknowledged"
|
||||
}
|
||||
item.Revision++
|
||||
occurrence := alert.Occurrence{ID: key, InstanceID: id, EvaluationKey: key, EventType: "acknowledge", From: alert.StateFiring, To: item.State, ObservedAt: at, Reason: item.Reason}
|
||||
if !acknowledge {
|
||||
occurrence.EventType = "unacknowledge"
|
||||
occurrence.From = alert.StateAcknowledged
|
||||
}
|
||||
s.items[id] = item
|
||||
s.occurrences[key] = occurrence
|
||||
return item.Instance, occurrence, false, nil
|
||||
}
|
||||
|
||||
func TestHandlerRoleMatrixIdempotenceAndAudit(t *testing.T) {
|
||||
store := newMemoryStore()
|
||||
store.items["instance-1"] = alert.Alert{Instance: alert.Instance{ID: "instance-1", State: alert.StateFiring, RetainedState: alert.StateFiring, Revision: 1, LastValue: 90, SourceHealth: map[string]any{}}, RuleName: "CPU", Severity: alert.SeverityCritical}
|
||||
auditStore := &audit.MemoryStore{}
|
||||
handler := Handler{Store: store, Audit: auditStore}
|
||||
viewer := requestWithPrincipal(http.MethodPost, "/api/v1/alerts/instance-1/acknowledge?revision=1", map[string]string{"evaluationKey": "ack-1"}, auth.RoleViewer)
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, viewer)
|
||||
if response.Code != http.StatusForbidden {
|
||||
t.Fatalf("viewer status = %d", response.Code)
|
||||
}
|
||||
operator := requestWithPrincipal(http.MethodPost, "/api/v1/alerts/instance-1/acknowledge?revision=1", map[string]string{"evaluationKey": "ack-1"}, auth.RoleOperator)
|
||||
response = httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, operator)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("ack status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
retry := requestWithPrincipal(http.MethodPost, "/api/v1/alerts/instance-1/acknowledge?revision=1", map[string]string{"evaluationKey": "ack-1"}, auth.RoleOperator)
|
||||
response = httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, retry)
|
||||
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"duplicate":true`) {
|
||||
t.Fatalf("duplicate ack = %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
unack := requestWithPrincipal(http.MethodPost, "/api/v1/alerts/instance-1/unacknowledge?revision=2", map[string]string{"evaluationKey": "unack-1"}, auth.RoleOperator)
|
||||
response = httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, unack)
|
||||
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"state":"firing"`) {
|
||||
t.Fatalf("unack = %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
list := requestWithPrincipal(http.MethodGet, "/api/v1/alerts?state=firing", nil, auth.RoleViewer)
|
||||
response = httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, list)
|
||||
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"ruleName":"CPU"`) {
|
||||
t.Fatalf("list = %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
if len(auditStore.Events) != 3 || auditStore.Events[1].Result != "idempotent" {
|
||||
t.Fatalf("audit = %#v", auditStore.Events)
|
||||
}
|
||||
}
|
||||
|
||||
func requestWithPrincipal(method, path string, body any, role auth.Role) *http.Request {
|
||||
encoded := ""
|
||||
if body != nil {
|
||||
value, _ := json.Marshal(body)
|
||||
encoded = string(value)
|
||||
}
|
||||
request := httptest.NewRequest(method, path, strings.NewReader(encoded)).WithContext(auth.WithPrincipal(context.Background(), auth.Principal{Subject: "operator-1", Role: role}))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
return request
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package alertworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
)
|
||||
|
||||
type memoryLease struct {
|
||||
lease Lease
|
||||
status string
|
||||
leaseUntil time.Time
|
||||
}
|
||||
|
||||
type MemoryLeaseStore struct {
|
||||
mu sync.Mutex
|
||||
records map[string]memoryLease
|
||||
}
|
||||
|
||||
func NewMemoryLeaseStore() *MemoryLeaseStore {
|
||||
return &MemoryLeaseStore{records: make(map[string]memoryLease)}
|
||||
}
|
||||
|
||||
func (s *MemoryLeaseStore) Acquire(_ context.Context, jobType, jobKey string, scheduledAt time.Time, owner string, now time.Time, ttl time.Duration) (Lease, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.records == nil {
|
||||
s.records = make(map[string]memoryLease)
|
||||
}
|
||||
key := leaseKey(jobType, jobKey, scheduledAt)
|
||||
if record, ok := s.records[key]; ok {
|
||||
if record.status != "running" || record.leaseUntil.After(now) {
|
||||
return Lease{}, false, nil
|
||||
}
|
||||
}
|
||||
lease := Lease{ID: alert.NewID(), JobKey: jobKey, ScheduledAt: scheduledAt.UTC(), Owner: owner}
|
||||
s.records[key] = memoryLease{lease: lease, status: "running", leaseUntil: now.Add(ttl)}
|
||||
return lease, true, nil
|
||||
}
|
||||
|
||||
func (s *MemoryLeaseStore) Complete(_ context.Context, lease Lease, status, _ string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
key := leaseKey(jobType, lease.JobKey, lease.ScheduledAt)
|
||||
record, ok := s.records[key]
|
||||
if !ok || record.lease.ID != lease.ID || record.lease.Owner != lease.Owner {
|
||||
return ErrLeaseLost
|
||||
}
|
||||
record.status = status
|
||||
record.leaseUntil = time.Time{}
|
||||
s.records[key] = record
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemoryLeaseStore) Status(jobKey string, scheduledAt time.Time) string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
record, ok := s.records[leaseKey(jobType, jobKey, scheduledAt)]
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return record.status
|
||||
}
|
||||
|
||||
func leaseKey(jobType, jobKey string, scheduledAt time.Time) string {
|
||||
return jobType + "|" + jobKey + "|" + scheduledAt.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package alertworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type PostgresLeaseStore struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func (s PostgresLeaseStore) Acquire(ctx context.Context, jobType, jobKey string, scheduledAt time.Time, owner string, now time.Time, ttl time.Duration) (Lease, bool, error) {
|
||||
if s.Pool == nil {
|
||||
return Lease{}, false, ErrInvalidConfig
|
||||
}
|
||||
tx, err := s.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Lease{}, false, fmt.Errorf("begin evaluator lease: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
leaseID := alert.NewID()
|
||||
leaseUntil := now.Add(ttl)
|
||||
var insertedID string
|
||||
err = tx.QueryRow(ctx, `INSERT INTO job_runs (id,job_type,job_key,scheduled_at,started_at,status,lease_owner,lease_until) VALUES ($1,$2,$3,$4,$5,'running',$6,$7) ON CONFLICT (job_type,job_key,scheduled_at) DO NOTHING RETURNING id`, leaseID, jobType, jobKey, scheduledAt.UTC(), now.UTC(), owner, leaseUntil.UTC()).Scan(&insertedID)
|
||||
if err == nil {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Lease{}, false, fmt.Errorf("commit evaluator lease: %w", err)
|
||||
}
|
||||
return Lease{ID: insertedID, JobKey: jobKey, ScheduledAt: scheduledAt.UTC(), Owner: owner}, true, nil
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Lease{}, false, fmt.Errorf("insert evaluator lease: %w", err)
|
||||
}
|
||||
var status string
|
||||
var existingUntil *time.Time
|
||||
if err := tx.QueryRow(ctx, `SELECT status,lease_until FROM job_runs WHERE job_type=$1 AND job_key=$2 AND scheduled_at=$3 FOR UPDATE`, jobType, jobKey, scheduledAt.UTC()).Scan(&status, &existingUntil); err != nil {
|
||||
return Lease{}, false, fmt.Errorf("read evaluator lease: %w", err)
|
||||
}
|
||||
if status != "running" || (existingUntil != nil && existingUntil.After(now)) {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Lease{}, false, err
|
||||
}
|
||||
return Lease{}, false, nil
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `UPDATE job_runs SET status='running',started_at=$1,completed_at=NULL,error_code=NULL,lease_owner=$2,lease_until=$3 WHERE job_type=$4 AND job_key=$5 AND scheduled_at=$6 AND status='running' AND (lease_until IS NULL OR lease_until <= $7)`, now.UTC(), owner, leaseUntil.UTC(), jobType, jobKey, scheduledAt.UTC(), now.UTC())
|
||||
if err != nil {
|
||||
return Lease{}, false, fmt.Errorf("renew evaluator lease: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Lease{}, false, err
|
||||
}
|
||||
return Lease{}, false, nil
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Lease{}, false, fmt.Errorf("commit evaluator lease renewal: %w", err)
|
||||
}
|
||||
return Lease{ID: leaseID, JobKey: jobKey, ScheduledAt: scheduledAt.UTC(), Owner: owner}, true, nil
|
||||
}
|
||||
|
||||
func (s PostgresLeaseStore) Complete(ctx context.Context, lease Lease, status, errorCode string) error {
|
||||
if s.Pool == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if status != "completed" && status != "failed" && status != "canceled" {
|
||||
return errors.New("invalid evaluator job status")
|
||||
}
|
||||
errorCode = strings.TrimSpace(errorCode)
|
||||
if len(errorCode) > 160 {
|
||||
errorCode = errorCode[:160]
|
||||
}
|
||||
counts, _ := json.Marshal(map[string]string{"status": status})
|
||||
tag, err := s.Pool.Exec(ctx, `UPDATE job_runs SET status=$1,completed_at=now(),counts=$2::jsonb,error_code=NULLIF($3,'') ,lease_owner=NULL,lease_until=NULL WHERE job_type=$4 AND job_key=$5 AND scheduled_at=$6 AND lease_owner=$7 AND status='running'`, status, counts, errorCode, jobType, lease.JobKey, lease.ScheduledAt.UTC(), lease.Owner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete evaluator job: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return ErrLeaseLost
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package alertworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
)
|
||||
|
||||
func TestPostgreSQLLeaseStoreCoordinatesAndReclaimsExpiredLease(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 4, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := PostgresLeaseStore{Pool: pool}
|
||||
now := time.Date(2026, 8, 2, 5, 0, 0, 0, time.UTC)
|
||||
key := alert.NewID()
|
||||
scheduled := now
|
||||
first, acquired, err := store.Acquire(ctx, jobType, key, scheduled, "worker-a", now, time.Minute)
|
||||
if err != nil || !acquired {
|
||||
t.Fatalf("first acquire lease=%#v acquired=%v err=%v", first, acquired, err)
|
||||
}
|
||||
if _, acquired, err := store.Acquire(ctx, jobType, key, scheduled, "worker-b", now, time.Minute); err != nil || acquired {
|
||||
t.Fatalf("duplicate acquire acquired=%v err=%v", acquired, err)
|
||||
}
|
||||
if err := store.Complete(ctx, first, "completed", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, acquired, err := store.Acquire(ctx, jobType, key, scheduled, "worker-b", now, time.Minute); err != nil || acquired {
|
||||
t.Fatalf("completed acquire acquired=%v err=%v", acquired, err)
|
||||
}
|
||||
expiredKey := alert.NewID()
|
||||
expired, acquired, err := store.Acquire(ctx, jobType, expiredKey, scheduled, "worker-a", now, time.Second)
|
||||
if err != nil || !acquired {
|
||||
t.Fatalf("expired first acquire=%#v acquired=%v err=%v", expired, acquired, err)
|
||||
}
|
||||
reclaimed, acquired, err := store.Acquire(ctx, jobType, expiredKey, scheduled, "worker-b", now.Add(2*time.Second), time.Minute)
|
||||
if err != nil || !acquired || reclaimed.JobKey != expiredKey {
|
||||
t.Fatalf("reclaim lease=%#v acquired=%v err=%v", reclaimed, acquired, err)
|
||||
}
|
||||
if err := store.Complete(ctx, reclaimed, "failed", "timeout"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package alertworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
)
|
||||
|
||||
const jobType = "alert-evaluation"
|
||||
|
||||
var (
|
||||
ErrInvalidConfig = errors.New("alert evaluator configuration is invalid")
|
||||
ErrLeaseLost = errors.New("alert evaluator lease was lost")
|
||||
)
|
||||
|
||||
type RuleSource interface {
|
||||
ListEnabled(context.Context, int) ([]alert.Rule, error)
|
||||
}
|
||||
|
||||
type Evaluator interface {
|
||||
Evaluate(context.Context, alert.Rule) error
|
||||
}
|
||||
|
||||
type EvaluateFunc func(context.Context, alert.Rule) error
|
||||
|
||||
func (f EvaluateFunc) Evaluate(ctx context.Context, rule alert.Rule) error {
|
||||
return f(ctx, rule)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
MaxConcurrent int
|
||||
MaxBatch int
|
||||
AttemptTimeout time.Duration
|
||||
LeaseTTL time.Duration
|
||||
Owner string
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (c Config) validate() error {
|
||||
if c.MaxConcurrent < 1 || c.MaxConcurrent > 64 || c.MaxBatch < 1 || c.MaxBatch > 100 || c.MaxConcurrent > c.MaxBatch {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if c.AttemptTimeout < time.Millisecond || c.AttemptTimeout > 2*time.Minute || c.LeaseTTL < c.AttemptTimeout || c.LeaseTTL > 10*time.Minute {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if c.Owner == "" || len(c.Owner) > 120 || c.Now == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Lease struct {
|
||||
ID string
|
||||
JobKey string
|
||||
ScheduledAt time.Time
|
||||
Owner string
|
||||
}
|
||||
|
||||
type LeaseStore interface {
|
||||
Acquire(context.Context, string, string, time.Time, string, time.Time, time.Duration) (Lease, bool, error)
|
||||
Complete(context.Context, Lease, string, string) error
|
||||
}
|
||||
|
||||
type JobResult struct {
|
||||
RuleID string
|
||||
JobKey string
|
||||
ScheduledAt time.Time
|
||||
StartedAt time.Time
|
||||
CompletedAt time.Time
|
||||
Status string
|
||||
ErrorCode string
|
||||
}
|
||||
|
||||
type RunReport struct {
|
||||
Scheduled int
|
||||
Started int
|
||||
Completed int
|
||||
Skipped int
|
||||
Failed int
|
||||
Canceled int
|
||||
Jobs []JobResult
|
||||
}
|
||||
|
||||
type Metrics struct {
|
||||
RunsStarted uint64
|
||||
RunsCompleted uint64
|
||||
JobsStarted uint64
|
||||
JobsCompleted uint64
|
||||
JobsFailed uint64
|
||||
JobsSkipped uint64
|
||||
LastRunDuration time.Duration
|
||||
}
|
||||
|
||||
type Worker struct {
|
||||
Source RuleSource
|
||||
Store LeaseStore
|
||||
Evaluator Evaluator
|
||||
Config Config
|
||||
mu sync.Mutex
|
||||
metrics Metrics
|
||||
}
|
||||
|
||||
func New(source RuleSource, store LeaseStore, evaluator Evaluator, config Config) (Worker, error) {
|
||||
if source == nil || store == nil || evaluator == nil {
|
||||
return Worker{}, ErrInvalidConfig
|
||||
}
|
||||
if config.Owner == "" {
|
||||
config.Owner = alert.NewID()
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
if err := config.validate(); err != nil {
|
||||
return Worker{}, err
|
||||
}
|
||||
return Worker{Source: source, Store: store, Evaluator: evaluator, Config: config}, nil
|
||||
}
|
||||
|
||||
func (w *Worker) RunOnce(ctx context.Context) (RunReport, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return RunReport{}, err
|
||||
}
|
||||
start := w.Config.Now().UTC()
|
||||
w.mu.Lock()
|
||||
w.metrics.RunsStarted++
|
||||
w.mu.Unlock()
|
||||
rules, err := w.Source.ListEnabled(ctx, w.Config.MaxBatch)
|
||||
if err != nil {
|
||||
return RunReport{}, fmt.Errorf("list enabled alert rules: %w", err)
|
||||
}
|
||||
report := RunReport{Scheduled: len(rules), Jobs: make([]JobResult, 0, len(rules))}
|
||||
type job struct {
|
||||
rule alert.Rule
|
||||
lease Lease
|
||||
}
|
||||
jobs := make([]job, 0, len(rules))
|
||||
for _, rule := range rules {
|
||||
interval := time.Duration(rule.EvaluationIntervalSeconds) * time.Second
|
||||
scheduledAt := start.Truncate(interval)
|
||||
key := rule.ID
|
||||
lease, acquired, err := w.Store.Acquire(ctx, jobType, key, scheduledAt, w.Config.Owner, start, w.Config.LeaseTTL)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("acquire alert evaluation lease: %w", err)
|
||||
}
|
||||
if !acquired {
|
||||
report.Skipped++
|
||||
w.mu.Lock()
|
||||
w.metrics.JobsSkipped++
|
||||
w.mu.Unlock()
|
||||
report.Jobs = append(report.Jobs, JobResult{RuleID: rule.ID, JobKey: key, ScheduledAt: scheduledAt, Status: "skipped"})
|
||||
continue
|
||||
}
|
||||
jobs = append(jobs, job{rule: rule, lease: lease})
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
w.finishRun(start)
|
||||
return report, nil
|
||||
}
|
||||
sem := make(chan struct{}, w.Config.MaxConcurrent)
|
||||
var wait sync.WaitGroup
|
||||
var reportMu sync.Mutex
|
||||
for _, item := range jobs {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
report.Canceled++
|
||||
report.Jobs = append(report.Jobs, JobResult{RuleID: item.rule.ID, JobKey: item.lease.JobKey, ScheduledAt: item.lease.ScheduledAt, Status: "canceled", ErrorCode: "shutdown"})
|
||||
_ = w.completeLease(item.lease, "canceled", "shutdown")
|
||||
case sem <- struct{}{}:
|
||||
wait.Add(1)
|
||||
report.Started++
|
||||
w.mu.Lock()
|
||||
w.metrics.JobsStarted++
|
||||
w.mu.Unlock()
|
||||
go func(item job) {
|
||||
defer wait.Done()
|
||||
defer func() { <-sem }()
|
||||
jobResult := w.evaluate(ctx, item.rule, item.lease)
|
||||
reportMu.Lock()
|
||||
report.Jobs = append(report.Jobs, jobResult)
|
||||
switch jobResult.Status {
|
||||
case "completed":
|
||||
report.Completed++
|
||||
case "failed":
|
||||
report.Failed++
|
||||
case "canceled":
|
||||
report.Canceled++
|
||||
}
|
||||
reportMu.Unlock()
|
||||
}(item)
|
||||
}
|
||||
}
|
||||
wait.Wait()
|
||||
sort.Slice(report.Jobs, func(i, j int) bool {
|
||||
if report.Jobs[i].ScheduledAt.Equal(report.Jobs[j].ScheduledAt) {
|
||||
return report.Jobs[i].RuleID < report.Jobs[j].RuleID
|
||||
}
|
||||
return report.Jobs[i].ScheduledAt.Before(report.Jobs[j].ScheduledAt)
|
||||
})
|
||||
w.finishRun(start)
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (w *Worker) RunLoop(ctx context.Context, interval time.Duration) error {
|
||||
if interval < time.Second || interval > time.Hour {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
for {
|
||||
_, err := w.RunOnce(ctx)
|
||||
if err != nil && !errors.Is(err, context.Canceled) {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(interval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) evaluate(ctx context.Context, rule alert.Rule, lease Lease) JobResult {
|
||||
started := w.Config.Now().UTC()
|
||||
jobResult := JobResult{RuleID: rule.ID, JobKey: lease.JobKey, ScheduledAt: lease.ScheduledAt, StartedAt: started}
|
||||
attemptCtx, cancel := context.WithTimeout(ctx, w.Config.AttemptTimeout)
|
||||
err := w.Evaluator.Evaluate(attemptCtx, rule)
|
||||
cancel()
|
||||
status, errorCode := "completed", ""
|
||||
if err != nil {
|
||||
status = "failed"
|
||||
errorCode = "evaluation_failed"
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(attemptCtx.Err(), context.DeadlineExceeded) {
|
||||
errorCode = "timeout"
|
||||
} else if errors.Is(err, context.Canceled) || errors.Is(ctx.Err(), context.Canceled) {
|
||||
status, errorCode = "canceled", "shutdown"
|
||||
}
|
||||
}
|
||||
jobResult.CompletedAt = w.Config.Now().UTC()
|
||||
jobResult.Status = status
|
||||
jobResult.ErrorCode = errorCode
|
||||
_ = w.completeLease(lease, status, errorCode)
|
||||
w.mu.Lock()
|
||||
switch status {
|
||||
case "completed":
|
||||
w.metrics.JobsCompleted++
|
||||
case "failed", "canceled":
|
||||
w.metrics.JobsFailed++
|
||||
}
|
||||
w.mu.Unlock()
|
||||
return jobResult
|
||||
}
|
||||
|
||||
func (w *Worker) completeLease(lease Lease, status, errorCode string) error {
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(context.Background()), 2*time.Second)
|
||||
defer cancel()
|
||||
return w.Store.Complete(ctx, lease, status, errorCode)
|
||||
}
|
||||
|
||||
func (w *Worker) finishRun(start time.Time) {
|
||||
w.mu.Lock()
|
||||
w.metrics.RunsCompleted++
|
||||
w.metrics.LastRunDuration = w.Config.Now().UTC().Sub(start)
|
||||
w.mu.Unlock()
|
||||
}
|
||||
|
||||
func (w *Worker) Metrics() Metrics {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.metrics
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package alertworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
)
|
||||
|
||||
type fakeSource struct {
|
||||
rules []alert.Rule
|
||||
}
|
||||
|
||||
func (s fakeSource) ListEnabled(_ context.Context, limit int) ([]alert.Rule, error) {
|
||||
if limit > len(s.rules) {
|
||||
limit = len(s.rules)
|
||||
}
|
||||
return s.rules[:limit], nil
|
||||
}
|
||||
|
||||
type trackingEvaluator struct {
|
||||
active atomic.Int32
|
||||
max atomic.Int32
|
||||
calls atomic.Int32
|
||||
wait time.Duration
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *trackingEvaluator) Evaluate(ctx context.Context, _ alert.Rule) error {
|
||||
e.calls.Add(1)
|
||||
active := e.active.Add(1)
|
||||
for {
|
||||
current := e.max.Load()
|
||||
if active <= current || e.max.CompareAndSwap(current, active) {
|
||||
break
|
||||
}
|
||||
}
|
||||
defer e.active.Add(-1)
|
||||
if e.wait > 0 {
|
||||
timer := time.NewTimer(e.wait)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return e.err
|
||||
}
|
||||
|
||||
func testRule(id string) alert.Rule {
|
||||
return alert.Rule{Document: alert.Document{ID: id, EvaluationIntervalSeconds: 30}}
|
||||
}
|
||||
|
||||
func testWorker(t *testing.T, source RuleSource, store LeaseStore, evaluator Evaluator, owner string, now time.Time, maxConcurrent int) *Worker {
|
||||
t.Helper()
|
||||
worker, err := New(source, store, evaluator, Config{MaxConcurrent: maxConcurrent, MaxBatch: 100, AttemptTimeout: 100 * time.Millisecond, LeaseTTL: time.Second, Owner: owner, Now: func() time.Time { return now }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &worker
|
||||
}
|
||||
|
||||
func TestDuplicateWorkersDoNotEvaluateSameSlotTwice(t *testing.T) {
|
||||
source := fakeSource{rules: []alert.Rule{testRule("rule-1")}}
|
||||
store := NewMemoryLeaseStore()
|
||||
evaluator := &trackingEvaluator{wait: 20 * time.Millisecond}
|
||||
now := time.Date(2026, 8, 2, 5, 0, 10, 0, time.UTC)
|
||||
first := testWorker(t, source, store, evaluator, "worker-1", now, 1)
|
||||
second := testWorker(t, source, store, evaluator, "worker-2", now, 1)
|
||||
var reports [2]RunReport
|
||||
var wait sync.WaitGroup
|
||||
wait.Add(2)
|
||||
go func() { defer wait.Done(); reports[0], _ = first.RunOnce(context.Background()) }()
|
||||
go func() { defer wait.Done(); reports[1], _ = second.RunOnce(context.Background()) }()
|
||||
wait.Wait()
|
||||
if evaluator.calls.Load() != 1 {
|
||||
t.Fatalf("evaluation calls = %d, want 1", evaluator.calls.Load())
|
||||
}
|
||||
if reports[0].Skipped+reports[1].Skipped != 1 {
|
||||
t.Fatalf("skips = %d, want 1", reports[0].Skipped+reports[1].Skipped)
|
||||
}
|
||||
third, err := first.RunOnce(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if third.Skipped != 1 || evaluator.calls.Load() != 1 {
|
||||
t.Fatalf("repeat report=%#v calls=%d", third, evaluator.calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeoutIsVisibleInJobResult(t *testing.T) {
|
||||
source := fakeSource{rules: []alert.Rule{testRule("rule-timeout")}}
|
||||
store := NewMemoryLeaseStore()
|
||||
evaluator := &trackingEvaluator{wait: time.Second}
|
||||
now := time.Date(2026, 8, 2, 5, 0, 10, 0, time.UTC)
|
||||
worker, err := New(source, store, evaluator, Config{MaxConcurrent: 1, MaxBatch: 1, AttemptTimeout: 10 * time.Millisecond, LeaseTTL: time.Second, Owner: "worker-timeout", Now: func() time.Time { return now }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, err := worker.RunOnce(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Failed != 1 || len(report.Jobs) != 1 || report.Jobs[0].ErrorCode != "timeout" {
|
||||
t.Fatalf("timeout report=%#v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShutdownCancelsOutstandingEvaluationSafely(t *testing.T) {
|
||||
source := fakeSource{rules: []alert.Rule{testRule("rule-shutdown")}}
|
||||
store := NewMemoryLeaseStore()
|
||||
started := make(chan struct{})
|
||||
evaluator := EvaluateFunc(func(ctx context.Context, _ alert.Rule) error {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
})
|
||||
now := time.Date(2026, 8, 2, 5, 0, 10, 0, time.UTC)
|
||||
worker := testWorker(t, source, store, evaluator, "worker-shutdown", now, 1)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
result := make(chan RunReport, 1)
|
||||
go func() {
|
||||
report, _ := worker.RunOnce(ctx)
|
||||
result <- report
|
||||
}()
|
||||
<-started
|
||||
cancel()
|
||||
report := <-result
|
||||
if report.Canceled != 1 || len(report.Jobs) != 1 || report.Jobs[0].ErrorCode != "shutdown" {
|
||||
t.Fatalf("shutdown report=%#v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadIsBoundedByMaxConcurrentAndBatch(t *testing.T) {
|
||||
rules := make([]alert.Rule, 20)
|
||||
for i := range rules {
|
||||
rules[i] = testRule(alert.NewID())
|
||||
}
|
||||
store := NewMemoryLeaseStore()
|
||||
evaluator := &trackingEvaluator{wait: 2 * time.Millisecond}
|
||||
now := time.Date(2026, 8, 2, 5, 0, 10, 0, time.UTC)
|
||||
worker := testWorker(t, fakeSource{rules: rules}, store, evaluator, "worker-scale", now, 3)
|
||||
report, err := worker.RunOnce(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Scheduled != 20 || report.Started != 20 || report.Completed != 20 || evaluator.max.Load() > 3 {
|
||||
t.Fatalf("bounded report=%#v max=%d", report, evaluator.max.Load())
|
||||
}
|
||||
if metrics := worker.Metrics(); metrics.JobsStarted != 20 || metrics.JobsCompleted != 20 {
|
||||
t.Fatalf("metrics=%#v", metrics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidWorkerConfigurationAndEvaluationError(t *testing.T) {
|
||||
_, err := New(fakeSource{}, NewMemoryLeaseStore(), EvaluateFunc(func(context.Context, alert.Rule) error { return nil }), Config{MaxConcurrent: 0, MaxBatch: 1, AttemptTimeout: time.Second, LeaseTTL: time.Second, Owner: "x", Now: time.Now})
|
||||
if !errors.Is(err, ErrInvalidConfig) {
|
||||
t.Fatalf("config error=%v", err)
|
||||
}
|
||||
source := fakeSource{rules: []alert.Rule{testRule("rule-error")}}
|
||||
store := NewMemoryLeaseStore()
|
||||
worker := testWorker(t, source, store, EvaluateFunc(func(context.Context, alert.Rule) error { return errors.New("upstream failed") }), "worker-error", time.Now().UTC(), 1)
|
||||
report, err := worker.RunOnce(context.Background())
|
||||
if err != nil || report.Failed != 1 || report.Jobs[0].ErrorCode != "evaluation_failed" {
|
||||
t.Fatalf("error report=%#v err=%v", report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedSlotIsVisibleAndNotRetried(t *testing.T) {
|
||||
source := fakeSource{rules: []alert.Rule{testRule("rule-failed")}}
|
||||
store := NewMemoryLeaseStore()
|
||||
evaluator := &trackingEvaluator{err: errors.New("upstream failed")}
|
||||
now := time.Date(2026, 8, 2, 5, 0, 10, 0, time.UTC)
|
||||
worker := testWorker(t, source, store, evaluator, "worker-failed", now, 1)
|
||||
first, err := worker.RunOnce(context.Background())
|
||||
if err != nil || first.Failed != 1 || first.Jobs[0].ErrorCode != "evaluation_failed" {
|
||||
t.Fatalf("first report=%#v err=%v", first, err)
|
||||
}
|
||||
second, err := worker.RunOnce(context.Background())
|
||||
if err != nil || second.Skipped != 1 || evaluator.calls.Load() != 1 {
|
||||
t.Fatalf("second report=%#v calls=%d err=%v", second, evaluator.calls.Load(), err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/reconciliation"
|
||||
)
|
||||
|
||||
const ContractVersion = "v1"
|
||||
|
||||
// SourceID is the stable logical source used for derived application IDs. It
|
||||
// is deliberately not a database datasource UUID: the observations remain
|
||||
// owned by the underlying container datasource while IDs stay stable across
|
||||
// API and worker projections.
|
||||
const SourceID = "applications"
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateHealthy State = "healthy"
|
||||
StateDegraded State = "degraded"
|
||||
StateUnknown State = "unknown"
|
||||
StateDown State = "down"
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
ReceivedAt time.Time `json:"receivedAt"`
|
||||
Freshness string `json:"freshness"`
|
||||
State string `json:"state"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
type ComponentInput struct {
|
||||
ID string
|
||||
Name string
|
||||
Kind string
|
||||
ContainerState State
|
||||
ServiceState State
|
||||
Critical bool
|
||||
}
|
||||
type DiscoveredApplication struct {
|
||||
ID string
|
||||
Name string
|
||||
Components []ComponentInput
|
||||
}
|
||||
type ApplicationOverride struct {
|
||||
ApplicationID string
|
||||
Name string
|
||||
CriticalByComponent map[string]bool
|
||||
}
|
||||
|
||||
type Reason struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
ComponentID string `json:"componentId,omitempty"`
|
||||
Critical bool `json:"critical"`
|
||||
}
|
||||
type Component struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Critical bool `json:"critical"`
|
||||
ContainerState State `json:"containerState"`
|
||||
ServiceState State `json:"serviceState"`
|
||||
Status State `json:"status"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
type Application struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Status State `json:"status"`
|
||||
Overridden bool `json:"overridden"`
|
||||
Components []Component `json:"components"`
|
||||
Reasons []Reason `json:"reasons,omitempty"`
|
||||
}
|
||||
type Snapshot struct {
|
||||
ContractVersion string `json:"contractVersion"`
|
||||
Source Source `json:"source"`
|
||||
Applications []Application `json:"applications"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
type Provider interface {
|
||||
Snapshot(context.Context) (Snapshot, error)
|
||||
}
|
||||
|
||||
func UnknownSnapshot(now time.Time, id, typ, reason string) Snapshot {
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
return Snapshot{ContractVersion: ContractVersion, Source: Source{ID: id, Type: typ, ReceivedAt: now.UTC(), Freshness: "unavailable", State: "unknown", Reason: reason}, Applications: []Application{}}
|
||||
}
|
||||
func StableApplicationID(sourceID, groupKey string) string {
|
||||
return reconciliation.StableEntityID(sourceID, "application", groupKey)
|
||||
}
|
||||
func BuildSnapshot(source Source, discovered []DiscoveredApplication, overrides []ApplicationOverride, now time.Time) (Snapshot, error) {
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
now = now.UTC()
|
||||
if strings.TrimSpace(source.ID) == "" {
|
||||
return Snapshot{}, errors.New("application source id is required")
|
||||
}
|
||||
if len(discovered) > 150 {
|
||||
return Snapshot{}, errors.New("application count exceeds bounds")
|
||||
}
|
||||
if source.ReceivedAt.IsZero() {
|
||||
source.ReceivedAt = now
|
||||
}
|
||||
if source.ObservedAt.IsZero() {
|
||||
source.ObservedAt = source.ReceivedAt
|
||||
}
|
||||
source.ObservedAt, source.ReceivedAt = source.ObservedAt.UTC(), source.ReceivedAt.UTC()
|
||||
if source.ObservedAt.After(now.Add(time.Minute)) {
|
||||
return Snapshot{}, errors.New("application observation is materially in the future")
|
||||
}
|
||||
source.Freshness, source.State = "fresh", "healthy"
|
||||
if now.Sub(source.ObservedAt) > time.Minute {
|
||||
source.Freshness, source.State, source.Reason = "stale", "unknown", "stale_source"
|
||||
}
|
||||
overrideByID := make(map[string]ApplicationOverride, len(overrides))
|
||||
for _, override := range overrides {
|
||||
if strings.TrimSpace(override.ApplicationID) == "" {
|
||||
return Snapshot{}, errors.New("application override id is required")
|
||||
}
|
||||
if _, exists := overrideByID[override.ApplicationID]; exists {
|
||||
return Snapshot{}, errors.New("duplicate application override")
|
||||
}
|
||||
overrideByID[override.ApplicationID] = override
|
||||
}
|
||||
apps := make([]Application, 0, len(discovered))
|
||||
seenApps := make(map[string]struct{}, len(discovered))
|
||||
componentCount := 0
|
||||
for _, group := range discovered {
|
||||
if strings.TrimSpace(group.ID) == "" || strings.TrimSpace(group.Name) == "" {
|
||||
return Snapshot{}, errors.New("application identity is required")
|
||||
}
|
||||
if _, exists := seenApps[group.ID]; exists {
|
||||
return Snapshot{}, errors.New("duplicate application identity")
|
||||
}
|
||||
seenApps[group.ID] = struct{}{}
|
||||
componentCount += len(group.Components)
|
||||
if componentCount > 1000 {
|
||||
return Snapshot{}, errors.New("application component count exceeds bounds")
|
||||
}
|
||||
override, overridden := overrideByID[group.ID]
|
||||
name := group.Name
|
||||
if strings.TrimSpace(override.Name) != "" {
|
||||
name = strings.TrimSpace(override.Name)
|
||||
}
|
||||
components := make([]Component, 0, len(group.Components))
|
||||
seenComponents := make(map[string]struct{}, len(group.Components))
|
||||
for _, input := range group.Components {
|
||||
if strings.TrimSpace(input.ID) == "" || strings.TrimSpace(input.Name) == "" {
|
||||
return Snapshot{}, errors.New("application component identity is required")
|
||||
}
|
||||
if _, exists := seenComponents[input.ID]; exists {
|
||||
return Snapshot{}, errors.New("duplicate application component")
|
||||
}
|
||||
seenComponents[input.ID] = struct{}{}
|
||||
critical := input.Critical
|
||||
if value, ok := override.CriticalByComponent[input.ID]; ok {
|
||||
critical = value
|
||||
}
|
||||
components = append(components, evaluateComponent(input, critical))
|
||||
}
|
||||
sort.Slice(components, func(i, j int) bool { return components[i].ID < components[j].ID })
|
||||
status, reasons := aggregate(components)
|
||||
apps = append(apps, Application{ID: group.ID, Name: name, Status: status, Overridden: overridden, Components: components, Reasons: reasons})
|
||||
}
|
||||
sort.Slice(apps, func(i, j int) bool { return apps[i].ID < apps[j].ID })
|
||||
if source.State == "unknown" {
|
||||
for i := range apps {
|
||||
apps[i].Status = StateUnknown
|
||||
apps[i].Reasons = append(apps[i].Reasons, Reason{Code: "source_unknown", Message: "Applicatiebron is onbekend.", Critical: true})
|
||||
}
|
||||
}
|
||||
return Snapshot{ContractVersion: ContractVersion, Source: source, Applications: apps, Total: len(apps)}, nil
|
||||
}
|
||||
func evaluateComponent(input ComponentInput, critical bool) Component {
|
||||
container, service := normalizeState(input.ContainerState), normalizeState(input.ServiceState)
|
||||
status, reason := container, ""
|
||||
if container == StateHealthy && service != "" {
|
||||
status = service
|
||||
}
|
||||
if status != StateHealthy {
|
||||
if service != "" && service != StateHealthy && container == StateHealthy {
|
||||
reason = "service_" + string(service)
|
||||
} else {
|
||||
reason = "container_" + string(status)
|
||||
}
|
||||
}
|
||||
return Component{ID: input.ID, Name: input.Name, Kind: input.Kind, Critical: critical, ContainerState: container, ServiceState: service, Status: status, Reason: reason}
|
||||
}
|
||||
func aggregate(components []Component) (State, []Reason) {
|
||||
reasons := make([]Reason, 0)
|
||||
criticalFailure, criticalUnknown, optionalFailure := false, false, false
|
||||
for _, component := range components {
|
||||
if component.Status == StateHealthy {
|
||||
continue
|
||||
}
|
||||
reasons = append(reasons, Reason{Code: component.Reason, Message: component.Name + " is " + string(component.Status) + ".", ComponentID: component.ID, Critical: component.Critical})
|
||||
if component.Critical {
|
||||
if component.Status == StateUnknown {
|
||||
criticalUnknown = true
|
||||
} else {
|
||||
criticalFailure = true
|
||||
}
|
||||
} else {
|
||||
optionalFailure = true
|
||||
}
|
||||
}
|
||||
sort.Slice(reasons, func(i, j int) bool {
|
||||
if reasons[i].ComponentID != reasons[j].ComponentID {
|
||||
return reasons[i].ComponentID < reasons[j].ComponentID
|
||||
}
|
||||
return reasons[i].Code < reasons[j].Code
|
||||
})
|
||||
if criticalFailure {
|
||||
return StateDegraded, reasons
|
||||
}
|
||||
if criticalUnknown {
|
||||
return StateUnknown, reasons
|
||||
}
|
||||
if optionalFailure {
|
||||
return StateDegraded, reasons
|
||||
}
|
||||
return StateHealthy, reasons
|
||||
}
|
||||
func normalizeState(state State) State {
|
||||
state = State(strings.ToLower(strings.TrimSpace(string(state))))
|
||||
switch state {
|
||||
case StateHealthy, StateDegraded, StateUnknown, StateDown:
|
||||
return state
|
||||
default:
|
||||
return StateUnknown
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func appSource(now time.Time) Source {
|
||||
return Source{ID: "agent", Type: "agent", ObservedAt: now, ReceivedAt: now}
|
||||
}
|
||||
|
||||
func TestCriticalAndOptionalAggregationAndReasons(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 22, 0, 0, 0, time.UTC)
|
||||
snapshot, err := BuildSnapshot(appSource(now), []DiscoveredApplication{{ID: "app-1", Name: "Pulse", Components: []ComponentInput{
|
||||
{ID: "db", Name: "Database", Kind: "container", ContainerState: StateHealthy, ServiceState: StateHealthy, Critical: true},
|
||||
{ID: "worker", Name: "Worker", Kind: "container", ContainerState: StateDown, ServiceState: StateDown, Critical: false},
|
||||
}}}, nil, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Applications[0].Status != StateDegraded || len(snapshot.Applications[0].Reasons) != 1 || snapshot.Applications[0].Reasons[0].Critical {
|
||||
t.Fatalf("unexpected optional aggregation: %+v", snapshot.Applications[0])
|
||||
}
|
||||
}
|
||||
func TestServiceDownWhileContainerRunningDegradesCriticalApplication(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 22, 0, 0, 0, time.UTC)
|
||||
snapshot, err := BuildSnapshot(appSource(now), []DiscoveredApplication{{ID: "app-1", Name: "Pulse", Components: []ComponentInput{
|
||||
{ID: "api", Name: "API", Kind: "container", ContainerState: StateHealthy, ServiceState: StateDown, Critical: true},
|
||||
}}}, nil, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
app := snapshot.Applications[0]
|
||||
if app.Status != StateDegraded || len(app.Reasons) != 1 || app.Reasons[0].Code != "service_down" || !app.Reasons[0].Critical {
|
||||
t.Fatalf("service failure not explained: %+v", app)
|
||||
}
|
||||
}
|
||||
func TestUnknownCriticalIsNotHealthy(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 22, 0, 0, 0, time.UTC)
|
||||
snapshot, err := BuildSnapshot(appSource(now), []DiscoveredApplication{{ID: "app-1", Name: "Pulse", Components: []ComponentInput{{ID: "db", Name: "Database", Critical: true, ContainerState: StateUnknown}}}}, nil, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Applications[0].Status != StateUnknown {
|
||||
t.Fatalf("unknown critical became healthy: %+v", snapshot.Applications[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestStateNormalizationIsCaseInsensitive(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 22, 0, 0, 0, time.UTC)
|
||||
snapshot, err := BuildSnapshot(appSource(now), []DiscoveredApplication{{ID: "app-1", Name: "Pulse", Components: []ComponentInput{{ID: "api", Name: "API", Critical: true, ContainerState: "HEALTHY", ServiceState: "HEALTHY"}}}}, nil, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Applications[0].Status != StateHealthy {
|
||||
t.Fatalf("uppercase healthy state was not normalized: %+v", snapshot.Applications[0])
|
||||
}
|
||||
}
|
||||
func TestUserOverridePersistsAcrossDiscoveryProjection(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 22, 0, 0, 0, time.UTC)
|
||||
discovered := []DiscoveredApplication{{ID: "app-1", Name: "Discovered name", Components: []ComponentInput{{ID: "worker", Name: "Worker", Critical: true, ContainerState: StateHealthy}}}}
|
||||
override := []ApplicationOverride{{ApplicationID: "app-1", Name: "Handmatige naam", CriticalByComponent: map[string]bool{"worker": false}}}
|
||||
first, err := BuildSnapshot(appSource(now), discovered, override, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := BuildSnapshot(appSource(now.Add(time.Minute)), discovered, override, now.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, snapshot := range []Snapshot{first, second} {
|
||||
if snapshot.Applications[0].Name != "Handmatige naam" || !snapshot.Applications[0].Overridden || snapshot.Applications[0].Components[0].Critical {
|
||||
t.Fatalf("override not retained: %+v", snapshot.Applications[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestStableApplicationIDIsDeterministic(t *testing.T) {
|
||||
if StableApplicationID("agent", "pulse") != StableApplicationID("agent", "pulse") {
|
||||
t.Fatal("stable application id changed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package applicationapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/application"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/problem"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
Provider interface {
|
||||
Snapshot(context.Context) (application.Snapshot, error)
|
||||
}
|
||||
}
|
||||
|
||||
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet || (r.URL.Path != "/api/v1/applications" && !strings.HasPrefix(r.URL.Path, "/api/v1/applications/")) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if _, ok := auth.PrincipalFromContext(r.Context()); !ok {
|
||||
problem.Write(w, r, http.StatusUnauthorized, "UNAUTHORIZED", "Authentication required", "Authentication is required to read applications.", nil)
|
||||
return
|
||||
}
|
||||
snapshot, err := h.snapshot(r)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return
|
||||
}
|
||||
problem.Write(w, r, http.StatusServiceUnavailable, "APPLICATIONS_UNAVAILABLE", "Applications not available", "Applicatiegegevens konden niet worden gelezen.", nil)
|
||||
return
|
||||
}
|
||||
if r.URL.Path != "/api/v1/applications" {
|
||||
id := strings.TrimPrefix(r.URL.Path, "/api/v1/applications/")
|
||||
for _, item := range snapshot.Applications {
|
||||
if item.ID == id {
|
||||
writeJSON(w, struct {
|
||||
Source application.Source `json:"source"`
|
||||
Application application.Application `json:"application"`
|
||||
}{Source: snapshot.Source, Application: item})
|
||||
return
|
||||
}
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if len(snapshot.Applications) > 100 {
|
||||
snapshot.Applications = snapshot.Applications[:100]
|
||||
}
|
||||
writeJSON(w, snapshot)
|
||||
}
|
||||
func (h Handler) snapshot(r *http.Request) (application.Snapshot, error) {
|
||||
if h.Provider == nil {
|
||||
return application.UnknownSnapshot(time.Now().UTC(), "applications", "agent", "source_unavailable"), nil
|
||||
}
|
||||
return h.Provider.Snapshot(r.Context())
|
||||
}
|
||||
func writeJSON(w http.ResponseWriter, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "private, max-age=5")
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package applicationapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/itworx/pulse/internal/application"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
)
|
||||
|
||||
type provider struct{ value application.Snapshot }
|
||||
|
||||
func (p provider) Snapshot(context.Context) (application.Snapshot, error) { return p.value, nil }
|
||||
func request(method, path string) *http.Request {
|
||||
r := httptest.NewRequest(method, path, nil)
|
||||
return r.WithContext(auth.WithPrincipal(r.Context(), auth.Principal{Subject: "viewer", Role: auth.RoleViewer}))
|
||||
}
|
||||
func TestHandlerListDetailAndAuth(t *testing.T) {
|
||||
snapshot := application.Snapshot{ContractVersion: application.ContractVersion, Applications: []application.Application{{ID: "app-1", Name: "Pulse", Status: application.StateHealthy}}}
|
||||
handler := Handler{Provider: provider{value: snapshot}}
|
||||
list := httptest.NewRecorder()
|
||||
handler.ServeHTTP(list, request(http.MethodGet, "/api/v1/applications"))
|
||||
if list.Code != http.StatusOK || !strings.Contains(list.Body.String(), "\"name\":\"Pulse\"") {
|
||||
t.Fatalf("list status=%d body=%s", list.Code, list.Body.String())
|
||||
}
|
||||
detail := httptest.NewRecorder()
|
||||
handler.ServeHTTP(detail, request(http.MethodGet, "/api/v1/applications/app-1"))
|
||||
if detail.Code != http.StatusOK || !strings.Contains(detail.Body.String(), "\"id\":\"app-1\"") {
|
||||
t.Fatalf("detail status=%d body=%s", detail.Code, detail.Body.String())
|
||||
}
|
||||
mutation := httptest.NewRecorder()
|
||||
handler.ServeHTTP(mutation, request(http.MethodPost, "/api/v1/applications/app-1"))
|
||||
if mutation.Code != http.StatusNotFound {
|
||||
t.Fatalf("mutation status=%d", mutation.Code)
|
||||
}
|
||||
}
|
||||
func TestHandlerRequiresAuthentication(t *testing.T) {
|
||||
response := httptest.NewRecorder()
|
||||
Handler{}.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/v1/applications", nil))
|
||||
if response.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status=%d", response.Code)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
package array
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const ContractVersion = "v1"
|
||||
|
||||
const (
|
||||
StateOperational = "operational"
|
||||
StateDegraded = "degraded"
|
||||
StateMissing = "missing"
|
||||
StateUnknown = "unknown"
|
||||
Fresh = "fresh"
|
||||
Stale = "stale"
|
||||
Unavailable = "unavailable"
|
||||
)
|
||||
|
||||
type Limits struct {
|
||||
MaxMembers int
|
||||
MaxHistory int
|
||||
MaxErrors int
|
||||
}
|
||||
|
||||
func (l Limits) withDefaults() Limits {
|
||||
if l.MaxMembers == 0 {
|
||||
l.MaxMembers = 64
|
||||
}
|
||||
if l.MaxHistory == 0 {
|
||||
l.MaxHistory = 64
|
||||
}
|
||||
if l.MaxErrors == 0 {
|
||||
l.MaxErrors = 20
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func (l Limits) Validate() error {
|
||||
if l.MaxMembers < 1 || l.MaxMembers > 256 || l.MaxHistory < 1 || l.MaxHistory > 256 || l.MaxErrors < 1 || l.MaxErrors > 100 {
|
||||
return errors.New("array limits are outside safe bounds")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Policy struct{ FreshnessMaxAge time.Duration }
|
||||
|
||||
func (p Policy) withDefaults() Policy {
|
||||
if p.FreshnessMaxAge == 0 {
|
||||
p.FreshnessMaxAge = 60 * time.Second
|
||||
}
|
||||
return p
|
||||
}
|
||||
func (p Policy) Validate() error {
|
||||
if p.FreshnessMaxAge <= 0 || p.FreshnessMaxAge > 24*time.Hour {
|
||||
return errors.New("array policy is outside safe bounds")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Source struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
CapabilityVersion string `json:"capabilityVersion"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
ReceivedAt time.Time `json:"receivedAt"`
|
||||
Freshness string `json:"freshness"`
|
||||
State string `json:"state"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
type RawMember struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Role string `json:"role"`
|
||||
State string `json:"state"`
|
||||
CapacityBytes uint64 `json:"capacityBytes"`
|
||||
ReadBytes uint64 `json:"readBytes"`
|
||||
WriteBytes uint64 `json:"writeBytes"`
|
||||
}
|
||||
|
||||
type Member struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Role string `json:"role"`
|
||||
State string `json:"state"`
|
||||
CapacityBytes uint64 `json:"capacityBytes"`
|
||||
ReadBytes uint64 `json:"readBytes"`
|
||||
WriteBytes uint64 `json:"writeBytes"`
|
||||
}
|
||||
|
||||
type RawParity struct {
|
||||
Present bool `json:"present"`
|
||||
State string `json:"state"`
|
||||
Errors uint64 `json:"errors"`
|
||||
}
|
||||
type Parity struct {
|
||||
Present bool `json:"present"`
|
||||
State string `json:"state"`
|
||||
Errors uint64 `json:"errors"`
|
||||
}
|
||||
|
||||
type RawCheck struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
ProgressPercent float64 `json:"progressPercent"`
|
||||
SpeedBytesPerSecond uint64 `json:"speedBytesPerSecond"`
|
||||
Errors uint64 `json:"errors"`
|
||||
StartedAt *time.Time `json:"startedAt,omitempty"`
|
||||
CompletedAt *time.Time `json:"completedAt,omitempty"`
|
||||
}
|
||||
type Check struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
ProgressPercent float64 `json:"progressPercent"`
|
||||
SpeedBytesPerSecond uint64 `json:"speedBytesPerSecond"`
|
||||
Errors uint64 `json:"errors"`
|
||||
StartedAt *time.Time `json:"startedAt,omitempty"`
|
||||
CompletedAt *time.Time `json:"completedAt,omitempty"`
|
||||
}
|
||||
|
||||
type RawSnapshot struct {
|
||||
Source Source `json:"source"`
|
||||
State string `json:"state"`
|
||||
Parity RawParity `json:"parity"`
|
||||
Members []RawMember `json:"members"`
|
||||
CurrentCheck *RawCheck `json:"currentCheck,omitempty"`
|
||||
History []RawCheck `json:"history,omitempty"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
ReceivedAt time.Time `json:"receivedAt"`
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
ContractVersion string `json:"contractVersion"`
|
||||
Source Source `json:"source"`
|
||||
State string `json:"state"`
|
||||
Parity Parity `json:"parity"`
|
||||
Members []Member `json:"members"`
|
||||
CurrentCheck *Check `json:"currentCheck,omitempty"`
|
||||
History []Check `json:"history"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
ReceivedAt time.Time `json:"receivedAt"`
|
||||
}
|
||||
|
||||
type Provider interface {
|
||||
Snapshot(context.Context) (Snapshot, error)
|
||||
}
|
||||
type RawProvider interface {
|
||||
Snapshot(context.Context) (RawSnapshot, error)
|
||||
}
|
||||
type Adapter struct {
|
||||
Source RawProvider
|
||||
Limits Limits
|
||||
Policy Policy
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (a Adapter) Snapshot(ctx context.Context) (Snapshot, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
if a.Source == nil {
|
||||
return UnknownSnapshot(time.Now().UTC(), "array", "unraid", "source_unavailable"), nil
|
||||
}
|
||||
raw, err := a.Source.Snapshot(ctx)
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if a.Now != nil {
|
||||
now = a.Now()
|
||||
}
|
||||
return Normalize(raw, now, a.Limits, a.Policy)
|
||||
}
|
||||
|
||||
func UnknownSnapshot(now time.Time, id, typ, reason string) Snapshot {
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
return Snapshot{ContractVersion: ContractVersion, Source: Source{ID: id, Type: typ, CapabilityVersion: ContractVersion, ReceivedAt: now.UTC(), Freshness: Unavailable, State: StateUnknown, Reason: reason}, State: StateUnknown, Members: []Member{}, History: []Check{}, ObservedAt: now.UTC(), ReceivedAt: now.UTC()}
|
||||
}
|
||||
|
||||
func Normalize(raw RawSnapshot, now time.Time, limits Limits, policy Policy) (Snapshot, error) {
|
||||
limits = limits.withDefaults()
|
||||
policy = policy.withDefaults()
|
||||
if err := limits.Validate(); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
if err := policy.Validate(); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
if raw.ReceivedAt.IsZero() {
|
||||
raw.ReceivedAt = now
|
||||
}
|
||||
if raw.ObservedAt.IsZero() {
|
||||
raw.ObservedAt = raw.ReceivedAt
|
||||
}
|
||||
if raw.ObservedAt.After(now.Add(time.Minute)) {
|
||||
return Snapshot{}, errors.New("array observation is materially in the future")
|
||||
}
|
||||
if len(raw.Members) > limits.MaxMembers || len(raw.History) > limits.MaxHistory {
|
||||
return Snapshot{}, errors.New("array payload exceeds bounds")
|
||||
}
|
||||
if raw.CurrentCheck != nil {
|
||||
if err := validateCheck(*raw.CurrentCheck); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
}
|
||||
for _, item := range raw.History {
|
||||
if err := validateCheck(item); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
}
|
||||
members := make([]Member, 0, len(raw.Members))
|
||||
seenMembers := make(map[string]RawMember, len(raw.Members))
|
||||
missing := 0
|
||||
for _, item := range raw.Members {
|
||||
item.ID = canonicalIdentity(item.ID)
|
||||
if previous, ok := seenMembers[item.ID]; ok {
|
||||
if reflect.DeepEqual(previous, item) {
|
||||
continue
|
||||
}
|
||||
return Snapshot{}, errors.New("conflicting duplicate array member identity")
|
||||
}
|
||||
seenMembers[item.ID] = item
|
||||
if strings.TrimSpace(item.ID) == "" || strings.TrimSpace(item.Name) == "" || len(item.ID) > 128 || len(item.Name) > 255 {
|
||||
return Snapshot{}, errors.New("array member identity is invalid")
|
||||
}
|
||||
state := bounded(item.State, StateUnknown)
|
||||
role := bounded(item.Role, "data")
|
||||
if state == "missing" || state == "disabled" || state == "emulated" {
|
||||
missing++
|
||||
}
|
||||
members = append(members, Member{ID: item.ID, Name: item.Name, Role: role, State: state, CapacityBytes: item.CapacityBytes, ReadBytes: item.ReadBytes, WriteBytes: item.WriteBytes})
|
||||
}
|
||||
sort.Slice(members, func(i, j int) bool {
|
||||
if members[i].Role != members[j].Role {
|
||||
return members[i].Role < members[j].Role
|
||||
}
|
||||
if members[i].Name != members[j].Name {
|
||||
return members[i].Name < members[j].Name
|
||||
}
|
||||
return members[i].ID < members[j].ID
|
||||
})
|
||||
parity := Parity{Present: raw.Parity.Present, State: bounded(raw.Parity.State, StateUnknown), Errors: raw.Parity.Errors}
|
||||
state := bounded(raw.State, StateUnknown)
|
||||
if state != StateOperational && state != StateDegraded && state != StateMissing && state != StateUnknown {
|
||||
state = StateUnknown
|
||||
}
|
||||
if missing > 0 && state == StateOperational {
|
||||
state = StateDegraded
|
||||
}
|
||||
if parity.Errors > 0 && state == StateOperational {
|
||||
state = StateDegraded
|
||||
}
|
||||
source := raw.Source
|
||||
if source.ID == "" {
|
||||
source.ID = "array"
|
||||
}
|
||||
if source.Type == "" {
|
||||
source.Type = "unraid"
|
||||
}
|
||||
if source.CapabilityVersion == "" {
|
||||
source.CapabilityVersion = ContractVersion
|
||||
}
|
||||
source.ObservedAt = raw.ObservedAt.UTC()
|
||||
source.ReceivedAt = raw.ReceivedAt.UTC()
|
||||
source.Freshness = Fresh
|
||||
source.State = state
|
||||
if now.Sub(raw.ObservedAt) > policy.FreshnessMaxAge {
|
||||
source.Freshness = Stale
|
||||
source.State = StateUnknown
|
||||
source.Reason = "stale_source"
|
||||
state = StateUnknown
|
||||
for i := range members {
|
||||
members[i].State = StateUnknown
|
||||
}
|
||||
}
|
||||
result := Snapshot{ContractVersion: ContractVersion, Source: source, State: state, Parity: parity, Members: members, ObservedAt: raw.ObservedAt.UTC(), ReceivedAt: raw.ReceivedAt.UTC()}
|
||||
if raw.CurrentCheck != nil {
|
||||
current := normalizeCheck(*raw.CurrentCheck)
|
||||
result.CurrentCheck = ¤t
|
||||
}
|
||||
result.History = make([]Check, 0, len(raw.History))
|
||||
for _, item := range raw.History {
|
||||
result.History = append(result.History, normalizeCheck(item))
|
||||
}
|
||||
sort.SliceStable(result.History, func(i, j int) bool { return checkTime(result.History[i]).After(checkTime(result.History[j])) })
|
||||
return result, nil
|
||||
}
|
||||
func canonicalIdentity(value string) string { return strings.ToLower(strings.TrimSpace(value)) }
|
||||
|
||||
func validateCheck(item RawCheck) error {
|
||||
if strings.TrimSpace(item.ID) == "" || len(item.ID) > 128 || item.ProgressPercent < 0 || item.ProgressPercent > 100 || math.IsNaN(item.ProgressPercent) || math.IsInf(item.ProgressPercent, 0) {
|
||||
return errors.New("invalid parity check")
|
||||
}
|
||||
if item.CompletedAt != nil && item.StartedAt != nil && item.CompletedAt.Before(*item.StartedAt) {
|
||||
return errors.New("parity check completed before start")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func normalizeCheck(item RawCheck) Check {
|
||||
result := Check{ID: item.ID, State: bounded(item.State, StateUnknown), ProgressPercent: item.ProgressPercent, SpeedBytesPerSecond: item.SpeedBytesPerSecond, Errors: item.Errors}
|
||||
if item.StartedAt != nil {
|
||||
value := item.StartedAt.UTC()
|
||||
result.StartedAt = &value
|
||||
}
|
||||
if item.CompletedAt != nil {
|
||||
value := item.CompletedAt.UTC()
|
||||
result.CompletedAt = &value
|
||||
}
|
||||
return result
|
||||
}
|
||||
func checkTime(item Check) time.Time {
|
||||
if item.CompletedAt != nil {
|
||||
return *item.CompletedAt
|
||||
}
|
||||
if item.StartedAt != nil {
|
||||
return *item.StartedAt
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
func bounded(value, fallback string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
if len(value) > 64 {
|
||||
return value[:64]
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Severity string `json:"severity"`
|
||||
EntityID string `json:"entityId"`
|
||||
OccurredAt time.Time `json:"occurredAt"`
|
||||
Attributes map[string]string `json:"attributes,omitempty"`
|
||||
}
|
||||
|
||||
func TransitionEvents(previous, current Snapshot) []Event {
|
||||
var events []Event
|
||||
at := current.ObservedAt
|
||||
if at.IsZero() {
|
||||
at = current.ReceivedAt
|
||||
}
|
||||
if at.IsZero() {
|
||||
at = time.Now().UTC()
|
||||
}
|
||||
at = at.UTC()
|
||||
add := func(kind, severity string, attrs map[string]string) {
|
||||
events = append(events, Event{ID: eventID(kind, current, at), Type: kind, Severity: severity, EntityID: "array", OccurredAt: at, Attributes: attrs})
|
||||
}
|
||||
if previous.State != current.State {
|
||||
switch current.State {
|
||||
case StateDegraded:
|
||||
add("array.degraded", "warning", map[string]string{"state": current.State})
|
||||
case StateMissing:
|
||||
add("array.missing", "critical", map[string]string{"state": current.State})
|
||||
case StateOperational:
|
||||
if previous.State == StateDegraded || previous.State == StateMissing {
|
||||
add("array.recovered", "info", map[string]string{"state": current.State})
|
||||
}
|
||||
}
|
||||
}
|
||||
if previous.Parity.State != current.Parity.State || previous.Parity.Errors != current.Parity.Errors {
|
||||
severity := "info"
|
||||
if current.Parity.Errors > 0 || current.Parity.State == "failed" {
|
||||
severity = "critical"
|
||||
}
|
||||
add("array.parity_changed", severity, map[string]string{"state": current.Parity.State, "errors": formatUint(current.Parity.Errors)})
|
||||
}
|
||||
return events
|
||||
}
|
||||
func eventID(kind string, snapshot Snapshot, at time.Time) string {
|
||||
return kind + ":" + snapshot.Source.ID + ":" + at.Format(time.RFC3339Nano)
|
||||
}
|
||||
func formatUint(value uint64) string {
|
||||
if value == 0 {
|
||||
return "0"
|
||||
}
|
||||
digits := make([]byte, 0, 20)
|
||||
for value > 0 {
|
||||
digits = append([]byte{byte('0' + value%10)}, digits...)
|
||||
value /= 10
|
||||
}
|
||||
return string(digits)
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package array
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type rawProvider struct {
|
||||
snapshot RawSnapshot
|
||||
err error
|
||||
}
|
||||
|
||||
func (p rawProvider) Snapshot(context.Context) (RawSnapshot, error) { return p.snapshot, p.err }
|
||||
|
||||
func baseRaw(now time.Time) RawSnapshot {
|
||||
return RawSnapshot{Source: Source{ID: "fixture-array", Type: "fixture"}, State: StateOperational, Parity: RawParity{Present: true, State: "idle"}, Members: []RawMember{{ID: "disk1", Name: "Disk 1", Role: "data", State: "online", CapacityBytes: 100}, {ID: "parity", Name: "Parity", Role: "parity", State: "online", CapacityBytes: 100}}, ObservedAt: now, ReceivedAt: now}
|
||||
}
|
||||
|
||||
func TestNormalizeOperationalDegradedAndMissingFixtures(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
|
||||
operational, err := Normalize(baseRaw(now), now, Limits{}, Policy{})
|
||||
if err != nil || operational.State != StateOperational || operational.Source.State != StateOperational {
|
||||
t.Fatalf("operational=%+v err=%v", operational, err)
|
||||
}
|
||||
degradedRaw := baseRaw(now)
|
||||
degradedRaw.Members[0].State = "emulated"
|
||||
degraded, err := Normalize(degradedRaw, now, Limits{}, Policy{})
|
||||
if err != nil || degraded.State != StateDegraded {
|
||||
t.Fatalf("degraded=%+v err=%v", degraded, err)
|
||||
}
|
||||
missingRaw := baseRaw(now)
|
||||
missingRaw.State = StateMissing
|
||||
missingRaw.Members[0].State = "missing"
|
||||
missing, err := Normalize(missingRaw, now, Limits{}, Policy{})
|
||||
if err != nil || missing.State != StateMissing {
|
||||
t.Fatalf("missing=%+v err=%v", missing, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeProgressSpeedHistoryAndDeterministicOrder(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
|
||||
start := now.Add(-10 * time.Minute)
|
||||
older := now.Add(-2 * time.Hour)
|
||||
raw := baseRaw(now)
|
||||
raw.Members = append(raw.Members, RawMember{ID: "disk0", Name: "Disk 0", Role: "data", State: "online"})
|
||||
raw.CurrentCheck = &RawCheck{ID: "check-current", State: "running", ProgressPercent: 42.5, SpeedBytesPerSecond: 123456, StartedAt: &start}
|
||||
raw.History = []RawCheck{{ID: "old", State: "completed", ProgressPercent: 100, CompletedAt: &older}, {ID: "new", State: "failed", ProgressPercent: 87, Errors: 2, CompletedAt: &start}}
|
||||
got, err := Normalize(raw, now, Limits{}, Policy{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.CurrentCheck == nil || got.CurrentCheck.ProgressPercent != 42.5 || got.CurrentCheck.SpeedBytesPerSecond != 123456 {
|
||||
t.Fatalf("current check=%+v", got.CurrentCheck)
|
||||
}
|
||||
if len(got.History) != 2 || got.History[0].ID != "new" {
|
||||
t.Fatalf("history=%+v", got.History)
|
||||
}
|
||||
if got.Members[0].ID != "disk0" || got.Members[1].Role != "data" {
|
||||
t.Fatalf("members=%+v", got.Members)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeStaleIsUnknownAndContextCancellation(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
|
||||
raw := baseRaw(now.Add(-2 * time.Minute))
|
||||
got, err := Normalize(raw, now, Limits{}, Policy{FreshnessMaxAge: time.Minute})
|
||||
if err != nil || got.State != StateUnknown || got.Source.State != StateUnknown || got.Source.Freshness != Stale || got.Members[0].State != StateUnknown {
|
||||
t.Fatalf("stale=%+v err=%v", got, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err = (Adapter{}).Snapshot(ctx)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransitionEventsAreBoundedAndDeterministic(t *testing.T) {
|
||||
at := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
|
||||
previous := UnknownSnapshot(at, "fixture-array", "fixture", "initial")
|
||||
current := previous
|
||||
current.ObservedAt = at
|
||||
current.Source.State = StateDegraded
|
||||
current.State = StateDegraded
|
||||
current.Parity.State = "failed"
|
||||
current.Parity.Errors = 3
|
||||
events := TransitionEvents(previous, current)
|
||||
if len(events) != 2 {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
if events[0].Type != "array.degraded" || events[1].Type != "array.parity_changed" {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
if events[0].ID == "" || !events[1].OccurredAt.Equal(at) {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdapterUnknownFallbackAndProviderError(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
|
||||
unknown, err := (Adapter{Now: func() time.Time { return now }}).Snapshot(context.Background())
|
||||
if err != nil || unknown.State != StateUnknown || unknown.Source.Reason != "source_unavailable" {
|
||||
t.Fatalf("unknown=%+v err=%v", unknown, err)
|
||||
}
|
||||
expected := errors.New("fixture")
|
||||
_, err = (Adapter{Source: rawProvider{err: expected}}).Snapshot(context.Background())
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArrayMembersDeduplicateByCanonicalPhysicalIdentity(t *testing.T) {
|
||||
now := time.Date(2026, 8, 11, 23, 45, 0, 0, time.UTC)
|
||||
raw := baseRaw(now)
|
||||
raw.Members = []RawMember{{ID: " DISK-1 ", Name: "disk1", Role: "data", State: "online", CapacityBytes: 100}, {ID: " DISK-1 ", Name: "disk1", Role: "data", State: "online", CapacityBytes: 100}}
|
||||
got, err := Normalize(raw, now, Limits{}, Policy{})
|
||||
if err != nil || len(got.Members) != 1 || got.Members[0].ID != "disk-1" {
|
||||
t.Fatalf("members=%+v err=%v", got.Members, err)
|
||||
}
|
||||
raw.Members[1].Role = "parity"
|
||||
if _, err := Normalize(raw, now, Limits{}, Policy{}); err == nil {
|
||||
t.Fatal("conflicting physical roles must fail closed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package arrayapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/array"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/problem"
|
||||
)
|
||||
|
||||
type Handler struct{ Provider array.Provider }
|
||||
|
||||
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet || r.URL.Path != "/api/v1/array" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if _, ok := auth.PrincipalFromContext(r.Context()); !ok {
|
||||
problem.Write(w, r, http.StatusUnauthorized, "UNAUTHORIZED", "Authentication required", "Authentication is required to read array status.", nil)
|
||||
return
|
||||
}
|
||||
if err := r.Context().Err(); err != nil {
|
||||
return
|
||||
}
|
||||
var snapshot array.Snapshot
|
||||
var err error
|
||||
if h.Provider == nil {
|
||||
snapshot = array.UnknownSnapshot(time.Now().UTC(), "array", "unraid", "source_unavailable")
|
||||
} else {
|
||||
snapshot, err = h.Provider.Snapshot(r.Context())
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || errors.Is(r.Context().Err(), context.Canceled) {
|
||||
return
|
||||
}
|
||||
problem.Write(w, r, http.StatusServiceUnavailable, "ARRAY_UNAVAILABLE", "Array status unavailable", "De arraystatus kon niet worden gelezen.", nil)
|
||||
return
|
||||
}
|
||||
if snapshot.Members == nil {
|
||||
snapshot.Members = []array.Member{}
|
||||
}
|
||||
if snapshot.History == nil {
|
||||
snapshot.History = []array.Check{}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "private, max-age=5")
|
||||
_ = json.NewEncoder(w).Encode(snapshot)
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package arrayapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/array"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
)
|
||||
|
||||
type provider struct{ snapshot array.Snapshot }
|
||||
|
||||
func (p provider) Snapshot(context.Context) (array.Snapshot, error) { return p.snapshot, nil }
|
||||
func authenticatedRequest(method, path string) *http.Request {
|
||||
request := httptest.NewRequest(method, path, nil)
|
||||
return request.WithContext(auth.WithPrincipal(request.Context(), auth.Principal{Subject: "viewer", Role: auth.RoleViewer}))
|
||||
}
|
||||
|
||||
func TestHandlerRequiresAuthenticationAndReturnsUnknown(t *testing.T) {
|
||||
unauthenticated := httptest.NewRecorder()
|
||||
Handler{}.ServeHTTP(unauthenticated, httptest.NewRequest(http.MethodGet, "/api/v1/array", nil))
|
||||
if unauthenticated.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status=%d", unauthenticated.Code)
|
||||
}
|
||||
response := httptest.NewRecorder()
|
||||
Handler{}.ServeHTTP(response, authenticatedRequest(http.MethodGet, "/api/v1/array"))
|
||||
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"state":"unknown"`) {
|
||||
t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
func TestHandlerReturnsSnapshotAndRejectsMutations(t *testing.T) {
|
||||
snapshot := array.UnknownSnapshot(time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC), "fixture-array", "fixture", "test")
|
||||
response := httptest.NewRecorder()
|
||||
Handler{Provider: provider{snapshot: snapshot}}.ServeHTTP(response, authenticatedRequest(http.MethodGet, "/api/v1/array"))
|
||||
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"id":"fixture-array"`) {
|
||||
t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
mutation := httptest.NewRecorder()
|
||||
Handler{}.ServeHTTP(mutation, authenticatedRequest(http.MethodPost, "/api/v1/array/check"))
|
||||
if mutation.Code != http.StatusNotFound {
|
||||
t.Fatalf("status=%d", mutation.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerEncodesEmptyCollectionsAsArrays(t *testing.T) {
|
||||
response := httptest.NewRecorder()
|
||||
Handler{Provider: provider{snapshot: array.Snapshot{}}}.ServeHTTP(response, authenticatedRequest(http.MethodGet, "/api/v1/array"))
|
||||
var body struct {
|
||||
Members []array.Member `json:"members"`
|
||||
History []array.Check `json:"history"`
|
||||
}
|
||||
if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body.Members == nil || body.History == nil {
|
||||
t.Fatalf("empty collections must be JSON arrays: %s", response.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package audit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type Event struct {
|
||||
ID string
|
||||
Actor string
|
||||
Action string
|
||||
ResourceType string
|
||||
ResourceID string
|
||||
Result string
|
||||
OccurredAt time.Time
|
||||
CorrelationID string
|
||||
Before map[string]any
|
||||
After map[string]any
|
||||
}
|
||||
|
||||
type Store interface {
|
||||
Append(context.Context, Event) error
|
||||
}
|
||||
|
||||
type MemoryStore struct {
|
||||
Events []Event
|
||||
}
|
||||
|
||||
func (store *MemoryStore) Append(_ context.Context, event Event) error {
|
||||
if event.ID == "" {
|
||||
event.ID = newID()
|
||||
}
|
||||
if event.OccurredAt.IsZero() {
|
||||
event.OccurredAt = time.Now().UTC()
|
||||
}
|
||||
store.Events = append(store.Events, event)
|
||||
return nil
|
||||
}
|
||||
|
||||
type PostgresStore struct{ Pool *pgxpool.Pool }
|
||||
|
||||
func (store PostgresStore) Append(ctx context.Context, event Event) error {
|
||||
if store.Pool == nil {
|
||||
return errors.New("audit database pool is nil")
|
||||
}
|
||||
if event.ID == "" {
|
||||
event.ID = newID()
|
||||
}
|
||||
if event.OccurredAt.IsZero() {
|
||||
event.OccurredAt = time.Now().UTC()
|
||||
}
|
||||
before, err := json.Marshal(event.Before)
|
||||
if err != nil {
|
||||
return errors.New("marshal audit before diff")
|
||||
}
|
||||
after, err := json.Marshal(event.After)
|
||||
if err != nil {
|
||||
return errors.New("marshal audit after diff")
|
||||
}
|
||||
_, err = store.Pool.Exec(ctx, `INSERT INTO audit_events (id, actor, action, resource_type, resource_id, result, occurred_at, correlation_id, before_diff, after_diff) VALUES ($1::uuid, $2, $3, $4, NULLIF($5, '')::uuid, $6, $7, $8, $9::jsonb, $10::jsonb)`, event.ID, event.Actor, event.Action, event.ResourceType, event.ResourceID, event.Result, event.OccurredAt, event.CorrelationID, before, after)
|
||||
if err != nil {
|
||||
return errors.New("write audit event")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newID() string {
|
||||
bytes := make([]byte, 16)
|
||||
if _, err := rand.Read(bytes); err != nil {
|
||||
return "00000000-0000-4000-8000-000000000000"
|
||||
}
|
||||
bytes[6] = (bytes[6] & 0x0f) | 0x40
|
||||
bytes[8] = (bytes[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", bytes[0:4], bytes[4:6], bytes[6:8], bytes[8:10], bytes[10:16])
|
||||
}
|
||||
|
||||
func RecordSecurityAction(ctx context.Context, store Store, actor, action, result, correlationID string) error {
|
||||
return store.Append(ctx, Event{Actor: actor, Action: action, ResourceType: "security", Result: result, CorrelationID: correlationID})
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package audit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSecurityActionsAreStoredWithCorrelation(t *testing.T) {
|
||||
store := &MemoryStore{}
|
||||
if err := RecordSecurityAction(context.Background(), store, "user-1", "config.update", "success", "corr-1234"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(store.Events) != 1 || store.Events[0].CorrelationID != "corr-1234" || store.Events[0].Action != "config.update" {
|
||||
t.Fatalf("unexpected audit event: %#v", store.Events)
|
||||
}
|
||||
if store.Events[0].ID == "" || store.Events[0].OccurredAt.IsZero() {
|
||||
t.Fatal("audit event lacks identity or timestamp")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-oidc/v3/oidc"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultFlowLifetime = 10 * time.Minute
|
||||
)
|
||||
|
||||
type OIDCConfig struct {
|
||||
Issuer string
|
||||
ClientID string
|
||||
ClientSecret string
|
||||
RedirectURL string
|
||||
Scopes []string
|
||||
}
|
||||
|
||||
type Authorization struct {
|
||||
URL string
|
||||
State string
|
||||
Nonce string
|
||||
CodeVerifier string
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
func BeginAuthorization(endpoint oauth2.Endpoint, config OIDCConfig, now time.Time) (Authorization, error) {
|
||||
if endpoint.AuthURL == "" || config.ClientID == "" || config.RedirectURL == "" {
|
||||
return Authorization{}, errors.New("OIDC authorization configuration is incomplete")
|
||||
}
|
||||
state, err := randomToken()
|
||||
if err != nil {
|
||||
return Authorization{}, errors.New("generate authorization state")
|
||||
}
|
||||
nonce, err := randomToken()
|
||||
if err != nil {
|
||||
return Authorization{}, errors.New("generate authorization nonce")
|
||||
}
|
||||
verifier, err := randomToken()
|
||||
if err != nil {
|
||||
return Authorization{}, errors.New("generate PKCE verifier")
|
||||
}
|
||||
scopes := config.Scopes
|
||||
if len(scopes) == 0 {
|
||||
scopes = []string{oidc.ScopeOpenID, "profile", "email"}
|
||||
}
|
||||
oauthConfig := oauth2.Config{
|
||||
ClientID: config.ClientID,
|
||||
ClientSecret: config.ClientSecret,
|
||||
Endpoint: endpoint,
|
||||
RedirectURL: config.RedirectURL,
|
||||
Scopes: scopes,
|
||||
}
|
||||
authURL := oauthConfig.AuthCodeURL(state,
|
||||
oauth2.SetAuthURLParam("nonce", nonce),
|
||||
oauth2.SetAuthURLParam("code_challenge", pkceChallenge(verifier)),
|
||||
oauth2.SetAuthURLParam("code_challenge_method", "S256"),
|
||||
)
|
||||
return Authorization{URL: authURL, State: state, Nonce: nonce, CodeVerifier: verifier, ExpiresAt: now.Add(defaultFlowLifetime)}, nil
|
||||
}
|
||||
|
||||
func ValidateCallback(flow Authorization, state, code string, now time.Time) error {
|
||||
if flow.State == "" || subtle.ConstantTimeCompare([]byte(flow.State), []byte(state)) != 1 {
|
||||
return errors.New("OIDC state validation failed")
|
||||
}
|
||||
if flow.CodeVerifier == "" || flow.Nonce == "" {
|
||||
return errors.New("OIDC flow is incomplete")
|
||||
}
|
||||
if now.After(flow.ExpiresAt) {
|
||||
return errors.New("OIDC authorization expired")
|
||||
}
|
||||
if strings.TrimSpace(code) == "" {
|
||||
return errors.New("OIDC authorization code is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func Exchange(ctx context.Context, flow Authorization, config OIDCConfig, endpoint oauth2.Endpoint, state, code string) (*oauth2.Token, error) {
|
||||
if err := ValidateCallback(flow, state, code, time.Now()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
oauthConfig := oauth2.Config{ClientID: config.ClientID, ClientSecret: config.ClientSecret, Endpoint: endpoint, RedirectURL: config.RedirectURL}
|
||||
return oauthConfig.Exchange(ctx, code, oauth2.SetAuthURLParam("code_verifier", flow.CodeVerifier))
|
||||
}
|
||||
|
||||
// Discovery is the provider metadata required to run one authorization code flow:
|
||||
// the authorization/token endpoints for BeginAuthorization and Exchange, and the
|
||||
// ID token verifier for VerifyIDToken. Resolve it once and reuse it.
|
||||
type Discovery struct {
|
||||
Endpoint oauth2.Endpoint
|
||||
Verifier *oidc.IDTokenVerifier
|
||||
}
|
||||
|
||||
func Discover(ctx context.Context, config OIDCConfig) (Discovery, error) {
|
||||
if config.Issuer == "" || config.ClientID == "" {
|
||||
return Discovery{}, errors.New("OIDC issuer and client ID are required")
|
||||
}
|
||||
provider, err := oidc.NewProvider(ctx, config.Issuer)
|
||||
if err != nil {
|
||||
return Discovery{}, fmt.Errorf("OIDC discovery failed")
|
||||
}
|
||||
return Discovery{Endpoint: provider.Endpoint(), Verifier: provider.Verifier(&oidc.Config{ClientID: config.ClientID})}, nil
|
||||
}
|
||||
|
||||
func NewVerifier(ctx context.Context, config OIDCConfig) (*oidc.IDTokenVerifier, error) {
|
||||
discovery, err := Discover(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return discovery.Verifier, nil
|
||||
}
|
||||
|
||||
func VerifyIDToken(ctx context.Context, verifier *oidc.IDTokenVerifier, rawToken, expectedNonce string) (*oidc.IDToken, error) {
|
||||
if verifier == nil || strings.TrimSpace(rawToken) == "" || expectedNonce == "" {
|
||||
return nil, errors.New("OIDC token verification input is incomplete")
|
||||
}
|
||||
token, err := verifier.Verify(ctx, rawToken)
|
||||
if err != nil {
|
||||
return nil, errors.New("OIDC token verification failed")
|
||||
}
|
||||
var claims struct {
|
||||
Nonce string `json:"nonce"`
|
||||
}
|
||||
if err := token.Claims(&claims); err != nil || subtle.ConstantTimeCompare([]byte(claims.Nonce), []byte(expectedNonce)) != 1 {
|
||||
return nil, errors.New("OIDC nonce validation failed")
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
const (
|
||||
defaultGroupsClaim = "groups"
|
||||
maxIdentityGroups = 128
|
||||
)
|
||||
|
||||
// Identity is the bounded subset of verified ID token claims Pulse consumes.
|
||||
type Identity struct {
|
||||
Subject string
|
||||
Groups []string
|
||||
}
|
||||
|
||||
// ExtractIdentity reads the subject and the configured role claim from an already
|
||||
// verified ID token. The claim may be a list of strings or a single string; values
|
||||
// are trimmed, empty values dropped and the list bounded.
|
||||
func ExtractIdentity(token *oidc.IDToken, groupsClaim string) (Identity, error) {
|
||||
if token == nil {
|
||||
return Identity{}, errors.New("OIDC identity token is required")
|
||||
}
|
||||
if groupsClaim == "" {
|
||||
groupsClaim = defaultGroupsClaim
|
||||
}
|
||||
subject := strings.TrimSpace(token.Subject)
|
||||
if subject == "" {
|
||||
return Identity{}, errors.New("OIDC subject claim is required")
|
||||
}
|
||||
var claims map[string]json.RawMessage
|
||||
if err := token.Claims(&claims); err != nil {
|
||||
return Identity{}, errors.New("OIDC claims could not be read")
|
||||
}
|
||||
raw, ok := claims[groupsClaim]
|
||||
if !ok {
|
||||
return Identity{Subject: subject}, nil
|
||||
}
|
||||
groups, err := normalizeGroupClaim(raw)
|
||||
if err != nil {
|
||||
return Identity{}, err
|
||||
}
|
||||
return Identity{Subject: subject, Groups: groups}, nil
|
||||
}
|
||||
|
||||
func normalizeGroupClaim(raw json.RawMessage) ([]string, error) {
|
||||
var values []string
|
||||
if err := json.Unmarshal(raw, &values); err != nil {
|
||||
var single string
|
||||
if err := json.Unmarshal(raw, &single); err != nil {
|
||||
return nil, errors.New("OIDC role claim is malformed")
|
||||
}
|
||||
values = []string{single}
|
||||
}
|
||||
groups := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" || len(groups) >= maxIdentityGroups {
|
||||
continue
|
||||
}
|
||||
groups = append(groups, trimmed)
|
||||
}
|
||||
return groups, nil
|
||||
}
|
||||
|
||||
func randomToken() (string, error) {
|
||||
bytes := make([]byte, 32)
|
||||
if _, err := rand.Read(bytes); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(bytes), nil
|
||||
}
|
||||
|
||||
func pkceChallenge(verifier string) string {
|
||||
digest := sha256.Sum256([]byte(verifier))
|
||||
return base64.RawURLEncoding.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
type Role string
|
||||
|
||||
const (
|
||||
RoleViewer Role = "viewer"
|
||||
RoleOperator Role = "operator"
|
||||
RoleEditor Role = "editor"
|
||||
RoleAdministrator Role = "administrator"
|
||||
)
|
||||
|
||||
type Permission string
|
||||
|
||||
const (
|
||||
PermissionView Permission = "view"
|
||||
PermissionOperate Permission = "operate"
|
||||
PermissionEdit Permission = "edit"
|
||||
PermissionAdmin Permission = "admin"
|
||||
)
|
||||
|
||||
type Principal struct {
|
||||
Subject string
|
||||
Role Role
|
||||
}
|
||||
|
||||
func MapRoles(claims []string, mapping map[string]Role) (Role, error) {
|
||||
priority := map[Role]int{RoleViewer: 1, RoleOperator: 2, RoleEditor: 3, RoleAdministrator: 4}
|
||||
var selected Role
|
||||
for _, claim := range claims {
|
||||
role, ok := mapping[claim]
|
||||
if !ok || priority[role] <= priority[selected] {
|
||||
continue
|
||||
}
|
||||
selected = role
|
||||
}
|
||||
if selected == "" {
|
||||
return "", errors.New("no authorized Pulse role")
|
||||
}
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
func Allows(role Role, permission Permission) bool {
|
||||
level := map[Role]int{RoleViewer: 1, RoleOperator: 2, RoleEditor: 3, RoleAdministrator: 4}[role]
|
||||
required := map[Permission]int{PermissionView: 1, PermissionOperate: 2, PermissionEdit: 3, PermissionAdmin: 4}[permission]
|
||||
return level > 0 && required > 0 && level >= required
|
||||
}
|
||||
|
||||
func Require(permission Permission, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
principal, ok := PrincipalFromContext(request.Context())
|
||||
if !ok {
|
||||
response.Header().Set("Cache-Control", "private, no-store")
|
||||
http.Error(response, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
if !Allows(principal.Role, permission) {
|
||||
response.Header().Set("Cache-Control", "private, no-store")
|
||||
http.Error(response, "forbidden", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(response, request)
|
||||
})
|
||||
}
|
||||
|
||||
type contextKey struct{}
|
||||
|
||||
func WithPrincipal(ctx context.Context, principal Principal) context.Context {
|
||||
return context.WithValue(ctx, contextKey{}, principal)
|
||||
}
|
||||
|
||||
func PrincipalFromContext(ctx context.Context) (Principal, bool) {
|
||||
principal, ok := ctx.Value(contextKey{}).(Principal)
|
||||
return principal, ok && principal.Subject != ""
|
||||
}
|
||||
|
||||
type BreakGlassPolicy struct {
|
||||
Enabled bool
|
||||
}
|
||||
|
||||
func (policy BreakGlassPolicy) Allows() bool { return policy.Enabled }
|
||||
@@ -0,0 +1,227 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
func TestBeginAuthorizationUsesStateNonceAndPKCE(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
|
||||
flow, err := BeginAuthorization(oauth2.Endpoint{AuthURL: "https://auth.example/authorize"}, OIDCConfig{ClientID: "pulse", RedirectURL: "https://pulse.example/callback"}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parsed, err := url.Parse(flow.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
query := parsed.Query()
|
||||
for _, key := range []string{"state", "nonce", "code_challenge", "code_challenge_method"} {
|
||||
if query.Get(key) == "" {
|
||||
t.Fatalf("authorization URL missing %s", key)
|
||||
}
|
||||
}
|
||||
if query.Get("state") != flow.State || query.Get("nonce") != flow.Nonce || query.Get("code_challenge_method") != "S256" {
|
||||
t.Fatalf("authorization URL does not match flow: %s", flow.URL)
|
||||
}
|
||||
if query.Get("code_challenge") != pkceChallenge(flow.CodeVerifier) {
|
||||
t.Fatal("authorization URL has incorrect PKCE challenge")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateCallbackRejectsStateNonceFlowAbuse(t *testing.T) {
|
||||
now := time.Now()
|
||||
flow := Authorization{State: "expected", Nonce: "nonce", CodeVerifier: "verifier", ExpiresAt: now.Add(time.Minute)}
|
||||
if err := ValidateCallback(flow, "wrong", "code", now); err == nil {
|
||||
t.Fatal("wrong state was accepted")
|
||||
}
|
||||
if err := ValidateCallback(flow, flow.State, "", now); err == nil {
|
||||
t.Fatal("empty code was accepted")
|
||||
}
|
||||
flow.ExpiresAt = now.Add(-time.Second)
|
||||
if err := ValidateCallback(flow, flow.State, "code", now); err == nil {
|
||||
t.Fatal("expired flow was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoleMappingAndAuthorizationMatrix(t *testing.T) {
|
||||
mapping := map[string]Role{"pulse-view": RoleViewer, "pulse-operator": RoleOperator, "pulse-admin": RoleAdministrator}
|
||||
role, err := MapRoles([]string{"unrelated", "pulse-operator", "pulse-view"}, mapping)
|
||||
if err != nil || role != RoleOperator {
|
||||
t.Fatalf("role mapping = %q, %v", role, err)
|
||||
}
|
||||
if _, err := MapRoles([]string{"unrelated"}, mapping); err == nil {
|
||||
t.Fatal("unmapped claims were authorized")
|
||||
}
|
||||
for _, test := range []struct {
|
||||
role Role
|
||||
permission Permission
|
||||
allowed bool
|
||||
}{
|
||||
{RoleViewer, PermissionView, true}, {RoleViewer, PermissionEdit, false},
|
||||
{RoleOperator, PermissionOperate, true}, {RoleOperator, PermissionAdmin, false},
|
||||
{RoleEditor, PermissionEdit, true}, {RoleEditor, PermissionAdmin, false},
|
||||
{RoleAdministrator, PermissionAdmin, true},
|
||||
} {
|
||||
if got := Allows(test.role, test.permission); got != test.allowed {
|
||||
t.Errorf("Allows(%s, %s) = %v, want %v", test.role, test.permission, got, test.allowed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnauthorizedPathsAreDenied(t *testing.T) {
|
||||
handler := Require(PermissionEdit, http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { response.WriteHeader(http.StatusNoContent) }))
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
status int
|
||||
}{
|
||||
{"anonymous", context.Background(), http.StatusUnauthorized},
|
||||
{"viewer", WithPrincipal(context.Background(), Principal{Subject: "user-1", Role: RoleViewer}), http.StatusForbidden},
|
||||
{"editor", WithPrincipal(context.Background(), Principal{Subject: "user-1", Role: RoleEditor}), http.StatusNoContent},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
request := httptest.NewRequest(http.MethodGet, "/protected", nil).WithContext(test.ctx)
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Code != test.status {
|
||||
t.Fatalf("status = %d, want %d", response.Code, test.status)
|
||||
}
|
||||
if test.status == http.StatusUnauthorized || test.status == http.StatusForbidden {
|
||||
if response.Header().Get("Cache-Control") != "private, no-store" {
|
||||
t.Fatalf("cache control = %q", response.Header().Get("Cache-Control"))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBreakGlassIsDisabledByDefault(t *testing.T) {
|
||||
if (BreakGlassPolicy{}).Allows() {
|
||||
t.Fatal("break-glass unexpectedly enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverResolvesEndpointAndVerifier(t *testing.T) {
|
||||
var issuer string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path != "/.well-known/openid-configuration" {
|
||||
response.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
response.Header().Set("Content-Type", "application/json")
|
||||
_, _ = response.Write([]byte(`{"issuer":"` + issuer + `","authorization_endpoint":"` + issuer + `/authorize","token_endpoint":"` + issuer + `/token","jwks_uri":"` + issuer + `/jwks","id_token_signing_alg_values_supported":["RS256"]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
issuer = server.URL
|
||||
|
||||
discovery, err := Discover(context.Background(), OIDCConfig{Issuer: issuer, ClientID: "pulse"})
|
||||
if err != nil {
|
||||
t.Fatalf("Discover: %v", err)
|
||||
}
|
||||
if discovery.Endpoint.AuthURL != issuer+"/authorize" || discovery.Endpoint.TokenURL != issuer+"/token" || discovery.Verifier == nil {
|
||||
t.Fatalf("discovery = %#v", discovery.Endpoint)
|
||||
}
|
||||
if _, err := NewVerifier(context.Background(), OIDCConfig{Issuer: issuer, ClientID: "pulse"}); err != nil {
|
||||
t.Fatalf("NewVerifier: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverRejectsIncompleteOrUnreachableIssuer(t *testing.T) {
|
||||
unreachable := httptest.NewServer(http.NewServeMux())
|
||||
unreachable.Close()
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
config OIDCConfig
|
||||
}{
|
||||
{"missing issuer", OIDCConfig{ClientID: "pulse"}},
|
||||
{"missing client id", OIDCConfig{Issuer: "https://idp.example"}},
|
||||
{"unreachable issuer", OIDCConfig{Issuer: unreachable.URL, ClientID: "pulse"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
discovery, err := Discover(context.Background(), test.config)
|
||||
if err == nil {
|
||||
t.Fatal("incomplete configuration was accepted")
|
||||
}
|
||||
if discovery.Verifier != nil {
|
||||
t.Fatal("a verifier was returned with an error")
|
||||
}
|
||||
if strings.Contains(err.Error(), test.config.Issuer) && test.config.Issuer != "" {
|
||||
t.Fatalf("error leaks the issuer: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractIdentityRequiresToken(t *testing.T) {
|
||||
if _, err := ExtractIdentity(nil, "groups"); err == nil {
|
||||
t.Fatal("nil token was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeGroupClaimBoundsAndShapes(t *testing.T) {
|
||||
many, err := json.Marshal(make([]string, maxIdentityGroups+50))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
raw string
|
||||
want []string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "list", raw: `["pulse-admin"," pulse-view ",""]`, want: []string{"pulse-admin", "pulse-view"}},
|
||||
{name: "single string", raw: `"pulse-admin"`, want: []string{"pulse-admin"}},
|
||||
{name: "empty list", raw: `[]`, want: []string{}},
|
||||
{name: "object", raw: `{"groups":["pulse-admin"]}`, wantErr: true},
|
||||
{name: "number", raw: `7`, wantErr: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
groups, err := normalizeGroupClaim(json.RawMessage(test.raw))
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("err = %v, wantErr = %v", err, test.wantErr)
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if len(groups) != len(test.want) {
|
||||
t.Fatalf("groups = %#v, want %#v", groups, test.want)
|
||||
}
|
||||
for index, value := range test.want {
|
||||
if groups[index] != value {
|
||||
t.Fatalf("groups = %#v, want %#v", groups, test.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
bounded, err := normalizeGroupClaim(many)
|
||||
if err != nil {
|
||||
t.Fatalf("normalizeGroupClaim: %v", err)
|
||||
}
|
||||
if len(bounded) != 0 {
|
||||
t.Fatalf("blank group values were kept: %d", len(bounded))
|
||||
}
|
||||
filled := make([]string, maxIdentityGroups+50)
|
||||
for index := range filled {
|
||||
filled[index] = "group"
|
||||
}
|
||||
encoded, err := json.Marshal(filled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
capped, err := normalizeGroupClaim(encoded)
|
||||
if err != nil {
|
||||
t.Fatalf("normalizeGroupClaim: %v", err)
|
||||
}
|
||||
if len(capped) != maxIdentityGroups {
|
||||
t.Fatalf("groups = %d, want %d", len(capped), maxIdentityGroups)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type session struct {
|
||||
principal Principal
|
||||
issuedAt time.Time
|
||||
expiresAt time.Time
|
||||
absoluteExpiresAt time.Time
|
||||
context context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
// SessionAuthentication carries the principal and the revocable lifetime of
|
||||
// the authenticated browser session. Long-lived transports must derive their
|
||||
// lifecycle from Context so logout and the absolute deadline remain effective
|
||||
// after an HTTP upgrade.
|
||||
type SessionAuthentication struct {
|
||||
Principal Principal
|
||||
Context context.Context
|
||||
}
|
||||
|
||||
type SessionManager struct {
|
||||
mu sync.Mutex
|
||||
sessions map[string]session
|
||||
CookieName string
|
||||
TTL time.Duration
|
||||
AbsoluteTTL time.Duration
|
||||
RenewBefore time.Duration
|
||||
Secure bool
|
||||
MaxSessions int
|
||||
MaxSessionsPerSubject int
|
||||
}
|
||||
|
||||
func NewSessionManager(cookieName string, ttl time.Duration, secure bool) *SessionManager {
|
||||
if cookieName == "" {
|
||||
cookieName = "pulse_session"
|
||||
}
|
||||
if ttl <= 0 {
|
||||
ttl = 8 * time.Hour
|
||||
}
|
||||
return &SessionManager{sessions: make(map[string]session), CookieName: cookieName, TTL: ttl, AbsoluteTTL: ttl, Secure: secure, MaxSessions: 4096, MaxSessionsPerSubject: 8}
|
||||
}
|
||||
|
||||
// NewSlidingSessionManager creates an idle-expiring browser session with a
|
||||
// separate absolute lifetime. Successful authenticated requests renew the idle
|
||||
// deadline once half of the idle lifetime has elapsed, but never beyond the
|
||||
// absolute deadline. Both lifetimes remain finite and the opaque token stays in
|
||||
// an HttpOnly cookie.
|
||||
func NewSlidingSessionManager(cookieName string, idleTTL, absoluteTTL time.Duration, secure bool) *SessionManager {
|
||||
manager := NewSessionManager(cookieName, idleTTL, secure)
|
||||
if absoluteTTL < idleTTL {
|
||||
absoluteTTL = idleTTL
|
||||
}
|
||||
manager.AbsoluteTTL = absoluteTTL
|
||||
manager.RenewBefore = idleTTL / 2
|
||||
return manager
|
||||
}
|
||||
|
||||
func (manager *SessionManager) Issue(response http.ResponseWriter, principal Principal, now time.Time) error {
|
||||
if principal.Subject == "" || !Allows(principal.Role, PermissionView) {
|
||||
return errors.New("session principal is invalid")
|
||||
}
|
||||
token, err := randomToken()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
absoluteExpiresAt := now.Add(manager.AbsoluteTTL)
|
||||
expiresAt := earliest(now.Add(manager.TTL), absoluteExpiresAt)
|
||||
sessionContext, cancel := context.WithDeadline(context.Background(), absoluteExpiresAt)
|
||||
manager.mu.Lock()
|
||||
manager.purgeExpiredLocked(now)
|
||||
manager.enforceSubjectLimitLocked(principal.Subject)
|
||||
if manager.MaxSessions > 0 && len(manager.sessions) >= manager.MaxSessions {
|
||||
manager.mu.Unlock()
|
||||
cancel()
|
||||
return errors.New("session capacity reached")
|
||||
}
|
||||
manager.sessions[hashToken(token)] = session{principal: principal, issuedAt: now, expiresAt: expiresAt, absoluteExpiresAt: absoluteExpiresAt, context: sessionContext, cancel: cancel}
|
||||
manager.mu.Unlock()
|
||||
manager.setCookie(response, token, now, expiresAt)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (manager *SessionManager) Principal(request *http.Request, now time.Time) (Principal, bool) {
|
||||
authentication, ok := manager.authenticate(nil, request, now)
|
||||
return authentication.Principal, ok
|
||||
}
|
||||
|
||||
// Authenticate validates the session and renews an active sliding session when
|
||||
// it enters its renewal window. The token is deliberately stable: concurrent
|
||||
// API requests cannot invalidate each other, while Clear still revokes it
|
||||
// immediately server-side.
|
||||
func (manager *SessionManager) Authenticate(response http.ResponseWriter, request *http.Request, now time.Time) (Principal, bool) {
|
||||
authentication, ok := manager.authenticate(response, request, now)
|
||||
return authentication.Principal, ok
|
||||
}
|
||||
|
||||
// AuthenticateSession validates and renews the cookie while exposing the
|
||||
// revocable session context to middleware that serves long-lived transports.
|
||||
func (manager *SessionManager) AuthenticateSession(response http.ResponseWriter, request *http.Request, now time.Time) (SessionAuthentication, bool) {
|
||||
return manager.authenticate(response, request, now)
|
||||
}
|
||||
|
||||
func (manager *SessionManager) authenticate(response http.ResponseWriter, request *http.Request, now time.Time) (SessionAuthentication, bool) {
|
||||
cookie, err := request.Cookie(manager.CookieName)
|
||||
if err != nil || cookie.Value == "" {
|
||||
return SessionAuthentication{}, false
|
||||
}
|
||||
manager.mu.Lock()
|
||||
defer manager.mu.Unlock()
|
||||
manager.purgeExpiredLocked(now)
|
||||
stored, ok := manager.sessions[hashToken(cookie.Value)]
|
||||
if !ok {
|
||||
return SessionAuthentication{}, false
|
||||
}
|
||||
if !now.Before(stored.expiresAt) || !now.Before(stored.absoluteExpiresAt) {
|
||||
stored.cancel()
|
||||
delete(manager.sessions, hashToken(cookie.Value))
|
||||
return SessionAuthentication{}, false
|
||||
}
|
||||
if response != nil && manager.RenewBefore > 0 && stored.expiresAt.Sub(now) <= manager.RenewBefore {
|
||||
renewed := earliest(now.Add(manager.TTL), stored.absoluteExpiresAt)
|
||||
if renewed.After(stored.expiresAt) {
|
||||
stored.expiresAt = renewed
|
||||
manager.sessions[hashToken(cookie.Value)] = stored
|
||||
manager.setCookie(response, cookie.Value, now, renewed)
|
||||
}
|
||||
}
|
||||
return SessionAuthentication{Principal: stored.principal, Context: stored.context}, true
|
||||
}
|
||||
|
||||
func (manager *SessionManager) Clear(response http.ResponseWriter, request *http.Request) {
|
||||
if cookie, err := request.Cookie(manager.CookieName); err == nil {
|
||||
manager.mu.Lock()
|
||||
key := hashToken(cookie.Value)
|
||||
if stored, ok := manager.sessions[key]; ok {
|
||||
stored.cancel()
|
||||
delete(manager.sessions, key)
|
||||
}
|
||||
manager.mu.Unlock()
|
||||
}
|
||||
http.SetCookie(response, &http.Cookie{Name: manager.CookieName, Value: "", Path: "/", MaxAge: -1, HttpOnly: true, Secure: manager.Secure, SameSite: http.SameSiteLaxMode})
|
||||
}
|
||||
|
||||
func (manager *SessionManager) purgeExpiredLocked(now time.Time) {
|
||||
for key, stored := range manager.sessions {
|
||||
if !now.Before(stored.expiresAt) || !now.Before(stored.absoluteExpiresAt) {
|
||||
stored.cancel()
|
||||
delete(manager.sessions, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (manager *SessionManager) enforceSubjectLimitLocked(subject string) {
|
||||
if manager.MaxSessionsPerSubject <= 0 {
|
||||
return
|
||||
}
|
||||
for {
|
||||
count := 0
|
||||
oldestKey := ""
|
||||
var oldest time.Time
|
||||
for key, stored := range manager.sessions {
|
||||
if stored.principal.Subject != subject {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
if oldestKey == "" || stored.issuedAt.Before(oldest) {
|
||||
oldestKey = key
|
||||
oldest = stored.issuedAt
|
||||
}
|
||||
}
|
||||
if count < manager.MaxSessionsPerSubject || oldestKey == "" {
|
||||
return
|
||||
}
|
||||
stored := manager.sessions[oldestKey]
|
||||
stored.cancel()
|
||||
delete(manager.sessions, oldestKey)
|
||||
}
|
||||
}
|
||||
|
||||
func hashToken(token string) string {
|
||||
digest := sha256.Sum256([]byte(token))
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func (manager *SessionManager) setCookie(response http.ResponseWriter, token string, now, expiresAt time.Time) {
|
||||
maxAge := int(expiresAt.Sub(now).Seconds())
|
||||
if maxAge < 1 {
|
||||
maxAge = 1
|
||||
}
|
||||
http.SetCookie(response, &http.Cookie{Name: manager.CookieName, Value: token, Path: "/", Expires: expiresAt, MaxAge: maxAge, HttpOnly: true, Secure: manager.Secure, SameSite: http.SameSiteLaxMode})
|
||||
}
|
||||
|
||||
func earliest(first, second time.Time) time.Time {
|
||||
if first.Before(second) {
|
||||
return first
|
||||
}
|
||||
return second
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSessionIssueReadAndClear(t *testing.T) {
|
||||
manager := NewSessionManager("pulse_test_session", time.Hour, true)
|
||||
now := time.Now()
|
||||
response := httptest.NewRecorder()
|
||||
principal := Principal{Subject: "subject-1", Role: RoleViewer}
|
||||
if err := manager.Issue(response, principal, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Header().Get("Set-Cookie") == "" {
|
||||
t.Fatal("session cookie was not set")
|
||||
}
|
||||
if !response.Result().Cookies()[0].HttpOnly || !response.Result().Cookies()[0].Secure {
|
||||
t.Fatal("session cookie is not hardened")
|
||||
}
|
||||
request := httptest.NewRequest("GET", "/", nil)
|
||||
for _, cookie := range response.Result().Cookies() {
|
||||
request.AddCookie(cookie)
|
||||
}
|
||||
got, ok := manager.Principal(request, now.Add(time.Minute))
|
||||
if !ok || got != principal {
|
||||
t.Fatalf("session principal = %#v, %v", got, ok)
|
||||
}
|
||||
clearResponse := httptest.NewRecorder()
|
||||
manager.Clear(clearResponse, request)
|
||||
if _, ok := manager.Principal(request, now.Add(time.Minute)); ok {
|
||||
t.Fatal("cleared session remained valid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExpires(t *testing.T) {
|
||||
manager := NewSessionManager("pulse_test_session", time.Minute, false)
|
||||
now := time.Now()
|
||||
response := httptest.NewRecorder()
|
||||
if err := manager.Issue(response, Principal{Subject: "subject-1", Role: RoleViewer}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := httptest.NewRequest("GET", "/", nil)
|
||||
request.AddCookie(response.Result().Cookies()[0])
|
||||
if _, ok := manager.Principal(request, now.Add(2*time.Minute)); ok {
|
||||
t.Fatal("expired session remained valid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlidingSessionRenewsIdleDeadlineButHonorsAbsoluteExpiry(t *testing.T) {
|
||||
manager := NewSlidingSessionManager("pulse_test_session", time.Minute, 3*time.Minute, true)
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
issued := httptest.NewRecorder()
|
||||
if err := manager.Issue(issued, Principal{Subject: "wallboard", Role: RoleViewer}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cookie := issued.Result().Cookies()[0]
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/system/status", nil)
|
||||
request.AddCookie(cookie)
|
||||
|
||||
beforeWindow := httptest.NewRecorder()
|
||||
if _, ok := manager.Authenticate(beforeWindow, request, now.Add(20*time.Second)); !ok {
|
||||
t.Fatal("active session was rejected before renewal window")
|
||||
}
|
||||
if beforeWindow.Header().Get("Set-Cookie") != "" {
|
||||
t.Fatal("session renewed before entering the bounded renewal window")
|
||||
}
|
||||
|
||||
for _, offset := range []time.Duration{40 * time.Second, 80 * time.Second, 130 * time.Second} {
|
||||
response := httptest.NewRecorder()
|
||||
if _, ok := manager.Authenticate(response, request, now.Add(offset)); !ok {
|
||||
t.Fatalf("active session was rejected at %s", offset)
|
||||
}
|
||||
renewed := response.Result().Cookies()
|
||||
if len(renewed) != 1 || renewed[0].Value != cookie.Value || renewed[0].Expires.After(now.Add(3*time.Minute)) {
|
||||
t.Fatalf("unsafe renewal at %s: %#v", offset, renewed)
|
||||
}
|
||||
}
|
||||
|
||||
if _, ok := manager.Authenticate(httptest.NewRecorder(), request, now.Add(3*time.Minute)); ok {
|
||||
t.Fatal("sliding session exceeded its absolute expiry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlidingSessionConcurrentRenewalKeepsTokenUsable(t *testing.T) {
|
||||
manager := NewSlidingSessionManager("pulse_test_session", time.Minute, time.Hour, false)
|
||||
now := time.Now().UTC()
|
||||
issued := httptest.NewRecorder()
|
||||
if err := manager.Issue(issued, Principal{Subject: "wallboard", Role: RoleViewer}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cookie := issued.Result().Cookies()[0]
|
||||
const workers = 24
|
||||
var wait sync.WaitGroup
|
||||
errors := make(chan string, workers)
|
||||
for index := 0; index < workers; index++ {
|
||||
wait.Add(1)
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/dashboards", nil)
|
||||
request.AddCookie(cookie)
|
||||
if _, ok := manager.Authenticate(httptest.NewRecorder(), request, now.Add(40*time.Second)); !ok {
|
||||
errors <- "concurrent renewal rejected a valid token"
|
||||
}
|
||||
}()
|
||||
}
|
||||
wait.Wait()
|
||||
close(errors)
|
||||
for message := range errors {
|
||||
t.Error(message)
|
||||
}
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/dashboards", nil)
|
||||
request.AddCookie(cookie)
|
||||
if _, ok := manager.Principal(request, now.Add(90*time.Second)); !ok {
|
||||
t.Fatal("stable token was invalidated by concurrent renewal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionAuthenticationContextIsRevokedByClear(t *testing.T) {
|
||||
manager := NewSlidingSessionManager("pulse_test_session", time.Minute, time.Hour, true)
|
||||
now := time.Now().UTC()
|
||||
issued := httptest.NewRecorder()
|
||||
if err := manager.Issue(issued, Principal{Subject: "viewer", Role: RoleViewer}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/live", nil)
|
||||
request.AddCookie(issued.Result().Cookies()[0])
|
||||
authentication, ok := manager.AuthenticateSession(httptest.NewRecorder(), request, now.Add(time.Second))
|
||||
if !ok || authentication.Context == nil {
|
||||
t.Fatal("session authentication context was not returned")
|
||||
}
|
||||
manager.Clear(httptest.NewRecorder(), request)
|
||||
select {
|
||||
case <-authentication.Context.Done():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("cleared session context remained active")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionAuthenticationContextEndsAtAbsoluteExpiry(t *testing.T) {
|
||||
manager := NewSlidingSessionManager("pulse_test_session", 25*time.Millisecond, 25*time.Millisecond, true)
|
||||
now := time.Now().UTC()
|
||||
issued := httptest.NewRecorder()
|
||||
if err := manager.Issue(issued, Principal{Subject: "viewer", Role: RoleViewer}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/live", nil)
|
||||
request.AddCookie(issued.Result().Cookies()[0])
|
||||
authentication, ok := manager.AuthenticateSession(httptest.NewRecorder(), request, now)
|
||||
if !ok {
|
||||
t.Fatal("new session was rejected")
|
||||
}
|
||||
select {
|
||||
case <-authentication.Context.Done():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("session context exceeded its absolute deadline")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionStoreEvictsOldestSessionsPerSubject(t *testing.T) {
|
||||
manager := NewSlidingSessionManager("pulse_test_session", time.Hour, 24*time.Hour, true)
|
||||
manager.MaxSessionsPerSubject = 3
|
||||
now := time.Now().UTC()
|
||||
for index := 0; index < 12; index++ {
|
||||
if err := manager.Issue(httptest.NewRecorder(), Principal{Subject: "viewer", Role: RoleViewer}, now.Add(time.Duration(index)*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if got := len(manager.sessions); got != 3 {
|
||||
t.Fatalf("session store size = %d, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionIssuePurgesExpiredEntriesAndHonorsGlobalCapacity(t *testing.T) {
|
||||
manager := NewSessionManager("pulse_test_session", time.Minute, true)
|
||||
manager.MaxSessions = 2
|
||||
manager.MaxSessionsPerSubject = 2
|
||||
now := time.Now().UTC()
|
||||
for _, subject := range []string{"viewer-1", "viewer-2"} {
|
||||
if err := manager.Issue(httptest.NewRecorder(), Principal{Subject: subject, Role: RoleViewer}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := manager.Issue(httptest.NewRecorder(), Principal{Subject: "viewer-3", Role: RoleViewer}, now); err == nil {
|
||||
t.Fatal("session capacity was not enforced")
|
||||
}
|
||||
if err := manager.Issue(httptest.NewRecorder(), Principal{Subject: "viewer-3", Role: RoleViewer}, now.Add(2*time.Minute)); err != nil {
|
||||
t.Fatalf("expired sessions were not purged: %v", err)
|
||||
}
|
||||
if got := len(manager.sessions); got != 1 {
|
||||
t.Fatalf("session store size after purge = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package authapi
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// signingKey is generated once per test binary; RSA generation is expensive.
|
||||
var signingKey = sync.OnceValue(func() *rsa.PrivateKey {
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return key
|
||||
})
|
||||
|
||||
// fakeIdP is a minimal OIDC provider: discovery document, JWKS and token endpoint.
|
||||
// Tests drive its behaviour through the exported fields before calling the callback.
|
||||
type fakeIdP struct {
|
||||
server *httptest.Server
|
||||
key *rsa.PrivateKey
|
||||
clientID string
|
||||
|
||||
mu sync.Mutex
|
||||
expectedChallenge string
|
||||
nonce string
|
||||
subject string
|
||||
groups []string
|
||||
tokenFails bool
|
||||
omitIDToken bool
|
||||
issuerOverride string
|
||||
verifierSeen string
|
||||
}
|
||||
|
||||
func newFakeIdP(t *testing.T, clientID string) *fakeIdP {
|
||||
t.Helper()
|
||||
idp := &fakeIdP{key: signingKey(), clientID: clientID, subject: "user-1", groups: []string{"pulse-operator"}}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/.well-known/openid-configuration", idp.discovery)
|
||||
mux.HandleFunc("/jwks", idp.jwks)
|
||||
mux.HandleFunc("/token", idp.token)
|
||||
mux.HandleFunc("/authorize", func(response http.ResponseWriter, _ *http.Request) {
|
||||
response.WriteHeader(http.StatusOK)
|
||||
})
|
||||
idp.server = httptest.NewServer(mux)
|
||||
t.Cleanup(idp.server.Close)
|
||||
return idp
|
||||
}
|
||||
|
||||
func (idp *fakeIdP) configure(mutate func(*fakeIdP)) {
|
||||
idp.mu.Lock()
|
||||
defer idp.mu.Unlock()
|
||||
mutate(idp)
|
||||
}
|
||||
|
||||
func (idp *fakeIdP) codeVerifier() string {
|
||||
idp.mu.Lock()
|
||||
defer idp.mu.Unlock()
|
||||
return idp.verifierSeen
|
||||
}
|
||||
|
||||
func (idp *fakeIdP) discovery(response http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(response, http.StatusOK, map[string]any{
|
||||
"issuer": idp.server.URL,
|
||||
"authorization_endpoint": idp.server.URL + "/authorize",
|
||||
"token_endpoint": idp.server.URL + "/token",
|
||||
"jwks_uri": idp.server.URL + "/jwks",
|
||||
"response_types_supported": []string{"code"},
|
||||
"subject_types_supported": []string{"public"},
|
||||
"id_token_signing_alg_values_supported": []string{"RS256"},
|
||||
})
|
||||
}
|
||||
|
||||
func (idp *fakeIdP) jwks(response http.ResponseWriter, _ *http.Request) {
|
||||
public := &idp.key.PublicKey
|
||||
writeJSON(response, http.StatusOK, map[string]any{"keys": []map[string]any{{
|
||||
"kty": "RSA",
|
||||
"kid": "test-key",
|
||||
"alg": "RS256",
|
||||
"use": "sig",
|
||||
"n": base64.RawURLEncoding.EncodeToString(public.N.Bytes()),
|
||||
"e": base64.RawURLEncoding.EncodeToString(big.NewInt(int64(public.E)).Bytes()),
|
||||
}}})
|
||||
}
|
||||
|
||||
func (idp *fakeIdP) token(response http.ResponseWriter, request *http.Request) {
|
||||
if err := request.ParseForm(); err != nil {
|
||||
writeJSON(response, http.StatusBadRequest, map[string]any{"error": "invalid_request"})
|
||||
return
|
||||
}
|
||||
idp.mu.Lock()
|
||||
defer idp.mu.Unlock()
|
||||
idp.verifierSeen = request.PostForm.Get("code_verifier")
|
||||
if idp.tokenFails {
|
||||
writeJSON(response, http.StatusBadRequest, map[string]any{"error": "invalid_grant"})
|
||||
return
|
||||
}
|
||||
if idp.expectedChallenge != "" {
|
||||
digest := sha256.Sum256([]byte(idp.verifierSeen))
|
||||
if base64.RawURLEncoding.EncodeToString(digest[:]) != idp.expectedChallenge {
|
||||
writeJSON(response, http.StatusBadRequest, map[string]any{"error": "invalid_grant"})
|
||||
return
|
||||
}
|
||||
}
|
||||
body := map[string]any{"access_token": "opaque-access-token", "token_type": "Bearer", "expires_in": 3600}
|
||||
if !idp.omitIDToken {
|
||||
issuer := idp.server.URL
|
||||
if idp.issuerOverride != "" {
|
||||
issuer = idp.issuerOverride
|
||||
}
|
||||
now := time.Now()
|
||||
body["id_token"] = idp.sign(map[string]any{
|
||||
"iss": issuer,
|
||||
"aud": idp.clientID,
|
||||
"sub": idp.subject,
|
||||
"iat": now.Unix(),
|
||||
"exp": now.Add(5 * time.Minute).Unix(),
|
||||
"nonce": idp.nonce,
|
||||
"groups": idp.groups,
|
||||
})
|
||||
}
|
||||
writeJSON(response, http.StatusOK, body)
|
||||
}
|
||||
|
||||
func (idp *fakeIdP) sign(claims map[string]any) string {
|
||||
segments := []string{encodeSegment(map[string]any{"alg": "RS256", "typ": "JWT", "kid": "test-key"}), encodeSegment(claims)}
|
||||
input := strings.Join(segments, ".")
|
||||
digest := sha256.Sum256([]byte(input))
|
||||
signature, err := rsa.SignPKCS1v15(rand.Reader, idp.key, crypto.SHA256, digest[:])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return input + "." + base64.RawURLEncoding.EncodeToString(signature)
|
||||
}
|
||||
|
||||
func encodeSegment(value map[string]any) string {
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(encoded)
|
||||
}
|
||||
|
||||
func writeJSON(response http.ResponseWriter, status int, body any) {
|
||||
response.Header().Set("Content-Type", "application/json")
|
||||
response.WriteHeader(status)
|
||||
_ = json.NewEncoder(response).Encode(body)
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package authapi
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultFlowTTL = 10 * time.Minute
|
||||
defaultMaxFlows = 1024
|
||||
flowIDByteLength = 32
|
||||
)
|
||||
|
||||
// flow is the server-side state of one in-progress authorization code flow. Only
|
||||
// an opaque identifier for it ever reaches the browser.
|
||||
type flow struct {
|
||||
authorization auth.Authorization
|
||||
redirect string
|
||||
createdAt time.Time
|
||||
}
|
||||
|
||||
// flowStore keeps pending flows in memory. It is bounded by TTL and by a maximum
|
||||
// entry count so an unauthenticated caller cannot grow it without limit, and it is
|
||||
// safe for concurrent use.
|
||||
type flowStore struct {
|
||||
mu sync.Mutex
|
||||
flows map[string]flow
|
||||
ttl time.Duration
|
||||
max int
|
||||
}
|
||||
|
||||
func newFlowStore(ttl time.Duration, max int) *flowStore {
|
||||
if ttl <= 0 {
|
||||
ttl = defaultFlowTTL
|
||||
}
|
||||
if max <= 0 {
|
||||
max = defaultMaxFlows
|
||||
}
|
||||
return &flowStore{flows: make(map[string]flow), ttl: ttl, max: max}
|
||||
}
|
||||
|
||||
// create stores one pending flow and returns its opaque identifier. Expired entries
|
||||
// are removed first; if the store is still at capacity the oldest entry is dropped
|
||||
// so a flood of abandoned flows cannot deny logins permanently.
|
||||
func (store *flowStore) create(entry flow, now time.Time) (string, error) {
|
||||
id, err := randomFlowID()
|
||||
if err != nil {
|
||||
return "", errors.New("generate authorization flow identifier")
|
||||
}
|
||||
entry.createdAt = now
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
store.purge(now)
|
||||
for len(store.flows) >= store.max && store.evictOldest() {
|
||||
}
|
||||
store.flows[id] = entry
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// take returns a pending flow and always removes it, so a flow identifier can be
|
||||
// used at most once. An unknown, replayed or expired identifier returns false.
|
||||
func (store *flowStore) take(id string, now time.Time) (flow, bool) {
|
||||
if id == "" {
|
||||
return flow{}, false
|
||||
}
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
entry, ok := store.flows[id]
|
||||
delete(store.flows, id)
|
||||
if !ok || !now.Before(entry.createdAt.Add(store.ttl)) {
|
||||
return flow{}, false
|
||||
}
|
||||
return entry, true
|
||||
}
|
||||
|
||||
func (store *flowStore) size() int {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
return len(store.flows)
|
||||
}
|
||||
|
||||
func (store *flowStore) purge(now time.Time) {
|
||||
for id, entry := range store.flows {
|
||||
if !now.Before(entry.createdAt.Add(store.ttl)) {
|
||||
delete(store.flows, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (store *flowStore) evictOldest() bool {
|
||||
oldest := ""
|
||||
var oldestAt time.Time
|
||||
for id, entry := range store.flows {
|
||||
if oldest == "" || entry.createdAt.Before(oldestAt) {
|
||||
oldest, oldestAt = id, entry.createdAt
|
||||
}
|
||||
}
|
||||
if oldest == "" {
|
||||
return false
|
||||
}
|
||||
delete(store.flows, oldest)
|
||||
return true
|
||||
}
|
||||
|
||||
func randomFlowID() (string, error) {
|
||||
buffer := make([]byte, flowIDByteLength)
|
||||
if _, err := rand.Read(buffer); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(buffer), nil
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package authapi
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
)
|
||||
|
||||
func testFlow(state string) flow {
|
||||
return flow{authorization: auth.Authorization{State: state, Nonce: "nonce", CodeVerifier: "verifier"}, redirect: "/"}
|
||||
}
|
||||
|
||||
func TestFlowStoreSingleUseAndExpiry(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 10, 0, 0, 0, time.UTC)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
takeAt time.Time
|
||||
twice bool
|
||||
wantOK bool
|
||||
wantAll int
|
||||
}{
|
||||
{name: "within ttl", takeAt: now.Add(time.Minute), wantOK: true},
|
||||
{name: "at ttl boundary", takeAt: now.Add(defaultFlowTTL), wantOK: false},
|
||||
{name: "after ttl", takeAt: now.Add(defaultFlowTTL + time.Second), wantOK: false},
|
||||
{name: "replayed", takeAt: now.Add(time.Minute), twice: true, wantOK: false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
store := newFlowStore(0, 0)
|
||||
id, err := store.create(testFlow("state-1"), now)
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
if test.twice {
|
||||
if _, ok := store.take(id, test.takeAt); !ok {
|
||||
t.Fatal("first take failed")
|
||||
}
|
||||
}
|
||||
entry, ok := store.take(id, test.takeAt)
|
||||
if ok != test.wantOK {
|
||||
t.Fatalf("take ok = %v, want %v", ok, test.wantOK)
|
||||
}
|
||||
if ok && entry.authorization.State != "state-1" {
|
||||
t.Fatalf("state = %q", entry.authorization.State)
|
||||
}
|
||||
if store.size() != 0 {
|
||||
t.Fatalf("take left %d entries behind", store.size())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlowStoreRejectsUnknownIdentifiers(t *testing.T) {
|
||||
store := newFlowStore(0, 0)
|
||||
for _, id := range []string{"", "unknown", " "} {
|
||||
if _, ok := store.take(id, time.Now()); ok {
|
||||
t.Fatalf("identifier %q was accepted", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlowStoreIsBounded(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
store := newFlowStore(time.Minute, 4)
|
||||
for index := range 50 {
|
||||
if _, err := store.create(testFlow("state"), now.Add(time.Duration(index)*time.Second)); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
}
|
||||
if store.size() != 4 {
|
||||
t.Fatalf("size = %d, want 4", store.size())
|
||||
}
|
||||
|
||||
expired, err := store.create(testFlow("expired"), now)
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
if _, err := store.create(testFlow("fresh"), now.Add(2*time.Minute)); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
if _, ok := store.take(expired, now.Add(2*time.Minute)); ok {
|
||||
t.Fatal("expired flow survived the purge")
|
||||
}
|
||||
if store.size() > 4 {
|
||||
t.Fatalf("size = %d, want at most 4", store.size())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlowStoreConcurrentAccess(t *testing.T) {
|
||||
const workers = 128
|
||||
store := newFlowStore(time.Minute, 64)
|
||||
now := time.Now().UTC()
|
||||
identifiers := make([]string, workers)
|
||||
var wait sync.WaitGroup
|
||||
for index := range workers {
|
||||
wait.Add(1)
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
id, err := store.create(testFlow("state"), now)
|
||||
if err != nil {
|
||||
t.Errorf("create: %v", err)
|
||||
return
|
||||
}
|
||||
identifiers[index] = id
|
||||
}()
|
||||
}
|
||||
wait.Wait()
|
||||
|
||||
unique := make(map[string]struct{}, workers)
|
||||
for _, id := range identifiers {
|
||||
if id == "" {
|
||||
t.Fatal("empty flow identifier")
|
||||
}
|
||||
unique[id] = struct{}{}
|
||||
}
|
||||
if len(unique) != workers {
|
||||
t.Fatalf("unique identifiers = %d, want %d", len(unique), workers)
|
||||
}
|
||||
if store.size() > 64 {
|
||||
t.Fatalf("size = %d, want at most 64", store.size())
|
||||
}
|
||||
|
||||
var taken sync.WaitGroup
|
||||
results := make(chan bool, 2*workers)
|
||||
for _, id := range identifiers {
|
||||
taken.Add(1)
|
||||
go func() {
|
||||
defer taken.Done()
|
||||
_, ok := store.take(id, now)
|
||||
results <- ok
|
||||
}()
|
||||
taken.Add(1)
|
||||
go func() {
|
||||
defer taken.Done()
|
||||
_, ok := store.take(id, now)
|
||||
results <- ok
|
||||
}()
|
||||
}
|
||||
taken.Wait()
|
||||
close(results)
|
||||
accepted := 0
|
||||
for ok := range results {
|
||||
if ok {
|
||||
accepted++
|
||||
}
|
||||
}
|
||||
if accepted > 64 {
|
||||
t.Fatalf("accepted %d flows, want at most the store capacity", accepted)
|
||||
}
|
||||
if store.size() != 0 {
|
||||
t.Fatalf("size = %d, want 0", store.size())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
// Package authapi exposes the two browser-facing OIDC endpoints that complete the
|
||||
// authorization code flow implemented in internal/auth: GET /auth/login starts a
|
||||
// flow and GET /auth/callback finishes it by issuing a Pulse session.
|
||||
//
|
||||
// Flow state (state, nonce, PKCE verifier and the post-login path) never leaves the
|
||||
// server; the browser only carries a short-lived opaque flow identifier cookie.
|
||||
// Every failure path destroys the identified flow, issues no session and redirects
|
||||
// to a fixed in-app error route with a reason code from a closed set. A flow whose
|
||||
// identifier never comes back simply expires. Provider-supplied
|
||||
// text is never reflected into a response, and tokens, codes and PKCE verifiers are
|
||||
// never logged.
|
||||
//
|
||||
// Wiring in cmd/api/main.go, after the session manager exists:
|
||||
//
|
||||
// oidcAuth, err := authapi.New(authapi.Options{
|
||||
// OIDC: auth.OIDCConfig{
|
||||
// Issuer: application.OIDCIssuer,
|
||||
// ClientID: application.OIDCClientID,
|
||||
// ClientSecret: application.OIDCClientSecret,
|
||||
// RedirectURL: application.OIDCRedirectURL,
|
||||
// },
|
||||
// RoleMapping: map[string]auth.Role{
|
||||
// "pulse-viewer": auth.RoleViewer, "pulse-operator": auth.RoleOperator,
|
||||
// "pulse-editor": auth.RoleEditor, "pulse-admin": auth.RoleAdministrator,
|
||||
// },
|
||||
// Sessions: sessions,
|
||||
// Secure: application.Environment == config.Production,
|
||||
// Logger: logger,
|
||||
// Audit: func(ctx context.Context, actor, result string) error {
|
||||
// if pool == nil {
|
||||
// return nil
|
||||
// }
|
||||
// return audit.RecordSecurityAction(ctx, audit.PostgresStore{Pool: pool}, actor, "auth.login", result, correlation.FromContext(ctx))
|
||||
// },
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// mux.Handle("/auth/login", oidcAuth.LoginHandler())
|
||||
// mux.Handle("/auth/callback", oidcAuth.CallbackHandler())
|
||||
//
|
||||
// New only fails on incomplete configuration, so registration is safe when
|
||||
// PULSE_AUTH_MODE is oidc; guard it with `if application.AuthMode == "oidc"` so a
|
||||
// mock-mode development run keeps working. The callback path registered here must
|
||||
// equal the path of PULSE_OIDC_REDIRECT_URL. Provider discovery happens lazily on
|
||||
// the first login and is cached, so a temporarily unreachable IdP does not prevent
|
||||
// the API from starting.
|
||||
//
|
||||
// The runtime mapping is supplied by PULSE_OIDC_ROLE_MAPPING through
|
||||
// internal/config. An empty mapping authorizes nobody, and production startup
|
||||
// rejects it before the handlers are registered.
|
||||
package authapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/correlation"
|
||||
"github.com/itworx/pulse/internal/problem"
|
||||
)
|
||||
|
||||
const (
|
||||
flowCookieName = "pulse_auth_flow"
|
||||
defaultErrorPath = "/login/error"
|
||||
defaultRedirect = "/"
|
||||
maxRedirectLength = 512
|
||||
discoveryTimeout = 10 * time.Second
|
||||
tokenTimeout = 15 * time.Second
|
||||
)
|
||||
|
||||
// Reason codes are a closed set; the provider never influences their value.
|
||||
const (
|
||||
reasonInvalidRequest = "invalid_request"
|
||||
reasonExpired = "expired"
|
||||
reasonDenied = "denied"
|
||||
reasonProviderUnavailable = "provider_unavailable"
|
||||
reasonNotAuthorized = "not_authorized"
|
||||
reasonUnavailable = "unavailable"
|
||||
)
|
||||
|
||||
// SessionIssuer is the part of *auth.SessionManager the callback needs.
|
||||
type SessionIssuer interface {
|
||||
Issue(response http.ResponseWriter, principal auth.Principal, now time.Time) error
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
// OIDC is the provider configuration; issuer, client ID and redirect URL are required.
|
||||
OIDC auth.OIDCConfig
|
||||
// RoleMapping maps IdP group claim values to Pulse roles. Empty means nobody can log in.
|
||||
RoleMapping map[string]auth.Role
|
||||
// GroupsClaim is the ID token claim holding role values; defaults to "groups".
|
||||
GroupsClaim string
|
||||
// Sessions issues the Pulse session cookie after a verified login.
|
||||
Sessions SessionIssuer
|
||||
// Secure marks the flow cookie Secure; set it in production.
|
||||
Secure bool
|
||||
// FlowTTL bounds how long a started flow stays valid; defaults to 10 minutes.
|
||||
FlowTTL time.Duration
|
||||
// MaxFlows caps concurrently pending flows; defaults to 1024.
|
||||
MaxFlows int
|
||||
// DefaultRedirect is the post-login path when none was requested; defaults to "/".
|
||||
DefaultRedirect string
|
||||
// ErrorPath is the in-app route failures redirect to; defaults to "/login/error".
|
||||
ErrorPath string
|
||||
// Logger receives structured, secret-free flow events; optional.
|
||||
Logger *slog.Logger
|
||||
// Now overrides the clock; defaults to time.Now().UTC(). It must stay close to
|
||||
// real time because the OIDC provider validates token freshness independently.
|
||||
Now func() time.Time
|
||||
// Audit records the security event before a session is issued. A returned error
|
||||
// fails the login closed; optional.
|
||||
Audit func(ctx context.Context, actor, result string) error
|
||||
}
|
||||
|
||||
// Handler serves the login and callback endpoints. Create it with New.
|
||||
type Handler struct {
|
||||
options Options
|
||||
flows *flowStore
|
||||
|
||||
mu sync.Mutex
|
||||
discovery auth.Discovery
|
||||
resolved bool
|
||||
}
|
||||
|
||||
func New(options Options) (*Handler, error) {
|
||||
if strings.TrimSpace(options.OIDC.Issuer) == "" || strings.TrimSpace(options.OIDC.ClientID) == "" || strings.TrimSpace(options.OIDC.RedirectURL) == "" {
|
||||
return nil, &configError{"OIDC issuer, client ID and redirect URL are required"}
|
||||
}
|
||||
if options.Sessions == nil {
|
||||
return nil, &configError{"session issuer is required"}
|
||||
}
|
||||
if options.GroupsClaim == "" {
|
||||
options.GroupsClaim = "groups"
|
||||
}
|
||||
options.DefaultRedirect = safePath(options.DefaultRedirect, defaultRedirect)
|
||||
if strings.ContainsAny(options.ErrorPath, "?#") {
|
||||
options.ErrorPath = ""
|
||||
}
|
||||
options.ErrorPath = safePath(options.ErrorPath, defaultErrorPath)
|
||||
if options.Logger == nil {
|
||||
options.Logger = slog.New(slog.DiscardHandler)
|
||||
}
|
||||
if options.Now == nil {
|
||||
options.Now = func() time.Time { return time.Now().UTC() }
|
||||
}
|
||||
return &Handler{options: options, flows: newFlowStore(options.FlowTTL, options.MaxFlows)}, nil
|
||||
}
|
||||
|
||||
type configError struct{ detail string }
|
||||
|
||||
func (e *configError) Error() string { return "authapi configuration invalid: " + e.detail }
|
||||
|
||||
// LoginHandler starts the authorization code flow. Register it on /auth/login.
|
||||
func (handler *Handler) LoginHandler() http.Handler { return http.HandlerFunc(handler.login) }
|
||||
|
||||
// CallbackHandler completes the flow. Register it on the path of the configured
|
||||
// OIDC redirect URL, normally /auth/callback.
|
||||
func (handler *Handler) CallbackHandler() http.Handler { return http.HandlerFunc(handler.callback) }
|
||||
|
||||
func (handler *Handler) login(response http.ResponseWriter, request *http.Request) {
|
||||
if request.Method != http.MethodGet {
|
||||
methodNotAllowed(response, request)
|
||||
return
|
||||
}
|
||||
now := handler.options.Now()
|
||||
discovery, err := handler.discover(request.Context())
|
||||
if err != nil {
|
||||
handler.reject(response, request, reasonProviderUnavailable, "discovery_failed")
|
||||
return
|
||||
}
|
||||
authorization, err := auth.BeginAuthorization(discovery.Endpoint, handler.options.OIDC, now)
|
||||
if err != nil {
|
||||
handler.reject(response, request, reasonProviderUnavailable, "authorization_start_failed")
|
||||
return
|
||||
}
|
||||
redirect := safePath(request.URL.Query().Get("redirect"), handler.options.DefaultRedirect)
|
||||
id, err := handler.flows.create(flow{authorization: authorization, redirect: redirect}, now)
|
||||
if err != nil {
|
||||
handler.reject(response, request, reasonUnavailable, "flow_not_stored")
|
||||
return
|
||||
}
|
||||
http.SetCookie(response, &http.Cookie{
|
||||
Name: flowCookieName,
|
||||
Value: id,
|
||||
Path: "/",
|
||||
MaxAge: int(handler.flows.ttl.Seconds()),
|
||||
Expires: now.Add(handler.flows.ttl),
|
||||
HttpOnly: true,
|
||||
Secure: handler.options.Secure,
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
})
|
||||
handler.options.Logger.Info("oidc login started", "correlation_id", correlation.FromContext(request.Context()), "pending_flows", handler.flows.size())
|
||||
http.Redirect(response, request, authorization.URL, http.StatusFound)
|
||||
}
|
||||
|
||||
func (handler *Handler) callback(response http.ResponseWriter, request *http.Request) {
|
||||
if request.Method != http.MethodGet {
|
||||
methodNotAllowed(response, request)
|
||||
return
|
||||
}
|
||||
now := handler.options.Now()
|
||||
cookie, err := request.Cookie(flowCookieName)
|
||||
handler.clearFlowCookie(response)
|
||||
if err != nil || cookie.Value == "" {
|
||||
handler.reject(response, request, reasonInvalidRequest, "flow_cookie_missing")
|
||||
return
|
||||
}
|
||||
pending, ok := handler.flows.take(cookie.Value, now)
|
||||
if !ok {
|
||||
handler.reject(response, request, reasonExpired, "flow_unknown_or_expired")
|
||||
return
|
||||
}
|
||||
query := request.URL.Query()
|
||||
if providerError := query.Get("error"); providerError != "" {
|
||||
reason := reasonProviderUnavailable
|
||||
if providerError == "access_denied" {
|
||||
reason = reasonDenied
|
||||
}
|
||||
handler.reject(response, request, reason, "provider_reported_error")
|
||||
return
|
||||
}
|
||||
state, code := query.Get("state"), query.Get("code")
|
||||
if err := auth.ValidateCallback(pending.authorization, state, code, now); err != nil {
|
||||
handler.reject(response, request, reasonInvalidRequest, "callback_validation_failed")
|
||||
return
|
||||
}
|
||||
discovery, err := handler.discover(request.Context())
|
||||
if err != nil {
|
||||
handler.reject(response, request, reasonProviderUnavailable, "discovery_failed")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(request.Context(), tokenTimeout)
|
||||
defer cancel()
|
||||
token, err := auth.Exchange(ctx, pending.authorization, handler.options.OIDC, discovery.Endpoint, state, code)
|
||||
if err != nil {
|
||||
handler.reject(response, request, reasonProviderUnavailable, "token_exchange_failed")
|
||||
return
|
||||
}
|
||||
rawIDToken, ok := token.Extra("id_token").(string)
|
||||
if !ok || rawIDToken == "" {
|
||||
handler.reject(response, request, reasonProviderUnavailable, "id_token_missing")
|
||||
return
|
||||
}
|
||||
idToken, err := auth.VerifyIDToken(ctx, discovery.Verifier, rawIDToken, pending.authorization.Nonce)
|
||||
if err != nil {
|
||||
handler.reject(response, request, reasonInvalidRequest, "id_token_rejected")
|
||||
return
|
||||
}
|
||||
identity, err := auth.ExtractIdentity(idToken, handler.options.GroupsClaim)
|
||||
if err != nil {
|
||||
handler.reject(response, request, reasonInvalidRequest, "identity_incomplete")
|
||||
return
|
||||
}
|
||||
role, err := auth.MapRoles(identity.Groups, handler.options.RoleMapping)
|
||||
if err != nil {
|
||||
handler.reject(response, request, reasonNotAuthorized, "no_authorized_role")
|
||||
return
|
||||
}
|
||||
principal := auth.Principal{Subject: identity.Subject, Role: role}
|
||||
if handler.options.Audit != nil {
|
||||
if err := handler.options.Audit(request.Context(), principal.Subject, "success"); err != nil {
|
||||
handler.reject(response, request, reasonUnavailable, "audit_unavailable")
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := handler.options.Sessions.Issue(response, principal, now); err != nil {
|
||||
handler.reject(response, request, reasonUnavailable, "session_not_issued")
|
||||
return
|
||||
}
|
||||
handler.options.Logger.Info("oidc login completed", "correlation_id", correlation.FromContext(request.Context()), "role", string(role))
|
||||
http.Redirect(response, request, safePath(pending.redirect, handler.options.DefaultRedirect), http.StatusFound)
|
||||
}
|
||||
|
||||
// discover resolves and caches the provider endpoints and verifier.
|
||||
func (handler *Handler) discover(ctx context.Context) (auth.Discovery, error) {
|
||||
handler.mu.Lock()
|
||||
defer handler.mu.Unlock()
|
||||
if handler.resolved {
|
||||
return handler.discovery, nil
|
||||
}
|
||||
discoveryContext, cancel := context.WithTimeout(ctx, discoveryTimeout)
|
||||
defer cancel()
|
||||
discovery, err := auth.Discover(discoveryContext, handler.options.OIDC)
|
||||
if err != nil {
|
||||
return auth.Discovery{}, err
|
||||
}
|
||||
handler.discovery, handler.resolved = discovery, true
|
||||
return discovery, nil
|
||||
}
|
||||
|
||||
// reject issues no session and sends the browser to the in-app error route with a
|
||||
// fixed reason code. The flow state is already removed by the time it is called.
|
||||
func (handler *Handler) reject(response http.ResponseWriter, request *http.Request, reason, event string) {
|
||||
handler.options.Logger.Warn("oidc flow rejected", "correlation_id", correlation.FromContext(request.Context()), "reason", reason, "event", event)
|
||||
target := handler.options.ErrorPath + "?" + url.Values{"reason": []string{reason}}.Encode()
|
||||
http.Redirect(response, request, target, http.StatusFound)
|
||||
}
|
||||
|
||||
func (handler *Handler) clearFlowCookie(response http.ResponseWriter) {
|
||||
http.SetCookie(response, &http.Cookie{
|
||||
Name: flowCookieName,
|
||||
Value: "",
|
||||
Path: "/",
|
||||
MaxAge: -1,
|
||||
HttpOnly: true,
|
||||
Secure: handler.options.Secure,
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
})
|
||||
}
|
||||
|
||||
func methodNotAllowed(response http.ResponseWriter, request *http.Request) {
|
||||
problem.Write(response, request, http.StatusMethodNotAllowed, "METHOD_NOT_ALLOWED", http.StatusText(http.StatusMethodNotAllowed), "This method is not supported.", nil)
|
||||
}
|
||||
|
||||
// safePath accepts only in-app absolute paths: one leading slash, no scheme, no
|
||||
// authority, no backslash and no control characters. Anything else falls back.
|
||||
func safePath(candidate, fallback string) string {
|
||||
target := strings.TrimSpace(candidate)
|
||||
if target == "" || len(target) > maxRedirectLength {
|
||||
return fallback
|
||||
}
|
||||
if !strings.HasPrefix(target, "/") || strings.HasPrefix(target, "//") {
|
||||
return fallback
|
||||
}
|
||||
if strings.Contains(target, "\\") {
|
||||
return fallback
|
||||
}
|
||||
for _, character := range target {
|
||||
if character < 0x20 || character == 0x7f {
|
||||
return fallback
|
||||
}
|
||||
}
|
||||
parsed, err := url.Parse(target)
|
||||
if err != nil || parsed.Scheme != "" || parsed.Host != "" || parsed.Opaque != "" || parsed.User != nil {
|
||||
return fallback
|
||||
}
|
||||
if !strings.HasPrefix(parsed.Path, "/") {
|
||||
return fallback
|
||||
}
|
||||
return target
|
||||
}
|
||||
@@ -0,0 +1,578 @@
|
||||
package authapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
)
|
||||
|
||||
const testClientID = "pulse-test-client"
|
||||
|
||||
type recordingSessions struct {
|
||||
mu sync.Mutex
|
||||
principals []auth.Principal
|
||||
failure error
|
||||
}
|
||||
|
||||
func (sessions *recordingSessions) Issue(response http.ResponseWriter, principal auth.Principal, _ time.Time) error {
|
||||
sessions.mu.Lock()
|
||||
defer sessions.mu.Unlock()
|
||||
if sessions.failure != nil {
|
||||
return sessions.failure
|
||||
}
|
||||
sessions.principals = append(sessions.principals, principal)
|
||||
http.SetCookie(response, &http.Cookie{Name: "pulse_session", Value: "issued", Path: "/", HttpOnly: true})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sessions *recordingSessions) issued() []auth.Principal {
|
||||
sessions.mu.Lock()
|
||||
defer sessions.mu.Unlock()
|
||||
return append([]auth.Principal(nil), sessions.principals...)
|
||||
}
|
||||
|
||||
type harness struct {
|
||||
handler *Handler
|
||||
idp *fakeIdP
|
||||
sessions *recordingSessions
|
||||
clock func() time.Time
|
||||
offset *time.Duration
|
||||
}
|
||||
|
||||
func newHarness(t *testing.T, mutate func(*Options)) *harness {
|
||||
t.Helper()
|
||||
idp := newFakeIdP(t, testClientID)
|
||||
sessions := &recordingSessions{}
|
||||
offset := time.Duration(0)
|
||||
options := Options{
|
||||
OIDC: auth.OIDCConfig{
|
||||
Issuer: idp.server.URL,
|
||||
ClientID: testClientID,
|
||||
ClientSecret: "test-secret",
|
||||
RedirectURL: "https://pulse.example/auth/callback",
|
||||
},
|
||||
RoleMapping: map[string]auth.Role{"pulse-viewer": auth.RoleViewer, "pulse-operator": auth.RoleOperator, "pulse-admin": auth.RoleAdministrator},
|
||||
Sessions: sessions,
|
||||
Now: func() time.Time { return time.Now().UTC().Add(offset) },
|
||||
}
|
||||
if mutate != nil {
|
||||
mutate(&options)
|
||||
}
|
||||
handler, err := New(options)
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
return &harness{handler: handler, idp: idp, sessions: sessions, clock: options.Now, offset: &offset}
|
||||
}
|
||||
|
||||
// begin runs GET /auth/login and returns the flow cookie value and the query the
|
||||
// browser would have sent to the IdP.
|
||||
func (h *harness) begin(t *testing.T, target string) (string, url.Values, *httptest.ResponseRecorder) {
|
||||
t.Helper()
|
||||
request := httptest.NewRequest(http.MethodGet, target, nil)
|
||||
response := httptest.NewRecorder()
|
||||
h.handler.LoginHandler().ServeHTTP(response, request)
|
||||
if response.Code != http.StatusFound {
|
||||
t.Fatalf("login status = %d, want %d", response.Code, http.StatusFound)
|
||||
}
|
||||
authorizationURL, err := url.Parse(response.Header().Get("Location"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse authorization URL: %v", err)
|
||||
}
|
||||
query := authorizationURL.Query()
|
||||
h.idp.configure(func(idp *fakeIdP) {
|
||||
idp.nonce = query.Get("nonce")
|
||||
idp.expectedChallenge = query.Get("code_challenge")
|
||||
})
|
||||
return flowCookie(t, response), query, response
|
||||
}
|
||||
|
||||
// complete runs GET /auth/callback with the supplied cookie and query.
|
||||
func (h *harness) complete(t *testing.T, cookie string, query url.Values) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
request := httptest.NewRequest(http.MethodGet, "/auth/callback?"+query.Encode(), nil)
|
||||
if cookie != "" {
|
||||
request.AddCookie(&http.Cookie{Name: flowCookieName, Value: cookie})
|
||||
}
|
||||
response := httptest.NewRecorder()
|
||||
h.handler.CallbackHandler().ServeHTTP(response, request)
|
||||
return response
|
||||
}
|
||||
|
||||
func flowCookie(t *testing.T, response *httptest.ResponseRecorder) string {
|
||||
t.Helper()
|
||||
for _, cookie := range response.Result().Cookies() {
|
||||
if cookie.Name == flowCookieName {
|
||||
return cookie.Value
|
||||
}
|
||||
}
|
||||
t.Fatal("flow cookie was not set")
|
||||
return ""
|
||||
}
|
||||
|
||||
func cookieByName(response *httptest.ResponseRecorder, name string) *http.Cookie {
|
||||
for _, cookie := range response.Result().Cookies() {
|
||||
if cookie.Name == name {
|
||||
return cookie
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func errorReason(t *testing.T, response *httptest.ResponseRecorder) string {
|
||||
t.Helper()
|
||||
if response.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d, want %d", response.Code, http.StatusFound)
|
||||
}
|
||||
location, err := url.Parse(response.Header().Get("Location"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse location: %v", err)
|
||||
}
|
||||
if location.Path != defaultErrorPath {
|
||||
t.Fatalf("location path = %q, want %q", location.Path, defaultErrorPath)
|
||||
}
|
||||
return location.Query().Get("reason")
|
||||
}
|
||||
|
||||
func TestLoginStartsBoundedServerSideFlow(t *testing.T) {
|
||||
harness := newHarness(t, func(options *Options) { options.Secure = true })
|
||||
cookie, query, response := harness.begin(t, "/auth/login")
|
||||
|
||||
for _, key := range []string{"state", "nonce", "code_challenge", "client_id", "redirect_uri"} {
|
||||
if query.Get(key) == "" {
|
||||
t.Fatalf("authorization URL missing %s", key)
|
||||
}
|
||||
}
|
||||
if query.Get("code_challenge_method") != "S256" {
|
||||
t.Fatalf("code_challenge_method = %q", query.Get("code_challenge_method"))
|
||||
}
|
||||
if strings.Contains(response.Header().Get("Location"), cookie) {
|
||||
t.Fatal("flow identifier leaked into the authorization URL")
|
||||
}
|
||||
if cookieByName(response, "pulse_session") != nil {
|
||||
t.Fatal("login issued a session")
|
||||
}
|
||||
flowCookie := cookieByName(response, flowCookieName)
|
||||
if !flowCookie.HttpOnly || !flowCookie.Secure || flowCookie.SameSite != http.SameSiteLaxMode {
|
||||
t.Fatalf("flow cookie attributes are unsafe: %#v", flowCookie)
|
||||
}
|
||||
if flowCookie.MaxAge <= 0 || flowCookie.MaxAge > int(defaultFlowTTL.Seconds()) {
|
||||
t.Fatalf("flow cookie MaxAge = %d", flowCookie.MaxAge)
|
||||
}
|
||||
if harness.handler.flows.size() != 1 {
|
||||
t.Fatalf("pending flows = %d, want 1", harness.handler.flows.size())
|
||||
}
|
||||
pending, ok := harness.handler.flows.take(cookie, harness.clock())
|
||||
if !ok || pending.authorization.State != query.Get("state") || pending.authorization.Nonce != query.Get("nonce") || pending.authorization.CodeVerifier == "" {
|
||||
t.Fatal("flow state was not stored server-side")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallbackHappyPathIssuesSession(t *testing.T) {
|
||||
harness := newHarness(t, nil)
|
||||
cookie, query, _ := harness.begin(t, "/auth/login?redirect=%2Fincidents")
|
||||
response := harness.complete(t, cookie, url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}})
|
||||
|
||||
if response.Code != http.StatusFound || response.Header().Get("Location") != "/incidents" {
|
||||
t.Fatalf("status = %d, location = %q", response.Code, response.Header().Get("Location"))
|
||||
}
|
||||
issued := harness.sessions.issued()
|
||||
if len(issued) != 1 || issued[0].Subject != "user-1" || issued[0].Role != auth.RoleOperator {
|
||||
t.Fatalf("issued sessions = %#v", issued)
|
||||
}
|
||||
if verifier := harness.idp.codeVerifier(); verifier == "" {
|
||||
t.Fatal("PKCE verifier was not sent to the token endpoint")
|
||||
}
|
||||
cleared := cookieByName(response, flowCookieName)
|
||||
if cleared == nil || cleared.MaxAge >= 0 || cleared.Value != "" {
|
||||
t.Fatalf("flow cookie was not cleared: %#v", cleared)
|
||||
}
|
||||
if harness.handler.flows.size() != 0 {
|
||||
t.Fatal("flow state survived the callback")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallbackWorksWithRealSessionManager(t *testing.T) {
|
||||
manager := auth.NewSessionManager("pulse_session", time.Hour, false)
|
||||
harness := newHarness(t, func(options *Options) { options.Sessions = manager })
|
||||
cookie, query, _ := harness.begin(t, "/auth/login")
|
||||
response := harness.complete(t, cookie, url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}})
|
||||
|
||||
sessionCookie := cookieByName(response, "pulse_session")
|
||||
if sessionCookie == nil {
|
||||
t.Fatal("session cookie was not set")
|
||||
}
|
||||
next := httptest.NewRequest(http.MethodGet, "/api/v1/system/status", nil)
|
||||
next.AddCookie(sessionCookie)
|
||||
principal, ok := manager.Principal(next, time.Now().UTC())
|
||||
if !ok || principal.Subject != "user-1" || principal.Role != auth.RoleOperator {
|
||||
t.Fatalf("principal = %#v, ok = %v", principal, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallbackFailurePathsIssueNoSession(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
// arrange starts a flow and returns the callback cookie and query.
|
||||
arrange func(t *testing.T, h *harness) (string, url.Values)
|
||||
reason string
|
||||
// pendingFlows is what may still be stored afterwards: an untouched flow
|
||||
// stays pending until it expires, an identified one is always destroyed.
|
||||
pendingFlows int
|
||||
}{
|
||||
{
|
||||
name: "missing flow cookie",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
_, query, _ := h.begin(t, "/auth/login")
|
||||
return "", url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}}
|
||||
},
|
||||
reason: reasonInvalidRequest,
|
||||
pendingFlows: 1,
|
||||
},
|
||||
{
|
||||
name: "unknown flow identifier",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
_, query, _ := h.begin(t, "/auth/login")
|
||||
return "not-a-known-flow", url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}}
|
||||
},
|
||||
reason: reasonExpired,
|
||||
pendingFlows: 1,
|
||||
},
|
||||
{
|
||||
name: "replayed flow identifier",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
cookie, query, _ := h.begin(t, "/auth/login")
|
||||
values := url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}}
|
||||
if first := h.complete(t, cookie, values); first.Header().Get("Location") != "/" {
|
||||
t.Fatalf("first callback did not succeed: %q", first.Header().Get("Location"))
|
||||
}
|
||||
h.sessions.mu.Lock()
|
||||
h.sessions.principals = nil
|
||||
h.sessions.mu.Unlock()
|
||||
return cookie, values
|
||||
},
|
||||
reason: reasonExpired,
|
||||
},
|
||||
{
|
||||
name: "expired flow",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
cookie, query, _ := h.begin(t, "/auth/login")
|
||||
*h.offset = defaultFlowTTL + time.Minute
|
||||
return cookie, url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}}
|
||||
},
|
||||
reason: reasonExpired,
|
||||
},
|
||||
{
|
||||
name: "state mismatch",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
cookie, _, _ := h.begin(t, "/auth/login")
|
||||
return cookie, url.Values{"state": {"forged-state"}, "code": {"authorization-code"}}
|
||||
},
|
||||
reason: reasonInvalidRequest,
|
||||
},
|
||||
{
|
||||
name: "missing authorization code",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
cookie, query, _ := h.begin(t, "/auth/login")
|
||||
return cookie, url.Values{"state": {query.Get("state")}}
|
||||
},
|
||||
reason: reasonInvalidRequest,
|
||||
},
|
||||
{
|
||||
name: "nonce mismatch",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
cookie, query, _ := h.begin(t, "/auth/login")
|
||||
h.idp.configure(func(idp *fakeIdP) { idp.nonce = "replayed-nonce" })
|
||||
return cookie, url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}}
|
||||
},
|
||||
reason: reasonInvalidRequest,
|
||||
},
|
||||
{
|
||||
name: "provider access denied",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
cookie, query, _ := h.begin(t, "/auth/login")
|
||||
return cookie, url.Values{"state": {query.Get("state")}, "error": {"access_denied"}, "error_description": {"<script>alert(1)</script> denied by policy"}}
|
||||
},
|
||||
reason: reasonDenied,
|
||||
},
|
||||
{
|
||||
name: "provider error response",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
cookie, query, _ := h.begin(t, "/auth/login")
|
||||
return cookie, url.Values{"state": {query.Get("state")}, "error": {"server_error"}}
|
||||
},
|
||||
reason: reasonProviderUnavailable,
|
||||
},
|
||||
{
|
||||
name: "token exchange failure",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
cookie, query, _ := h.begin(t, "/auth/login")
|
||||
h.idp.configure(func(idp *fakeIdP) { idp.tokenFails = true })
|
||||
return cookie, url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}}
|
||||
},
|
||||
reason: reasonProviderUnavailable,
|
||||
},
|
||||
{
|
||||
name: "id token missing",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
cookie, query, _ := h.begin(t, "/auth/login")
|
||||
h.idp.configure(func(idp *fakeIdP) { idp.omitIDToken = true })
|
||||
return cookie, url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}}
|
||||
},
|
||||
reason: reasonProviderUnavailable,
|
||||
},
|
||||
{
|
||||
name: "id token from wrong issuer",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
cookie, query, _ := h.begin(t, "/auth/login")
|
||||
h.idp.configure(func(idp *fakeIdP) { idp.issuerOverride = "https://attacker.example" })
|
||||
return cookie, url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}}
|
||||
},
|
||||
reason: reasonInvalidRequest,
|
||||
},
|
||||
{
|
||||
name: "no mapped role",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
cookie, query, _ := h.begin(t, "/auth/login")
|
||||
h.idp.configure(func(idp *fakeIdP) { idp.groups = []string{"some-other-group"} })
|
||||
return cookie, url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}}
|
||||
},
|
||||
reason: reasonNotAuthorized,
|
||||
},
|
||||
{
|
||||
name: "session issue failure",
|
||||
arrange: func(t *testing.T, h *harness) (string, url.Values) {
|
||||
cookie, query, _ := h.begin(t, "/auth/login")
|
||||
h.sessions.mu.Lock()
|
||||
h.sessions.failure = errors.New("session store unavailable")
|
||||
h.sessions.mu.Unlock()
|
||||
return cookie, url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}}
|
||||
},
|
||||
reason: reasonUnavailable,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
harness := newHarness(t, nil)
|
||||
cookie, query := test.arrange(t, harness)
|
||||
response := harness.complete(t, cookie, query)
|
||||
|
||||
if reason := errorReason(t, response); reason != test.reason {
|
||||
t.Fatalf("reason = %q, want %q", reason, test.reason)
|
||||
}
|
||||
if issued := harness.sessions.issued(); len(issued) != 0 {
|
||||
t.Fatalf("a session was issued on a failure path: %#v", issued)
|
||||
}
|
||||
if cookieByName(response, "pulse_session") != nil {
|
||||
t.Fatal("a session cookie was set on a failure path")
|
||||
}
|
||||
if harness.handler.flows.size() != test.pendingFlows {
|
||||
t.Fatalf("pending flows = %d, want %d", harness.handler.flows.size(), test.pendingFlows)
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, forbidden := range []string{"<script>", "denied by policy", "server_error", "access_denied", "authorization-code"} {
|
||||
if strings.Contains(body, forbidden) {
|
||||
t.Fatalf("response body reflected untrusted text %q: %s", forbidden, body)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuditFailureBlocksSession(t *testing.T) {
|
||||
harness := newHarness(t, func(options *Options) {
|
||||
options.Audit = func(context.Context, string, string) error { return errors.New("audit unavailable") }
|
||||
})
|
||||
cookie, query, _ := harness.begin(t, "/auth/login")
|
||||
response := harness.complete(t, cookie, url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}})
|
||||
if reason := errorReason(t, response); reason != reasonUnavailable {
|
||||
t.Fatalf("reason = %q, want %q", reason, reasonUnavailable)
|
||||
}
|
||||
if issued := harness.sessions.issued(); len(issued) != 0 {
|
||||
t.Fatalf("session issued despite audit failure: %#v", issued)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuditRecordsSuccessfulLogin(t *testing.T) {
|
||||
var actor, result string
|
||||
harness := newHarness(t, func(options *Options) {
|
||||
options.Audit = func(_ context.Context, recordedActor, recordedResult string) error {
|
||||
actor, result = recordedActor, recordedResult
|
||||
return nil
|
||||
}
|
||||
})
|
||||
cookie, query, _ := harness.begin(t, "/auth/login")
|
||||
harness.complete(t, cookie, url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}})
|
||||
if actor != "user-1" || result != "success" {
|
||||
t.Fatalf("audit actor = %q, result = %q", actor, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostLoginRedirectRejectsUnsafeTargets(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
redirect string
|
||||
want string
|
||||
}{
|
||||
{"in-app path", "/incidents", "/incidents"},
|
||||
{"in-app path with query and fragment", "/dashboards/1?tab=live#top", "/dashboards/1?tab=live#top"},
|
||||
{"absent", "", "/"},
|
||||
{"protocol relative", "//evil.example/phish", "/"},
|
||||
{"triple slash", "///evil.example", "/"},
|
||||
{"absolute http", "http://evil.example", "/"},
|
||||
{"absolute https", "https://evil.example/x", "/"},
|
||||
{"scheme relative backslash", "/\\evil.example", "/"},
|
||||
{"backslashes", "\\\\evil.example", "/"},
|
||||
{"relative path", "incidents", "/"},
|
||||
{"javascript scheme", "javascript:alert(1)", "/"},
|
||||
{"data scheme", "data:text/html,<script>alert(1)</script>", "/"},
|
||||
{"newline injection", "/incidents\r\nSet-Cookie: x=1", "/"},
|
||||
{"userinfo authority", "//user:pass@evil.example/", "/"},
|
||||
{"overlong", "/" + strings.Repeat("a", maxRedirectLength), "/"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
harness := newHarness(t, nil)
|
||||
cookie, query, _ := harness.begin(t, "/auth/login?redirect="+url.QueryEscape(test.redirect))
|
||||
response := harness.complete(t, cookie, url.Values{"state": {query.Get("state")}, "code": {"authorization-code"}})
|
||||
if response.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d", response.Code)
|
||||
}
|
||||
if location := response.Header().Get("Location"); location != test.want {
|
||||
t.Fatalf("location = %q, want %q", location, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentFlowCreationIsSafeAndBounded(t *testing.T) {
|
||||
const workers = 64
|
||||
harness := newHarness(t, func(options *Options) { options.MaxFlows = 16 })
|
||||
var wait sync.WaitGroup
|
||||
cookies := make([]string, workers)
|
||||
for index := range workers {
|
||||
wait.Add(1)
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
request := httptest.NewRequest(http.MethodGet, "/auth/login", nil)
|
||||
response := httptest.NewRecorder()
|
||||
harness.handler.LoginHandler().ServeHTTP(response, request)
|
||||
for _, cookie := range response.Result().Cookies() {
|
||||
if cookie.Name == flowCookieName {
|
||||
cookies[index] = cookie.Value
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wait.Wait()
|
||||
|
||||
unique := make(map[string]struct{}, workers)
|
||||
for _, cookie := range cookies {
|
||||
if cookie == "" {
|
||||
t.Fatal("a concurrent login produced no flow cookie")
|
||||
}
|
||||
unique[cookie] = struct{}{}
|
||||
}
|
||||
if len(unique) != workers {
|
||||
t.Fatalf("unique flow identifiers = %d, want %d", len(unique), workers)
|
||||
}
|
||||
if size := harness.handler.flows.size(); size > 16 {
|
||||
t.Fatalf("pending flows = %d, want at most 16", size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveryFailureRedirectsSafely(t *testing.T) {
|
||||
closed := httptest.NewServer(http.NewServeMux())
|
||||
issuer := closed.URL
|
||||
closed.Close()
|
||||
handler, err := New(Options{
|
||||
OIDC: auth.OIDCConfig{Issuer: issuer, ClientID: testClientID, RedirectURL: "https://pulse.example/auth/callback"},
|
||||
Sessions: &recordingSessions{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
response := httptest.NewRecorder()
|
||||
handler.LoginHandler().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/auth/login", nil))
|
||||
if reason := errorReason(t, response); reason != reasonProviderUnavailable {
|
||||
t.Fatalf("reason = %q, want %q", reason, reasonProviderUnavailable)
|
||||
}
|
||||
if body := response.Body.String(); strings.Contains(body, issuer) {
|
||||
t.Fatalf("response leaked the issuer: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonGetMethodsAreRejected(t *testing.T) {
|
||||
harness := newHarness(t, nil)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
handler http.Handler
|
||||
target string
|
||||
}{
|
||||
{"login", harness.handler.LoginHandler(), "/auth/login"},
|
||||
{"callback", harness.handler.CallbackHandler(), "/auth/callback"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
response := httptest.NewRecorder()
|
||||
test.handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, test.target, nil))
|
||||
if response.Code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("status = %d, want %d", response.Code, http.StatusMethodNotAllowed)
|
||||
}
|
||||
if contentType := response.Header().Get("Content-Type"); contentType != "application/problem+json" {
|
||||
t.Fatalf("content type = %q", contentType)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewValidatesOptions(t *testing.T) {
|
||||
valid := auth.OIDCConfig{Issuer: "https://idp.example", ClientID: "pulse", RedirectURL: "https://pulse.example/auth/callback"}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
options Options
|
||||
wantErr bool
|
||||
}{
|
||||
{"complete", Options{OIDC: valid, Sessions: &recordingSessions{}}, false},
|
||||
{"missing issuer", Options{OIDC: auth.OIDCConfig{ClientID: "pulse", RedirectURL: valid.RedirectURL}, Sessions: &recordingSessions{}}, true},
|
||||
{"missing client id", Options{OIDC: auth.OIDCConfig{Issuer: valid.Issuer, RedirectURL: valid.RedirectURL}, Sessions: &recordingSessions{}}, true},
|
||||
{"missing redirect url", Options{OIDC: auth.OIDCConfig{Issuer: valid.Issuer, ClientID: "pulse"}, Sessions: &recordingSessions{}}, true},
|
||||
{"missing sessions", Options{OIDC: valid}, true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
handler, err := New(test.options)
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("err = %v, wantErr = %v", err, test.wantErr)
|
||||
}
|
||||
if err != nil {
|
||||
if handler != nil {
|
||||
t.Fatal("handler returned with an error")
|
||||
}
|
||||
return
|
||||
}
|
||||
if handler.options.ErrorPath != defaultErrorPath || handler.options.DefaultRedirect != defaultRedirect || handler.options.GroupsClaim != "groups" {
|
||||
t.Fatalf("defaults not applied: %#v", handler.options)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewNormalizesUnsafePaths(t *testing.T) {
|
||||
handler, err := New(Options{
|
||||
OIDC: auth.OIDCConfig{Issuer: "https://idp.example", ClientID: "pulse", RedirectURL: "https://pulse.example/auth/callback"},
|
||||
Sessions: &recordingSessions{},
|
||||
DefaultRedirect: "//evil.example",
|
||||
ErrorPath: "/login/error?reason=spoofed",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
if handler.options.DefaultRedirect != defaultRedirect || handler.options.ErrorPath != defaultErrorPath {
|
||||
t.Fatalf("unsafe paths were kept: %#v", handler.options)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package authapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/audit"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/authapi"
|
||||
"github.com/itworx/pulse/internal/config"
|
||||
"github.com/itworx/pulse/internal/correlation"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// TestDocumentedWiringCompilesAndRoutes mirrors the registration snippet in the
|
||||
// package documentation so cmd/api/main.go can copy it verbatim.
|
||||
func TestDocumentedWiringCompilesAndRoutes(t *testing.T) {
|
||||
// The issuer points at a closed local server so the test stays offline: both
|
||||
// endpoints then answer with the safe error redirect instead of a session.
|
||||
unreachable := httptest.NewServer(http.NewServeMux())
|
||||
unreachable.Close()
|
||||
application := config.Config{
|
||||
Environment: config.Development,
|
||||
AuthMode: "oidc",
|
||||
OIDCIssuer: unreachable.URL,
|
||||
OIDCClientID: "pulse",
|
||||
OIDCRedirectURL: "https://pulse.example/auth/callback",
|
||||
}
|
||||
sessions := auth.NewSessionManager("pulse_session", 8*time.Hour, application.Environment == config.Production)
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
var pool *pgxpool.Pool
|
||||
|
||||
oidcAuth, err := authapi.New(authapi.Options{
|
||||
OIDC: auth.OIDCConfig{
|
||||
Issuer: application.OIDCIssuer,
|
||||
ClientID: application.OIDCClientID,
|
||||
ClientSecret: application.OIDCClientSecret,
|
||||
RedirectURL: application.OIDCRedirectURL,
|
||||
},
|
||||
RoleMapping: map[string]auth.Role{
|
||||
"pulse-viewer": auth.RoleViewer, "pulse-operator": auth.RoleOperator,
|
||||
"pulse-editor": auth.RoleEditor, "pulse-admin": auth.RoleAdministrator,
|
||||
},
|
||||
Sessions: sessions,
|
||||
Secure: application.Environment == config.Production,
|
||||
Logger: logger,
|
||||
Audit: func(ctx context.Context, actor, result string) error {
|
||||
if pool == nil {
|
||||
return nil
|
||||
}
|
||||
return audit.RecordSecurityAction(ctx, audit.PostgresStore{Pool: pool}, actor, "auth.login", result, correlation.FromContext(ctx))
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/auth/login", oidcAuth.LoginHandler())
|
||||
mux.Handle("/auth/callback", oidcAuth.CallbackHandler())
|
||||
|
||||
for _, path := range []string{"/auth/login", "/auth/callback"} {
|
||||
response := httptest.NewRecorder()
|
||||
mux.ServeHTTP(response, httptest.NewRequest(http.MethodGet, path, nil))
|
||||
if response.Code != http.StatusFound {
|
||||
t.Fatalf("%s status = %d, want %d", path, response.Code, http.StatusFound)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,573 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const (
|
||||
formatVersion = 2
|
||||
defaultRetention = 5
|
||||
manifestName = "manifest.json"
|
||||
)
|
||||
|
||||
var ErrNotConfigured = errors.New("backup destination is not configured")
|
||||
|
||||
type Manager struct {
|
||||
Pool *pgxpool.Pool
|
||||
Directory string
|
||||
Retention int
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type Manifest struct {
|
||||
FormatVersion int `json:"formatVersion"`
|
||||
BackupID string `json:"backupId"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
SchemaVersion int64 `json:"schemaVersion"`
|
||||
Tables []TableManifest `json:"tables"`
|
||||
Excluded []string `json:"excluded"`
|
||||
}
|
||||
|
||||
type TableManifest struct {
|
||||
Name string `json:"name"`
|
||||
File string `json:"file"`
|
||||
Rows int64 `json:"rows"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
BackupID string `json:"backupId"`
|
||||
Path string `json:"path"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Rows int64 `json:"rows"`
|
||||
Created time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
type tableSpec struct {
|
||||
name string
|
||||
columns string
|
||||
restoreCols string
|
||||
orderBy string
|
||||
}
|
||||
|
||||
var tableSpecs = []tableSpec{
|
||||
{name: "roles", columns: "id,name,created_at", restoreCols: "id,name,created_at", orderBy: "id"},
|
||||
{name: "users", columns: "id,external_subject,display_name,email,status,created_at,updated_at,last_login_at", restoreCols: "id,external_subject,display_name,email,status,created_at,updated_at,last_login_at", orderBy: "id"},
|
||||
{name: "user_roles", columns: "user_id,role_id,created_at", restoreCols: "user_id,role_id,created_at", orderBy: "user_id,role_id"},
|
||||
{name: "data_sources", columns: "id,type,name,enabled,configuration_ref,capability_document,health_state,last_success_at,last_error_code,last_error_message,freshness_policy,created_at,updated_at", restoreCols: "id,type,name,enabled,configuration_ref,capability_document,health_state,last_success_at,last_error_code,last_error_message,freshness_policy,created_at,updated_at", orderBy: "id"},
|
||||
{name: "collectors", columns: "id,datasource_id,kind,version,heartbeat,capabilities,status", restoreCols: "id,datasource_id,kind,version,heartbeat,capabilities,status", orderBy: "id"},
|
||||
{name: "entities", columns: "id,entity_type,canonical_name,display_name,status,status_reasons,first_seen_at,last_seen_at,tombstoned_at,attributes", restoreCols: "id,entity_type,canonical_name,display_name,status,status_reasons,first_seen_at,last_seen_at,tombstoned_at,attributes", orderBy: "id"},
|
||||
{name: "entity_aliases", columns: "entity_id,source_id,external_type,external_id", restoreCols: "entity_id,source_id,external_type,external_id", orderBy: "source_id,external_type,external_id"},
|
||||
{name: "container_aliases", columns: "source_id,runtime_id,entity_id,name,project,service,image_digest,observed_state,observed_health,restart_count,intentional_stop,first_seen_at,last_seen_at,tombstoned_at", restoreCols: "source_id,runtime_id,entity_id,name,project,service,image_digest,observed_state,observed_health,restart_count,intentional_stop,first_seen_at,last_seen_at,tombstoned_at", orderBy: "source_id,runtime_id"},
|
||||
{name: "entity_facts", columns: "entity_id,field_name,source_id,value,observed_at,confidence,valid_until", restoreCols: "entity_id,field_name,source_id,value,observed_at,confidence,valid_until", orderBy: "entity_id,field_name,source_id"},
|
||||
{name: "entity_overrides", columns: "entity_id,field_name,value,user_id,updated_at", restoreCols: "entity_id,field_name,value,user_id,updated_at", orderBy: "entity_id,field_name"},
|
||||
{name: "entity_relations", columns: "id,source_entity_id,relation_type,target_entity_id,source_id,confidence,confirmed,first_seen_at,last_seen_at,tombstoned_at", restoreCols: "id,source_entity_id,relation_type,target_entity_id,source_id,confidence,confirmed,first_seen_at,last_seen_at,tombstoned_at", orderBy: "id"},
|
||||
{name: "dashboards", columns: "id,slug,name,description,owner_user_id,scope,archived_at,current_version_id,revision,created_at,updated_at", restoreCols: "id,slug,name,description,owner_user_id,scope,archived_at,revision,created_at,updated_at", orderBy: "id"},
|
||||
{name: "dashboard_versions", columns: "id,dashboard_id,version_number,schema_version,document,change_summary,created_by,created_at", restoreCols: "id,dashboard_id,version_number,schema_version,document,change_summary,created_by,created_at", orderBy: "dashboard_id,version_number"},
|
||||
{name: "events", columns: "id,event_type,severity,entity_id,source_id,occurred_at,received_at,dedup_key,summary,attributes,correlation_id", restoreCols: "id,event_type,severity,entity_id,source_id,occurred_at,received_at,dedup_key,summary,attributes,correlation_id", orderBy: "id"},
|
||||
{name: "audit_events", columns: "id,actor,action,resource_type,resource_id,result,occurred_at,correlation_id,before_diff,after_diff", restoreCols: "id,actor,action,resource_type,resource_id,result,occurred_at,correlation_id,before_diff,after_diff", orderBy: "occurred_at,id"},
|
||||
{name: "job_runs", columns: "id,job_type,job_key,scheduled_at,started_at,completed_at,status,counts,error_code,correlation_id,lease_owner,lease_until", restoreCols: "id,job_type,job_key,scheduled_at,started_at,completed_at,status,counts,error_code,correlation_id,lease_owner,lease_until", orderBy: "scheduled_at,id"},
|
||||
{name: "services", columns: "id,entity_id,source_id,name,description,state,labels,revision,archived_at,created_by,created_at,updated_at", restoreCols: "id,entity_id,source_id,name,description,state,labels,revision,archived_at,created_by,created_at,updated_at", orderBy: "id"},
|
||||
{name: "service_endpoints", columns: "id,service_id,source_id,name,endpoint_type,target,enabled,revision,archived_at,created_at,updated_at", restoreCols: "id,service_id,source_id,name,endpoint_type,target,enabled,revision,archived_at,created_at,updated_at", orderBy: "id"},
|
||||
{name: "probes", columns: "id,service_id,endpoint_id,source_id,name,probe_type,target,interval_seconds,timeout_seconds,enabled,expected_status_codes,follow_redirects,verify_tls,content_assertion,network_policy_id,revision,archived_at,created_by,created_at,updated_at", restoreCols: "id,service_id,endpoint_id,source_id,name,probe_type,target,interval_seconds,timeout_seconds,enabled,expected_status_codes,follow_redirects,verify_tls,content_assertion,network_policy_id,revision,archived_at,created_by,created_at,updated_at", orderBy: "id"},
|
||||
{name: "probe_results", columns: "id,probe_id,source_id,observed_at,completed_at,state,response_time_ms,status_code,error_class,error_message,attributes", restoreCols: "id,probe_id,source_id,observed_at,completed_at,state,response_time_ms,status_code,error_class,error_message,attributes", orderBy: "probe_id,observed_at"},
|
||||
{name: "service_certificates", columns: "id,service_id,endpoint_id,source_id,observed_at,expires_at,issuer,subject,hostname_valid,verification_state,attributes", restoreCols: "id,service_id,endpoint_id,source_id,observed_at,expires_at,issuer,subject,hostname_valid,verification_state,attributes", orderBy: "id"},
|
||||
{name: "service_dependencies", columns: "id,service_id,depends_on_service_id,source_id,relation_type,confidence,confirmed,first_seen_at,last_seen_at,archived_at", restoreCols: "id,service_id,depends_on_service_id,source_id,relation_type,confidence,confirmed,first_seen_at,last_seen_at,archived_at", orderBy: "id"},
|
||||
{name: "service_permissions", columns: "service_id,role_id,permission,created_at", restoreCols: "service_id,role_id,permission,created_at", orderBy: "service_id,role_id,permission"},
|
||||
{name: "alert_rules", columns: "id,schema_version,name,enabled,severity,scope,condition,evaluation_interval_seconds,pending_seconds,resolve_seconds,unknown_behavior,group_by,suppress_when,message,current_version_id,revision,created_by,created_at,updated_at,cooldown_seconds", restoreCols: "id,schema_version,name,enabled,severity,scope,condition,evaluation_interval_seconds,pending_seconds,resolve_seconds,unknown_behavior,group_by,suppress_when,message,revision,created_by,created_at,updated_at,cooldown_seconds", orderBy: "id"},
|
||||
{name: "alert_rule_versions", columns: "id,rule_id,version_number,document,change_summary,created_by,created_at", restoreCols: "id,rule_id,version_number,document,change_summary,created_by,created_at", orderBy: "rule_id,version_number"},
|
||||
{name: "alert_instances", columns: "id,rule_id,rule_version_id,fingerprint,entity_id,current_state,retained_state,active_since,recovery_since,last_evaluated_at,last_known_at,last_value,reason,source_health,acknowledged_by,acknowledged_at,revision,created_at,updated_at,cooldown_until", restoreCols: "id,rule_id,rule_version_id,fingerprint,entity_id,current_state,retained_state,active_since,recovery_since,last_evaluated_at,last_known_at,last_value,reason,source_health,acknowledged_by,acknowledged_at,revision,created_at,updated_at,cooldown_until", orderBy: "id"},
|
||||
{name: "alert_occurrences", columns: "id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at", restoreCols: "id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at", orderBy: "instance_id,observed_at,id"},
|
||||
{name: "alert_silences", columns: "id,name,reason,owner,matchers,starts_at,expires_at,status,created_by,created_at,revoked_by,revoked_at,expired_at,revision", restoreCols: "id,name,reason,owner,matchers,starts_at,expires_at,status,created_by,created_at,revoked_by,revoked_at,expired_at,revision", orderBy: "id"},
|
||||
{name: "maintenance_windows", columns: "id,name,reason,selector,starts_at,ends_at,status,created_by,created_at,revoked_by,revoked_at,expired_at,revision", restoreCols: "id,name,reason,selector,starts_at,ends_at,status,created_by,created_at,revoked_by,revoked_at,expired_at,revision", orderBy: "id"},
|
||||
{name: "incidents", columns: "id,correlation_key,title,summary,severity,status,started_at,resolved_at,owner_user_id,correlation_method,confidence,revision,created_at,updated_at", restoreCols: "id,correlation_key,title,summary,severity,status,started_at,resolved_at,owner_user_id,correlation_method,confidence,revision,created_at,updated_at", orderBy: "id"},
|
||||
{name: "incident_alerts", columns: "incident_id,alert_id,rationale,confidence,correlation_method,is_manual,added_by,created_at", restoreCols: "incident_id,alert_id,rationale,confidence,correlation_method,is_manual,added_by,created_at", orderBy: "incident_id,alert_id"},
|
||||
{name: "incident_entities", columns: "incident_id,entity_id,rationale,confidence,created_at", restoreCols: "incident_id,entity_id,rationale,confidence,created_at", orderBy: "incident_id,entity_id"},
|
||||
{name: "incident_notes", columns: "id,incident_id,author,body,created_at", restoreCols: "id,incident_id,author,body,created_at", orderBy: "incident_id,created_at,id"},
|
||||
}
|
||||
|
||||
// backupExcludedTables classifies application tables that deliberately do not
|
||||
// belong in a portable archive. The PostgreSQL integration test requires every
|
||||
// migrated application table to appear either here or in tableSpecs, so adding a
|
||||
// migration cannot silently make restore incomplete.
|
||||
var backupExcludedTables = map[string]string{
|
||||
"agent_snapshots": "bounded runtime telemetry is republished by the agent after restart",
|
||||
"capacity_samples": "bounded forecast telemetry is republished by the agent after restart",
|
||||
"notification_channels": "secret references and channel configuration must be reattached",
|
||||
"notification_deliveries": "runtime notification delivery history is intentionally excluded",
|
||||
"notification_outbox": "runtime notification delivery state is intentionally excluded",
|
||||
"system_settings": "runtime configuration and secret-bearing values must be reattached",
|
||||
}
|
||||
|
||||
func backupExclusions() []string {
|
||||
names := make([]string, 0, len(backupExcludedTables))
|
||||
for name := range backupExcludedTables {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
result := make([]string, 0, len(names))
|
||||
for _, name := range names {
|
||||
result = append(result, name+" ("+backupExcludedTables[name]+")")
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (m Manager) Create(ctx context.Context) (Result, error) {
|
||||
if m.Pool == nil || strings.TrimSpace(m.Directory) == "" {
|
||||
return Result{}, ErrNotConfigured
|
||||
}
|
||||
if err := os.MkdirAll(m.Directory, 0o700); err != nil {
|
||||
return Result{}, fmt.Errorf("create backup directory: %w", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if m.Now != nil {
|
||||
now = m.Now().UTC()
|
||||
}
|
||||
id, err := newID()
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("generate backup id: %w", err)
|
||||
}
|
||||
schemaVersion, err := currentSchemaVersion(ctx, m.Pool)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
temp, err := os.CreateTemp(m.Directory, ".pulse-backup-*.tmp")
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("create temporary backup: %w", err)
|
||||
}
|
||||
tempName := temp.Name()
|
||||
defer os.Remove(tempName)
|
||||
archive := zip.NewWriter(temp)
|
||||
manifest := Manifest{FormatVersion: formatVersion, BackupID: id, CreatedAt: now, SchemaVersion: schemaVersion, Excluded: backupExclusions()}
|
||||
for _, spec := range tableSpecs {
|
||||
entry, err := archive.Create("data/" + spec.name + ".jsonl")
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("create archive entry %s: %w", spec.name, err)
|
||||
}
|
||||
hash := sha256.New()
|
||||
writer := io.MultiWriter(entry, hash)
|
||||
rows, err := exportTable(ctx, m.Pool, spec, writer)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("export %s: %w", spec.name, err)
|
||||
}
|
||||
manifest.Tables = append(manifest.Tables, TableManifest{Name: spec.name, File: "data/" + spec.name + ".jsonl", Rows: rows, SHA256: hex.EncodeToString(hash.Sum(nil))})
|
||||
}
|
||||
manifestEntry, err := archive.Create(manifestName)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("create manifest: %w", err)
|
||||
}
|
||||
if err := json.NewEncoder(manifestEntry).Encode(manifest); err != nil {
|
||||
return Result{}, fmt.Errorf("write manifest: %w", err)
|
||||
}
|
||||
if err := archive.Close(); err != nil {
|
||||
return Result{}, fmt.Errorf("close backup archive: %w", err)
|
||||
}
|
||||
if err := temp.Sync(); err != nil {
|
||||
return Result{}, fmt.Errorf("sync backup archive: %w", err)
|
||||
}
|
||||
if err := temp.Close(); err != nil {
|
||||
return Result{}, fmt.Errorf("close backup file: %w", err)
|
||||
}
|
||||
finalPath := filepath.Join(m.Directory, "pulse-backup-"+id+".zip")
|
||||
if err := os.Rename(tempName, finalPath); err != nil {
|
||||
return Result{}, fmt.Errorf("finalize backup: %w", err)
|
||||
}
|
||||
checksum, size, err := fileChecksum(finalPath)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := os.WriteFile(finalPath+".sha256", []byte(checksum+" "+filepath.Base(finalPath)+"\n"), 0o600); err != nil {
|
||||
return Result{}, fmt.Errorf("write backup checksum: %w", err)
|
||||
}
|
||||
if err := m.prune(ctx, finalPath); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
var rows int64
|
||||
for _, table := range manifest.Tables {
|
||||
rows += table.Rows
|
||||
}
|
||||
return Result{BackupID: id, Path: finalPath, SHA256: checksum, Bytes: size, Rows: rows, Created: now}, nil
|
||||
}
|
||||
|
||||
func (m Manager) Verify(ctx context.Context, path string) (Manifest, error) {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return Manifest{}, errors.New("backup path is required")
|
||||
}
|
||||
archive, err := zip.OpenReader(path)
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("open backup: %w", err)
|
||||
}
|
||||
defer archive.Close()
|
||||
entries := make(map[string]*zip.File, len(archive.File))
|
||||
for _, entry := range archive.File {
|
||||
if _, exists := entries[entry.Name]; exists {
|
||||
return Manifest{}, fmt.Errorf("duplicate archive entry %q", entry.Name)
|
||||
}
|
||||
entries[entry.Name] = entry
|
||||
}
|
||||
manifestFile, ok := entries[manifestName]
|
||||
if !ok {
|
||||
return Manifest{}, errors.New("backup manifest is missing")
|
||||
}
|
||||
manifestReader, err := manifestFile.Open()
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("open manifest: %w", err)
|
||||
}
|
||||
var manifest Manifest
|
||||
err = json.NewDecoder(manifestReader).Decode(&manifest)
|
||||
_ = manifestReader.Close()
|
||||
if err != nil || manifest.FormatVersion != formatVersion || manifest.BackupID == "" || manifest.SchemaVersion <= 0 {
|
||||
return Manifest{}, errors.New("backup manifest is invalid")
|
||||
}
|
||||
if len(manifest.Tables) != len(tableSpecs) {
|
||||
return Manifest{}, fmt.Errorf("backup table set is incomplete: got %d, want %d", len(manifest.Tables), len(tableSpecs))
|
||||
}
|
||||
expectedFiles := map[string]bool{manifestName: true}
|
||||
for _, spec := range tableSpecs {
|
||||
expectedFiles["data/"+spec.name+".jsonl"] = true
|
||||
}
|
||||
for name := range entries {
|
||||
if !expectedFiles[name] {
|
||||
return Manifest{}, fmt.Errorf("unexpected backup archive entry %q", name)
|
||||
}
|
||||
}
|
||||
seen := make(map[string]bool, len(manifest.Tables))
|
||||
for _, table := range manifest.Tables {
|
||||
if seen[table.Name] || table.Rows < 0 || table.SHA256 == "" || table.File != "data/"+table.Name+".jsonl" || !expectedFiles[table.File] {
|
||||
return Manifest{}, fmt.Errorf("backup table entry %q is invalid", table.Name)
|
||||
}
|
||||
seen[table.Name] = true
|
||||
entry, ok := entries[table.File]
|
||||
if !ok {
|
||||
return Manifest{}, fmt.Errorf("backup table file %q is missing", table.File)
|
||||
}
|
||||
if err := verifyTable(entry, table); err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
}
|
||||
for _, spec := range tableSpecs {
|
||||
if !seen[spec.name] {
|
||||
return Manifest{}, fmt.Errorf("backup table %q is missing", spec.name)
|
||||
}
|
||||
}
|
||||
checksum, _, err := fileChecksum(path)
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
sidecar, err := os.ReadFile(path + ".sha256")
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("read backup checksum: %w", err)
|
||||
}
|
||||
if !strings.HasPrefix(string(sidecar), checksum+" ") {
|
||||
return Manifest{}, errors.New("backup archive checksum does not match sidecar")
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func (m Manager) List(ctx context.Context) ([]Result, error) {
|
||||
if strings.TrimSpace(m.Directory) == "" {
|
||||
return nil, ErrNotConfigured
|
||||
}
|
||||
entries, err := os.ReadDir(m.Directory)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []Result{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("list backups: %w", err)
|
||||
}
|
||||
results := make([]Result, 0)
|
||||
for _, entry := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if entry.IsDir() || !strings.HasPrefix(entry.Name(), "pulse-backup-") || !strings.HasSuffix(entry.Name(), ".zip") {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(m.Directory, entry.Name())
|
||||
manifest, err := m.Verify(ctx, path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("verify listed backup %s: %w", entry.Name(), err)
|
||||
}
|
||||
checksum, size, err := fileChecksum(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var rows int64
|
||||
for _, table := range manifest.Tables {
|
||||
rows += table.Rows
|
||||
}
|
||||
results = append(results, Result{BackupID: manifest.BackupID, Path: path, SHA256: checksum, Bytes: size, Rows: rows, Created: manifest.CreatedAt})
|
||||
}
|
||||
sort.Slice(results, func(i, j int) bool { return results[i].Created.After(results[j].Created) })
|
||||
return results, nil
|
||||
}
|
||||
func (m Manager) Restore(ctx context.Context, path string) (Manifest, error) {
|
||||
if m.Pool == nil {
|
||||
return Manifest{}, ErrNotConfigured
|
||||
}
|
||||
manifest, err := m.Verify(ctx, path)
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
for _, spec := range tableSpecs {
|
||||
var count int64
|
||||
if err := m.Pool.QueryRow(ctx, "SELECT count(*) FROM "+spec.name).Scan(&count); err != nil {
|
||||
return Manifest{}, fmt.Errorf("check restore target %s: %w", spec.name, err)
|
||||
}
|
||||
if count != 0 {
|
||||
return Manifest{}, fmt.Errorf("restore target is not empty: %s has %d rows", spec.name, count)
|
||||
}
|
||||
}
|
||||
archive, err := zip.OpenReader(path)
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("open restore archive: %w", err)
|
||||
}
|
||||
defer archive.Close()
|
||||
entries := make(map[string]*zip.File, len(archive.File))
|
||||
for _, entry := range archive.File {
|
||||
entries[entry.Name] = entry
|
||||
}
|
||||
tx, err := m.Pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("begin restore: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
for _, spec := range tableSpecs {
|
||||
entry := entries["data/"+spec.name+".jsonl"]
|
||||
reader, err := entry.Open()
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("open restore table %s: %w", spec.name, err)
|
||||
}
|
||||
decoder := json.NewDecoder(bufio.NewReader(reader))
|
||||
for {
|
||||
var row json.RawMessage
|
||||
if err := decoder.Decode(&row); errors.Is(err, io.EOF) {
|
||||
break
|
||||
} else if err != nil {
|
||||
_ = reader.Close()
|
||||
return Manifest{}, fmt.Errorf("decode restore table %s: %w", spec.name, err)
|
||||
}
|
||||
selectCols := spec.restoreCols
|
||||
if spec.name == "alert_instances" {
|
||||
selectCols = strings.Replace(selectCols, "last_value", "COALESCE(last_value, 'null'::jsonb)", 1)
|
||||
} else if spec.name == "alert_occurrences" {
|
||||
selectCols = strings.Replace(selectCols, "value", "COALESCE(value, 'null'::jsonb)", 1)
|
||||
}
|
||||
query := "INSERT INTO " + spec.name + " (" + spec.restoreCols + ") SELECT " + selectCols + " FROM jsonb_populate_record(NULL::" + spec.name + ", $1::jsonb)"
|
||||
if _, err := tx.Exec(ctx, query, []byte(row)); err != nil {
|
||||
_ = reader.Close()
|
||||
return Manifest{}, fmt.Errorf("restore %s: %w", spec.name, err)
|
||||
}
|
||||
}
|
||||
_ = reader.Close()
|
||||
}
|
||||
for _, deferred := range []struct{ table, id, column string }{{"dashboards", "id", "current_version_id"}, {"alert_rules", "id", "current_version_id"}} {
|
||||
entry := entries["data/"+deferred.table+".jsonl"]
|
||||
reader, err := entry.Open()
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("open deferred restore %s: %w", deferred.table, err)
|
||||
}
|
||||
decoder := json.NewDecoder(bufio.NewReader(reader))
|
||||
for {
|
||||
var row map[string]json.RawMessage
|
||||
if err := decoder.Decode(&row); errors.Is(err, io.EOF) {
|
||||
break
|
||||
} else if err != nil {
|
||||
_ = reader.Close()
|
||||
return Manifest{}, fmt.Errorf("decode deferred restore %s: %w", deferred.table, err)
|
||||
}
|
||||
id, ok := row[deferred.id]
|
||||
value, valueOK := row[deferred.column]
|
||||
if !ok || !valueOK || string(value) == "null" {
|
||||
continue
|
||||
}
|
||||
var valueID, rowID string
|
||||
if err := json.Unmarshal(value, &valueID); err != nil {
|
||||
_ = reader.Close()
|
||||
return Manifest{}, fmt.Errorf("decode deferred %s id: %w", deferred.table, err)
|
||||
}
|
||||
if err := json.Unmarshal(id, &rowID); err != nil {
|
||||
_ = reader.Close()
|
||||
return Manifest{}, fmt.Errorf("decode deferred %s row id: %w", deferred.table, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, "UPDATE "+deferred.table+" SET "+deferred.column+"=$1::uuid WHERE "+deferred.id+"=$2::uuid", valueID, rowID); err != nil {
|
||||
_ = reader.Close()
|
||||
return Manifest{}, fmt.Errorf("restore deferred %s: %w", deferred.table, err)
|
||||
}
|
||||
}
|
||||
_ = reader.Close()
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Manifest{}, fmt.Errorf("commit restore: %w", err)
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func exportTable(ctx context.Context, pool *pgxpool.Pool, spec tableSpec, writer io.Writer) (int64, error) {
|
||||
rows, err := pool.Query(ctx, "SELECT row_to_json(t) FROM (SELECT "+spec.columns+" FROM "+spec.name+" ORDER BY "+spec.orderBy+") t")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var count int64
|
||||
for rows.Next() {
|
||||
var raw []byte
|
||||
if err := rows.Scan(&raw); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if containsSensitiveKey(raw) {
|
||||
return 0, fmt.Errorf("sensitive key detected in %s export", spec.name)
|
||||
}
|
||||
if _, err := writer.Write(append(raw, '\n')); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
count++
|
||||
}
|
||||
return count, rows.Err()
|
||||
}
|
||||
|
||||
func verifyTable(entry *zip.File, expected TableManifest) error {
|
||||
reader, err := entry.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("open table %s: %w", expected.Name, err)
|
||||
}
|
||||
defer reader.Close()
|
||||
hash := sha256.New()
|
||||
decoder := json.NewDecoder(io.TeeReader(reader, hash))
|
||||
var rows int64
|
||||
for {
|
||||
var raw json.RawMessage
|
||||
if err := decoder.Decode(&raw); errors.Is(err, io.EOF) {
|
||||
break
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("validate table %s: %w", expected.Name, err)
|
||||
}
|
||||
if containsSensitiveKey(raw) {
|
||||
return fmt.Errorf("sensitive key detected in %s archive", expected.Name)
|
||||
}
|
||||
rows++
|
||||
}
|
||||
if rows != expected.Rows || hex.EncodeToString(hash.Sum(nil)) != expected.SHA256 {
|
||||
return fmt.Errorf("table %s checksum or row count mismatch", expected.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func currentSchemaVersion(ctx context.Context, pool *pgxpool.Pool) (int64, error) {
|
||||
var version int64
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM schema_migrations`).Scan(&version); err != nil {
|
||||
return 0, fmt.Errorf("read schema version: %w", err)
|
||||
}
|
||||
return version, nil
|
||||
}
|
||||
|
||||
func fileChecksum(path string) (string, int64, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("open backup for checksum: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("stat backup: %w", err)
|
||||
}
|
||||
hash := sha256.New()
|
||||
if _, err := io.Copy(hash, file); err != nil {
|
||||
return "", 0, fmt.Errorf("checksum backup: %w", err)
|
||||
}
|
||||
return hex.EncodeToString(hash.Sum(nil)), info.Size(), nil
|
||||
}
|
||||
|
||||
func (m Manager) prune(ctx context.Context, keepPath string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
retention := m.Retention
|
||||
if retention <= 0 {
|
||||
retention = defaultRetention
|
||||
}
|
||||
entries, err := os.ReadDir(m.Directory)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list backups: %w", err)
|
||||
}
|
||||
var archives []os.DirEntry
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() && strings.HasPrefix(entry.Name(), "pulse-backup-") && strings.HasSuffix(entry.Name(), ".zip") {
|
||||
archives = append(archives, entry)
|
||||
}
|
||||
}
|
||||
sort.Slice(archives, func(i, j int) bool { return archives[i].Name() > archives[j].Name() })
|
||||
if len(archives) <= retention {
|
||||
return nil
|
||||
}
|
||||
for _, entry := range archives[retention:] {
|
||||
path := filepath.Join(m.Directory, entry.Name())
|
||||
if path == keepPath {
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
return fmt.Errorf("prune backup %s: %w", entry.Name(), err)
|
||||
}
|
||||
_ = os.Remove(path + ".sha256")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func containsSensitiveKey(raw []byte) bool {
|
||||
var value any
|
||||
if json.Unmarshal(raw, &value) != nil {
|
||||
return true
|
||||
}
|
||||
return sensitiveValue(value)
|
||||
}
|
||||
|
||||
func sensitiveValue(value any) bool {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
for key, nested := range typed {
|
||||
lower := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(key, "-", "_"), " ", "_"))
|
||||
for _, part := range []string{"authorization", "cookie", "password", "passwd", "secret", "token", "api_key", "apikey", "client_secret"} {
|
||||
if strings.Contains(lower, part) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if sensitiveValue(nested) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, nested := range typed {
|
||||
if sensitiveValue(nested) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func newID() (string, error) {
|
||||
var bytes [16]byte
|
||||
if _, err := rand.Read(bytes[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
bytes[6] = (bytes[6] & 0x0f) | 0x40
|
||||
bytes[8] = (bytes[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%s-%s-%s-%s-%s", hex.EncodeToString(bytes[0:4]), hex.EncodeToString(bytes[4:6]), hex.EncodeToString(bytes[6:8]), hex.EncodeToString(bytes[8:10]), hex.EncodeToString(bytes[10:16])), nil
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestPostgreSQLBackupRestoreCleanRoom(t *testing.T) {
|
||||
sourceDSN := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
targetDSN := os.Getenv("PULSE_TEST_RESTORE_DATABASE_URL")
|
||||
if sourceDSN == "" || targetDSN == "" {
|
||||
if os.Getenv("PULSE_REQUIRE_BACKUP_INTEGRATION") == "true" {
|
||||
t.Fatal("backup integration is required but both PostgreSQL DSNs are not configured")
|
||||
}
|
||||
t.Skip("PULSE_TEST_DATABASE_URL and PULSE_TEST_RESTORE_DATABASE_URL are required")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
source, err := database.NewPool(ctx, database.Config{URL: sourceDSN})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer source.Close()
|
||||
target, err := database.NewPool(ctx, database.Config{URL: targetDSN})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer target.Close()
|
||||
if err := database.Migrate(ctx, source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.Migrate(ctx, target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertBackupSchemaCoverage(t, ctx, source)
|
||||
ids := testIDs()
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
seed := []struct {
|
||||
query string
|
||||
args []any
|
||||
}{
|
||||
{`INSERT INTO roles (id,name) VALUES ($1,'administrator')`, []any{ids.role}},
|
||||
{`INSERT INTO users (id,external_subject,display_name,email) VALUES ($1,$2,'Backup Test','backup@example.invalid')`, []any{ids.user, ids.user}},
|
||||
{`INSERT INTO user_roles (user_id,role_id) VALUES ($1,$2)`, []any{ids.user, ids.role}},
|
||||
{`INSERT INTO data_sources (id,type,name,configuration_ref) VALUES ($1,'exporter','Backup source','source/ref')`, []any{ids.source}},
|
||||
{`INSERT INTO entities (id,entity_type,canonical_name,display_name,first_seen_at) VALUES ($1,'container','backup-test','Backup test',$2)`, []any{ids.entity, now}},
|
||||
{`INSERT INTO container_aliases (source_id,runtime_id,entity_id,name,project,service,image_digest,observed_state,observed_health,restart_count,first_seen_at,last_seen_at) VALUES ($1,'runtime-backup-test',$2,'backup-test','pulse','api','sha256:test','running','healthy',2,$3,$3)`, []any{ids.source, ids.entity, now}},
|
||||
{`INSERT INTO dashboards (id,slug,name,description,owner_user_id,scope,current_version_id) VALUES ($1,'backup-test','Backup test dashboard','portable',$2,'shared',NULL)`, []any{ids.dashboard, ids.user}},
|
||||
{`INSERT INTO dashboard_versions (id,dashboard_id,version_number,schema_version,document,created_by) VALUES ($1,$2,1,1,'{}',$3)`, []any{ids.dashboardVersion, ids.dashboard, ids.user}},
|
||||
{`UPDATE dashboards SET current_version_id=$1 WHERE id=$2`, []any{ids.dashboardVersion, ids.dashboard}},
|
||||
{`INSERT INTO alert_rules (id,schema_version,name,severity,condition,evaluation_interval_seconds,pending_seconds,resolve_seconds,unknown_behavior,group_by,suppress_when,message,current_version_id,created_by) VALUES ($1,1,'Backup test rule','critical','{}',60,0,0,'become-unknown','[]','[]','{}',NULL,$2)`, []any{ids.rule, ids.user}},
|
||||
{`INSERT INTO alert_rule_versions (id,rule_id,version_number,document,created_by) VALUES ($1,$2,1,'{}',$3)`, []any{ids.ruleVersion, ids.rule, ids.user}},
|
||||
{`UPDATE alert_rules SET current_version_id=$1 WHERE id=$2`, []any{ids.ruleVersion, ids.rule}},
|
||||
{`INSERT INTO alert_instances (id,rule_id,rule_version_id,fingerprint,entity_id,last_evaluated_at) VALUES ($1,$2,$3,'backup-fingerprint',$4,$5)`, []any{ids.instance, ids.rule, ids.ruleVersion, ids.entity, now}},
|
||||
{`INSERT INTO incidents (id,correlation_key,title,severity,started_at,owner_user_id,correlation_method,confidence) VALUES ($1,'backup-correlation','Backup test incident','critical',$2,$3,'deterministic',0.900)`, []any{ids.incident, now, ids.user}},
|
||||
{`INSERT INTO incident_alerts (incident_id,alert_id,rationale,confidence,correlation_method,added_by) VALUES ($1,$2,'backup test rationale',0.900,'deterministic','test')`, []any{ids.incident, ids.instance}},
|
||||
{`INSERT INTO incident_entities (incident_id,entity_id,rationale,confidence) VALUES ($1,$2,'backup test entity',0.900)`, []any{ids.incident, ids.entity}},
|
||||
{`INSERT INTO incident_notes (id,incident_id,author,body) VALUES ($1,$2,'test','backup note')`, []any{ids.note, ids.incident}},
|
||||
{`INSERT INTO audit_events (id,actor,action,resource_type,resource_id,result,after_diff) VALUES ($1,'backup-test','backup.seed','dashboard',$2,'success','{}')`, []any{ids.audit, ids.dashboard}},
|
||||
}
|
||||
for _, statement := range seed {
|
||||
if _, err := source.Exec(ctx, statement.query, statement.args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
directory := t.TempDir()
|
||||
manager := Manager{Pool: source, Directory: directory, Retention: 2, Now: func() time.Time { return now }}
|
||||
created, err := manager.Create(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if created.Rows <= 0 || created.SHA256 == "" {
|
||||
t.Fatalf("unexpected backup result: %#v", created)
|
||||
}
|
||||
verified, err := manager.Verify(ctx, created.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if verified.FormatVersion != formatVersion {
|
||||
t.Fatalf("backup format = %d, want %d", verified.FormatVersion, formatVersion)
|
||||
}
|
||||
containerAliasManifest := false
|
||||
for _, table := range verified.Tables {
|
||||
if table.Name == "container_aliases" && table.Rows == 1 && table.SHA256 != "" {
|
||||
containerAliasManifest = true
|
||||
}
|
||||
}
|
||||
if !containerAliasManifest {
|
||||
t.Fatal("container_aliases is missing from the checksummed manifest")
|
||||
}
|
||||
if _, err := manager.List(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restored, err := (Manager{Pool: target}).Restore(ctx, created.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if restored.BackupID != created.BackupID {
|
||||
t.Fatalf("restored manifest = %s, want %s", restored.BackupID, created.BackupID)
|
||||
}
|
||||
for _, check := range []struct {
|
||||
table string
|
||||
want int
|
||||
}{
|
||||
{"container_aliases", 1}, {"dashboards", 1}, {"dashboard_versions", 1}, {"alert_rules", 1}, {"alert_rule_versions", 1}, {"incidents", 1}, {"incident_alerts", 1}, {"incident_entities", 1}, {"incident_notes", 1}, {"audit_events", 1},
|
||||
} {
|
||||
var got int
|
||||
if err := target.QueryRow(ctx, "SELECT count(*) FROM "+check.table).Scan(&got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != check.want {
|
||||
t.Fatalf("%s count = %d, want %d", check.table, got, check.want)
|
||||
}
|
||||
}
|
||||
var dashboardVersion, ruleVersion string
|
||||
if err := target.QueryRow(ctx, `SELECT current_version_id::text FROM dashboards WHERE id=$1`, ids.dashboard).Scan(&dashboardVersion); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := target.QueryRow(ctx, `SELECT current_version_id::text FROM alert_rules WHERE id=$1`, ids.rule).Scan(&ruleVersion); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if dashboardVersion != ids.dashboardVersion || ruleVersion != ids.ruleVersion {
|
||||
t.Fatalf("deferred links = %s/%s", dashboardVersion, ruleVersion)
|
||||
}
|
||||
var runtimeEntity string
|
||||
if err := target.QueryRow(ctx, `SELECT entity_id::text FROM container_aliases WHERE source_id=$1 AND runtime_id='runtime-backup-test'`, ids.source).Scan(&runtimeEntity); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runtimeEntity != ids.entity {
|
||||
t.Fatalf("restored container alias entity = %s, want %s", runtimeEntity, ids.entity)
|
||||
}
|
||||
}
|
||||
|
||||
func assertBackupSchemaCoverage(t *testing.T, ctx context.Context, pool *pgxpool.Pool) {
|
||||
t.Helper()
|
||||
rows, err := pool.Query(ctx, `SELECT c.table_name, c.column_name
|
||||
FROM information_schema.columns c
|
||||
JOIN information_schema.tables t
|
||||
ON t.table_schema = c.table_schema AND t.table_name = c.table_name
|
||||
WHERE c.table_schema = 'public' AND t.table_type = 'BASE TABLE'
|
||||
ORDER BY c.table_name, c.ordinal_position`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
columnsByTable := map[string]map[string]bool{}
|
||||
for rows.Next() {
|
||||
var table, column string
|
||||
if err := rows.Scan(&table, &column); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if columnsByTable[table] == nil {
|
||||
columnsByTable[table] = map[string]bool{}
|
||||
}
|
||||
columnsByTable[table][column] = true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
classified := map[string]string{"schema_migrations": "migration metadata"}
|
||||
for name, reason := range backupExcludedTables {
|
||||
classified[name] = reason
|
||||
}
|
||||
for _, spec := range tableSpecs {
|
||||
if previous, duplicate := classified[spec.name]; duplicate {
|
||||
t.Fatalf("backup table %q is classified more than once (previous: %s)", spec.name, previous)
|
||||
}
|
||||
classified[spec.name] = "portable backup"
|
||||
tableColumns, exists := columnsByTable[spec.name]
|
||||
if !exists {
|
||||
t.Fatalf("backup table %q does not exist after migrations", spec.name)
|
||||
}
|
||||
for _, list := range []string{spec.columns, spec.restoreCols} {
|
||||
for _, column := range strings.Split(list, ",") {
|
||||
column = strings.TrimSpace(column)
|
||||
if column == "" || !tableColumns[column] {
|
||||
t.Fatalf("backup table %q references missing column %q", spec.name, column)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var unclassified, missing []string
|
||||
for table := range columnsByTable {
|
||||
if _, ok := classified[table]; !ok {
|
||||
unclassified = append(unclassified, table)
|
||||
}
|
||||
}
|
||||
for table := range classified {
|
||||
if _, ok := columnsByTable[table]; !ok {
|
||||
missing = append(missing, table)
|
||||
}
|
||||
}
|
||||
sort.Strings(unclassified)
|
||||
sort.Strings(missing)
|
||||
if len(unclassified) > 0 || len(missing) > 0 {
|
||||
t.Fatalf("backup schema drift: unclassified=%v missing=%v", unclassified, missing)
|
||||
}
|
||||
}
|
||||
|
||||
type testIDSet struct {
|
||||
role, user, source, entity, dashboard, dashboardVersion, rule, ruleVersion, instance, incident, note, audit string
|
||||
}
|
||||
|
||||
func testIDs() testIDSet {
|
||||
id := func() string {
|
||||
value, err := newID()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return value
|
||||
}
|
||||
return testIDSet{
|
||||
role: id(), user: id(), source: id(), entity: id(), dashboard: id(), dashboardVersion: id(),
|
||||
rule: id(), ruleVersion: id(), instance: id(), incident: id(), note: id(), audit: id(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSensitiveArchiveKeysAreRejected(t *testing.T) {
|
||||
for _, raw := range []string{`{"before_diff":{"api_token":"value"}}`, `{"configuration":{"password":"value"}}`, `{"authorization":"Bearer value"}`} {
|
||||
if !containsSensitiveKey([]byte(raw)) {
|
||||
t.Fatalf("sensitive key was not detected in %s", raw)
|
||||
}
|
||||
}
|
||||
if containsSensitiveKey([]byte(`{"display_name":"Pulse","configuration_ref":"source/ref"}`)) {
|
||||
t.Fatal("safe reference fields were rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerListReturnsEmptyForMissingDirectory(t *testing.T) {
|
||||
directory := filepath.Join(t.TempDir(), "backups")
|
||||
results, err := (Manager{Directory: directory}).List(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(results) != 0 {
|
||||
t.Fatalf("results = %d, want 0", len(results))
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerCreateRequiresConfiguredPoolAndDirectory(t *testing.T) {
|
||||
if _, err := (Manager{}).Create(context.Background()); err != ErrNotConfigured {
|
||||
t.Fatalf("error = %v, want ErrNotConfigured", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneKeepsConfiguredRetentionAndSidecars(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
for _, name := range []string{"pulse-backup-00000001.zip", "pulse-backup-00000002.zip", "pulse-backup-00000003.zip"} {
|
||||
if err := os.WriteFile(filepath.Join(directory, name), []byte(name), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(directory, name+".sha256"), []byte("checksum"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := (Manager{Directory: directory, Retention: 2}).prune(context.Background(), filepath.Join(directory, "pulse-backup-00000003.zip")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(directory, "pulse-backup-00000001.zip")); !os.IsNotExist(err) {
|
||||
t.Fatalf("old backup still exists: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(directory, "pulse-backup-00000001.zip.sha256")); !os.IsNotExist(err) {
|
||||
t.Fatalf("old sidecar still exists: %v", err)
|
||||
}
|
||||
if strings.TrimSpace(string(mustRead(t, filepath.Join(directory, "pulse-backup-00000003.zip.sha256")))) != "checksum" {
|
||||
t.Fatal("kept sidecar was changed")
|
||||
}
|
||||
}
|
||||
|
||||
func mustRead(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
value, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package backupapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/backup"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
Manager *backup.Manager
|
||||
Audit func(context.Context, string, string) error
|
||||
// OnCreated invalidates derived status caches after the archive and its
|
||||
// checksum have both been written successfully.
|
||||
OnCreated func()
|
||||
}
|
||||
|
||||
type publicResult struct {
|
||||
BackupID string `json:"backupId"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Rows int64 `json:"rows"`
|
||||
Created time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if h.Manager == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "BACKUP_UNAVAILABLE", "Backup is not configured")
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
if r.URL.Path != "/api/v1/system/backups" {
|
||||
writeError(w, http.StatusNotFound, "NOT_FOUND", "Not found")
|
||||
return
|
||||
}
|
||||
result, err := h.Manager.List(r.Context())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "BACKUP_UNAVAILABLE", "Backups are not available")
|
||||
return
|
||||
}
|
||||
public := make([]publicResult, 0, len(result))
|
||||
for _, item := range result {
|
||||
public = append(public, toPublic(item))
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"backups": public})
|
||||
case http.MethodPost:
|
||||
if r.URL.Path != "/api/v1/system/backups" {
|
||||
writeError(w, http.StatusNotFound, "NOT_FOUND", "Not found")
|
||||
return
|
||||
}
|
||||
result, err := h.Manager.Create(r.Context())
|
||||
principal, _ := auth.PrincipalFromContext(r.Context())
|
||||
if err != nil {
|
||||
if h.Audit != nil {
|
||||
_ = h.Audit(r.Context(), principal.Subject, "failure")
|
||||
}
|
||||
status := http.StatusInternalServerError
|
||||
code := "BACKUP_FAILED"
|
||||
if errors.Is(err, backup.ErrNotConfigured) {
|
||||
status = http.StatusServiceUnavailable
|
||||
code = "BACKUP_UNAVAILABLE"
|
||||
}
|
||||
writeError(w, status, code, "Backup could not be created")
|
||||
return
|
||||
}
|
||||
if h.Audit != nil {
|
||||
_ = h.Audit(r.Context(), principal.Subject, "success")
|
||||
}
|
||||
if h.OnCreated != nil {
|
||||
h.OnCreated()
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, toPublic(result))
|
||||
default:
|
||||
writeError(w, http.StatusMethodNotAllowed, "METHOD_NOT_ALLOWED", "Method not allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func toPublic(result backup.Result) publicResult {
|
||||
return publicResult{BackupID: result.BackupID, SHA256: result.SHA256, Bytes: result.Bytes, Rows: result.Rows, Created: result.Created}
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "private, no-store")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, code, detail string) {
|
||||
writeJSON(w, status, map[string]string{"code": code, "detail": detail})
|
||||
}
|
||||
|
||||
var _ http.Handler = Handler{}
|
||||
@@ -0,0 +1,44 @@
|
||||
package backupapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/itworx/pulse/internal/backup"
|
||||
)
|
||||
|
||||
func TestHandlerRejectsUnconfiguredBackupWithoutDisclosure(t *testing.T) {
|
||||
handler := Handler{}
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/system/backups", nil)
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusServiceUnavailable || !strings.Contains(response.Body.String(), "BACKUP_UNAVAILABLE") {
|
||||
t.Fatalf("response = %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
if strings.Contains(response.Body.String(), "PULSE_") || strings.Contains(response.Body.String(), "password") {
|
||||
t.Fatal("configuration detail leaked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerRejectsUnsupportedMethod(t *testing.T) {
|
||||
handler := Handler{Manager: &backup.Manager{}}
|
||||
request := httptest.NewRequest(http.MethodDelete, "/api/v1/system/backups", nil)
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("response = %d, want method not allowed", response.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicResultOmitsServerPath(t *testing.T) {
|
||||
payload, err := json.Marshal(toPublic(backup.Result{BackupID: "id", Path: "C:/private/backups/pulse.zip", SHA256: "checksum"}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(payload), "private/backups") || strings.Contains(string(payload), "path") {
|
||||
t.Fatalf("server path leaked: %s", payload)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package buildinfo
|
||||
|
||||
import "time"
|
||||
|
||||
// These values are replaced with -ldflags by reproducible release builds.
|
||||
var (
|
||||
Version = "development"
|
||||
Commit = "unknown"
|
||||
BuildTime = "unknown"
|
||||
)
|
||||
|
||||
func BuiltAt() *time.Time {
|
||||
value, err := time.Parse(time.RFC3339, BuildTime)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
value = value.UTC()
|
||||
return &value
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package buildinfo
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestVersionIsNonEmpty(t *testing.T) {
|
||||
if Version == "" {
|
||||
t.Fatal("version must not be empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltAtRejectsPlaceholderAndNormalizesUTC(t *testing.T) {
|
||||
original := BuildTime
|
||||
t.Cleanup(func() { BuildTime = original })
|
||||
BuildTime = "unknown"
|
||||
if BuiltAt() != nil {
|
||||
t.Fatal("placeholder build time must be absent")
|
||||
}
|
||||
BuildTime = "2026-08-12T04:00:00+02:00"
|
||||
if got := BuiltAt(); got == nil || got.Format(time.RFC3339) != "2026-08-12T02:00:00Z" {
|
||||
t.Fatalf("built at = %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,466 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Environment string
|
||||
|
||||
const (
|
||||
Development Environment = "development"
|
||||
Test Environment = "test"
|
||||
Production Environment = "production"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Environment Environment
|
||||
Timezone string
|
||||
DefaultLocale string
|
||||
LogLevel string
|
||||
PublicURL string
|
||||
DatabaseURL string
|
||||
PrometheusURL string
|
||||
PrometheusTimeout time.Duration
|
||||
UnraidURL string
|
||||
UnraidAPIToken string
|
||||
AuthMode string
|
||||
OIDCIssuer string
|
||||
OIDCClientID string
|
||||
OIDCClientSecret string
|
||||
OIDCRedirectURL string
|
||||
OIDCGroupsClaim string
|
||||
OIDCRoleMapping map[string]string
|
||||
SessionIdleTTL time.Duration
|
||||
SessionAbsoluteTTL time.Duration
|
||||
BreakGlassEnabled bool
|
||||
BackupDirectory string
|
||||
BackupRetention int
|
||||
// ContainerSourceID is the data_sources UUID the worker attributes container
|
||||
// discovery to. Discovery stays disabled until a source is registered, so no
|
||||
// inventory is ever written against an unknown origin.
|
||||
ContainerSourceID string
|
||||
// ProbeAllowedNetworks are the private/loopback CIDRs service probes may
|
||||
// reach. The probe network policy blocks private space unless it is
|
||||
// explicitly allowlisted here; link-local, multicast and cloud metadata
|
||||
// addresses stay blocked regardless.
|
||||
ProbeAllowedNetworks []string
|
||||
NotificationWebhookURL string
|
||||
NotificationWebhookToken string
|
||||
NotificationWebhookTimeout time.Duration
|
||||
}
|
||||
|
||||
type ValidationError struct {
|
||||
Fields []string
|
||||
}
|
||||
|
||||
func (e *ValidationError) Error() string {
|
||||
return "invalid configuration: " + strings.Join(e.Fields, "; ")
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
return LoadFrom(os.LookupEnv)
|
||||
}
|
||||
|
||||
func LoadFrom(lookup func(string) (string, bool)) (Config, error) {
|
||||
config, err := parseFrom(lookup)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return config, Validate(config)
|
||||
}
|
||||
|
||||
// LoadWorker loads only the settings consumed by the background worker. API
|
||||
// authentication credentials are intentionally not part of that container's
|
||||
// privilege boundary.
|
||||
func LoadWorker() (Config, error) {
|
||||
return LoadWorkerFrom(os.LookupEnv)
|
||||
}
|
||||
|
||||
func LoadWorkerFrom(lookup func(string) (string, bool)) (Config, error) {
|
||||
config, err := parseFrom(lookup)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return config, ValidateWorker(config)
|
||||
}
|
||||
|
||||
func parseFrom(lookup func(string) (string, bool)) (Config, error) {
|
||||
get := func(key, fallback string) string {
|
||||
if value, ok := lookup(key); ok {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
config := Config{
|
||||
Environment: Environment(get("PULSE_ENV", string(Development))),
|
||||
Timezone: get("PULSE_TIMEZONE", "Europe/Brussels"),
|
||||
DefaultLocale: get("PULSE_DEFAULT_LOCALE", "nl-BE"),
|
||||
LogLevel: get("PULSE_LOG_LEVEL", "info"),
|
||||
PublicURL: get("PULSE_PUBLIC_URL", ""),
|
||||
DatabaseURL: get("PULSE_DATABASE_URL", ""),
|
||||
PrometheusURL: get("PULSE_PROMETHEUS_URL", ""),
|
||||
UnraidURL: get("PULSE_UNRAID_URL", ""),
|
||||
UnraidAPIToken: get("PULSE_UNRAID_API_TOKEN", ""),
|
||||
AuthMode: get("PULSE_AUTH_MODE", "oidc"),
|
||||
OIDCIssuer: get("PULSE_OIDC_ISSUER", ""),
|
||||
OIDCClientID: get("PULSE_OIDC_CLIENT_ID", ""),
|
||||
OIDCClientSecret: get("PULSE_OIDC_CLIENT_SECRET", ""),
|
||||
OIDCRedirectURL: get("PULSE_OIDC_REDIRECT_URL", ""),
|
||||
OIDCGroupsClaim: get("PULSE_OIDC_GROUPS_CLAIM", "groups"),
|
||||
SessionIdleTTL: 8 * time.Hour,
|
||||
SessionAbsoluteTTL: 7 * 24 * time.Hour,
|
||||
BackupDirectory: get("PULSE_BACKUP_DIR", ""),
|
||||
BackupRetention: 5,
|
||||
|
||||
ContainerSourceID: strings.TrimSpace(get("PULSE_CONTAINER_SOURCE_ID", "")),
|
||||
NotificationWebhookURL: strings.TrimSpace(get("PULSE_NOTIFICATION_WEBHOOK_URL", "")),
|
||||
NotificationWebhookToken: get("PULSE_NOTIFICATION_WEBHOOK_TOKEN", ""),
|
||||
}
|
||||
networks, networksErr := parseAllowedNetworks(get("PULSE_PROBE_ALLOWED_NETWORKS", ""))
|
||||
if networksErr != nil {
|
||||
return Config{}, &ValidationError{Fields: []string{networksErr.Error()}}
|
||||
}
|
||||
config.ProbeAllowedNetworks = networks
|
||||
mapping, mappingErr := parseRoleMapping(get("PULSE_OIDC_ROLE_MAPPING", ""))
|
||||
if mappingErr != nil {
|
||||
return Config{}, &ValidationError{Fields: []string{mappingErr.Error()}}
|
||||
}
|
||||
config.OIDCRoleMapping = mapping
|
||||
config.PrometheusTimeout = 10 * time.Second
|
||||
config.NotificationWebhookTimeout = 10 * time.Second
|
||||
if raw := get("PULSE_SESSION_IDLE_TTL", ""); raw != "" {
|
||||
parsed, err := time.ParseDuration(raw)
|
||||
if err != nil {
|
||||
return Config{}, &ValidationError{Fields: []string{"PULSE_SESSION_IDLE_TTL must be a duration"}}
|
||||
}
|
||||
config.SessionIdleTTL = parsed
|
||||
}
|
||||
if raw := get("PULSE_SESSION_ABSOLUTE_TTL", ""); raw != "" {
|
||||
parsed, err := time.ParseDuration(raw)
|
||||
if err != nil {
|
||||
return Config{}, &ValidationError{Fields: []string{"PULSE_SESSION_ABSOLUTE_TTL must be a duration"}}
|
||||
}
|
||||
config.SessionAbsoluteTTL = parsed
|
||||
}
|
||||
if raw := get("PULSE_PROMETHEUS_TIMEOUT", ""); raw != "" {
|
||||
parsed, err := time.ParseDuration(raw)
|
||||
if err != nil {
|
||||
return Config{}, &ValidationError{Fields: []string{"PULSE_PROMETHEUS_TIMEOUT must be a duration"}}
|
||||
}
|
||||
config.PrometheusTimeout = parsed
|
||||
}
|
||||
if raw := get("PULSE_NOTIFICATION_WEBHOOK_TIMEOUT", ""); raw != "" {
|
||||
parsed, err := time.ParseDuration(raw)
|
||||
if err != nil {
|
||||
return Config{}, &ValidationError{Fields: []string{"PULSE_NOTIFICATION_WEBHOOK_TIMEOUT must be a duration"}}
|
||||
}
|
||||
config.NotificationWebhookTimeout = parsed
|
||||
}
|
||||
if raw := get("PULSE_BREAK_GLASS_ENABLED", "false"); raw != "" {
|
||||
parsed, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return Config{}, &ValidationError{Fields: []string{"PULSE_BREAK_GLASS_ENABLED must be true or false"}}
|
||||
}
|
||||
config.BreakGlassEnabled = parsed
|
||||
}
|
||||
if raw := get("PULSE_BACKUP_RETENTION", ""); raw != "" {
|
||||
parsed, err := strconv.Atoi(raw)
|
||||
if err != nil {
|
||||
return Config{}, &ValidationError{Fields: []string{"PULSE_BACKUP_RETENTION must be an integer"}}
|
||||
}
|
||||
config.BackupRetention = parsed
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// ValidateWorker validates the worker's actual source and sink boundary. OIDC,
|
||||
// session, backup and Unraid settings belong to the API or agent and must not
|
||||
// be copied into the worker merely to satisfy unrelated validation.
|
||||
func ValidateWorker(config Config) error {
|
||||
var fields []string
|
||||
if config.Environment != Development && config.Environment != Test && config.Environment != Production {
|
||||
fields = append(fields, "PULSE_ENV must be development, test, or production")
|
||||
}
|
||||
if strings.TrimSpace(config.DatabaseURL) == "" {
|
||||
fields = append(fields, "PULSE_DATABASE_URL is required: every worker job is database-coordinated")
|
||||
} else {
|
||||
validateDatabaseURL(&fields, config.DatabaseURL)
|
||||
}
|
||||
if config.PrometheusTimeout <= 0 || config.PrometheusTimeout > time.Minute {
|
||||
fields = append(fields, "PULSE_PROMETHEUS_TIMEOUT must be between 1ns and 1m")
|
||||
}
|
||||
if config.PrometheusURL != "" {
|
||||
validateURL(&fields, "PULSE_PROMETHEUS_URL", config.PrometheusURL, config.Environment, false)
|
||||
}
|
||||
if config.ContainerSourceID != "" && !uuidPattern.MatchString(config.ContainerSourceID) {
|
||||
fields = append(fields, "PULSE_CONTAINER_SOURCE_ID must be a UUID")
|
||||
}
|
||||
if config.NotificationWebhookTimeout < time.Second || config.NotificationWebhookTimeout > 30*time.Second {
|
||||
fields = append(fields, "PULSE_NOTIFICATION_WEBHOOK_TIMEOUT must be between 1s and 30s")
|
||||
}
|
||||
if config.NotificationWebhookURL != "" {
|
||||
validateURL(&fields, "PULSE_NOTIFICATION_WEBHOOK_URL", config.NotificationWebhookURL, config.Environment, true)
|
||||
parsed, err := url.Parse(config.NotificationWebhookURL)
|
||||
if err == nil && (parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "") {
|
||||
fields = append(fields, "PULSE_NOTIFICATION_WEBHOOK_URL may not contain credentials, query parameters, or a fragment")
|
||||
}
|
||||
if strings.TrimSpace(config.NotificationWebhookToken) == "" {
|
||||
fields = append(fields, "PULSE_NOTIFICATION_WEBHOOK_TOKEN is required when the webhook is configured")
|
||||
}
|
||||
}
|
||||
if len(fields) > 0 {
|
||||
return &ValidationError{Fields: fields}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func Validate(config Config) error {
|
||||
var fields []string
|
||||
if config.Environment != Development && config.Environment != Test && config.Environment != Production {
|
||||
fields = append(fields, "PULSE_ENV must be development, test, or production")
|
||||
}
|
||||
if config.Timezone == "" {
|
||||
fields = append(fields, "PULSE_TIMEZONE is required")
|
||||
} else if _, err := time.LoadLocation(config.Timezone); err != nil {
|
||||
fields = append(fields, "PULSE_TIMEZONE must be a valid IANA timezone")
|
||||
}
|
||||
if config.DefaultLocale == "" {
|
||||
fields = append(fields, "PULSE_DEFAULT_LOCALE is required")
|
||||
}
|
||||
if !contains([]string{"debug", "info", "warn", "error"}, config.LogLevel) {
|
||||
fields = append(fields, "PULSE_LOG_LEVEL must be debug, info, warn, or error")
|
||||
}
|
||||
if config.PrometheusTimeout <= 0 || config.PrometheusTimeout > time.Minute {
|
||||
fields = append(fields, "PULSE_PROMETHEUS_TIMEOUT must be between 1ns and 1m")
|
||||
}
|
||||
if config.NotificationWebhookTimeout < time.Second || config.NotificationWebhookTimeout > 30*time.Second {
|
||||
fields = append(fields, "PULSE_NOTIFICATION_WEBHOOK_TIMEOUT must be between 1s and 30s")
|
||||
}
|
||||
minimumIdle := 5 * time.Second
|
||||
minimumAbsolute := config.SessionIdleTTL
|
||||
if config.Environment == Production {
|
||||
minimumIdle = 10 * time.Minute
|
||||
minimumAbsolute = 24 * time.Hour
|
||||
}
|
||||
if config.SessionIdleTTL < minimumIdle || config.SessionIdleTTL > 24*time.Hour {
|
||||
fields = append(fields, fmt.Sprintf("PULSE_SESSION_IDLE_TTL must be between %s and 24h", minimumIdle))
|
||||
}
|
||||
if config.SessionAbsoluteTTL < minimumAbsolute || config.SessionAbsoluteTTL > 30*24*time.Hour || config.SessionAbsoluteTTL < config.SessionIdleTTL {
|
||||
fields = append(fields, fmt.Sprintf("PULSE_SESSION_ABSOLUTE_TTL must be between %s and 720h and not shorter than PULSE_SESSION_IDLE_TTL", minimumAbsolute))
|
||||
}
|
||||
if config.NotificationWebhookURL != "" {
|
||||
validateURL(&fields, "PULSE_NOTIFICATION_WEBHOOK_URL", config.NotificationWebhookURL, config.Environment, true)
|
||||
parsed, err := url.Parse(config.NotificationWebhookURL)
|
||||
if err == nil && (parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "") {
|
||||
fields = append(fields, "PULSE_NOTIFICATION_WEBHOOK_URL may not contain credentials, query parameters, or a fragment")
|
||||
}
|
||||
if strings.TrimSpace(config.NotificationWebhookToken) == "" {
|
||||
fields = append(fields, "PULSE_NOTIFICATION_WEBHOOK_TOKEN is required when the webhook is configured")
|
||||
}
|
||||
}
|
||||
if config.BackupRetention < 1 || config.BackupRetention > 100 {
|
||||
fields = append(fields, "PULSE_BACKUP_RETENTION must be between 1 and 100")
|
||||
}
|
||||
if config.PublicURL != "" {
|
||||
validateURL(&fields, "PULSE_PUBLIC_URL", config.PublicURL, config.Environment, true)
|
||||
}
|
||||
if config.DatabaseURL != "" {
|
||||
validateDatabaseURL(&fields, config.DatabaseURL)
|
||||
}
|
||||
if config.PrometheusURL != "" {
|
||||
validateURL(&fields, "PULSE_PROMETHEUS_URL", config.PrometheusURL, config.Environment, false)
|
||||
}
|
||||
if config.UnraidURL != "" {
|
||||
validateURL(&fields, "PULSE_UNRAID_URL", config.UnraidURL, config.Environment, false)
|
||||
}
|
||||
if config.ContainerSourceID != "" && !uuidPattern.MatchString(config.ContainerSourceID) {
|
||||
fields = append(fields, "PULSE_CONTAINER_SOURCE_ID must be a UUID")
|
||||
}
|
||||
if config.AuthMode != "oidc" && config.AuthMode != "mock" {
|
||||
fields = append(fields, "PULSE_AUTH_MODE must be oidc or mock")
|
||||
}
|
||||
if config.OIDCIssuer != "" {
|
||||
validateURL(&fields, "PULSE_OIDC_ISSUER", config.OIDCIssuer, config.Environment, true)
|
||||
}
|
||||
if config.OIDCRedirectURL != "" {
|
||||
validateURL(&fields, "PULSE_OIDC_REDIRECT_URL", config.OIDCRedirectURL, config.Environment, true)
|
||||
}
|
||||
if config.Environment == Production {
|
||||
for key, value := range map[string]string{
|
||||
"PULSE_PUBLIC_URL": config.PublicURL,
|
||||
"PULSE_DATABASE_URL": config.DatabaseURL,
|
||||
"PULSE_OIDC_ISSUER": config.OIDCIssuer,
|
||||
"PULSE_OIDC_CLIENT_ID": config.OIDCClientID,
|
||||
"PULSE_OIDC_CLIENT_SECRET": config.OIDCClientSecret,
|
||||
"PULSE_OIDC_REDIRECT_URL": config.OIDCRedirectURL,
|
||||
} {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
fields = append(fields, key+" is required in production")
|
||||
}
|
||||
}
|
||||
if config.AuthMode != "oidc" {
|
||||
fields = append(fields, "PULSE_AUTH_MODE=mock is forbidden in production")
|
||||
}
|
||||
if len(config.OIDCRoleMapping) == 0 {
|
||||
fields = append(fields, "PULSE_OIDC_ROLE_MAPPING is required in production; without it no identity can be granted a role")
|
||||
}
|
||||
if config.BreakGlassEnabled {
|
||||
fields = append(fields, "PULSE_BREAK_GLASS_ENABLED must remain false in production until secure initialization exists")
|
||||
}
|
||||
}
|
||||
if len(fields) > 0 {
|
||||
return &ValidationError{Fields: fields}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c Config) String() string {
|
||||
return fmt.Sprintf("Config{environment=%s, public_url=%s, database_url=%s, oidc_issuer=%s, oidc_client_id=%s, oidc_client_secret=%s, unraid_api_token=%s, notification_webhook_url=%s, notification_webhook_token=%s}", c.Environment, redacted(c.PublicURL), redacted(c.DatabaseURL), redacted(c.OIDCIssuer), redacted(c.OIDCClientID), redacted(c.OIDCClientSecret), redacted(c.UnraidAPIToken), redacted(c.NotificationWebhookURL), redacted(c.NotificationWebhookToken))
|
||||
}
|
||||
|
||||
func (c Config) Redacted() Config {
|
||||
c.DatabaseURL = redacted(c.DatabaseURL)
|
||||
c.OIDCClientSecret = redacted(c.OIDCClientSecret)
|
||||
c.UnraidAPIToken = redacted(c.UnraidAPIToken)
|
||||
c.NotificationWebhookToken = redacted(c.NotificationWebhookToken)
|
||||
return c
|
||||
}
|
||||
|
||||
func redacted(value string) string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return "<unset>"
|
||||
}
|
||||
return "<set>"
|
||||
}
|
||||
|
||||
func validateDatabaseURL(fields *[]string, raw string) {
|
||||
if err := ValidateDatabaseURL(raw); err != nil {
|
||||
*fields = append(*fields, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateDatabaseURL reports whether raw is a usable PostgreSQL URL. It is exported so
|
||||
// services that take only the database URL from the environment — pulse-agent, which
|
||||
// must not require the API's public URL or OIDC settings — apply the same rule as the
|
||||
// full configuration loader instead of inventing a second one.
|
||||
func ValidateDatabaseURL(raw string) error {
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil || (parsed.Scheme != "postgres" && parsed.Scheme != "postgresql") || parsed.Host == "" {
|
||||
return errors.New("PULSE_DATABASE_URL must be a PostgreSQL URL")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateURL is the reusable strict URL boundary for narrowly scoped runtime
|
||||
// components. The full application validation keeps field-specific messages; callers
|
||||
// such as pulse-agent need the same HTTPS/absolute-url policy without copying it.
|
||||
func ValidateURL(raw string, requireHTTPS bool) error {
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
return errors.New("must be an absolute URL")
|
||||
}
|
||||
if requireHTTPS && parsed.Scheme != "https" {
|
||||
return errors.New("must use https")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateURL(fields *[]string, key, raw string, environment Environment, requireHTTPS bool) {
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
*fields = append(*fields, key+" must be an absolute URL")
|
||||
return
|
||||
}
|
||||
if requireHTTPS && environment == Production && parsed.Scheme != "https" {
|
||||
*fields = append(*fields, key+" must use https in production")
|
||||
}
|
||||
}
|
||||
|
||||
func contains(values []string, target string) bool {
|
||||
for _, value := range values {
|
||||
if value == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// uuidPattern bounds identifiers that must reference a database row.
|
||||
var uuidPattern = regexp.MustCompile("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$")
|
||||
|
||||
// maxAllowedNetworks mirrors the bound enforced by the probe network policy.
|
||||
const maxAllowedNetworks = 32
|
||||
|
||||
// parseAllowedNetworks reads a comma-separated CIDR list. An invalid or
|
||||
// unbounded list fails startup rather than silently widening or narrowing what
|
||||
// probes may reach.
|
||||
func parseAllowedNetworks(raw string) ([]string, error) {
|
||||
trimmed := strings.TrimSpace(raw)
|
||||
if trimmed == "" {
|
||||
return nil, nil
|
||||
}
|
||||
entries := strings.Split(trimmed, ",")
|
||||
networks := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
entry = strings.TrimSpace(entry)
|
||||
if entry == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := netip.ParsePrefix(entry); err != nil {
|
||||
return nil, errors.New("PULSE_PROBE_ALLOWED_NETWORKS entries must be CIDR prefixes")
|
||||
}
|
||||
networks = append(networks, entry)
|
||||
}
|
||||
if len(networks) == 0 || len(networks) > maxAllowedNetworks {
|
||||
return nil, fmt.Errorf("PULSE_PROBE_ALLOWED_NETWORKS must contain between 1 and %d CIDR prefixes", maxAllowedNetworks)
|
||||
}
|
||||
return networks, nil
|
||||
}
|
||||
|
||||
// knownRoles bounds the Pulse role names accepted in PULSE_OIDC_ROLE_MAPPING. It
|
||||
// mirrors the roles defined in internal/auth without importing that package, so
|
||||
// configuration stays free of runtime dependencies.
|
||||
var knownRoles = []string{"viewer", "operator", "editor", "administrator"}
|
||||
|
||||
// parseRoleMapping reads a comma-separated "claim=role" list mapping identity
|
||||
// provider group claim values onto Pulse roles, for example
|
||||
// "pulse-admin=administrator,pulse-staff=viewer". An empty value yields a nil map,
|
||||
// which means no group grants access.
|
||||
func parseRoleMapping(raw string) (map[string]string, error) {
|
||||
trimmed := strings.TrimSpace(raw)
|
||||
if trimmed == "" {
|
||||
return nil, nil
|
||||
}
|
||||
mapping := make(map[string]string)
|
||||
for _, entry := range strings.Split(trimmed, ",") {
|
||||
entry = strings.TrimSpace(entry)
|
||||
if entry == "" {
|
||||
continue
|
||||
}
|
||||
claim, role, found := strings.Cut(entry, "=")
|
||||
claim = strings.TrimSpace(claim)
|
||||
role = strings.ToLower(strings.TrimSpace(role))
|
||||
if !found || claim == "" || role == "" {
|
||||
return nil, errors.New("PULSE_OIDC_ROLE_MAPPING entries must use claim=role")
|
||||
}
|
||||
if !contains(knownRoles, role) {
|
||||
return nil, errors.New("PULSE_OIDC_ROLE_MAPPING role must be one of " + strings.Join(knownRoles, ", "))
|
||||
}
|
||||
if _, duplicate := mapping[claim]; duplicate {
|
||||
return nil, errors.New("PULSE_OIDC_ROLE_MAPPING contains a duplicate claim")
|
||||
}
|
||||
mapping[claim] = role
|
||||
}
|
||||
if len(mapping) == 0 {
|
||||
return nil, errors.New("PULSE_OIDC_ROLE_MAPPING must contain at least one claim=role entry")
|
||||
}
|
||||
return mapping, nil
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestProductionListsAllMissingMandatoryValues(t *testing.T) {
|
||||
values := map[string]string{"PULSE_ENV": "production"}
|
||||
_, err := LoadFrom(mapLookup(values))
|
||||
if err == nil {
|
||||
t.Fatal("expected production validation error")
|
||||
}
|
||||
message := err.Error()
|
||||
for _, key := range []string{"PULSE_PUBLIC_URL", "PULSE_DATABASE_URL", "PULSE_OIDC_ISSUER", "PULSE_OIDC_CLIENT_ID", "PULSE_OIDC_CLIENT_SECRET", "PULSE_OIDC_REDIRECT_URL"} {
|
||||
if !strings.Contains(message, key) {
|
||||
t.Errorf("error %q does not mention %s", message, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebhookConfigurationIsSecureAndRedacted(t *testing.T) {
|
||||
secret := "webhook-runtime-secret"
|
||||
configuration, err := LoadFrom(mapLookup(map[string]string{
|
||||
"PULSE_ENV": "development",
|
||||
"PULSE_NOTIFICATION_WEBHOOK_URL": "http://127.0.0.1:8080/pulse",
|
||||
"PULSE_NOTIFICATION_WEBHOOK_TOKEN": secret,
|
||||
"PULSE_NOTIFICATION_WEBHOOK_TIMEOUT": "3s",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if configuration.NotificationWebhookTimeout != 3*time.Second {
|
||||
t.Fatalf("timeout = %s", configuration.NotificationWebhookTimeout)
|
||||
}
|
||||
if strings.Contains(configuration.String(), secret) || strings.Contains(configuration.Redacted().NotificationWebhookToken, secret) {
|
||||
t.Fatal("webhook credential leaked through config rendering")
|
||||
}
|
||||
for name, values := range map[string]map[string]string{
|
||||
"missing token": {"PULSE_NOTIFICATION_WEBHOOK_URL": "https://receiver.example/hook"},
|
||||
"query token": {"PULSE_NOTIFICATION_WEBHOOK_URL": "https://receiver.example/hook?token=value", "PULSE_NOTIFICATION_WEBHOOK_TOKEN": secret},
|
||||
"production http": {"PULSE_ENV": "production", "PULSE_NOTIFICATION_WEBHOOK_URL": "http://receiver.example/hook", "PULSE_NOTIFICATION_WEBHOOK_TOKEN": secret},
|
||||
"unbounded timeout": {"PULSE_NOTIFICATION_WEBHOOK_URL": "https://receiver.example/hook", "PULSE_NOTIFICATION_WEBHOOK_TOKEN": secret, "PULSE_NOTIFICATION_WEBHOOK_TIMEOUT": "31s"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := LoadFrom(mapLookup(values)); err == nil {
|
||||
t.Fatal("expected webhook configuration rejection")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionRejectsMockAuth(t *testing.T) {
|
||||
values := map[string]string{
|
||||
"PULSE_ENV": "production", "PULSE_PUBLIC_URL": "https://pulse.example",
|
||||
"PULSE_DATABASE_URL": "postgres://pulse@db/pulse", "PULSE_OIDC_ISSUER": "https://auth.example",
|
||||
"PULSE_OIDC_CLIENT_ID": "pulse", "PULSE_OIDC_CLIENT_SECRET": "secret-value",
|
||||
"PULSE_OIDC_REDIRECT_URL": "https://pulse.example/auth/callback", "PULSE_AUTH_MODE": "mock",
|
||||
}
|
||||
_, err := LoadFrom(mapLookup(values))
|
||||
if err == nil || !strings.Contains(err.Error(), "mock") {
|
||||
t.Fatalf("expected mock-auth rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationErrorsAndStringNeverExposeSecrets(t *testing.T) {
|
||||
secret := "super-secret-token"
|
||||
values := map[string]string{
|
||||
"PULSE_ENV": "production", "PULSE_DATABASE_URL": "not-a-url", "PULSE_OIDC_CLIENT_SECRET": secret,
|
||||
"PULSE_UNRAID_API_TOKEN": secret, "PULSE_AUTH_MODE": "mock",
|
||||
}
|
||||
config, err := LoadFrom(mapLookup(values))
|
||||
if err == nil {
|
||||
t.Fatal("expected validation error")
|
||||
}
|
||||
if strings.Contains(err.Error(), secret) {
|
||||
t.Fatal("validation error leaked a secret")
|
||||
}
|
||||
if strings.Contains(config.String(), secret) {
|
||||
t.Fatal("config String leaked a secret")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDevelopmentDefaultsAllowExplicitMockMode(t *testing.T) {
|
||||
values := map[string]string{"PULSE_ENV": "development", "PULSE_AUTH_MODE": "mock"}
|
||||
config, err := LoadFrom(mapLookup(values))
|
||||
if err != nil {
|
||||
t.Fatalf("development defaults rejected: %v", err)
|
||||
}
|
||||
if config.Environment != Development || config.AuthMode != "mock" {
|
||||
t.Fatalf("unexpected config: %s", config)
|
||||
}
|
||||
if config.SessionIdleTTL != 8*time.Hour || config.SessionAbsoluteTTL != 7*24*time.Hour {
|
||||
t.Fatalf("unexpected session defaults: idle=%s absolute=%s", config.SessionIdleTTL, config.SessionAbsoluteTTL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionLifetimeConfigurationIsBounded(t *testing.T) {
|
||||
configured, err := LoadFrom(mapLookup(map[string]string{
|
||||
"PULSE_ENV": "development", "PULSE_AUTH_MODE": "mock",
|
||||
"PULSE_SESSION_IDLE_TTL": "20s", "PULSE_SESSION_ABSOLUTE_TTL": "2m",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if configured.SessionIdleTTL != 20*time.Second || configured.SessionAbsoluteTTL != 2*time.Minute {
|
||||
t.Fatalf("unexpected session lifetimes: %#v", configured)
|
||||
}
|
||||
|
||||
for name, values := range map[string]map[string]string{
|
||||
"invalid duration": {"PULSE_SESSION_IDLE_TTL": "later"},
|
||||
"idle too short": {"PULSE_SESSION_IDLE_TTL": "4s"},
|
||||
"absolute shorter than idle": {"PULSE_SESSION_IDLE_TTL": "20s", "PULSE_SESSION_ABSOLUTE_TTL": "10s"},
|
||||
"absolute unbounded": {"PULSE_SESSION_ABSOLUTE_TTL": "721h"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := LoadFrom(mapLookup(values)); err == nil {
|
||||
t.Fatal("expected bounded session configuration rejection")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionRequiresWallboardCapableAbsoluteSessionLifetime(t *testing.T) {
|
||||
config := Config{
|
||||
Environment: Production, Timezone: "Europe/Brussels", DefaultLocale: "nl-BE", LogLevel: "info",
|
||||
PublicURL: "https://pulse.example", DatabaseURL: "postgres://pulse@db/pulse", AuthMode: "oidc",
|
||||
OIDCIssuer: "https://auth.example", OIDCClientID: "pulse", OIDCClientSecret: "secret",
|
||||
OIDCRedirectURL: "https://pulse.example/auth/callback", OIDCRoleMapping: map[string]string{"viewer": "viewer"},
|
||||
PrometheusTimeout: 10 * time.Second, NotificationWebhookTimeout: 10 * time.Second, BackupRetention: 5,
|
||||
SessionIdleTTL: 8 * time.Hour, SessionAbsoluteTTL: 23 * time.Hour,
|
||||
}
|
||||
if err := Validate(config); err == nil || !strings.Contains(err.Error(), "PULSE_SESSION_ABSOLUTE_TTL") {
|
||||
t.Fatalf("production accepted a session unable to cover the wallboard budget: %v", err)
|
||||
}
|
||||
config.SessionAbsoluteTTL = 24 * time.Hour
|
||||
config.SessionIdleTTL = 5 * time.Minute
|
||||
if err := Validate(config); err == nil || !strings.Contains(err.Error(), "PULSE_SESSION_IDLE_TTL") {
|
||||
t.Fatalf("production accepted an idle TTL that can race the five-minute wallboard refresh: %v", err)
|
||||
}
|
||||
config.SessionIdleTTL = 10 * time.Minute
|
||||
if err := Validate(config); err != nil {
|
||||
t.Fatalf("bounded 24-hour production session rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupConfigurationIsBoundedAndOptional(t *testing.T) {
|
||||
values := map[string]string{"PULSE_ENV": "development", "PULSE_AUTH_MODE": "mock", "PULSE_BACKUP_DIR": "C:/pulse-backups", "PULSE_BACKUP_RETENTION": "12"}
|
||||
config, err := LoadFrom(mapLookup(values))
|
||||
if err != nil {
|
||||
t.Fatalf("backup config rejected: %v", err)
|
||||
}
|
||||
if config.BackupDirectory != values["PULSE_BACKUP_DIR"] || config.BackupRetention != 12 {
|
||||
t.Fatalf("unexpected backup config: %#v", config)
|
||||
}
|
||||
values["PULSE_BACKUP_RETENTION"] = "101"
|
||||
if _, err := LoadFrom(mapLookup(values)); err == nil || !strings.Contains(err.Error(), "PULSE_BACKUP_RETENTION") {
|
||||
t.Fatalf("expected bounded retention error, got %v", err)
|
||||
}
|
||||
}
|
||||
func mapLookup(values map[string]string) func(string) (string, bool) {
|
||||
return func(key string) (string, bool) { value, ok := values[key]; return value, ok }
|
||||
}
|
||||
|
||||
func TestRoleMappingParsesClaimsOntoRoles(t *testing.T) {
|
||||
config, err := LoadFrom(mapLookup(map[string]string{
|
||||
"PULSE_OIDC_ROLE_MAPPING": "pulse-admin=administrator, pulse-staff =viewer,pulse-ops=Operator",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
expected := map[string]string{"pulse-admin": "administrator", "pulse-staff": "viewer", "pulse-ops": "operator"}
|
||||
if len(config.OIDCRoleMapping) != len(expected) {
|
||||
t.Fatalf("expected %d mapped claims, got %d", len(expected), len(config.OIDCRoleMapping))
|
||||
}
|
||||
for claim, role := range expected {
|
||||
if config.OIDCRoleMapping[claim] != role {
|
||||
t.Fatalf("claim %q mapped to %q, want %q", claim, config.OIDCRoleMapping[claim], role)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoleMappingDefaultsToNoAccess(t *testing.T) {
|
||||
config, err := LoadFrom(mapLookup(nil))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if config.OIDCRoleMapping != nil {
|
||||
t.Fatal("an unset role mapping must grant nobody a role")
|
||||
}
|
||||
if config.OIDCGroupsClaim != "groups" {
|
||||
t.Fatalf("groups claim defaulted to %q, want groups", config.OIDCGroupsClaim)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoleMappingRejectsMalformedInput(t *testing.T) {
|
||||
for name, raw := range map[string]string{
|
||||
"missing separator": "pulse-admin",
|
||||
"empty claim": "=administrator",
|
||||
"empty role": "pulse-admin=",
|
||||
"unknown role": "pulse-admin=superuser",
|
||||
"duplicate claim": "pulse-admin=viewer,pulse-admin=editor",
|
||||
"only separators": ",,",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := LoadFrom(mapLookup(map[string]string{"PULSE_OIDC_ROLE_MAPPING": raw})); err == nil {
|
||||
t.Fatalf("expected %q to be rejected", raw)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionRequiresARoleMapping(t *testing.T) {
|
||||
base := map[string]string{
|
||||
"PULSE_ENV": "production",
|
||||
"PULSE_PUBLIC_URL": "https://pulse.example.test",
|
||||
"PULSE_DATABASE_URL": "postgres://pulse@db:5432/pulse",
|
||||
"PULSE_OIDC_ISSUER": "https://id.example.test",
|
||||
"PULSE_OIDC_CLIENT_ID": "pulse",
|
||||
"PULSE_OIDC_CLIENT_SECRET": "secret",
|
||||
"PULSE_OIDC_REDIRECT_URL": "https://pulse.example.test/auth/callback",
|
||||
}
|
||||
if _, err := LoadFrom(mapLookup(base)); err == nil {
|
||||
t.Fatal("production without a role mapping must fail: no identity could obtain a role")
|
||||
}
|
||||
base["PULSE_OIDC_ROLE_MAPPING"] = "pulse-admin=administrator"
|
||||
if _, err := LoadFrom(mapLookup(base)); err != nil {
|
||||
t.Fatalf("production with a role mapping must succeed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadWorkerProductionDoesNotRequireAPICredentials(t *testing.T) {
|
||||
configuration, err := LoadWorkerFrom(mapLookup(map[string]string{
|
||||
"PULSE_ENV": "production",
|
||||
"PULSE_DATABASE_URL": "postgres://pulse:secret@pulse-postgres:5432/pulse?sslmode=disable",
|
||||
"PULSE_PROMETHEUS_URL": "http://192.0.2.10:9090",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadWorkerFrom returned API-only validation error: %v", err)
|
||||
}
|
||||
if configuration.Environment != Production || configuration.DatabaseURL == "" {
|
||||
t.Fatalf("unexpected worker config: %#v", configuration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadWorkerStillRequiresDatabaseAndValidatesSources(t *testing.T) {
|
||||
_, err := LoadWorkerFrom(mapLookup(map[string]string{
|
||||
"PULSE_ENV": "production",
|
||||
"PULSE_PROMETHEUS_URL": "://invalid",
|
||||
}))
|
||||
if err == nil {
|
||||
t.Fatal("LoadWorkerFrom accepted missing database and malformed Prometheus URL")
|
||||
}
|
||||
message := err.Error()
|
||||
if !strings.Contains(message, "PULSE_DATABASE_URL") || !strings.Contains(message, "PULSE_PROMETHEUS_URL") {
|
||||
t.Fatalf("worker validation error = %q", message)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
ContractVersion = "v1"
|
||||
DefaultMaxContainers = 250
|
||||
)
|
||||
|
||||
type Limits struct {
|
||||
MaxContainers, MaxPorts, MaxVolumes, MaxNetworks, MaxLabels, MaxPageSize int
|
||||
FreshnessMaxAge time.Duration
|
||||
}
|
||||
|
||||
func (l Limits) withDefaults() Limits {
|
||||
if l.MaxContainers == 0 {
|
||||
// The v1 performance target remains 150 containers. Bounded growth margin
|
||||
// keeps a modestly larger host from invalidating the complete snapshot.
|
||||
l.MaxContainers = DefaultMaxContainers
|
||||
}
|
||||
if l.MaxPorts == 0 {
|
||||
l.MaxPorts = 32
|
||||
}
|
||||
if l.MaxVolumes == 0 {
|
||||
l.MaxVolumes = 32
|
||||
}
|
||||
if l.MaxNetworks == 0 {
|
||||
l.MaxNetworks = 32
|
||||
}
|
||||
if l.MaxLabels == 0 {
|
||||
l.MaxLabels = 64
|
||||
}
|
||||
if l.MaxPageSize == 0 {
|
||||
l.MaxPageSize = 100
|
||||
}
|
||||
if l.FreshnessMaxAge == 0 {
|
||||
l.FreshnessMaxAge = 60 * time.Second
|
||||
}
|
||||
return l
|
||||
}
|
||||
func (l Limits) Validate() error {
|
||||
if l.MaxContainers < 1 || l.MaxContainers > 1000 || l.MaxPorts < 1 || l.MaxPorts > 128 || l.MaxVolumes < 1 || l.MaxVolumes > 128 || l.MaxNetworks < 1 || l.MaxNetworks > 128 || l.MaxLabels < 1 || l.MaxLabels > 256 || l.MaxPageSize < 1 || l.MaxPageSize > 500 || l.FreshnessMaxAge <= 0 || l.FreshnessMaxAge > 24*time.Hour {
|
||||
return errors.New("container limits are outside safe bounds")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Source struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
ReceivedAt time.Time `json:"receivedAt"`
|
||||
Freshness string `json:"freshness"`
|
||||
State string `json:"state"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
type Port struct {
|
||||
ContainerPort int `json:"containerPort"`
|
||||
HostPort int `json:"hostPort,omitempty"`
|
||||
Protocol string `json:"protocol"`
|
||||
}
|
||||
type RawContainer struct {
|
||||
ID, Name, Image, ImageDigest, State, Health string
|
||||
IntentionalStop bool
|
||||
MetricsAvailable, LifecycleAvailable bool
|
||||
UptimeSeconds float64
|
||||
RestartCount int
|
||||
ExitCode int
|
||||
CPUPercent float64
|
||||
MemoryBytes, MemoryLimitBytes uint64
|
||||
NetworkRxBytes, NetworkTxBytes uint64
|
||||
BlockReadBytes, BlockWriteBytes uint64
|
||||
Ports []Port
|
||||
Volumes, Networks []string
|
||||
Project string
|
||||
Labels map[string]string
|
||||
}
|
||||
type Container struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Image string `json:"image,omitempty"`
|
||||
ImageDigest string `json:"imageDigest,omitempty"`
|
||||
State string `json:"state"`
|
||||
Health string `json:"health"`
|
||||
IntentionalStop bool `json:"intentionalStop"`
|
||||
MetricsAvailable bool `json:"metricsAvailable"`
|
||||
LifecycleAvailable bool `json:"lifecycleAvailable"`
|
||||
UptimeSeconds float64 `json:"uptimeSeconds"`
|
||||
RestartCount int `json:"restartCount"`
|
||||
ExitCode int `json:"exitCode"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
MemoryBytes uint64 `json:"memoryBytes"`
|
||||
MemoryLimitBytes uint64 `json:"memoryLimitBytes"`
|
||||
NetworkRxBytes uint64 `json:"networkRxBytes"`
|
||||
NetworkTxBytes uint64 `json:"networkTxBytes"`
|
||||
BlockReadBytes uint64 `json:"blockReadBytes"`
|
||||
BlockWriteBytes uint64 `json:"blockWriteBytes"`
|
||||
Ports []Port `json:"ports"`
|
||||
Volumes []string `json:"volumes"`
|
||||
Networks []string `json:"networks"`
|
||||
Project string `json:"project,omitempty"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
}
|
||||
type RawSnapshot struct {
|
||||
Source Source
|
||||
Containers []RawContainer
|
||||
ObservedAt, ReceivedAt time.Time
|
||||
}
|
||||
type Snapshot struct {
|
||||
ContractVersion string `json:"contractVersion"`
|
||||
Source Source `json:"source"`
|
||||
Containers []Container `json:"containers"`
|
||||
Total int `json:"total"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
type Provider interface {
|
||||
Snapshot(context.Context) (Snapshot, error)
|
||||
}
|
||||
type Adapter struct {
|
||||
Source interface {
|
||||
Snapshot(context.Context) (RawSnapshot, error)
|
||||
}
|
||||
Limits Limits
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (a Adapter) Snapshot(ctx context.Context) (Snapshot, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
if a.Source == nil {
|
||||
return UnknownSnapshot(time.Now().UTC(), "container", "agent", "source_unavailable"), nil
|
||||
}
|
||||
raw, err := a.Source.Snapshot(ctx)
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if a.Now != nil {
|
||||
now = a.Now()
|
||||
}
|
||||
return Normalize(raw, now, a.Limits)
|
||||
}
|
||||
func UnknownSnapshot(now time.Time, id, typ, reason string) Snapshot {
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
return Snapshot{ContractVersion: ContractVersion, Source: Source{ID: id, Type: typ, ReceivedAt: now, Freshness: "unavailable", State: "unknown", Reason: reason}, Containers: []Container{}, Total: 0}
|
||||
}
|
||||
func Normalize(raw RawSnapshot, now time.Time, limits Limits) (Snapshot, error) {
|
||||
limits = limits.withDefaults()
|
||||
if err := limits.Validate(); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
if raw.ReceivedAt.IsZero() {
|
||||
raw.ReceivedAt = now
|
||||
}
|
||||
if raw.ObservedAt.IsZero() {
|
||||
raw.ObservedAt = raw.ReceivedAt
|
||||
}
|
||||
if raw.ObservedAt.After(now.Add(time.Minute)) {
|
||||
return Snapshot{}, errors.New("container observation is materially in the future")
|
||||
}
|
||||
if len(raw.Containers) > limits.MaxContainers {
|
||||
return Snapshot{}, errors.New("container count exceeds bounds")
|
||||
}
|
||||
source := raw.Source
|
||||
if source.ID == "" {
|
||||
source.ID = "container"
|
||||
}
|
||||
if source.Type == "" {
|
||||
source.Type = "agent"
|
||||
}
|
||||
source.ObservedAt = raw.ObservedAt.UTC()
|
||||
source.ReceivedAt = raw.ReceivedAt.UTC()
|
||||
source.Freshness = "fresh"
|
||||
source.State = "healthy"
|
||||
if now.Sub(raw.ObservedAt) > limits.FreshnessMaxAge {
|
||||
source.Freshness = "stale"
|
||||
source.State = "unknown"
|
||||
source.Reason = "stale_source"
|
||||
}
|
||||
items := make([]Container, 0, len(raw.Containers))
|
||||
for _, r := range raw.Containers {
|
||||
if strings.TrimSpace(r.ID) == "" || strings.TrimSpace(r.Name) == "" || len(r.Name) > 255 || r.RestartCount < 0 || r.UptimeSeconds < 0 || r.CPUPercent < 0 || r.CPUPercent > 10000 || math.IsNaN(r.CPUPercent) || math.IsInf(r.CPUPercent, 0) {
|
||||
return Snapshot{}, errors.New("invalid container identity or metrics")
|
||||
}
|
||||
if len(r.Ports) > limits.MaxPorts || len(r.Volumes) > limits.MaxVolumes || len(r.Networks) > limits.MaxNetworks || len(r.Labels) > limits.MaxLabels {
|
||||
return Snapshot{}, errors.New("container detail exceeds bounds")
|
||||
}
|
||||
ports := append([]Port(nil), r.Ports...)
|
||||
sort.Slice(ports, func(i, j int) bool {
|
||||
if ports[i].ContainerPort != ports[j].ContainerPort {
|
||||
return ports[i].ContainerPort < ports[j].ContainerPort
|
||||
}
|
||||
return ports[i].Protocol < ports[j].Protocol
|
||||
})
|
||||
volumes := sortedStrings(r.Volumes)
|
||||
networks := sortedStrings(r.Networks)
|
||||
labels := make(map[string]string, len(r.Labels))
|
||||
for k, v := range r.Labels {
|
||||
if len(k) <= 128 && len(v) <= 512 {
|
||||
labels[k] = v
|
||||
}
|
||||
}
|
||||
items = append(items, Container{ID: r.ID, Name: r.Name, Image: r.Image, ImageDigest: r.ImageDigest, State: normalizeRuntimeState(r.State), Health: normalizeHealth(r.Health), IntentionalStop: r.IntentionalStop, MetricsAvailable: r.MetricsAvailable, LifecycleAvailable: r.LifecycleAvailable, UptimeSeconds: r.UptimeSeconds, RestartCount: r.RestartCount, ExitCode: r.ExitCode, CPUPercent: r.CPUPercent, MemoryBytes: r.MemoryBytes, MemoryLimitBytes: r.MemoryLimitBytes, NetworkRxBytes: r.NetworkRxBytes, NetworkTxBytes: r.NetworkTxBytes, BlockReadBytes: r.BlockReadBytes, BlockWriteBytes: r.BlockWriteBytes, Ports: ports, Volumes: volumes, Networks: networks, Project: r.Project, Labels: labels})
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
if items[i].Name != items[j].Name {
|
||||
return items[i].Name < items[j].Name
|
||||
}
|
||||
return items[i].ID < items[j].ID
|
||||
})
|
||||
return Snapshot{ContractVersion: ContractVersion, Source: source, Containers: items, Total: len(items)}, nil
|
||||
}
|
||||
func Page(snapshot Snapshot, limit int, after string, limits Limits) (Snapshot, error) {
|
||||
return FilteredPage(snapshot, limit, after, limits, "", "", "", "name")
|
||||
}
|
||||
|
||||
func FilteredPage(snapshot Snapshot, limit int, after string, limits Limits, query, state, health, order string) (Snapshot, error) {
|
||||
limits = limits.withDefaults()
|
||||
if err := limits.Validate(); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
if limit < 1 || limit > limits.MaxPageSize {
|
||||
return Snapshot{}, errors.New("container page limit is outside bounds")
|
||||
}
|
||||
query, state, health, order = strings.ToLower(strings.TrimSpace(query)), strings.ToLower(strings.TrimSpace(state)), strings.ToLower(strings.TrimSpace(health)), strings.ToLower(strings.TrimSpace(order))
|
||||
if len(query) > 100 || (state != "" && normalizeRuntimeState(state) != state) || (health != "" && normalizeHealth(health) != health) || (order != "name" && order != "cpu" && order != "memory" && order != "state") {
|
||||
return Snapshot{}, errors.New("container filters are invalid")
|
||||
}
|
||||
items := make([]Container, 0, len(snapshot.Containers))
|
||||
for _, item := range snapshot.Containers {
|
||||
if query != "" && !strings.Contains(strings.ToLower(item.Name+" "+item.Project+" "+item.Image), query) {
|
||||
continue
|
||||
}
|
||||
if state != "" && item.State != state {
|
||||
continue
|
||||
}
|
||||
if health != "" && item.Health != health {
|
||||
continue
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool {
|
||||
switch order {
|
||||
case "cpu":
|
||||
if items[i].CPUPercent != items[j].CPUPercent {
|
||||
return items[i].CPUPercent > items[j].CPUPercent
|
||||
}
|
||||
case "memory":
|
||||
if items[i].MemoryBytes != items[j].MemoryBytes {
|
||||
return items[i].MemoryBytes > items[j].MemoryBytes
|
||||
}
|
||||
case "state":
|
||||
if items[i].State != items[j].State {
|
||||
return items[i].State < items[j].State
|
||||
}
|
||||
}
|
||||
if items[i].Name != items[j].Name {
|
||||
return items[i].Name < items[j].Name
|
||||
}
|
||||
return items[i].ID < items[j].ID
|
||||
})
|
||||
start := 0
|
||||
if after != "" {
|
||||
n, err := strconv.Atoi(after)
|
||||
if err != nil || n < 0 || n > len(items) {
|
||||
return Snapshot{}, errors.New("invalid container cursor")
|
||||
}
|
||||
start = n
|
||||
}
|
||||
end := start + limit
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
result := snapshot
|
||||
result.Containers = items[start:end]
|
||||
result.Total = len(items)
|
||||
result.NextCursor = ""
|
||||
if end < len(items) {
|
||||
result.NextCursor = strconv.Itoa(end)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
func sortedStrings(values []string) []string {
|
||||
r := append([]string(nil), values...)
|
||||
sort.Strings(r)
|
||||
return r
|
||||
}
|
||||
func normalizeRuntimeState(value string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
switch value {
|
||||
case "running", "restarting", "paused", "exited", "dead", "stopped", "created", "removing":
|
||||
return value
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeHealth(value string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
switch value {
|
||||
case "healthy", "unhealthy", "starting":
|
||||
return value
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNormalizeAcceptsBoundedOperationalContainerHeadroom(t *testing.T) {
|
||||
now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC)
|
||||
raw := RawSnapshot{Source: Source{ID: "agent"}, ObservedAt: now, ReceivedAt: now}
|
||||
for index := 0; index < DefaultMaxContainers; index++ {
|
||||
raw.Containers = append(raw.Containers, RawContainer{ID: fmt.Sprintf("id-%03d", index), Name: fmt.Sprintf("container-%03d", index), State: "running"})
|
||||
}
|
||||
if snapshot, err := Normalize(raw, now, Limits{}); err != nil || snapshot.Total != DefaultMaxContainers {
|
||||
t.Fatalf("bounded headroom snapshot total=%d err=%v", snapshot.Total, err)
|
||||
}
|
||||
raw.Containers = append(raw.Containers, RawContainer{ID: "overflow", Name: "overflow", State: "running"})
|
||||
if _, err := Normalize(raw, now, Limits{}); err == nil {
|
||||
t.Fatal("container inventory beyond bounded headroom was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalize150ContainerFixtureAndSeparateStateHealth(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
|
||||
raw := RawSnapshot{Source: Source{ID: "agent-1"}, ObservedAt: now, ReceivedAt: now}
|
||||
for i := 0; i < 150; i++ {
|
||||
raw.Containers = append(raw.Containers, RawContainer{ID: string(rune('a'+i/26)) + string(rune('a'+i%26)), Name: "container-" + string(rune('a'+i%26)) + "-" + string(rune('0'+i/26)), State: "running", Health: "unhealthy", CPUPercent: 1})
|
||||
}
|
||||
raw.Containers[0].IntentionalStop = true
|
||||
raw.Containers[0].State = "exited"
|
||||
raw.Containers[0].Health = "healthy"
|
||||
snapshot, err := Normalize(raw, now, Limits{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Total != 150 || len(snapshot.Containers) != 150 || snapshot.Containers[0].State == snapshot.Containers[0].Health {
|
||||
t.Fatalf("container state and health were conflated: %+v", snapshot.Containers[0])
|
||||
}
|
||||
foundStopped := false
|
||||
for _, item := range snapshot.Containers {
|
||||
if item.IntentionalStop {
|
||||
foundStopped = item.State == "exited" && item.Health == "healthy"
|
||||
}
|
||||
}
|
||||
if !foundStopped {
|
||||
t.Fatal("intentional stop was not preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageIsBoundedAndDeterministic(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
raw := RawSnapshot{Source: Source{ID: "agent"}, ObservedAt: now, ReceivedAt: now, Containers: []RawContainer{{ID: "b", Name: "zeta", State: "running"}, {ID: "a", Name: "alpha", State: "running"}}}
|
||||
snapshot, err := Normalize(raw, now, Limits{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
page, err := Page(snapshot, 1, "", Limits{})
|
||||
if err != nil || len(page.Containers) != 1 || page.Containers[0].Name != "alpha" || page.NextCursor != "1" {
|
||||
t.Fatalf("page=%+v err=%v", page, err)
|
||||
}
|
||||
if _, err := Page(snapshot, 101, "", Limits{}); err == nil {
|
||||
t.Fatal("oversized page accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilteredPageFiltersBeforeCursorAndSortsDeterministically(t *testing.T) {
|
||||
snapshot := Snapshot{Containers: []Container{{ID: "b", Name: "Beta", State: "running", Health: "healthy", CPUPercent: 5}, {ID: "a", Name: "Alpha", State: "running", Health: "healthy", CPUPercent: 10}, {ID: "c", Name: "Other", State: "exited", Health: "unknown"}}}
|
||||
page, err := FilteredPage(snapshot, 1, "", Limits{}, "a", "running", "healthy", "cpu")
|
||||
if err != nil || page.Total != 2 || len(page.Containers) != 1 || page.Containers[0].ID != "a" || page.NextCursor != "1" {
|
||||
t.Fatalf("filtered page=%+v err=%v", page, err)
|
||||
}
|
||||
next, err := FilteredPage(snapshot, 1, page.NextCursor, Limits{}, "a", "running", "healthy", "cpu")
|
||||
if err != nil || len(next.Containers) != 1 || next.Containers[0].ID != "b" {
|
||||
t.Fatalf("next filtered page=%+v err=%v", next, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaleSourceAndContextCancellation(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
raw := RawSnapshot{Source: Source{ID: "agent"}, ObservedAt: now.Add(-2 * time.Minute), ReceivedAt: now, Containers: []RawContainer{{ID: "a", Name: "alpha", State: "running"}}}
|
||||
snapshot, err := Normalize(raw, now, Limits{FreshnessMaxAge: time.Minute})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Source.State != "unknown" || snapshot.Source.Freshness != "stale" {
|
||||
t.Fatalf("source=%+v", snapshot.Source)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err = (Adapter{}).Snapshot(ctx)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeCanonicalizesRuntimeAndPreservesAvailability(t *testing.T) {
|
||||
now := time.Date(2026, 8, 12, 0, 0, 0, 0, time.UTC)
|
||||
snapshot, err := Normalize(RawSnapshot{
|
||||
Source: Source{ID: "agent"}, ObservedAt: now, ReceivedAt: now,
|
||||
Containers: []RawContainer{
|
||||
{ID: "a", Name: "alpha", State: " RUNNING ", Health: " HEALTHY ", MetricsAvailable: true, LifecycleAvailable: true},
|
||||
{ID: "b", Name: "beta", State: "RUNNING", Health: "Up 34 hours"},
|
||||
},
|
||||
}, now, Limits{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Containers[0].State != "running" || snapshot.Containers[0].Health != "healthy" || !snapshot.Containers[0].MetricsAvailable || !snapshot.Containers[0].LifecycleAvailable {
|
||||
t.Fatalf("canonical container = %+v", snapshot.Containers[0])
|
||||
}
|
||||
if snapshot.Containers[1].State != "running" || snapshot.Containers[1].Health != "unknown" || snapshot.Containers[1].MetricsAvailable || snapshot.Containers[1].LifecycleAvailable {
|
||||
t.Fatalf("missing telemetry was fabricated: %+v", snapshot.Containers[1])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package containerapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/problem"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
Provider interface {
|
||||
Snapshot(context.Context) (container.Snapshot, error)
|
||||
}
|
||||
Limits container.Limits
|
||||
}
|
||||
|
||||
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet || (r.URL.Path != "/api/v1/containers" && !strings.HasPrefix(r.URL.Path, "/api/v1/containers/")) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if _, ok := auth.PrincipalFromContext(r.Context()); !ok {
|
||||
problem.Write(w, r, http.StatusUnauthorized, "UNAUTHORIZED", "Authentication required", "Authentication is required to read containers.", nil)
|
||||
return
|
||||
}
|
||||
snapshot, err := h.snapshot(r)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return
|
||||
}
|
||||
problem.Write(w, r, http.StatusServiceUnavailable, "CONTAINERS_UNAVAILABLE", "Containers not available", "Containergegevens konden niet worden gelezen.", nil)
|
||||
return
|
||||
}
|
||||
if r.URL.Path != "/api/v1/containers" {
|
||||
id := strings.TrimPrefix(r.URL.Path, "/api/v1/containers/")
|
||||
for _, item := range snapshot.Containers {
|
||||
if item.ID == id {
|
||||
writeJSON(w, struct {
|
||||
Source container.Source `json:"source"`
|
||||
Container container.Container `json:"container"`
|
||||
}{Source: snapshot.Source, Container: item})
|
||||
return
|
||||
}
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
limit := 50
|
||||
if value := r.URL.Query().Get("limit"); value != "" {
|
||||
parsed, parseErr := strconv.Atoi(value)
|
||||
if parseErr != nil {
|
||||
problem.Write(w, r, http.StatusBadRequest, "CONTAINER_QUERY_INVALID", "Invalid container query", "De containerlimiet is ongeldig.", nil)
|
||||
return
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
order := r.URL.Query().Get("sort")
|
||||
if order == "" {
|
||||
order = "name"
|
||||
}
|
||||
page, err := container.FilteredPage(snapshot, limit, r.URL.Query().Get("after"), h.Limits, r.URL.Query().Get("q"), r.URL.Query().Get("state"), r.URL.Query().Get("health"), order)
|
||||
if err != nil {
|
||||
problem.Write(w, r, http.StatusBadRequest, "CONTAINER_QUERY_INVALID", "Invalid container query", "De containerlimiet of cursor is ongeldig.", nil)
|
||||
return
|
||||
}
|
||||
writeJSON(w, page)
|
||||
}
|
||||
func (h Handler) snapshot(r *http.Request) (container.Snapshot, error) {
|
||||
if h.Provider == nil {
|
||||
return container.UnknownSnapshot(time.Now().UTC(), "container", "agent", "source_unavailable"), nil
|
||||
}
|
||||
return h.Provider.Snapshot(r.Context())
|
||||
}
|
||||
func writeJSON(w http.ResponseWriter, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "private, max-age=5")
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package containerapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
)
|
||||
|
||||
type provider struct{ value container.Snapshot }
|
||||
|
||||
func (p provider) Snapshot(context.Context) (container.Snapshot, error) { return p.value, nil }
|
||||
func authRequest(method, path string) *http.Request {
|
||||
r := httptest.NewRequest(method, path, nil)
|
||||
return r.WithContext(auth.WithPrincipal(r.Context(), auth.Principal{Subject: "viewer", Role: auth.RoleViewer}))
|
||||
}
|
||||
|
||||
func TestHandlerListDetailAndNoMutation(t *testing.T) {
|
||||
snapshot := container.Snapshot{ContractVersion: container.ContractVersion, Source: container.Source{ID: "agent", State: "healthy"}, Containers: []container.Container{{ID: "abc", Name: "media", State: "running", Health: "unhealthy"}}, Total: 1}
|
||||
h := Handler{Provider: provider{value: snapshot}}
|
||||
list := httptest.NewRecorder()
|
||||
h.ServeHTTP(list, authRequest(http.MethodGet, "/api/v1/containers?limit=1"))
|
||||
if list.Code != http.StatusOK || !strings.Contains(list.Body.String(), `"name":"media"`) {
|
||||
t.Fatalf("status=%d body=%s", list.Code, list.Body.String())
|
||||
}
|
||||
detail := httptest.NewRecorder()
|
||||
h.ServeHTTP(detail, authRequest(http.MethodGet, "/api/v1/containers/abc"))
|
||||
if detail.Code != http.StatusOK || !strings.Contains(detail.Body.String(), `"container":{"id":"abc"`) {
|
||||
t.Fatalf("status=%d body=%s", detail.Code, detail.Body.String())
|
||||
}
|
||||
mutate := httptest.NewRecorder()
|
||||
h.ServeHTTP(mutate, authRequest(http.MethodPost, "/api/v1/containers/abc/restart"))
|
||||
if mutate.Code != http.StatusNotFound {
|
||||
t.Fatalf("mutation=%d", mutate.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerRequiresAuthentication(t *testing.T) {
|
||||
response := httptest.NewRecorder()
|
||||
Handler{}.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/v1/containers", nil))
|
||||
if response.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status=%d", response.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerAppliesFiltersBeforePagination(t *testing.T) {
|
||||
snapshot := container.Snapshot{Source: container.Source{ID: "agent", State: "healthy"}, Containers: []container.Container{{ID: "a", Name: "api", State: "running", Health: "healthy"}, {ID: "b", Name: "database", State: "running", Health: "healthy"}}, Total: 2}
|
||||
response := httptest.NewRecorder()
|
||||
Handler{Provider: provider{value: snapshot}}.ServeHTTP(response, authRequest(http.MethodGet, "/api/v1/containers?limit=1&q=database&state=running&health=healthy&sort=name"))
|
||||
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"name":"database"`) || !strings.Contains(response.Body.String(), `"total":1`) {
|
||||
t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package correlation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
const Header = "X-Correlation-ID"
|
||||
|
||||
type contextKey struct{}
|
||||
|
||||
var validID = regexp.MustCompile(`^[A-Za-z0-9._:-]{8,64}$`)
|
||||
|
||||
func New() string {
|
||||
bytes := make([]byte, 16)
|
||||
if _, err := rand.Read(bytes); err != nil {
|
||||
return "correlation-unavailable"
|
||||
}
|
||||
return hex.EncodeToString(bytes)
|
||||
}
|
||||
|
||||
func FromContext(ctx context.Context) string {
|
||||
value, _ := ctx.Value(contextKey{}).(string)
|
||||
return value
|
||||
}
|
||||
|
||||
func WithContext(ctx context.Context, id string) context.Context {
|
||||
return context.WithValue(ctx, contextKey{}, id)
|
||||
}
|
||||
|
||||
func Middleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
id := request.Header.Get(Header)
|
||||
if !validID.MatchString(id) {
|
||||
id = New()
|
||||
}
|
||||
response.Header().Set(Header, id)
|
||||
next.ServeHTTP(response, request.WithContext(WithContext(request.Context(), id)))
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package correlation
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMiddlewarePreservesValidCorrelationID(t *testing.T) {
|
||||
handler := Middleware(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
if got := FromContext(request.Context()); got != "request-123" {
|
||||
t.Errorf("context correlation ID = %q", got)
|
||||
}
|
||||
response.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
request := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
request.Header.Set(Header, "request-123")
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Header().Get(Header) != "request-123" {
|
||||
t.Fatalf("response correlation ID = %q", response.Header().Get(Header))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMiddlewareReplacesInvalidCorrelationID(t *testing.T) {
|
||||
handler := Middleware(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
if len(FromContext(request.Context())) < 8 {
|
||||
t.Error("generated correlation ID is too short")
|
||||
}
|
||||
response.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
request := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
request.Header.Set(Header, "secret\nforged")
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Header().Get(Header) == "secret\nforged" || response.Header().Get(Header) == "" {
|
||||
t.Fatalf("invalid correlation ID was not replaced: %q", response.Header().Get(Header))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
const CurrentSchemaVersion = 2
|
||||
|
||||
var slugPattern = regexp.MustCompile("^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||
|
||||
type Document map[string]any
|
||||
|
||||
func Validate(document Document) error {
|
||||
if document == nil {
|
||||
return errors.New("dashboard document is required")
|
||||
}
|
||||
if version, ok := number(document["schemaVersion"]); !ok || version < 1 || version > CurrentSchemaVersion {
|
||||
return errors.New("unsupported dashboard schema version")
|
||||
}
|
||||
for _, field := range []string{"id", "slug", "name", "scope", "variables", "widgets", "settings"} {
|
||||
if _, ok := document[field]; !ok {
|
||||
return fmt.Errorf("dashboard field %q is required", field)
|
||||
}
|
||||
}
|
||||
slug, ok := document["slug"].(string)
|
||||
if !ok || !slugPattern.MatchString(slug) || len(slug) > 80 {
|
||||
return errors.New("invalid dashboard slug")
|
||||
}
|
||||
name, ok := document["name"].(string)
|
||||
if !ok || name == "" || len(name) > 120 {
|
||||
return errors.New("invalid dashboard name")
|
||||
}
|
||||
scope, ok := document["scope"].(string)
|
||||
if !ok || scope != "personal" && scope != "shared" && scope != "system" {
|
||||
return errors.New("invalid dashboard scope")
|
||||
}
|
||||
if _, ok := document["widgets"].([]any); !ok {
|
||||
return errors.New("dashboard widgets must be an array")
|
||||
}
|
||||
if _, ok := document["variables"].([]any); !ok {
|
||||
return errors.New("dashboard variables must be an array")
|
||||
}
|
||||
if _, ok := document["settings"].(map[string]any); !ok {
|
||||
return errors.New("dashboard settings must be an object")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func Migrate(document Document) (Document, error) {
|
||||
if err := Validate(document); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
version, _ := number(document["schemaVersion"])
|
||||
if version == CurrentSchemaVersion {
|
||||
return clone(document), nil
|
||||
}
|
||||
migrated := clone(document)
|
||||
migrated["schemaVersion"] = CurrentSchemaVersion
|
||||
settings := migrated["settings"].(map[string]any)
|
||||
if _, ok := settings["live"]; !ok {
|
||||
settings["live"] = false
|
||||
}
|
||||
if _, ok := settings["refreshSeconds"]; !ok {
|
||||
settings["refreshSeconds"] = 30
|
||||
}
|
||||
return migrated, nil
|
||||
}
|
||||
func clone(document Document) Document {
|
||||
encoded, _ := json.Marshal(document)
|
||||
var result Document
|
||||
_ = json.Unmarshal(encoded, &result)
|
||||
return result
|
||||
}
|
||||
func number(value any) (int, bool) {
|
||||
switch v := value.(type) {
|
||||
case int:
|
||||
return v, true
|
||||
case float64:
|
||||
return int(v), v == float64(int(v))
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package dashboard
|
||||
|
||||
import "testing"
|
||||
|
||||
func valid() Document {
|
||||
return Document{"schemaVersion": 1, "id": "00000000-0000-0000-0000-000000000001", "slug": "overview", "name": "Overview", "scope": "system", "variables": []any{}, "widgets": []any{}, "settings": map[string]any{}}
|
||||
}
|
||||
func TestInvalidDocumentRejected(t *testing.T) {
|
||||
doc := valid()
|
||||
delete(doc, "widgets")
|
||||
if err := Validate(doc); err == nil {
|
||||
t.Fatal("expected invalid document rejection")
|
||||
}
|
||||
}
|
||||
func TestMigrationIsDeterministicAndPreservesCustomSettings(t *testing.T) {
|
||||
doc := valid()
|
||||
doc["settings"].(map[string]any)["refreshSeconds"] = 90
|
||||
first, err := Migrate(doc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := Migrate(doc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first["schemaVersion"] != 2 || first["settings"].(map[string]any)["refreshSeconds"] != float64(90) {
|
||||
t.Fatalf("migration overwrote custom value: %+v", first)
|
||||
}
|
||||
if len(first["widgets"].([]any)) != len(second["widgets"].([]any)) {
|
||||
t.Fatal("migration not deterministic")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
func mapDatabaseError(err error) error {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return ErrConflict
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
)
|
||||
|
||||
func TestDashboardVersionIsImmutableInPostgreSQL(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const dashboardID = "00000000-0000-0000-0000-0000000000d1"
|
||||
const versionID = "00000000-0000-0000-0000-0000000000f1"
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM dashboards WHERE id=$1`, dashboardID)
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO dashboards (id,slug,name,scope) VALUES ($1,'m3-test','M3 test','system')`, dashboardID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO dashboard_versions (id,dashboard_id,version_number,schema_version,document) VALUES ($1,$2,1,1,'{}')`, versionID, dashboardID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE dashboard_versions SET change_summary='mutated' WHERE id=$1`, versionID); err == nil {
|
||||
t.Fatal("expected immutable update failure")
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `DELETE FROM dashboards WHERE id=$1`, dashboardID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var ErrConflict = errors.New("dashboard revision conflict")
|
||||
var ErrNotFound = errors.New("dashboard not found")
|
||||
var ErrForbidden = errors.New("dashboard access denied")
|
||||
|
||||
type Summary struct {
|
||||
ID, Slug, Name, Description, OwnerID, Scope string
|
||||
ArchivedAt *time.Time
|
||||
Revision int64
|
||||
CurrentVersion int
|
||||
}
|
||||
type Version struct {
|
||||
ID, DashboardID string
|
||||
Number, SchemaVersion int
|
||||
Document Document
|
||||
ChangeSummary, CreatedBy string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
type Repository struct{ Pool *pgxpool.Pool }
|
||||
|
||||
func (r Repository) CanAccess(ctx context.Context, id, actor string) (bool, error) {
|
||||
if r.Pool == nil {
|
||||
return false, errors.New("dashboard repository is not configured")
|
||||
}
|
||||
var allowed bool
|
||||
err := r.Pool.QueryRow(ctx, `SELECT EXISTS (
|
||||
SELECT 1 FROM dashboards
|
||||
WHERE id=$1 AND archived_at IS NULL
|
||||
AND (scope IN ('shared','system') OR owner_user_id=(SELECT id FROM users WHERE external_subject=$2))
|
||||
)`, id, actor).Scan(&allowed)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("check dashboard access: %w", err)
|
||||
}
|
||||
return allowed, nil
|
||||
}
|
||||
|
||||
func (r Repository) Create(ctx context.Context, actor string, document Document, changeSummary string) (Summary, Version, error) {
|
||||
if err := Validate(document); err != nil {
|
||||
return Summary{}, Version{}, err
|
||||
}
|
||||
id, ok := document["id"].(string)
|
||||
if !ok || id == "" {
|
||||
return Summary{}, Version{}, errors.New("dashboard id is required")
|
||||
}
|
||||
slug, _ := document["slug"].(string)
|
||||
name, _ := document["name"].(string)
|
||||
scope, _ := document["scope"].(string)
|
||||
description, _ := document["description"].(string)
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Summary{}, Version{}, fmt.Errorf("begin dashboard create: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
docJSON, _ := json.Marshal(document)
|
||||
versionID := idForVersion(id, 1)
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO dashboards (id,slug,name,description,owner_user_id,scope) VALUES ($1,$2,$3,$4,(SELECT id FROM users WHERE external_subject=$5),$6)`, id, slug, name, description, actor, scope); err != nil {
|
||||
err = mapDatabaseError(err)
|
||||
return Summary{}, Version{}, fmt.Errorf("create dashboard: %w", err)
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO dashboard_versions (id,dashboard_id,version_number,schema_version,document,change_summary,created_by) VALUES ($1,$2,1,$3,$4,$5,(SELECT id FROM users WHERE external_subject=$6))`, versionID, id, CurrentSchemaVersion, docJSON, changeSummary, actor); err != nil {
|
||||
return Summary{}, Version{}, fmt.Errorf("create dashboard version: %w", err)
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE dashboards SET current_version_id=$1 WHERE id=$2`, versionID, id); err != nil {
|
||||
return Summary{}, Version{}, fmt.Errorf("set current dashboard version: %w", err)
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return Summary{}, Version{}, fmt.Errorf("commit dashboard create: %w", err)
|
||||
}
|
||||
createdSummary, createdVersion, err := r.Get(ctx, id, actor)
|
||||
if err != nil {
|
||||
return Summary{}, Version{}, err
|
||||
}
|
||||
return createdSummary, createdVersion, nil
|
||||
}
|
||||
|
||||
func (r Repository) UpdateDocument(ctx context.Context, id, actor string, expected int64, document Document, summary string) (Summary, error) {
|
||||
if err := Validate(document); err != nil {
|
||||
return Summary{}, err
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Summary{}, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var currentVersionID string
|
||||
var currentVersion int
|
||||
var currentJSON []byte
|
||||
var current Summary
|
||||
var canEdit bool
|
||||
err = tx.QueryRow(ctx, `SELECT id,slug,name,description,COALESCE(owner_user_id::text,''),scope,archived_at,revision,current_version_id,(owner_user_id=(SELECT id FROM users WHERE external_subject=$2) OR scope IN ('shared','system')) FROM dashboards WHERE id=$1 FOR UPDATE`, id, actor).Scan(¤t.ID, ¤t.Slug, ¤t.Name, ¤t.Description, ¤t.OwnerID, ¤t.Scope, ¤t.ArchivedAt, ¤t.Revision, ¤tVersionID, &canEdit)
|
||||
if err != nil {
|
||||
return Summary{}, err
|
||||
}
|
||||
if !canEdit {
|
||||
return Summary{}, ErrForbidden
|
||||
}
|
||||
if current.Revision != expected {
|
||||
return Summary{}, ErrConflict
|
||||
}
|
||||
if err = tx.QueryRow(ctx, `SELECT version_number,document FROM dashboard_versions WHERE id=$1`, currentVersionID).Scan(¤tVersion, ¤tJSON); err != nil {
|
||||
return Summary{}, err
|
||||
}
|
||||
newJSON, _ := json.Marshal(document)
|
||||
var stored, normalized Document
|
||||
_ = json.Unmarshal(currentJSON, &stored)
|
||||
_ = json.Unmarshal(newJSON, &normalized)
|
||||
if reflect.DeepEqual(stored, normalized) {
|
||||
current.CurrentVersion = currentVersion
|
||||
return current, nil
|
||||
}
|
||||
next := currentVersion + 1
|
||||
versionID := idForVersion(id, next)
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO dashboard_versions (id,dashboard_id,version_number,schema_version,document,change_summary,created_by) VALUES ($1,$2,$3,$4,$5,$6,(SELECT id FROM users WHERE external_subject=$7))`, versionID, id, next, CurrentSchemaVersion, newJSON, summary, actor); err != nil {
|
||||
return Summary{}, err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `UPDATE dashboards SET current_version_id=$1,revision=revision+1,updated_at=now() WHERE id=$2 AND revision=$3`, versionID, id, expected)
|
||||
if err != nil {
|
||||
return Summary{}, err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return Summary{}, ErrConflict
|
||||
}
|
||||
current.Revision++
|
||||
current.CurrentVersion = next
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return Summary{}, err
|
||||
}
|
||||
return current, nil
|
||||
}
|
||||
|
||||
func (r Repository) Restore(ctx context.Context, id, actor string, expected int64, versionNumber int) (Summary, error) {
|
||||
version, err := r.GetVersion(ctx, id, actor, versionNumber)
|
||||
if err != nil {
|
||||
return Summary{}, err
|
||||
}
|
||||
return r.UpdateDocument(ctx, id, actor, expected, version.Document, fmt.Sprintf("restore version %d", versionNumber))
|
||||
}
|
||||
|
||||
func (r Repository) Get(ctx context.Context, id, actor string) (Summary, Version, error) {
|
||||
var s Summary
|
||||
var versionID string
|
||||
err := r.Pool.QueryRow(ctx, `SELECT id,slug,name,description,COALESCE(owner_user_id::text,''),scope,archived_at,revision,current_version_id FROM dashboards WHERE id=$1 AND (scope IN ('shared','system') OR owner_user_id=(SELECT id FROM users WHERE external_subject=$2))`, id, actor).Scan(&s.ID, &s.Slug, &s.Name, &s.Description, &s.OwnerID, &s.Scope, &s.ArchivedAt, &s.Revision, &versionID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Summary{}, Version{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Summary{}, Version{}, err
|
||||
}
|
||||
var raw []byte
|
||||
var v Version
|
||||
err = r.Pool.QueryRow(ctx, `SELECT id,dashboard_id,version_number,schema_version,document,change_summary,COALESCE(created_by::text,''),created_at FROM dashboard_versions WHERE id=$1`, versionID).Scan(&v.ID, &v.DashboardID, &v.Number, &v.SchemaVersion, &raw, &v.ChangeSummary, &v.CreatedBy, &v.CreatedAt)
|
||||
if err != nil {
|
||||
return Summary{}, Version{}, err
|
||||
}
|
||||
if err = json.Unmarshal(raw, &v.Document); err != nil {
|
||||
return Summary{}, Version{}, errors.New("invalid stored dashboard document")
|
||||
}
|
||||
s.CurrentVersion = v.Number
|
||||
return s, v, nil
|
||||
}
|
||||
|
||||
func (r Repository) List(ctx context.Context, actor string, limit int) ([]Summary, error) {
|
||||
if limit < 1 || limit > 100 {
|
||||
return nil, errors.New("dashboard page limit is invalid")
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, `SELECT d.id,d.slug,d.name,d.description,COALESCE(d.owner_user_id::text,''),d.scope,d.archived_at,d.revision,v.version_number FROM dashboards d JOIN dashboard_versions v ON v.id=d.current_version_id WHERE d.scope <> 'personal' OR d.owner_user_id=(SELECT id FROM users WHERE external_subject=$1) ORDER BY d.name ASC,d.id ASC LIMIT $2`, actor, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list dashboards: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
result := make([]Summary, 0, limit)
|
||||
for rows.Next() {
|
||||
var s Summary
|
||||
if err := rows.Scan(&s.ID, &s.Slug, &s.Name, &s.Description, &s.OwnerID, &s.Scope, &s.ArchivedAt, &s.Revision, &s.CurrentVersion); err != nil {
|
||||
return nil, fmt.Errorf("scan dashboard: %w", err)
|
||||
}
|
||||
result = append(result, s)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func (r Repository) UpdateMetadata(ctx context.Context, id, actor string, expected int64, name, description string) (Summary, error) {
|
||||
tag, err := r.Pool.Exec(ctx, `UPDATE dashboards SET name=$1,description=$2,revision=revision+1,updated_at=now() WHERE id=$3 AND revision=$4 AND (owner_user_id=(SELECT id FROM users WHERE external_subject=$5) OR scope IN ('shared','system'))`, name, description, id, expected, actor)
|
||||
if err != nil {
|
||||
return Summary{}, fmt.Errorf("update dashboard metadata: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return Summary{}, ErrConflict
|
||||
}
|
||||
s, _, err := r.Get(ctx, id, actor)
|
||||
return s, err
|
||||
}
|
||||
|
||||
func (r Repository) Archive(ctx context.Context, id, actor string, expected int64) (Summary, error) {
|
||||
tag, err := r.Pool.Exec(ctx, `UPDATE dashboards SET archived_at=now(),revision=revision+1,updated_at=now() WHERE id=$1 AND revision=$2 AND (owner_user_id=(SELECT id FROM users WHERE external_subject=$3) OR scope IN ('shared','system'))`, id, expected, actor)
|
||||
if err != nil {
|
||||
return Summary{}, fmt.Errorf("archive dashboard: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return Summary{}, ErrConflict
|
||||
}
|
||||
s, _, err := r.Get(ctx, id, actor)
|
||||
return s, err
|
||||
}
|
||||
|
||||
func (r Repository) Versions(ctx context.Context, id, actor string, limit int) ([]Version, error) {
|
||||
if limit < 1 || limit > 100 {
|
||||
return nil, errors.New("version page limit is invalid")
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, `SELECT v.id,v.dashboard_id,v.version_number,v.schema_version,v.document,v.change_summary,COALESCE(v.created_by::text,''),v.created_at FROM dashboard_versions v JOIN dashboards d ON d.id=v.dashboard_id WHERE v.dashboard_id=$1 AND (d.scope IN ('shared','system') OR d.owner_user_id=(SELECT id FROM users WHERE external_subject=$2)) ORDER BY v.version_number DESC LIMIT $3`, id, actor, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list dashboard versions: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
result := make([]Version, 0, limit)
|
||||
for rows.Next() {
|
||||
var v Version
|
||||
var raw []byte
|
||||
if err := rows.Scan(&v.ID, &v.DashboardID, &v.Number, &v.SchemaVersion, &raw, &v.ChangeSummary, &v.CreatedBy, &v.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &v.Document); err != nil {
|
||||
return nil, errors.New("invalid stored dashboard document")
|
||||
}
|
||||
result = append(result, v)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
func (r Repository) Clone(ctx context.Context, actor, sourceID, slug, name string) (Summary, Version, error) {
|
||||
summary, version, err := r.Get(ctx, sourceID, actor)
|
||||
if err != nil {
|
||||
return Summary{}, Version{}, err
|
||||
}
|
||||
var allowed bool
|
||||
if err := r.Pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM dashboards WHERE id=$1 AND (scope <> 'personal' OR owner_user_id=(SELECT id FROM users WHERE external_subject=$2)))`, sourceID, actor).Scan(&allowed); err != nil {
|
||||
return Summary{}, Version{}, err
|
||||
}
|
||||
if !allowed {
|
||||
return Summary{}, Version{}, ErrForbidden
|
||||
}
|
||||
if slug == "" {
|
||||
slug = summary.Slug + "-copy"
|
||||
}
|
||||
if name == "" {
|
||||
name = summary.Name + " (kopie)"
|
||||
}
|
||||
copyDoc := clone(version.Document)
|
||||
copyDoc["id"] = newUUID()
|
||||
copyDoc["slug"] = slug
|
||||
copyDoc["name"] = name
|
||||
copyDoc["scope"] = "personal"
|
||||
return r.Create(ctx, actor, copyDoc, "cloned dashboard")
|
||||
}
|
||||
|
||||
func newUUID() string {
|
||||
b := make([]byte, 16)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "00000000-0000-4000-8000-000000000000"
|
||||
}
|
||||
b[6] = (b[6] & 0x0f) | 0x40
|
||||
b[8] = (b[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
|
||||
}
|
||||
func idForVersion(id string, number int) string {
|
||||
digest := sha256.Sum256([]byte(fmt.Sprintf("%s:%d", id, number)))
|
||||
digest[6] = (digest[6] & 0x0f) | 0x50
|
||||
digest[8] = (digest[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", digest[0:4], digest[4:6], digest[6:8], digest[8:10], digest[10:16])
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
)
|
||||
|
||||
func TestDashboardRepositoryPostgreSQL(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 10, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
actor := "00000000-0000-0000-0000-0000000003a1"
|
||||
otherActor := "00000000-0000-0000-0000-0000000003a2"
|
||||
if _, err := pool.Exec(ctx, "INSERT INTO users (id, external_subject, display_name) VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING", actor, actor, "M3 actor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, "INSERT INTO users (id, external_subject, display_name) VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING", otherActor, otherActor, "M3 other actor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repo := Repository{Pool: pool}
|
||||
id := "00000000-0000-0000-0000-0000000003d1"
|
||||
_, _ = pool.Exec(ctx, "DELETE FROM dashboards WHERE id=$1 OR slug IN ('m3-repo','m3-repo-copy')", id)
|
||||
doc := Document{"schemaVersion": 2, "id": id, "slug": "m3-repo", "name": "M3 Repo", "scope": "personal", "variables": []any{}, "widgets": []any{}, "settings": map[string]any{"refreshSeconds": 30}}
|
||||
|
||||
summary, version, err := repo.Create(ctx, actor, doc, "initial")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version.Number != 1 || summary.Revision != 1 {
|
||||
t.Fatalf("create=%+v/%+v", summary, version)
|
||||
}
|
||||
same, err := repo.UpdateDocument(ctx, id, actor, 1, doc, "noop")
|
||||
if err != nil || same.Revision != 1 {
|
||||
t.Fatalf("noop=%+v err=%v", same, err)
|
||||
}
|
||||
doc["name"] = "Changed"
|
||||
updated, err := repo.UpdateDocument(ctx, id, actor, 1, doc, "edit")
|
||||
if err != nil || updated.Revision != 2 || updated.CurrentVersion != 2 {
|
||||
t.Fatalf("update=%+v err=%v", updated, err)
|
||||
}
|
||||
if _, err := repo.UpdateDocument(ctx, id, actor, 1, doc, "stale"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("expected conflict, got %v", err)
|
||||
}
|
||||
restored, err := repo.Restore(ctx, id, actor, 2, 1)
|
||||
if err != nil || restored.Revision != 3 || restored.CurrentVersion != 3 {
|
||||
t.Fatalf("restore=%+v err=%v", restored, err)
|
||||
}
|
||||
|
||||
versions, err := repo.Versions(ctx, id, actor, 10)
|
||||
if err != nil || len(versions) != 3 {
|
||||
t.Fatalf("versions=%d err=%v", len(versions), err)
|
||||
}
|
||||
historical, err := repo.GetVersion(ctx, id, actor, 1)
|
||||
if err != nil || historical.Number != 1 {
|
||||
t.Fatalf("historical=%+v err=%v", historical, err)
|
||||
}
|
||||
if _, _, err := repo.Get(ctx, id, otherActor); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("personal dashboard leaked through direct read: %v", err)
|
||||
}
|
||||
if otherVersions, err := repo.Versions(ctx, id, otherActor, 10); err != nil || len(otherVersions) != 0 {
|
||||
t.Fatalf("personal dashboard versions leaked: count=%d err=%v", len(otherVersions), err)
|
||||
}
|
||||
if _, err := repo.GetVersion(ctx, id, otherActor, 1); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("personal dashboard version leaked through direct read: %v", err)
|
||||
}
|
||||
if _, _, err := repo.Create(ctx, actor, doc, "duplicate"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("expected duplicate conflict, got %v", err)
|
||||
}
|
||||
|
||||
cloned, clonedVersion, err := repo.Clone(ctx, actor, id, "m3-repo-copy", "M3 Repo Copy")
|
||||
if err != nil || cloned.ID == id || clonedVersion.Number != 1 || cloned.Scope != "personal" {
|
||||
t.Fatalf("clone=%+v/%+v err=%v", cloned, clonedVersion, err)
|
||||
}
|
||||
listed, err := repo.List(ctx, actor, 10)
|
||||
if err != nil || len(listed) < 2 {
|
||||
t.Fatalf("list=%d err=%v", len(listed), err)
|
||||
}
|
||||
edited, err := repo.UpdateMetadata(ctx, id, actor, 3, "M3 Repo Renamed", "updated")
|
||||
if err != nil || edited.Revision != 4 {
|
||||
t.Fatalf("metadata=%+v err=%v", edited, err)
|
||||
}
|
||||
archived, err := repo.Archive(ctx, id, actor, 4)
|
||||
if err != nil || archived.ArchivedAt == nil || archived.Revision != 5 {
|
||||
t.Fatalf("archive=%+v err=%v", archived, err)
|
||||
}
|
||||
|
||||
if _, err := pool.Exec(ctx, "UPDATE dashboard_versions SET change_summary='bad' WHERE dashboard_id=$1", id); err == nil {
|
||||
t.Fatal("expected immutable version failure")
|
||||
}
|
||||
if _, err := pool.Exec(ctx, "DELETE FROM dashboards WHERE id=$1", id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, "DELETE FROM dashboards WHERE id=$1", cloned.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
)
|
||||
|
||||
func TestDashboardListTargetScalePostgreSQL(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 10, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
actor := "00000000-0000-0000-0000-0000000003a1"
|
||||
if _, err := pool.Exec(ctx, "INSERT INTO users (id, external_subject, display_name) VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING", actor, actor, "M3 scale actor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _, _ = pool.Exec(context.Background(), "DELETE FROM dashboards WHERE slug LIKE 'm3-scale-%'") }()
|
||||
repo := Repository{Pool: pool}
|
||||
for i := 0; i < 150; i++ {
|
||||
id := fmt.Sprintf("00000000-0000-0000-0000-%012x", 0x5000+i)
|
||||
doc := Document{"schemaVersion": 2, "id": id, "slug": fmt.Sprintf("m3-scale-%03d", i), "name": fmt.Sprintf("M3 Scale %03d", i), "scope": "personal", "variables": []any{}, "widgets": []any{}, "settings": map[string]any{"refreshSeconds": 30}}
|
||||
if _, _, err := repo.Create(ctx, actor, doc, "scale fixture"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
samples := make([]time.Duration, 20)
|
||||
for i := range samples {
|
||||
start := time.Now()
|
||||
items, err := repo.List(ctx, actor, 100)
|
||||
samples[i] = time.Since(start)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(items) < 100 {
|
||||
t.Fatalf("list returned %d items", len(items))
|
||||
}
|
||||
}
|
||||
sort.Slice(samples, func(i, j int) bool { return samples[i] < samples[j] })
|
||||
p95 := samples[len(samples)*95/100-1]
|
||||
t.Logf("target-scale dashboards=150 list_limit=100 p95=%s max=%s", p95, samples[len(samples)-1])
|
||||
if p95 > 250*time.Millisecond {
|
||||
t.Fatalf("dashboard list p95=%s exceeds 250ms budget", p95)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func (r Repository) GetVersion(ctx context.Context, id, actor string, number int) (Version, error) {
|
||||
var v Version
|
||||
var raw []byte
|
||||
err := r.Pool.QueryRow(ctx, `SELECT v.id,v.dashboard_id,v.version_number,v.schema_version,v.document,v.change_summary,COALESCE(v.created_by::text,''),v.created_at FROM dashboard_versions v JOIN dashboards d ON d.id=v.dashboard_id WHERE v.dashboard_id=$1 AND v.version_number=$2 AND (d.scope IN ('shared','system') OR d.owner_user_id=(SELECT id FROM users WHERE external_subject=$3))`, id, number, actor).Scan(&v.ID, &v.DashboardID, &v.Number, &v.SchemaVersion, &raw, &v.ChangeSummary, &v.CreatedBy, &v.CreatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Version{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Version{}, err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &v.Document); err != nil {
|
||||
return Version{}, errors.New("invalid stored dashboard document")
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
@@ -0,0 +1,407 @@
|
||||
package dashboardapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/itworx/pulse/internal/audit"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/correlation"
|
||||
"github.com/itworx/pulse/internal/dashboard"
|
||||
"github.com/itworx/pulse/internal/problem"
|
||||
"github.com/itworx/pulse/internal/widgetpreview"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
Repository dashboard.Repository
|
||||
Audit audit.Store
|
||||
}
|
||||
|
||||
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
principal, ok := auth.PrincipalFromContext(r.Context())
|
||||
if !ok {
|
||||
fail(w, r, http.StatusUnauthorized, "UNAUTHORIZED", "Authentication required.")
|
||||
return
|
||||
}
|
||||
path := strings.TrimPrefix(r.URL.Path, "/api/v1/dashboards")
|
||||
if path == "" || path == "/" {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
h.list(w, r, principal.Subject)
|
||||
case http.MethodPost:
|
||||
if !auth.Allows(principal.Role, auth.PermissionEdit) {
|
||||
fail(w, r, http.StatusForbidden, "FORBIDDEN", "Dashboard editing is not allowed for this role.")
|
||||
return
|
||||
}
|
||||
h.create(w, r, principal.Subject)
|
||||
default:
|
||||
fail(w, r, http.StatusMethodNotAllowed, "METHOD_NOT_ALLOWED", "This method is not supported.")
|
||||
}
|
||||
return
|
||||
}
|
||||
parts := strings.Split(strings.Trim(path, "/"), "/")
|
||||
if len(parts) == 0 || parts[0] == "" {
|
||||
fail(w, r, http.StatusNotFound, "NOT_FOUND", "Dashboard not found.")
|
||||
return
|
||||
}
|
||||
id := parts[0]
|
||||
if len(parts) == 2 && parts[1] == "versions" && r.Method == http.MethodGet {
|
||||
h.versions(w, r, id, principal.Subject)
|
||||
return
|
||||
}
|
||||
if len(parts) == 3 && parts[1] == "versions" && r.Method == http.MethodGet {
|
||||
h.version(w, r, id, parts[2], principal.Subject)
|
||||
return
|
||||
}
|
||||
if len(parts) == 2 && parts[1] == "document" && r.Method == http.MethodPut {
|
||||
if !requireEdit(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.update(w, r, id, principal.Subject)
|
||||
return
|
||||
}
|
||||
if len(parts) == 2 && parts[1] == "preview" && r.Method == http.MethodPost {
|
||||
if !requireEdit(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.preview(w, r, id, principal.Subject)
|
||||
return
|
||||
}
|
||||
if len(parts) == 2 && parts[1] == "clone" && r.Method == http.MethodPost {
|
||||
if !requireEdit(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.clone(w, r, id, principal.Subject)
|
||||
return
|
||||
}
|
||||
if len(parts) == 2 && parts[1] == "restore" && r.Method == http.MethodPost {
|
||||
if !requireEdit(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.restore(w, r, id, principal.Subject, "")
|
||||
return
|
||||
}
|
||||
if len(parts) == 3 && parts[1] == "restore" && r.Method == http.MethodPost {
|
||||
if !requireEdit(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.restore(w, r, id, principal.Subject, parts[2])
|
||||
return
|
||||
}
|
||||
if len(parts) == 1 && r.Method == http.MethodGet {
|
||||
h.get(w, r, id, principal.Subject)
|
||||
return
|
||||
}
|
||||
if len(parts) == 1 && r.Method == http.MethodPatch {
|
||||
if !requireEdit(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.metadata(w, r, id, principal.Subject)
|
||||
return
|
||||
}
|
||||
if len(parts) == 1 && r.Method == http.MethodDelete {
|
||||
if !requireEdit(w, r, principal.Role) {
|
||||
return
|
||||
}
|
||||
h.archive(w, r, id, principal.Subject)
|
||||
return
|
||||
}
|
||||
fail(w, r, http.StatusNotFound, "NOT_FOUND", "Dashboard route not found.")
|
||||
}
|
||||
|
||||
func requireEdit(w http.ResponseWriter, r *http.Request, role auth.Role) bool {
|
||||
if auth.Allows(role, auth.PermissionEdit) {
|
||||
return true
|
||||
}
|
||||
fail(w, r, http.StatusForbidden, "FORBIDDEN", "Dashboard editing is not allowed for this role.")
|
||||
return false
|
||||
}
|
||||
|
||||
func (h Handler) create(w http.ResponseWriter, r *http.Request, actor string) {
|
||||
var doc dashboard.Document
|
||||
if err := decode(r, &doc); err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_DOCUMENT", "The dashboard document is invalid.")
|
||||
return
|
||||
}
|
||||
s, v, err := h.Repository.Create(r.Context(), actor, doc, "initial version")
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Dashboard could not be created.")
|
||||
return
|
||||
}
|
||||
if err := h.record(r, actor, "dashboard.create", s.ID, nil, map[string]any{"revision": s.Revision}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusCreated, map[string]any{"dashboard": s, "version": v})
|
||||
}
|
||||
|
||||
func (h Handler) list(w http.ResponseWriter, r *http.Request, actor string) {
|
||||
limit := 100
|
||||
if value := r.URL.Query().Get("limit"); value != "" {
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed < 1 || parsed > 100 {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_LIMIT", "The dashboard limit must be between 1 and 100.")
|
||||
return
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
items, err := h.Repository.List(r.Context(), actor, limit)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Dashboard list unavailable.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"items": items})
|
||||
}
|
||||
|
||||
func (h Handler) get(w http.ResponseWriter, r *http.Request, id, actor string) {
|
||||
s, v, err := h.Repository.Get(r.Context(), id, actor)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Dashboard not found.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"dashboard": s, "version": v})
|
||||
}
|
||||
|
||||
func (h Handler) preview(w http.ResponseWriter, r *http.Request, id, actor string) {
|
||||
allowed, err := h.Repository.CanAccess(r.Context(), id, actor)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Dashboard preview unavailable.")
|
||||
return
|
||||
}
|
||||
if !allowed {
|
||||
fail(w, r, http.StatusForbidden, "FORBIDDEN", "You cannot preview this dashboard.")
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Widget map[string]any
|
||||
State string
|
||||
}
|
||||
if err := decode(r, &body); err != nil || body.Widget == nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_PREVIEW", "A widget configuration is required.")
|
||||
return
|
||||
}
|
||||
result, err := widgetpreview.Preview(body.Widget, body.State)
|
||||
if err != nil {
|
||||
var invalid widgetpreview.InvalidConfig
|
||||
if errors.As(err, &invalid) {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_WIDGET_CONFIG", "The widget configuration is invalid.", invalid.Fields)
|
||||
return
|
||||
}
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_PREVIEW", "The widget preview is invalid.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"preview": result})
|
||||
}
|
||||
|
||||
func (h Handler) update(w http.ResponseWriter, r *http.Request, id, actor string) {
|
||||
expected, err := revision(r)
|
||||
if err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_REVISION", "A valid If-Match or revision value is required.")
|
||||
return
|
||||
}
|
||||
var doc dashboard.Document
|
||||
if err := decode(r, &doc); err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_DOCUMENT", "The dashboard document is invalid.")
|
||||
return
|
||||
}
|
||||
s, err := h.Repository.UpdateDocument(r.Context(), id, actor, expected, doc, "document update")
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Dashboard update failed.")
|
||||
return
|
||||
}
|
||||
if err := h.record(r, actor, "dashboard.update_document", s.ID, map[string]any{"revision": expected}, map[string]any{"revision": s.Revision}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"dashboard": s})
|
||||
}
|
||||
|
||||
func (h Handler) metadata(w http.ResponseWriter, r *http.Request, id, actor string) {
|
||||
expected, err := revision(r)
|
||||
if err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_REVISION", "A valid If-Match or revision value is required.")
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Name string
|
||||
Description string
|
||||
}
|
||||
if err := decode(r, &body); err != nil || strings.TrimSpace(body.Name) == "" {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_METADATA", "A dashboard name is required.")
|
||||
return
|
||||
}
|
||||
s, err := h.Repository.UpdateMetadata(r.Context(), id, actor, expected, body.Name, body.Description)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Metadata update failed.")
|
||||
return
|
||||
}
|
||||
if err := h.record(r, actor, "dashboard.update_metadata", s.ID, map[string]any{"revision": expected}, map[string]any{"revision": s.Revision}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"dashboard": s})
|
||||
}
|
||||
|
||||
func (h Handler) archive(w http.ResponseWriter, r *http.Request, id, actor string) {
|
||||
expected, err := revision(r)
|
||||
if err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_REVISION", "A valid If-Match or revision value is required.")
|
||||
return
|
||||
}
|
||||
s, err := h.Repository.Archive(r.Context(), id, actor, expected)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Dashboard archive failed.")
|
||||
return
|
||||
}
|
||||
if err := h.record(r, actor, "dashboard.archive", s.ID, map[string]any{"revision": expected}, map[string]any{"revision": s.Revision}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"dashboard": s})
|
||||
}
|
||||
|
||||
func (h Handler) versions(w http.ResponseWriter, r *http.Request, id, actor string) {
|
||||
items, err := h.Repository.Versions(r.Context(), id, actor, 100)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Version history unavailable.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"items": items})
|
||||
}
|
||||
|
||||
func (h Handler) version(w http.ResponseWriter, r *http.Request, id, value, actor string) {
|
||||
number, err := strconv.Atoi(value)
|
||||
if err != nil || number < 1 {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_VERSION", "The version number is invalid.")
|
||||
return
|
||||
}
|
||||
item, err := h.Repository.GetVersion(r.Context(), id, actor, number)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Dashboard version not found.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"version": item})
|
||||
}
|
||||
|
||||
func (h Handler) clone(w http.ResponseWriter, r *http.Request, id, actor string) {
|
||||
var body struct {
|
||||
Slug string
|
||||
Name string
|
||||
}
|
||||
if err := decode(r, &body); err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_CLONE", "The clone options are invalid.")
|
||||
return
|
||||
}
|
||||
s, v, err := h.Repository.Clone(r.Context(), actor, id, body.Slug, body.Name)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Dashboard could not be cloned.")
|
||||
return
|
||||
}
|
||||
if err := h.record(r, actor, "dashboard.clone", s.ID, nil, map[string]any{"revision": s.Revision}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusCreated, map[string]any{"dashboard": s, "version": v})
|
||||
}
|
||||
|
||||
func (h Handler) restore(w http.ResponseWriter, r *http.Request, id, actor, value string) {
|
||||
number := value
|
||||
if number == "" {
|
||||
var body struct{ Version int }
|
||||
if err := decode(r, &body); err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_VERSION", "A version number is required.")
|
||||
return
|
||||
}
|
||||
number = strconv.Itoa(body.Version)
|
||||
}
|
||||
versionNumber, err := strconv.Atoi(number)
|
||||
if err != nil || versionNumber < 1 {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_VERSION", "The version number is invalid.")
|
||||
return
|
||||
}
|
||||
expected, err := revision(r)
|
||||
if err != nil {
|
||||
fail(w, r, http.StatusBadRequest, "INVALID_REVISION", "A valid If-Match or revision value is required.")
|
||||
return
|
||||
}
|
||||
s, err := h.Repository.Restore(r.Context(), id, actor, expected, versionNumber)
|
||||
if err != nil {
|
||||
h.repositoryFailure(w, r, err, "Dashboard could not be restored.")
|
||||
return
|
||||
}
|
||||
if err := h.record(r, actor, "dashboard.restore", s.ID, map[string]any{"version": versionNumber}, map[string]any{"revision": s.Revision}); err != nil {
|
||||
fail(w, r, http.StatusServiceUnavailable, "AUDIT_UNAVAILABLE", "The audit event could not be recorded.")
|
||||
return
|
||||
}
|
||||
write(w, http.StatusOK, map[string]any{"dashboard": s})
|
||||
}
|
||||
|
||||
func (h Handler) record(r *http.Request, actor, action, resourceID string, before, after map[string]any) error {
|
||||
if h.Audit == nil {
|
||||
return nil
|
||||
}
|
||||
return h.Audit.Append(r.Context(), audit.Event{Actor: actor, Action: action, ResourceType: "dashboard", ResourceID: resourceID, Result: "success", CorrelationID: correlation.FromContext(r.Context()), Before: before, After: after})
|
||||
}
|
||||
|
||||
func (h Handler) repositoryFailure(w http.ResponseWriter, r *http.Request, err error, fallback string) {
|
||||
switch {
|
||||
case errors.Is(err, dashboard.ErrConflict):
|
||||
fail(w, r, http.StatusConflict, "REVISION_CONFLICT", "The dashboard was changed by another request.")
|
||||
case errors.Is(err, dashboard.ErrForbidden):
|
||||
fail(w, r, http.StatusForbidden, "FORBIDDEN", "You cannot change this dashboard.")
|
||||
case errors.Is(err, dashboard.ErrNotFound):
|
||||
fail(w, r, http.StatusNotFound, "NOT_FOUND", fallback)
|
||||
default:
|
||||
fail(w, r, http.StatusBadRequest, "DASHBOARD_REQUEST_FAILED", fallback)
|
||||
}
|
||||
}
|
||||
|
||||
func decode(r *http.Request, target any) error {
|
||||
contentType := strings.ToLower(strings.TrimSpace(strings.Split(r.Header.Get("Content-Type"), ";")[0]))
|
||||
if contentType != "" && contentType != "application/json" {
|
||||
return errors.New("unsupported content type")
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, 2<<20+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer r.Body.Close()
|
||||
if len(body) > 2<<20 {
|
||||
return errors.New("request too large")
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(string(body)))
|
||||
if err := decoder.Decode(target); err != nil {
|
||||
return err
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return errors.New("multiple JSON values")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func revision(r *http.Request) (int64, error) {
|
||||
value := r.Header.Get("If-Match")
|
||||
if value == "" {
|
||||
value = r.URL.Query().Get("revision")
|
||||
}
|
||||
return strconv.ParseInt(strings.Trim(value, "\""), 10, 64)
|
||||
}
|
||||
|
||||
func fail(w http.ResponseWriter, r *http.Request, status int, code, detail string, fields ...map[string]string) {
|
||||
var extra map[string]string
|
||||
if len(fields) > 0 {
|
||||
extra = fields[0]
|
||||
}
|
||||
problem.Write(w, r, status, code, http.StatusText(status), detail, extra)
|
||||
}
|
||||
|
||||
func write(w http.ResponseWriter, status int, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package dashboardapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/itworx/pulse/internal/audit"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/correlation"
|
||||
"github.com/itworx/pulse/internal/dashboard"
|
||||
)
|
||||
|
||||
func requestWithPrincipal(method, path, body string, principal *auth.Principal) *httptest.ResponseRecorder {
|
||||
request := httptest.NewRequest(method, path, strings.NewReader(body))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request = request.WithContext(correlation.WithContext(request.Context(), "m3-03-test-correlation"))
|
||||
if principal != nil {
|
||||
request = request.WithContext(auth.WithPrincipal(request.Context(), *principal))
|
||||
}
|
||||
response := httptest.NewRecorder()
|
||||
(Handler{Repository: dashboard.Repository{}}).ServeHTTP(response, request)
|
||||
return response
|
||||
}
|
||||
|
||||
func problemCode(t *testing.T, response *httptest.ResponseRecorder) string {
|
||||
t.Helper()
|
||||
var body struct{ Code string }
|
||||
if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("problem response is not JSON: %v", err)
|
||||
}
|
||||
return body.Code
|
||||
}
|
||||
|
||||
func TestHandlerRequiresAuthenticationWithProblemResponse(t *testing.T) {
|
||||
response := requestWithPrincipal(http.MethodGet, "/api/v1/dashboards", "", nil)
|
||||
if response.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status=%d", response.Code)
|
||||
}
|
||||
if got := response.Header().Get("Content-Type"); got != "application/problem+json" {
|
||||
t.Fatalf("content type=%q", got)
|
||||
}
|
||||
if got := problemCode(t, response); got != "UNAUTHORIZED" {
|
||||
t.Fatalf("code=%q", got)
|
||||
}
|
||||
if got := response.Header().Get(correlation.Header); got != "m3-03-test-correlation" {
|
||||
t.Fatalf("correlation=%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerEnforcesEditorForMutation(t *testing.T) {
|
||||
viewer := auth.Principal{Subject: "viewer", Role: auth.RoleViewer}
|
||||
response := requestWithPrincipal(http.MethodPost, "/api/v1/dashboards", "{}", &viewer)
|
||||
if response.Code != http.StatusForbidden {
|
||||
t.Fatalf("status=%d", response.Code)
|
||||
}
|
||||
if got := problemCode(t, response); got != "FORBIDDEN" {
|
||||
t.Fatalf("code=%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerEnforcesEditorForPreview(t *testing.T) {
|
||||
viewer := auth.Principal{Subject: "viewer", Role: auth.RoleViewer}
|
||||
response := requestWithPrincipal(http.MethodPost, "/api/v1/dashboards/00000000-0000-0000-0000-000000000001/preview", `{"widget":{}}`, &viewer)
|
||||
if response.Code != http.StatusForbidden || problemCode(t, response) != "FORBIDDEN" {
|
||||
t.Fatalf("response=%d %s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerRejectsInvalidContentTypeAndVersion(t *testing.T) {
|
||||
editor := auth.Principal{Subject: "editor", Role: auth.RoleEditor}
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/dashboards", strings.NewReader("{}"))
|
||||
request.Header.Set("Content-Type", "text/plain")
|
||||
request = request.WithContext(auth.WithPrincipal(context.Background(), editor))
|
||||
response := httptest.NewRecorder()
|
||||
(Handler{Repository: dashboard.Repository{}}).ServeHTTP(response, request)
|
||||
if response.Code != http.StatusBadRequest || problemCode(t, response) != "INVALID_DOCUMENT" {
|
||||
t.Fatalf("response=%d %s", response.Code, response.Body.String())
|
||||
}
|
||||
|
||||
response = requestWithPrincipal(http.MethodPost, "/api/v1/dashboards/00000000-0000-0000-0000-000000000001/restore/not-a-number", "{}", &editor)
|
||||
if response.Code != http.StatusBadRequest || problemCode(t, response) != "INVALID_VERSION" {
|
||||
t.Fatalf("response=%d %s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerRejectsUnsupportedMethod(t *testing.T) {
|
||||
viewer := auth.Principal{Subject: "viewer", Role: auth.RoleViewer}
|
||||
response := requestWithPrincipal(http.MethodPut, "/api/v1/dashboards", "{}", &viewer)
|
||||
if response.Code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("status=%d", response.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerRecordsAuditedSafeDiff(t *testing.T) {
|
||||
store := &audit.MemoryStore{}
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/dashboards", strings.NewReader("{}"))
|
||||
request = request.WithContext(correlation.WithContext(request.Context(), "audit-correlation"))
|
||||
handler := Handler{Audit: store}
|
||||
if err := handler.record(request, "subject-1", "dashboard.update_document", "00000000-0000-0000-0000-000000000001", map[string]any{"revision": 1}, map[string]any{"revision": 2}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(store.Events) != 1 || store.Events[0].Action != "dashboard.update_document" || store.Events[0].CorrelationID != "audit-correlation" {
|
||||
t.Fatalf("events=%+v", store.Events)
|
||||
}
|
||||
if _, ok := store.Events[0].After["document"]; ok {
|
||||
t.Fatal("audit event unexpectedly contains full document")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
//go:embed migrations/*.sql
|
||||
var migrationFiles embed.FS
|
||||
|
||||
const (
|
||||
defaultMaxConns = 10
|
||||
defaultMinConns = 1
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
URL string
|
||||
MaxConns int32
|
||||
MinConns int32
|
||||
MaxConnIdle time.Duration
|
||||
}
|
||||
|
||||
func NewPool(ctx context.Context, config Config) (*pgxpool.Pool, error) {
|
||||
if strings.TrimSpace(config.URL) == "" {
|
||||
return nil, errors.New("database URL is required")
|
||||
}
|
||||
poolConfig, err := pgxpool.ParseConfig(config.URL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse database URL: %w", err)
|
||||
}
|
||||
if config.MaxConns == 0 {
|
||||
config.MaxConns = defaultMaxConns
|
||||
}
|
||||
if config.MinConns == 0 {
|
||||
config.MinConns = defaultMinConns
|
||||
}
|
||||
if config.MaxConns < config.MinConns || config.MinConns < 0 {
|
||||
return nil, errors.New("database pool limits are invalid")
|
||||
}
|
||||
poolConfig.MaxConns = config.MaxConns
|
||||
poolConfig.MinConns = config.MinConns
|
||||
if config.MaxConnIdle > 0 {
|
||||
poolConfig.MaxConnIdleTime = config.MaxConnIdle
|
||||
}
|
||||
pool, err := pgxpool.NewWithConfig(ctx, poolConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create database pool: %w", err)
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
func Ping(ctx context.Context, pool *pgxpool.Pool) error {
|
||||
if pool == nil {
|
||||
return errors.New("database pool is nil")
|
||||
}
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return fmt.Errorf("database ping: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func Migrate(ctx context.Context, pool *pgxpool.Pool) error {
|
||||
if pool == nil {
|
||||
return errors.New("database pool is nil")
|
||||
}
|
||||
entries, err := fs.Glob(migrationFiles, "migrations/*.sql")
|
||||
if err != nil {
|
||||
return fmt.Errorf("list migrations: %w", err)
|
||||
}
|
||||
sort.Strings(entries)
|
||||
for _, entry := range entries {
|
||||
if err := applyMigration(ctx, pool, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyMigration(ctx context.Context, pool *pgxpool.Pool, entry string) error {
|
||||
migrationID := strings.TrimSuffix(path.Base(entry), path.Ext(entry))
|
||||
sqlBytes, err := migrationFiles.ReadFile(entry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read migration %s: %w", migrationID, err)
|
||||
}
|
||||
tx, err := pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin migration %s: %w", migrationID, err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtext('itworx-pulse:schema-migrations'))`); err != nil {
|
||||
return fmt.Errorf("lock migrations: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
id text PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT now()
|
||||
)`); err != nil {
|
||||
return fmt.Errorf("create migration table: %w", err)
|
||||
}
|
||||
var exists bool
|
||||
if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE id = $1)`, migrationID).Scan(&exists); err != nil {
|
||||
return fmt.Errorf("check migration %s: %w", migrationID, err)
|
||||
}
|
||||
if exists {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit migration check %s: %w", migrationID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if _, err := tx.Exec(ctx, string(sqlBytes)); err != nil {
|
||||
return fmt.Errorf("apply migration %s: %w", migrationID, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO schema_migrations (id) VALUES ($1)`, migrationID); err != nil {
|
||||
return fmt.Errorf("record migration %s: %w", migrationID, err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit migration %s: %w", migrationID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMigrationsAreEmbeddedAndOrdered(t *testing.T) {
|
||||
entries, err := migrationFiles.ReadDir("migrations")
|
||||
if err != nil {
|
||||
t.Fatalf("read embedded migrations: %v", err)
|
||||
}
|
||||
expected := []string{
|
||||
"0001_foundation.sql", "0002_inventory.sql", "0003_dashboard_immutability.sql", "0004_dashboard_revision.sql",
|
||||
"0005_services_probes.sql", "0006_alert_rules.sql", "0007_alert_evaluator_leases.sql", "0008_alert_state.sql",
|
||||
"0009_alert_hysteresis.sql", "0010_alert_controls.sql", "0011_alert_unacknowledge.sql", "0012_notifications.sql",
|
||||
"0013_incidents.sql", "0014_incident_notes.sql", "0015_entity_listing_index.sql", "0016_agent_snapshots.sql",
|
||||
"0017_worker_runtime.sql", "0018_inventory_read_indexes.sql", "0019_capacity_samples.sql",
|
||||
"0020_service_certificate_history_index.sql",
|
||||
}
|
||||
if len(entries) != len(expected) {
|
||||
t.Fatalf("migration count = %d, want %d: %#v", len(entries), len(expected), entries)
|
||||
}
|
||||
for index, name := range expected {
|
||||
if entries[index].Name() != name {
|
||||
t.Fatalf("migration %d = %q, want %q", index, entries[index].Name(), name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceCertificateHistoryMigrationMatchesStatusQuery(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0020_service_certificate_history_index.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, fragment := range []string{"service_certificates_service_history_idx", "service_id", "observed_at DESC", "id ASC"} {
|
||||
if !strings.Contains(sql, fragment) {
|
||||
t.Fatalf("service certificate history migration is missing %q", fragment)
|
||||
}
|
||||
}
|
||||
if strings.Contains(sql, "CONCURRENTLY") {
|
||||
t.Fatal("migrations run inside a transaction and cannot create indexes concurrently")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerRuntimeMigrationContainsAliasAndJobLookupSafety(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0017_worker_runtime.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, fragment := range []string{"CREATE TABLE container_aliases", "PRIMARY KEY (source_id, runtime_id)", "tombstoned_at", "ON DELETE CASCADE", "container_aliases_active_idx", "CREATE INDEX job_runs_recent_idx"} {
|
||||
if !strings.Contains(sql, fragment) {
|
||||
t.Fatalf("worker runtime migration is missing %q", fragment)
|
||||
}
|
||||
}
|
||||
if strings.Contains(sql, "CONCURRENTLY") {
|
||||
t.Fatal("migrations run inside a transaction and cannot create indexes concurrently")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertControlsMigrationContainsExpiryAndIndexes(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0010_alert_controls.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, fragment := range []string{"CREATE TABLE alert_silences", "CREATE TABLE maintenance_windows", "expires_at", "ends_at", "status", "alert_silences_active_expiry_idx", "maintenance_windows_active_expiry_idx"} {
|
||||
if !strings.Contains(sql, fragment) {
|
||||
t.Fatalf("control migration is missing %q", fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestNotificationsMigrationContainsOutboxAndAuditConstraints(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0012_notifications.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, fragment := range []string{"CREATE TABLE notification_channels", "CREATE TABLE notification_outbox", "CREATE TABLE notification_deliveries", "UNIQUE (outbox_id, attempt)", "ON DELETE RESTRICT", "notification_outbox_due_idx", "notification_deliveries_history_idx"} {
|
||||
if !strings.Contains(sql, fragment) {
|
||||
t.Fatalf("notification migration is missing %q", fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestIncidentsMigrationContainsCorrelationAndAssociationSafety(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0013_incidents.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, fragment := range []string{"CREATE TABLE incidents", "incidents_active_correlation_key_uq", "CREATE TABLE incident_alerts", "CREATE TABLE incident_entities", "ON DELETE RESTRICT", "confidence", "rationale"} {
|
||||
if !strings.Contains(sql, fragment) {
|
||||
t.Fatalf("incident migration is missing %q", fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestIncidentNotesMigrationContainsBoundedNotes(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0014_incident_notes.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, fragment := range []string{"CREATE TABLE incident_notes", "incident_notes_history_idx", "ON DELETE CASCADE", "char_length(body) BETWEEN 1 AND 2000"} {
|
||||
if !strings.Contains(sql, fragment) {
|
||||
t.Fatalf("incident notes migration is missing %q", fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestEntityListingMigrationIndexesKeysetPagination(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0015_entity_listing_index.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, fragment := range []string{"CREATE INDEX IF NOT EXISTS entities_canonical_name_idx", "ON entities (canonical_name ASC, id ASC)"} {
|
||||
if !strings.Contains(sql, fragment) {
|
||||
t.Fatalf("entity listing migration is missing %q", fragment)
|
||||
}
|
||||
}
|
||||
if strings.Contains(sql, "CONCURRENTLY") {
|
||||
t.Fatal("migrations run inside a transaction and cannot create indexes concurrently")
|
||||
}
|
||||
}
|
||||
func TestPostgreSQLMigrationsAreRestartSafe(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
pool, err := NewPool(ctx, Config{URL: dsn})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := Ping(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := Migrate(ctx, pool); err != nil {
|
||||
t.Fatalf("repeated migration: %v", err)
|
||||
}
|
||||
var count int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM schema_migrations WHERE id = '0001_foundation'`).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("migration count = %d, want 1", count)
|
||||
}
|
||||
var inventoryCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM schema_migrations WHERE id = '0002_inventory'`).Scan(&inventoryCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inventoryCount != 1 {
|
||||
t.Fatalf("inventory migration count = %d, want 1", inventoryCount)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO system_settings (key, value) VALUES ('test.persistence', '{"ok":true}') ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var value bool
|
||||
if err := pool.QueryRow(ctx, `SELECT value->>'ok' = 'true' FROM system_settings WHERE key = 'test.persistence'`).Scan(&value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !value {
|
||||
t.Fatal("persisted setting was not retained")
|
||||
}
|
||||
pool.Close()
|
||||
restartedPool, err := NewPool(ctx, Config{URL: dsn})
|
||||
if err != nil {
|
||||
t.Fatalf("reopen database pool: %v", err)
|
||||
}
|
||||
defer restartedPool.Close()
|
||||
var afterRestart bool
|
||||
if err := restartedPool.QueryRow(ctx, `SELECT value->>'ok' = 'true' FROM system_settings WHERE key = 'test.persistence'`).Scan(&afterRestart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !afterRestart {
|
||||
t.Fatal("persisted setting was not retained after pool restart")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertRuleMigrationContainsVersionSafety(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0006_alert_rules.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, table := range []string{"alert_rules", "alert_rule_versions"} {
|
||||
if !strings.Contains(sql, "CREATE TABLE "+table) {
|
||||
t.Fatalf("migration is missing table %s", table)
|
||||
}
|
||||
}
|
||||
for _, constraint := range []string{"UNIQUE (rule_id, version_number)", "ON DELETE RESTRICT", "DEFERRABLE INITIALLY DEFERRED", "WHERE enabled = true"} {
|
||||
if !strings.Contains(sql, constraint) {
|
||||
t.Fatalf("migration is missing safety constraint %q", constraint)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertEvaluatorLeaseMigrationContainsExpiryIndex(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0007_alert_evaluator_leases.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, fragment := range []string{"ADD COLUMN lease_owner text", "ADD COLUMN lease_until timestamptz", "CREATE INDEX job_runs_lease_idx", "status IN ('queued', 'running')"} {
|
||||
if !strings.Contains(sql, fragment) {
|
||||
t.Fatalf("lease migration is missing %q", fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertStateMigrationContainsLifecycleAndHistorySafety(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0008_alert_state.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, table := range []string{"alert_instances", "alert_occurrences"} {
|
||||
if !strings.Contains(sql, "CREATE TABLE "+table) {
|
||||
t.Fatalf("migration is missing table %s", table)
|
||||
}
|
||||
}
|
||||
for _, fragment := range []string{"UNIQUE (rule_id, fingerprint)", "UNIQUE (instance_id, evaluation_key)", "ON DELETE RESTRICT", "current_state text NOT NULL", "CREATE INDEX alert_occurrences_history_idx"} {
|
||||
if !strings.Contains(sql, fragment) {
|
||||
t.Fatalf("state migration is missing %q", fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestAlertHysteresisMigrationContainsCooldownSafety(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0009_alert_hysteresis.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, fragment := range []string{"ADD COLUMN cooldown_seconds", "ADD COLUMN cooldown_until", "cooldown_seconds BETWEEN 0 AND 2592000", "CREATE INDEX alert_instances_cooldown_idx"} {
|
||||
if !strings.Contains(sql, fragment) {
|
||||
t.Fatalf("hysteresis migration is missing %q", fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestAlertUnacknowledgeMigrationContainsConstraintSafety(t *testing.T) {
|
||||
sqlBytes, err := migrationFiles.ReadFile("migrations/0011_alert_unacknowledge.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(sqlBytes)
|
||||
for _, fragment := range []string{"DROP CONSTRAINT alert_occurrences_event_type_check", "unacknowledge", "alert_instances_acknowledged_idx"} {
|
||||
if !strings.Contains(sql, fragment) {
|
||||
t.Fatalf("unacknowledge migration is missing %q", fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestServiceProbeMigrationContainsHistoryAndAccessSafety(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0005_services_probes.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, table := range []string{"services", "service_endpoints", "probes", "probe_results", "service_certificates", "service_dependencies", "service_permissions"} {
|
||||
if !strings.Contains(sql, "CREATE TABLE "+table) {
|
||||
t.Fatalf("migration is missing table %s", table)
|
||||
}
|
||||
}
|
||||
for _, constraint := range []string{"revision bigint NOT NULL DEFAULT 1 CHECK (revision > 0)", "ON DELETE RESTRICT", "WHERE archived_at IS NULL", "UNIQUE (probe_id, observed_at)", "permission IN ('view', 'operate', 'edit', 'admin')"} {
|
||||
if !strings.Contains(sql, constraint) {
|
||||
t.Fatalf("migration is missing safety constraint %q", constraint)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentSnapshotMigrationBoundsPayloadAndCapability(t *testing.T) {
|
||||
content, err := migrationFiles.ReadFile("migrations/0016_agent_snapshots.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(content)
|
||||
for _, fragment := range []string{
|
||||
"CREATE TABLE agent_snapshots",
|
||||
"observed_at timestamptz NOT NULL",
|
||||
"received_at timestamptz NOT NULL",
|
||||
"jsonb_typeof(payload) = 'object'",
|
||||
"pg_column_size(payload) <= 2097152",
|
||||
"capability IN ('host', 'processes', 'containers', 'array', 'disks', 'pools', 'shares')",
|
||||
"PRIMARY KEY (agent_id, capability)",
|
||||
"agent_snapshots_capability_freshness_idx",
|
||||
} {
|
||||
if !strings.Contains(sql, fragment) {
|
||||
t.Fatalf("agent snapshot migration is missing %q", fragment)
|
||||
}
|
||||
}
|
||||
if strings.Contains(sql, "CONCURRENTLY") {
|
||||
t.Fatal("migrations run inside a transaction and cannot create indexes concurrently")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
external_subject text NOT NULL UNIQUE,
|
||||
display_name text NOT NULL,
|
||||
email text,
|
||||
status text NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'disabled')),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
last_login_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE roles (
|
||||
id uuid PRIMARY KEY,
|
||||
name text NOT NULL UNIQUE CHECK (name IN ('viewer', 'operator', 'editor', 'administrator')),
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE user_roles (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
role_id uuid NOT NULL REFERENCES roles(id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, role_id)
|
||||
);
|
||||
|
||||
CREATE TABLE data_sources (
|
||||
id uuid PRIMARY KEY,
|
||||
type text NOT NULL,
|
||||
name text NOT NULL,
|
||||
enabled boolean NOT NULL DEFAULT true,
|
||||
configuration_ref text NOT NULL,
|
||||
capability_document jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
health_state text NOT NULL DEFAULT 'unknown' CHECK (health_state IN ('healthy', 'degraded', 'unhealthy', 'unknown')),
|
||||
last_success_at timestamptz,
|
||||
last_error_code text,
|
||||
last_error_message text,
|
||||
freshness_policy jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE collectors (
|
||||
id uuid PRIMARY KEY,
|
||||
datasource_id uuid NOT NULL REFERENCES data_sources(id) ON DELETE CASCADE,
|
||||
kind text NOT NULL,
|
||||
version text NOT NULL,
|
||||
heartbeat timestamptz,
|
||||
capabilities jsonb NOT NULL DEFAULT '[]'::jsonb,
|
||||
status text NOT NULL DEFAULT 'unknown',
|
||||
UNIQUE (datasource_id, kind)
|
||||
);
|
||||
|
||||
CREATE TABLE entities (
|
||||
id uuid PRIMARY KEY,
|
||||
entity_type text NOT NULL,
|
||||
canonical_name text NOT NULL,
|
||||
display_name text NOT NULL,
|
||||
status text NOT NULL DEFAULT 'unknown',
|
||||
status_reasons jsonb NOT NULL DEFAULT '[]'::jsonb,
|
||||
first_seen_at timestamptz NOT NULL,
|
||||
last_seen_at timestamptz,
|
||||
tombstoned_at timestamptz,
|
||||
attributes jsonb NOT NULL DEFAULT '{}'::jsonb
|
||||
);
|
||||
|
||||
CREATE TABLE entity_aliases (
|
||||
entity_id uuid NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
|
||||
source_id uuid NOT NULL REFERENCES data_sources(id) ON DELETE CASCADE,
|
||||
external_type text NOT NULL,
|
||||
external_id text NOT NULL,
|
||||
PRIMARY KEY (source_id, external_type, external_id)
|
||||
);
|
||||
|
||||
CREATE TABLE dashboards (
|
||||
id uuid PRIMARY KEY,
|
||||
slug text NOT NULL UNIQUE,
|
||||
name text NOT NULL,
|
||||
description text NOT NULL DEFAULT '',
|
||||
owner_user_id uuid REFERENCES users(id) ON DELETE SET NULL,
|
||||
scope text NOT NULL CHECK (scope IN ('personal', 'shared', 'system')),
|
||||
archived_at timestamptz,
|
||||
current_version_id uuid,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE dashboard_versions (
|
||||
id uuid PRIMARY KEY,
|
||||
dashboard_id uuid NOT NULL REFERENCES dashboards(id) ON DELETE CASCADE,
|
||||
version_number integer NOT NULL CHECK (version_number > 0),
|
||||
schema_version integer NOT NULL CHECK (schema_version > 0),
|
||||
document jsonb NOT NULL,
|
||||
change_summary text NOT NULL DEFAULT '',
|
||||
created_by uuid REFERENCES users(id) ON DELETE SET NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (dashboard_id, version_number)
|
||||
);
|
||||
|
||||
ALTER TABLE dashboards
|
||||
ADD CONSTRAINT dashboards_current_version_fk
|
||||
FOREIGN KEY (current_version_id) REFERENCES dashboard_versions(id) ON DELETE SET NULL;
|
||||
|
||||
CREATE TABLE events (
|
||||
id uuid PRIMARY KEY,
|
||||
event_type text NOT NULL,
|
||||
severity text NOT NULL,
|
||||
entity_id uuid REFERENCES entities(id) ON DELETE SET NULL,
|
||||
source_id uuid REFERENCES data_sources(id) ON DELETE SET NULL,
|
||||
occurred_at timestamptz NOT NULL,
|
||||
received_at timestamptz NOT NULL DEFAULT now(),
|
||||
dedup_key text NOT NULL,
|
||||
summary text NOT NULL,
|
||||
attributes jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
correlation_id text,
|
||||
UNIQUE (source_id, dedup_key, occurred_at)
|
||||
);
|
||||
|
||||
CREATE TABLE audit_events (
|
||||
id uuid PRIMARY KEY,
|
||||
actor text NOT NULL,
|
||||
action text NOT NULL,
|
||||
resource_type text NOT NULL,
|
||||
resource_id uuid,
|
||||
result text NOT NULL,
|
||||
occurred_at timestamptz NOT NULL DEFAULT now(),
|
||||
correlation_id text,
|
||||
before_diff jsonb,
|
||||
after_diff jsonb
|
||||
);
|
||||
|
||||
CREATE TABLE job_runs (
|
||||
id uuid PRIMARY KEY,
|
||||
job_type text NOT NULL,
|
||||
job_key text NOT NULL,
|
||||
scheduled_at timestamptz NOT NULL,
|
||||
started_at timestamptz,
|
||||
completed_at timestamptz,
|
||||
status text NOT NULL,
|
||||
counts jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
error_code text,
|
||||
correlation_id text,
|
||||
UNIQUE (job_type, job_key, scheduled_at)
|
||||
);
|
||||
|
||||
CREATE TABLE system_settings (
|
||||
key text PRIMARY KEY,
|
||||
value jsonb NOT NULL,
|
||||
version bigint NOT NULL DEFAULT 1 CHECK (version > 0),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX entities_type_status_idx ON entities (entity_type, status);
|
||||
CREATE INDEX events_occurred_at_idx ON events (occurred_at DESC);
|
||||
CREATE INDEX audit_events_occurred_at_idx ON audit_events (occurred_at DESC);
|
||||
CREATE INDEX job_runs_status_idx ON job_runs (status, scheduled_at);
|
||||
@@ -0,0 +1,36 @@
|
||||
CREATE TABLE entity_facts (
|
||||
entity_id uuid NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
|
||||
field_name text NOT NULL,
|
||||
source_id uuid NOT NULL REFERENCES data_sources(id) ON DELETE CASCADE,
|
||||
value jsonb NOT NULL,
|
||||
observed_at timestamptz NOT NULL,
|
||||
confidence numeric(5,4) NOT NULL CHECK (confidence >= 0 AND confidence <= 1),
|
||||
valid_until timestamptz,
|
||||
PRIMARY KEY (entity_id, field_name, source_id)
|
||||
);
|
||||
|
||||
CREATE TABLE entity_overrides (
|
||||
entity_id uuid NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
|
||||
field_name text NOT NULL,
|
||||
value jsonb NOT NULL,
|
||||
user_id uuid REFERENCES users(id) ON DELETE SET NULL,
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (entity_id, field_name)
|
||||
);
|
||||
|
||||
CREATE TABLE entity_relations (
|
||||
id uuid PRIMARY KEY,
|
||||
source_entity_id uuid NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
|
||||
relation_type text NOT NULL,
|
||||
target_entity_id uuid NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
|
||||
source_id uuid NOT NULL REFERENCES data_sources(id) ON DELETE CASCADE,
|
||||
confidence numeric(5,4) NOT NULL CHECK (confidence >= 0 AND confidence <= 1),
|
||||
confirmed boolean NOT NULL DEFAULT false,
|
||||
first_seen_at timestamptz NOT NULL,
|
||||
last_seen_at timestamptz,
|
||||
tombstoned_at timestamptz,
|
||||
UNIQUE (source_entity_id, relation_type, target_entity_id, source_id)
|
||||
);
|
||||
|
||||
CREATE INDEX entity_facts_source_observed_idx ON entity_facts (source_id, observed_at DESC);
|
||||
CREATE INDEX entity_relations_source_idx ON entity_relations (source_id, last_seen_at DESC);
|
||||
@@ -0,0 +1,11 @@
|
||||
CREATE OR REPLACE FUNCTION prevent_dashboard_version_mutation() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
RAISE EXCEPTION 'dashboard versions are immutable';
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE OR REPLACE TRIGGER dashboard_versions_immutable_update
|
||||
BEFORE UPDATE ON dashboard_versions
|
||||
FOR EACH ROW EXECUTE FUNCTION prevent_dashboard_version_mutation();
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dashboard_versions_created_at_idx ON dashboard_versions (dashboard_id, created_at DESC, version_number DESC);
|
||||
@@ -0,0 +1,6 @@
|
||||
ALTER TABLE dashboards ADD COLUMN IF NOT EXISTS revision bigint NOT NULL DEFAULT 1 CHECK (revision > 0);
|
||||
CREATE INDEX IF NOT EXISTS dashboards_owner_revision_idx ON dashboards (owner_user_id, revision DESC, id ASC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dashboards_name_id_idx ON dashboards (name ASC, id ASC);
|
||||
CREATE INDEX IF NOT EXISTS dashboards_scope_name_id_idx ON dashboards (scope, name ASC, id ASC);
|
||||
CREATE INDEX IF NOT EXISTS dashboards_owner_name_id_idx ON dashboards (owner_user_id, name ASC, id ASC);
|
||||
@@ -0,0 +1,124 @@
|
||||
CREATE TABLE services (
|
||||
id uuid PRIMARY KEY,
|
||||
entity_id uuid REFERENCES entities(id) ON DELETE SET NULL,
|
||||
source_id uuid REFERENCES data_sources(id) ON DELETE SET NULL,
|
||||
name text NOT NULL CHECK (length(name) BETWEEN 1 AND 160),
|
||||
description text NOT NULL DEFAULT '',
|
||||
state text NOT NULL DEFAULT 'unknown' CHECK (state IN ('up', 'degraded', 'down', 'unknown')),
|
||||
labels jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
revision bigint NOT NULL DEFAULT 1 CHECK (revision > 0),
|
||||
archived_at timestamptz,
|
||||
created_by uuid REFERENCES users(id) ON DELETE SET NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX services_entity_idx ON services (entity_id) WHERE archived_at IS NULL;
|
||||
CREATE INDEX services_source_state_idx ON services (source_id, state, updated_at DESC);
|
||||
|
||||
CREATE TABLE service_endpoints (
|
||||
id uuid PRIMARY KEY,
|
||||
service_id uuid NOT NULL REFERENCES services(id) ON DELETE RESTRICT,
|
||||
source_id uuid REFERENCES data_sources(id) ON DELETE SET NULL,
|
||||
name text NOT NULL CHECK (length(name) BETWEEN 1 AND 160),
|
||||
endpoint_type text NOT NULL CHECK (endpoint_type IN ('http', 'tcp', 'dns', 'icmp', 'tls')),
|
||||
target jsonb NOT NULL,
|
||||
enabled boolean NOT NULL DEFAULT true,
|
||||
revision bigint NOT NULL DEFAULT 1 CHECK (revision > 0),
|
||||
archived_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX service_endpoints_active_idx ON service_endpoints (service_id, enabled) WHERE archived_at IS NULL;
|
||||
CREATE UNIQUE INDEX service_endpoints_active_name_idx ON service_endpoints (service_id, name) WHERE archived_at IS NULL;
|
||||
|
||||
CREATE TABLE probes (
|
||||
id uuid PRIMARY KEY,
|
||||
service_id uuid NOT NULL REFERENCES services(id) ON DELETE RESTRICT,
|
||||
endpoint_id uuid REFERENCES service_endpoints(id) ON DELETE SET NULL,
|
||||
source_id uuid REFERENCES data_sources(id) ON DELETE SET NULL,
|
||||
name text NOT NULL CHECK (length(name) BETWEEN 1 AND 160),
|
||||
probe_type text NOT NULL CHECK (probe_type IN ('http', 'tcp', 'dns', 'icmp', 'tls')),
|
||||
target jsonb NOT NULL,
|
||||
interval_seconds integer NOT NULL CHECK (interval_seconds BETWEEN 5 AND 86400),
|
||||
timeout_seconds integer NOT NULL CHECK (timeout_seconds BETWEEN 1 AND 120),
|
||||
enabled boolean NOT NULL DEFAULT true,
|
||||
expected_status_codes jsonb NOT NULL DEFAULT '[]'::jsonb,
|
||||
follow_redirects boolean NOT NULL DEFAULT false,
|
||||
verify_tls boolean NOT NULL DEFAULT true,
|
||||
content_assertion jsonb,
|
||||
secret_reference text,
|
||||
network_policy_id uuid,
|
||||
revision bigint NOT NULL DEFAULT 1 CHECK (revision > 0),
|
||||
archived_at timestamptz,
|
||||
created_by uuid REFERENCES users(id) ON DELETE SET NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX probes_schedule_idx ON probes (enabled, interval_seconds, updated_at) WHERE archived_at IS NULL;
|
||||
CREATE INDEX probes_service_idx ON probes (service_id, updated_at DESC);
|
||||
CREATE UNIQUE INDEX probes_active_name_idx ON probes (service_id, name) WHERE archived_at IS NULL;
|
||||
|
||||
CREATE TABLE probe_results (
|
||||
id uuid PRIMARY KEY,
|
||||
probe_id uuid NOT NULL REFERENCES probes(id) ON DELETE RESTRICT,
|
||||
source_id uuid REFERENCES data_sources(id) ON DELETE SET NULL,
|
||||
observed_at timestamptz NOT NULL,
|
||||
completed_at timestamptz NOT NULL,
|
||||
state text NOT NULL CHECK (state IN ('up', 'degraded', 'down', 'unknown')),
|
||||
response_time_ms integer CHECK (response_time_ms IS NULL OR response_time_ms >= 0),
|
||||
status_code integer CHECK (status_code IS NULL OR status_code BETWEEN 100 AND 599),
|
||||
error_class text,
|
||||
error_message text,
|
||||
attributes jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
UNIQUE (probe_id, observed_at)
|
||||
);
|
||||
|
||||
CREATE INDEX probe_results_history_idx ON probe_results (probe_id, observed_at DESC);
|
||||
CREATE INDEX probe_results_state_idx ON probe_results (state, observed_at DESC);
|
||||
|
||||
CREATE TABLE service_certificates (
|
||||
id uuid PRIMARY KEY,
|
||||
service_id uuid NOT NULL REFERENCES services(id) ON DELETE RESTRICT,
|
||||
endpoint_id uuid REFERENCES service_endpoints(id) ON DELETE SET NULL,
|
||||
source_id uuid REFERENCES data_sources(id) ON DELETE SET NULL,
|
||||
observed_at timestamptz NOT NULL,
|
||||
expires_at timestamptz,
|
||||
issuer text,
|
||||
subject text,
|
||||
hostname_valid boolean,
|
||||
verification_state text NOT NULL CHECK (verification_state IN ('valid', 'attention', 'invalid', 'unknown')),
|
||||
attributes jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
UNIQUE (service_id, endpoint_id, observed_at)
|
||||
);
|
||||
|
||||
CREATE INDEX service_certificates_expiry_idx ON service_certificates (expires_at, observed_at DESC);
|
||||
CREATE UNIQUE INDEX service_certificates_without_endpoint_unique_idx ON service_certificates (service_id, observed_at) WHERE endpoint_id IS NULL;
|
||||
|
||||
CREATE TABLE service_dependencies (
|
||||
id uuid PRIMARY KEY,
|
||||
service_id uuid NOT NULL REFERENCES services(id) ON DELETE RESTRICT,
|
||||
depends_on_service_id uuid NOT NULL REFERENCES services(id) ON DELETE RESTRICT,
|
||||
source_id uuid REFERENCES data_sources(id) ON DELETE SET NULL,
|
||||
relation_type text NOT NULL CHECK (relation_type IN ('depends_on', 'backs', 'exposes')),
|
||||
confidence numeric(5,4) NOT NULL CHECK (confidence >= 0 AND confidence <= 1),
|
||||
confirmed boolean NOT NULL DEFAULT false,
|
||||
first_seen_at timestamptz NOT NULL,
|
||||
last_seen_at timestamptz,
|
||||
archived_at timestamptz,
|
||||
UNIQUE (service_id, depends_on_service_id, relation_type, source_id),
|
||||
CHECK (service_id <> depends_on_service_id)
|
||||
);
|
||||
|
||||
CREATE INDEX service_dependencies_source_idx ON service_dependencies (source_id, last_seen_at DESC);
|
||||
CREATE UNIQUE INDEX service_dependencies_manual_unique_idx ON service_dependencies (service_id, depends_on_service_id, relation_type) WHERE source_id IS NULL;
|
||||
|
||||
CREATE TABLE service_permissions (
|
||||
service_id uuid NOT NULL REFERENCES services(id) ON DELETE CASCADE,
|
||||
role_id uuid NOT NULL REFERENCES roles(id) ON DELETE CASCADE,
|
||||
permission text NOT NULL CHECK (permission IN ('view', 'operate', 'edit', 'admin')),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (service_id, role_id, permission)
|
||||
);
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user