ITWorx Pulse release export
bd774932d5
Public source validation / validate (push) Failing after 3m8s
705 lines
26 KiB
Go
705 lines
26 KiB
Go
package workerruntime
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/itworx/pulse/internal/alert"
|
|
"github.com/itworx/pulse/internal/container"
|
|
"github.com/itworx/pulse/internal/discovery"
|
|
"github.com/itworx/pulse/internal/inventory"
|
|
"github.com/itworx/pulse/internal/notification"
|
|
"github.com/itworx/pulse/internal/probe"
|
|
"github.com/itworx/pulse/internal/reconciliation"
|
|
)
|
|
|
|
const testSourceID = "11111111-1111-4111-8111-111111111111"
|
|
|
|
// --- discovery job ---------------------------------------------------------
|
|
|
|
type fakeContainerProvider struct {
|
|
mu sync.Mutex
|
|
snapshot container.Snapshot
|
|
err error
|
|
}
|
|
|
|
func (p *fakeContainerProvider) Snapshot(context.Context) (container.Snapshot, error) {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
return p.snapshot, p.err
|
|
}
|
|
func (p *fakeContainerProvider) set(snapshot container.Snapshot) {
|
|
p.mu.Lock()
|
|
p.snapshot = snapshot
|
|
p.mu.Unlock()
|
|
}
|
|
|
|
type memoryAliasStore struct {
|
|
mu sync.Mutex
|
|
records []ContainerAliasRecord
|
|
saves int
|
|
}
|
|
|
|
func (s *memoryAliasStore) List(context.Context, string) ([]ContainerAliasRecord, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return append([]ContainerAliasRecord(nil), s.records...), nil
|
|
}
|
|
func (s *memoryAliasStore) Save(_ context.Context, _ string, records []ContainerAliasRecord) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.records = append([]ContainerAliasRecord(nil), records...)
|
|
s.saves++
|
|
return nil
|
|
}
|
|
|
|
type memoryInventory struct {
|
|
mu sync.Mutex
|
|
persisted map[string]inventory.Entity
|
|
facts map[string][]inventory.Fact
|
|
relations map[string][]inventory.Relation
|
|
tombstoned map[string]inventory.Entity
|
|
}
|
|
|
|
func newMemoryInventory() *memoryInventory {
|
|
return &memoryInventory{persisted: map[string]inventory.Entity{}, facts: map[string][]inventory.Fact{}, relations: map[string][]inventory.Relation{}, tombstoned: map[string]inventory.Entity{}}
|
|
}
|
|
func (s *memoryInventory) PersistDiscovery(_ context.Context, entity inventory.Entity, _ inventory.Alias, facts []inventory.Fact, relations []inventory.Relation) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.persisted[entity.ID] = entity
|
|
s.facts[entity.ID] = append([]inventory.Fact(nil), facts...)
|
|
s.relations[entity.ID] = append([]inventory.Relation(nil), relations...)
|
|
if entity.TombstonedAt != nil {
|
|
s.tombstoned[entity.ID] = entity
|
|
}
|
|
return nil
|
|
}
|
|
func (s *memoryInventory) UpsertEntity(_ context.Context, entity inventory.Entity) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.tombstoned[entity.ID] = entity
|
|
return nil
|
|
}
|
|
|
|
func healthySnapshot(observedAt time.Time, containers ...container.Container) container.Snapshot {
|
|
return container.Snapshot{
|
|
ContractVersion: container.ContractVersion,
|
|
Source: container.Source{ID: "container", Type: "agent", ObservedAt: observedAt, ReceivedAt: observedAt, Freshness: "fresh", State: "healthy"},
|
|
Containers: containers, Total: len(containers),
|
|
}
|
|
}
|
|
|
|
func newDiscoveryJob(provider container.Provider, aliases ContainerAliasStore, store discovery.Store, inv InventoryStore) DiscoveryJob {
|
|
return DiscoveryJob{SourceID: testSourceID, Provider: provider, Aliases: aliases, Inventory: inv,
|
|
Runner: discovery.Runner{Store: store, MaxAttempts: 1}}
|
|
}
|
|
|
|
func TestDiscoveryJobSkipsAnUnavailableSourceWithoutTombstoning(t *testing.T) {
|
|
observedAt := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
|
aliases := &memoryAliasStore{records: []ContainerAliasRecord{{ContainerAlias: aliasFixture("runtime-1", observedAt)}}}
|
|
inv := newMemoryInventory()
|
|
store := discovery.NewMemoryStore()
|
|
provider := &fakeContainerProvider{snapshot: container.UnknownSnapshot(observedAt, "container", "agent", "source_unavailable")}
|
|
job := newDiscoveryJob(provider, aliases, store, inv)
|
|
|
|
outcome, err := job.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !outcome.Skipped || outcome.Reason != "source_unavailable" {
|
|
t.Fatalf("outcome = %#v", outcome)
|
|
}
|
|
if aliases.saves != 0 || len(inv.tombstoned) != 0 || len(store.Events) != 0 {
|
|
t.Fatalf("an unavailable source changed inventory: saves=%d tombstoned=%d events=%d", aliases.saves, len(inv.tombstoned), len(store.Events))
|
|
}
|
|
}
|
|
|
|
func TestDiscoveryJobDerivesLifecycleEventsAndIsIdempotent(t *testing.T) {
|
|
first := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
|
aliases := &memoryAliasStore{}
|
|
inv := newMemoryInventory()
|
|
store := discovery.NewMemoryStore()
|
|
provider := &fakeContainerProvider{snapshot: healthySnapshot(first,
|
|
container.Container{ID: "runtime-1", Name: "pulse-api", State: "running", Health: "healthy"})}
|
|
job := newDiscoveryJob(provider, aliases, store, inv)
|
|
|
|
outcome, err := job.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome.Counts["added"] != 1 || outcome.Counts["events"] != 0 {
|
|
t.Fatalf("first pass counts = %#v", outcome.Counts)
|
|
}
|
|
if len(inv.persisted) != 2 || aliases.saves != 1 {
|
|
t.Fatalf("first pass persisted=%d saves=%d", len(inv.persisted), aliases.saves)
|
|
}
|
|
containerID := aliases.records[0].EntityID
|
|
if len(inv.facts[containerID]) < 6 || len(inv.relations[containerID]) != 1 {
|
|
t.Fatalf("container provenance facts=%d relations=%d", len(inv.facts[containerID]), len(inv.relations[containerID]))
|
|
}
|
|
relation := inv.relations[containerID][0]
|
|
if relation.RelationType != "member_of" || inv.persisted[relation.TargetEntityID].EntityType != "application" {
|
|
t.Fatalf("application relation = %+v target = %+v", relation, inv.persisted[relation.TargetEntityID])
|
|
}
|
|
|
|
// A repeat of the same window is claimed once, so nothing runs twice.
|
|
if _, err := job.Run(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if aliases.saves != 1 {
|
|
t.Fatalf("repeated window saves = %d, want 1", aliases.saves)
|
|
}
|
|
|
|
stateAfterFirstPass, err := aliases.List(context.Background(), testSourceID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
second := first.Add(time.Minute)
|
|
provider.set(healthySnapshot(second,
|
|
container.Container{ID: "runtime-1", Name: "pulse-api", State: "exited", Health: "unhealthy", RestartCount: 1}))
|
|
outcome, err = job.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome.Counts["events"] != 3 {
|
|
t.Fatalf("second pass counts = %#v, want 3 lifecycle events", outcome.Counts)
|
|
}
|
|
types := map[string]bool{}
|
|
for _, event := range store.Events {
|
|
types[event.Type] = true
|
|
if event.EntityID == "" || event.SourceID != testSourceID || event.Severity == "" {
|
|
t.Fatalf("event is missing identity: %#v", event)
|
|
}
|
|
}
|
|
for _, expected := range []string{"container.state_changed", "container.health_changed", "container.restart"} {
|
|
if !types[expected] {
|
|
t.Fatalf("missing event %s in %#v", expected, types)
|
|
}
|
|
}
|
|
emitted := len(store.Events)
|
|
|
|
// The same observation replayed emits the same deduplicated events.
|
|
store2 := discovery.NewMemoryStore()
|
|
job2 := newDiscoveryJob(provider, &memoryAliasStore{records: stateAfterFirstPass}, store2, newMemoryInventory())
|
|
if _, err := job2.Run(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(store2.Events) != emitted {
|
|
t.Fatalf("replayed events = %d, want %d", len(store2.Events), emitted)
|
|
}
|
|
for key := range store.Events {
|
|
if _, ok := store2.Events[key]; !ok {
|
|
t.Fatalf("replay produced a different dedup key set: %q missing", key)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDiscoveryJobTombstonesMissingContainers(t *testing.T) {
|
|
first := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
|
aliases := &memoryAliasStore{records: aliasSnapshotBefore(first)}
|
|
inv := newMemoryInventory()
|
|
job := newDiscoveryJob(&fakeContainerProvider{snapshot: healthySnapshot(first.Add(time.Minute))}, aliases, discovery.NewMemoryStore(), inv)
|
|
outcome, err := job.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome.Counts["tombstoned"] != 1 || len(inv.tombstoned) != 2 {
|
|
t.Fatalf("counts = %#v tombstoned = %#v", outcome.Counts, inv.tombstoned)
|
|
}
|
|
for entityID, relations := range inv.relations {
|
|
if inv.persisted[entityID].EntityType == "container" && (len(relations) != 1 || relations[0].TombstonedAt == nil) {
|
|
t.Fatalf("missing container relation was not tombstoned: %+v", relations)
|
|
}
|
|
}
|
|
}
|
|
|
|
func aliasFixture(runtimeID string, observedAt time.Time) reconciliation.ContainerAlias {
|
|
return reconciliation.ContainerAlias{
|
|
EntityID: "22222222-2222-4222-8222-222222222222", SourceID: testSourceID, RuntimeID: runtimeID,
|
|
Name: "pulse-api", FirstSeenAt: observedAt, LastSeenAt: observedAt, Active: true,
|
|
}
|
|
}
|
|
|
|
func aliasSnapshotBefore(observedAt time.Time) []ContainerAliasRecord {
|
|
return []ContainerAliasRecord{{ContainerAlias: aliasFixture("runtime-1", observedAt), State: "running", Health: "healthy"}}
|
|
}
|
|
|
|
// --- probe job -------------------------------------------------------------
|
|
|
|
type fakeProbeStore struct {
|
|
mu sync.Mutex
|
|
due []probe.Definition
|
|
saved []probe.Result
|
|
enabled int
|
|
listErr error
|
|
}
|
|
|
|
func (s *fakeProbeStore) CountEnabled(context.Context) (int, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return s.enabled, nil
|
|
}
|
|
|
|
func (s *fakeProbeStore) ListDue(context.Context, time.Time, int) ([]probe.Definition, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return append([]probe.Definition(nil), s.due...), s.listErr
|
|
}
|
|
func (s *fakeProbeStore) SaveResults(_ context.Context, results []probe.Result) (int, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.saved = append(s.saved, results...)
|
|
return len(results), nil
|
|
}
|
|
|
|
type fakeExecutor struct {
|
|
state string
|
|
err error
|
|
}
|
|
|
|
func (e fakeExecutor) Execute(_ context.Context, definition probe.Definition) (probe.Result, error) {
|
|
if e.err != nil {
|
|
return probe.Result{ProbeID: definition.ID, State: "unknown"}, e.err
|
|
}
|
|
return probe.Result{ProbeID: definition.ID, State: e.state, ObservedAt: time.Now().UTC(), CompletedAt: time.Now().UTC()}, nil
|
|
}
|
|
|
|
func probeDefinition(id string, interval, timeout time.Duration) probe.Definition {
|
|
return probe.Definition{ID: id, ServiceID: "service", Name: id, Type: probe.TypeTCP,
|
|
Target: probe.Target{Host: "example.internal", Port: 443}, Interval: interval, Timeout: timeout, Enabled: true, Revision: 1}
|
|
}
|
|
|
|
func TestProbeJobExecutesDueProbesAndPersistsResults(t *testing.T) {
|
|
store := &fakeProbeStore{due: []probe.Definition{
|
|
probeDefinition("probe-a", time.Minute, 5*time.Second),
|
|
// An individually invalid probe must not stop the batch.
|
|
probeDefinition("probe-b", 10*time.Second, 30*time.Second),
|
|
}}
|
|
scheduler, err := probe.NewScheduler(fakeExecutor{state: "up"}, probe.SchedulerConfig{MaxConcurrent: 4, MaxAttempts: 1, AttemptTimeout: time.Second})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
job := &ProbeJob{Store: store, Scheduler: scheduler, Logger: quietLogger()}
|
|
outcome, err := job.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome.Counts["due"] != 2 || outcome.Counts["invalid"] != 1 || outcome.Counts["executed"] != 1 || outcome.Counts["saved"] != 1 {
|
|
t.Fatalf("counts = %#v", outcome.Counts)
|
|
}
|
|
if len(store.saved) != 1 || store.saved[0].ProbeID != "probe-a" || store.saved[0].State != "up" {
|
|
t.Fatalf("saved = %#v", store.saved)
|
|
}
|
|
}
|
|
|
|
func TestProbeJobWithoutConfiguredProbesIsDisabled(t *testing.T) {
|
|
scheduler, err := probe.NewScheduler(fakeExecutor{state: "up"}, probe.SchedulerConfig{MaxConcurrent: 1, MaxAttempts: 1, AttemptTimeout: time.Second})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
job := &ProbeJob{Store: &fakeProbeStore{}, Scheduler: scheduler, Logger: quietLogger()}
|
|
outcome, err := job.Run(context.Background())
|
|
if err != nil || !outcome.Disabled || outcome.Reason != "no_probes_configured" {
|
|
t.Fatalf("outcome = %#v err = %v", outcome, err)
|
|
}
|
|
// Probes exist but none are due: the scan itself is the successful unit of work.
|
|
job = &ProbeJob{Store: &fakeProbeStore{enabled: 3}, Scheduler: scheduler, Logger: quietLogger()}
|
|
outcome, err = job.Run(context.Background())
|
|
if err != nil || outcome.Disabled || outcome.Counts["enabled"] != 3 {
|
|
t.Fatalf("outcome = %#v err = %v", outcome, err)
|
|
}
|
|
}
|
|
|
|
func TestProbeJobReportsStoreFailure(t *testing.T) {
|
|
store := &fakeProbeStore{listErr: errors.New("probes table unavailable")}
|
|
scheduler, err := probe.NewScheduler(fakeExecutor{state: "up"}, probe.SchedulerConfig{MaxConcurrent: 1, MaxAttempts: 1, AttemptTimeout: time.Second})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
job := &ProbeJob{Store: store, Scheduler: scheduler, Logger: quietLogger()}
|
|
if _, err := job.Run(context.Background()); err == nil {
|
|
t.Fatal("a store failure must surface as a job failure")
|
|
}
|
|
}
|
|
|
|
func TestProbeJobWithoutAStoreIsDisabled(t *testing.T) {
|
|
job := &ProbeJob{}
|
|
outcome, err := job.Run(context.Background())
|
|
if err != nil || !outcome.Disabled {
|
|
t.Fatalf("outcome = %#v err = %v", outcome, err)
|
|
}
|
|
}
|
|
|
|
// --- notification drain ----------------------------------------------------
|
|
|
|
type fakeOutboxStore struct {
|
|
mu sync.Mutex
|
|
items map[string]*notification.Outbox
|
|
}
|
|
|
|
func newFakeOutboxStore(items ...notification.Outbox) *fakeOutboxStore {
|
|
store := &fakeOutboxStore{items: map[string]*notification.Outbox{}}
|
|
for index := range items {
|
|
item := items[index]
|
|
store.items[item.ID] = &item
|
|
}
|
|
return store
|
|
}
|
|
func (s *fakeOutboxStore) Enqueue(_ context.Context, item notification.Outbox) (notification.Outbox, bool, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
for _, existing := range s.items {
|
|
if existing.IdempotencyKey == item.IdempotencyKey {
|
|
return *existing, true, nil
|
|
}
|
|
}
|
|
if item.ID == "" {
|
|
item.ID = item.IdempotencyKey
|
|
}
|
|
item.Status = notification.StatusPending
|
|
s.items[item.ID] = &item
|
|
return item, false, nil
|
|
}
|
|
func (s *fakeOutboxStore) GetOutbox(_ context.Context, id string) (notification.Outbox, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
item, ok := s.items[id]
|
|
if !ok {
|
|
return notification.Outbox{}, notification.ErrNotFound
|
|
}
|
|
return *item, nil
|
|
}
|
|
func (s *fakeOutboxStore) ClaimDue(_ context.Context, now time.Time, limit int) ([]notification.Outbox, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
claimed := make([]notification.Outbox, 0, limit)
|
|
for _, item := range s.items {
|
|
if len(claimed) >= limit {
|
|
break
|
|
}
|
|
if item.Status != notification.StatusPending && item.Status != notification.StatusRetry {
|
|
continue
|
|
}
|
|
if item.NextAttemptAt.After(now) {
|
|
continue
|
|
}
|
|
item.Status = notification.StatusDelivering
|
|
item.Attempts++
|
|
claimed = append(claimed, *item)
|
|
}
|
|
return claimed, nil
|
|
}
|
|
func (s *fakeOutboxStore) Complete(_ context.Context, id string, attempt int, success bool, _ error, now time.Time) (notification.Outbox, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
item, ok := s.items[id]
|
|
if !ok {
|
|
return notification.Outbox{}, notification.ErrNotFound
|
|
}
|
|
if item.Status != notification.StatusDelivering || item.Attempts != attempt {
|
|
return *item, nil
|
|
}
|
|
if success {
|
|
item.Status = notification.StatusDelivered
|
|
item.DeliveredAt = &now
|
|
} else {
|
|
item.Status = notification.StatusRetry
|
|
item.NextAttemptAt = now.Add(notification.RetryDelay(attempt))
|
|
}
|
|
return *item, nil
|
|
}
|
|
|
|
type fakeChannelLister struct{ channels []notification.Channel }
|
|
|
|
func (l fakeChannelLister) ListChannels(context.Context, int) ([]notification.Channel, error) {
|
|
return l.channels, nil
|
|
}
|
|
|
|
func TestNotificationDrainDeliversOnceAndRetriesWithoutDuplicating(t *testing.T) {
|
|
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
|
store := newFakeOutboxStore(notification.Outbox{ID: "out-1", IdempotencyKey: "key-1", ChannelID: "channel-1",
|
|
EventType: notification.EventFiring, Subject: "firing", Body: "body", Status: notification.StatusPending, NextAttemptAt: now})
|
|
channel := ¬ification.MemoryChannel{}
|
|
job := NotificationDrainJob{
|
|
Store: store,
|
|
Channels: fakeChannelLister{channels: []notification.Channel{{ID: "channel-1", Type: "memory", Enabled: true}}},
|
|
Senders: map[string]notification.ChannelSender{"memory": channel},
|
|
Logger: quietLogger(), Now: func() time.Time { return now },
|
|
}
|
|
outcome, err := job.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome.Counts["delivered"] != 1 {
|
|
t.Fatalf("counts = %#v", outcome.Counts)
|
|
}
|
|
// A second drain must not deliver the same item again.
|
|
if _, err := job.Run(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(channel.Deliveries) != 1 {
|
|
t.Fatalf("deliveries = %d, want 1", len(channel.Deliveries))
|
|
}
|
|
}
|
|
|
|
func TestNotificationDrainBuildsChannelSpecificTransport(t *testing.T) {
|
|
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
|
store := newFakeOutboxStore(notification.Outbox{ID: "out-1", IdempotencyKey: "key-1", ChannelID: "channel-1",
|
|
EventType: notification.EventFiring, Subject: "firing", Body: "body", Status: notification.StatusPending, NextAttemptAt: now})
|
|
memory := ¬ification.MemoryChannel{}
|
|
factoryCalls := 0
|
|
job := NotificationDrainJob{
|
|
Store: store, Channels: fakeChannelLister{channels: []notification.Channel{{ID: "channel-1", Type: "webhook", Enabled: true}}},
|
|
Factories: map[string]notification.ChannelSenderFactory{"webhook": notification.ChannelSenderFactoryFunc(func(context.Context, notification.Channel) (notification.ChannelSender, error) {
|
|
factoryCalls++
|
|
return memory, nil
|
|
})}, Logger: quietLogger(), Now: func() time.Time { return now },
|
|
}
|
|
outcome, err := job.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome.Counts["delivered"] != 1 || factoryCalls != 1 || len(memory.Deliveries) != 1 {
|
|
t.Fatalf("outcome=%#v factoryCalls=%d deliveries=%d", outcome, factoryCalls, len(memory.Deliveries))
|
|
}
|
|
}
|
|
|
|
func TestNotificationDrainStatesWhyItCannotDeliver(t *testing.T) {
|
|
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
|
store := newFakeOutboxStore(notification.Outbox{ID: "out-1", IdempotencyKey: "key-1", ChannelID: "channel-1",
|
|
EventType: notification.EventFiring, Subject: "firing", Body: "body", Status: notification.StatusPending, NextAttemptAt: now})
|
|
for name, testCase := range map[string]struct {
|
|
channels []notification.Channel
|
|
senders map[string]notification.ChannelSender
|
|
disabled bool
|
|
wantErr bool
|
|
}{
|
|
"no channels": {channels: nil, disabled: true},
|
|
"all disabled": {channels: []notification.Channel{{ID: "c", Type: "webhook"}}, disabled: true},
|
|
"no transport": {channels: []notification.Channel{{ID: "c", Type: "webhook", Enabled: true}}, wantErr: true},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
job := NotificationDrainJob{Store: store, Channels: fakeChannelLister{channels: testCase.channels},
|
|
Senders: testCase.senders, Logger: quietLogger(), Now: func() time.Time { return now }}
|
|
outcome, err := job.Run(context.Background())
|
|
if testCase.wantErr && err == nil {
|
|
t.Fatal("an undeliverable channel must be reported as a failure")
|
|
}
|
|
if !testCase.wantErr && err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome.Disabled != testCase.disabled {
|
|
t.Fatalf("outcome = %#v", outcome)
|
|
}
|
|
if item, _ := store.GetOutbox(context.Background(), "out-1"); item.Attempts != 0 {
|
|
t.Fatalf("an undeliverable drain consumed attempt %d", item.Attempts)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// --- alert evaluation ------------------------------------------------------
|
|
|
|
type fakeStateWriter struct {
|
|
mu sync.Mutex
|
|
observations []alert.Observation
|
|
to alert.State
|
|
duplicate bool
|
|
}
|
|
|
|
func (s *fakeStateWriter) ApplyObservation(_ context.Context, input alert.StateInput) (alert.Instance, alert.Occurrence, bool, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.observations = append(s.observations, input.Observation)
|
|
to := s.to
|
|
if to == "" {
|
|
to = alert.StateInactive
|
|
}
|
|
instance := alert.Instance{ID: "instance-1", RuleID: input.RuleID, Fingerprint: input.Fingerprint, State: to}
|
|
occurrence := alert.Occurrence{InstanceID: instance.ID, EvaluationKey: input.Observation.EvaluationKey,
|
|
From: alert.StateInactive, To: to, ObservedAt: input.Observation.ObservedAt}
|
|
return instance, occurrence, s.duplicate, nil
|
|
}
|
|
func (s *fakeStateWriter) last() alert.Observation {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return s.observations[len(s.observations)-1]
|
|
}
|
|
|
|
type fakePrior struct{ state PriorAlertState }
|
|
|
|
func (p fakePrior) PriorState(context.Context, string, string) (PriorAlertState, error) {
|
|
return p.state, nil
|
|
}
|
|
|
|
type fakeVersions struct{}
|
|
|
|
func (fakeVersions) Versions(context.Context, string, int) ([]alert.Version, error) {
|
|
return []alert.Version{{ID: "33333333-3333-4333-8333-333333333333", VersionNumber: 1}}, nil
|
|
}
|
|
|
|
type fakeMetricSource struct {
|
|
value MetricValue
|
|
err error
|
|
}
|
|
|
|
func (s fakeMetricSource) Value(context.Context, alert.Rule) (MetricValue, error) {
|
|
return s.value, s.err
|
|
}
|
|
|
|
func ruleFixture() alert.Rule {
|
|
return alert.Rule{Document: alert.Document{
|
|
SchemaVersion: 1, ID: "44444444-4444-4444-8444-444444444444", Name: "CPU high", Enabled: true, Severity: alert.SeverityDegraded,
|
|
Condition: alert.Condition{InputType: "metric", Metric: "host.cpu.utilization", Operator: ">", Threshold: float64(80)},
|
|
EvaluationIntervalSeconds: 60, UnknownBehavior: alert.UnknownRetain, Message: alert.Message{TitleKey: "t", BodyKey: "b"},
|
|
}, CurrentVersion: 1}
|
|
}
|
|
|
|
func TestAlertEvaluatorRecordsUnknownWhenTheInputIsUnavailable(t *testing.T) {
|
|
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
|
for name, testCase := range map[string]struct {
|
|
source MetricSource
|
|
reason string
|
|
}{
|
|
"no source": {source: nil, reason: "metric_source_not_configured"},
|
|
"query failure": {source: fakeMetricSource{err: errors.New("prometheus is down")}, reason: "metric_query_failed"},
|
|
"no series": {source: fakeMetricSource{value: MetricValue{Reason: "metric_no_series"}}, reason: "metric_no_series"},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
states := &fakeStateWriter{to: alert.StateUnknown}
|
|
evaluator := &AlertEvaluator{Metrics: testCase.source, States: states, Prior: fakePrior{}, Versions: fakeVersions{},
|
|
Logger: quietLogger(), Now: func() time.Time { return now }}
|
|
if err := evaluator.Evaluate(context.Background(), ruleFixture()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
observation := states.last()
|
|
if !observation.Unknown || observation.Reason != testCase.reason {
|
|
t.Fatalf("observation = %#v", observation)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAlertEvaluatorEvaluatesConditionAndQueuesOneNotificationPerChannel(t *testing.T) {
|
|
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
|
states := &fakeStateWriter{to: alert.StateFiring}
|
|
outbox := newFakeOutboxStore()
|
|
queue := &fakeNotificationQueue{outbox: outbox, channels: []notification.Channel{
|
|
{ID: "channel-1", Type: "memory", Enabled: true}, {ID: "channel-2", Type: "memory", Enabled: true}, {ID: "channel-3", Type: "memory"},
|
|
}}
|
|
evaluator := &AlertEvaluator{Metrics: fakeMetricSource{value: MetricValue{Value: 91, Known: true, ObservedAt: now}},
|
|
States: states, Prior: fakePrior{}, Versions: fakeVersions{}, Notifications: queue, Logger: quietLogger(), Now: func() time.Time { return now }}
|
|
|
|
if err := evaluator.Evaluate(context.Background(), ruleFixture()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
observation := states.last()
|
|
if observation.Unknown || !observation.ConditionTrue {
|
|
t.Fatalf("observation = %#v", observation)
|
|
}
|
|
if queued := outbox.count(); queued != 2 {
|
|
t.Fatalf("queued notifications = %d, want one per enabled channel", queued)
|
|
}
|
|
for _, item := range outbox.all() {
|
|
if strings.Contains(item.Body, "secret") || strings.Contains(item.Body, "token") {
|
|
t.Fatalf("notification body leaks configuration: %q", item.Body)
|
|
}
|
|
}
|
|
// Re-running the same evaluation window must not queue a second delivery.
|
|
if err := evaluator.Evaluate(context.Background(), ruleFixture()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if queued := outbox.count(); queued != 2 {
|
|
t.Fatalf("queued notifications after repeat = %d, want 2", queued)
|
|
}
|
|
}
|
|
|
|
func TestAlertEvaluatorHonoursCooldownAndDuplicateWindows(t *testing.T) {
|
|
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
|
cooldown := now.Add(5 * time.Minute)
|
|
for name, testCase := range map[string]struct {
|
|
prior PriorAlertState
|
|
duplicate bool
|
|
want int
|
|
}{
|
|
"fresh firing": {prior: PriorAlertState{State: alert.StateInactive}, want: 1},
|
|
"cooldown active": {prior: PriorAlertState{State: alert.StateResolved, CooldownUntil: &cooldown, Found: true}, want: 0},
|
|
"already firing": {prior: PriorAlertState{State: alert.StateFiring, Found: true}, want: 0},
|
|
"duplicate window": {prior: PriorAlertState{State: alert.StateInactive}, duplicate: true, want: 0},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
outbox := newFakeOutboxStore()
|
|
queue := &fakeNotificationQueue{outbox: outbox, channels: []notification.Channel{{ID: "channel-1", Type: "memory", Enabled: true}}}
|
|
evaluator := &AlertEvaluator{Metrics: fakeMetricSource{value: MetricValue{Value: 91, Known: true}},
|
|
States: &fakeStateWriter{to: alert.StateFiring, duplicate: testCase.duplicate}, Prior: fakePrior{state: testCase.prior},
|
|
Versions: fakeVersions{}, Notifications: queue, Logger: quietLogger(), Now: func() time.Time { return now }}
|
|
if err := evaluator.Evaluate(context.Background(), ruleFixture()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if queued := outbox.count(); queued != testCase.want {
|
|
t.Fatalf("queued = %d, want %d", queued, testCase.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEvaluationKeyAndFingerprintAreStable(t *testing.T) {
|
|
rule := ruleFixture()
|
|
base := time.Date(2026, 8, 4, 12, 0, 30, 0, time.UTC)
|
|
if EvaluationKey(rule, base) != EvaluationKey(rule, base.Add(20*time.Second)) {
|
|
t.Fatal("evaluation key changed inside one interval")
|
|
}
|
|
if EvaluationKey(rule, base) == EvaluationKey(rule, base.Add(time.Minute)) {
|
|
t.Fatal("evaluation key did not change between intervals")
|
|
}
|
|
scoped := ruleFixture()
|
|
scoped.Scope = map[string]any{"host": "tower"}
|
|
if RuleFingerprint(rule) == RuleFingerprint(scoped) {
|
|
t.Fatal("scope is not part of the fingerprint")
|
|
}
|
|
repeated, again := RuleFingerprint(scoped), RuleFingerprint(scoped)
|
|
if repeated != again || len(repeated) == 0 || len(repeated) > 160 {
|
|
t.Fatal("fingerprint is unstable or unbounded")
|
|
}
|
|
}
|
|
|
|
type fakeNotificationQueue struct {
|
|
outbox *fakeOutboxStore
|
|
channels []notification.Channel
|
|
}
|
|
|
|
func (q *fakeNotificationQueue) Enqueue(ctx context.Context, item notification.Outbox) (notification.Outbox, bool, error) {
|
|
return q.outbox.Enqueue(ctx, item)
|
|
}
|
|
func (q *fakeNotificationQueue) ListChannels(context.Context, int) ([]notification.Channel, error) {
|
|
return q.channels, nil
|
|
}
|
|
|
|
func (s *fakeOutboxStore) count() int {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return len(s.items)
|
|
}
|
|
func (s *fakeOutboxStore) all() []notification.Outbox {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
items := make([]notification.Outbox, 0, len(s.items))
|
|
for _, item := range s.items {
|
|
items = append(items, *item)
|
|
}
|
|
return items
|
|
}
|
|
|
|
func TestAlertEvaluationJobReportsFailuresAndDisabledState(t *testing.T) {
|
|
outcome, err := AlertEvaluationJob{}.Run(context.Background())
|
|
if err != nil || !outcome.Disabled || outcome.Reason != "alert_evaluation_not_configured" {
|
|
t.Fatalf("outcome = %#v err = %v", outcome, err)
|
|
}
|
|
}
|