This commit is contained in:
ITWorx Pulse release export
@@ -0,0 +1,704 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/discovery"
|
||||
"github.com/itworx/pulse/internal/inventory"
|
||||
"github.com/itworx/pulse/internal/notification"
|
||||
"github.com/itworx/pulse/internal/probe"
|
||||
"github.com/itworx/pulse/internal/reconciliation"
|
||||
)
|
||||
|
||||
const testSourceID = "11111111-1111-4111-8111-111111111111"
|
||||
|
||||
// --- discovery job ---------------------------------------------------------
|
||||
|
||||
type fakeContainerProvider struct {
|
||||
mu sync.Mutex
|
||||
snapshot container.Snapshot
|
||||
err error
|
||||
}
|
||||
|
||||
func (p *fakeContainerProvider) Snapshot(context.Context) (container.Snapshot, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.snapshot, p.err
|
||||
}
|
||||
func (p *fakeContainerProvider) set(snapshot container.Snapshot) {
|
||||
p.mu.Lock()
|
||||
p.snapshot = snapshot
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
type memoryAliasStore struct {
|
||||
mu sync.Mutex
|
||||
records []ContainerAliasRecord
|
||||
saves int
|
||||
}
|
||||
|
||||
func (s *memoryAliasStore) List(context.Context, string) ([]ContainerAliasRecord, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]ContainerAliasRecord(nil), s.records...), nil
|
||||
}
|
||||
func (s *memoryAliasStore) Save(_ context.Context, _ string, records []ContainerAliasRecord) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.records = append([]ContainerAliasRecord(nil), records...)
|
||||
s.saves++
|
||||
return nil
|
||||
}
|
||||
|
||||
type memoryInventory struct {
|
||||
mu sync.Mutex
|
||||
persisted map[string]inventory.Entity
|
||||
facts map[string][]inventory.Fact
|
||||
relations map[string][]inventory.Relation
|
||||
tombstoned map[string]inventory.Entity
|
||||
}
|
||||
|
||||
func newMemoryInventory() *memoryInventory {
|
||||
return &memoryInventory{persisted: map[string]inventory.Entity{}, facts: map[string][]inventory.Fact{}, relations: map[string][]inventory.Relation{}, tombstoned: map[string]inventory.Entity{}}
|
||||
}
|
||||
func (s *memoryInventory) PersistDiscovery(_ context.Context, entity inventory.Entity, _ inventory.Alias, facts []inventory.Fact, relations []inventory.Relation) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.persisted[entity.ID] = entity
|
||||
s.facts[entity.ID] = append([]inventory.Fact(nil), facts...)
|
||||
s.relations[entity.ID] = append([]inventory.Relation(nil), relations...)
|
||||
if entity.TombstonedAt != nil {
|
||||
s.tombstoned[entity.ID] = entity
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *memoryInventory) UpsertEntity(_ context.Context, entity inventory.Entity) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.tombstoned[entity.ID] = entity
|
||||
return nil
|
||||
}
|
||||
|
||||
func healthySnapshot(observedAt time.Time, containers ...container.Container) container.Snapshot {
|
||||
return container.Snapshot{
|
||||
ContractVersion: container.ContractVersion,
|
||||
Source: container.Source{ID: "container", Type: "agent", ObservedAt: observedAt, ReceivedAt: observedAt, Freshness: "fresh", State: "healthy"},
|
||||
Containers: containers, Total: len(containers),
|
||||
}
|
||||
}
|
||||
|
||||
func newDiscoveryJob(provider container.Provider, aliases ContainerAliasStore, store discovery.Store, inv InventoryStore) DiscoveryJob {
|
||||
return DiscoveryJob{SourceID: testSourceID, Provider: provider, Aliases: aliases, Inventory: inv,
|
||||
Runner: discovery.Runner{Store: store, MaxAttempts: 1}}
|
||||
}
|
||||
|
||||
func TestDiscoveryJobSkipsAnUnavailableSourceWithoutTombstoning(t *testing.T) {
|
||||
observedAt := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
aliases := &memoryAliasStore{records: []ContainerAliasRecord{{ContainerAlias: aliasFixture("runtime-1", observedAt)}}}
|
||||
inv := newMemoryInventory()
|
||||
store := discovery.NewMemoryStore()
|
||||
provider := &fakeContainerProvider{snapshot: container.UnknownSnapshot(observedAt, "container", "agent", "source_unavailable")}
|
||||
job := newDiscoveryJob(provider, aliases, store, inv)
|
||||
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !outcome.Skipped || outcome.Reason != "source_unavailable" {
|
||||
t.Fatalf("outcome = %#v", outcome)
|
||||
}
|
||||
if aliases.saves != 0 || len(inv.tombstoned) != 0 || len(store.Events) != 0 {
|
||||
t.Fatalf("an unavailable source changed inventory: saves=%d tombstoned=%d events=%d", aliases.saves, len(inv.tombstoned), len(store.Events))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveryJobDerivesLifecycleEventsAndIsIdempotent(t *testing.T) {
|
||||
first := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
aliases := &memoryAliasStore{}
|
||||
inv := newMemoryInventory()
|
||||
store := discovery.NewMemoryStore()
|
||||
provider := &fakeContainerProvider{snapshot: healthySnapshot(first,
|
||||
container.Container{ID: "runtime-1", Name: "pulse-api", State: "running", Health: "healthy"})}
|
||||
job := newDiscoveryJob(provider, aliases, store, inv)
|
||||
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Counts["added"] != 1 || outcome.Counts["events"] != 0 {
|
||||
t.Fatalf("first pass counts = %#v", outcome.Counts)
|
||||
}
|
||||
if len(inv.persisted) != 2 || aliases.saves != 1 {
|
||||
t.Fatalf("first pass persisted=%d saves=%d", len(inv.persisted), aliases.saves)
|
||||
}
|
||||
containerID := aliases.records[0].EntityID
|
||||
if len(inv.facts[containerID]) < 6 || len(inv.relations[containerID]) != 1 {
|
||||
t.Fatalf("container provenance facts=%d relations=%d", len(inv.facts[containerID]), len(inv.relations[containerID]))
|
||||
}
|
||||
relation := inv.relations[containerID][0]
|
||||
if relation.RelationType != "member_of" || inv.persisted[relation.TargetEntityID].EntityType != "application" {
|
||||
t.Fatalf("application relation = %+v target = %+v", relation, inv.persisted[relation.TargetEntityID])
|
||||
}
|
||||
|
||||
// A repeat of the same window is claimed once, so nothing runs twice.
|
||||
if _, err := job.Run(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if aliases.saves != 1 {
|
||||
t.Fatalf("repeated window saves = %d, want 1", aliases.saves)
|
||||
}
|
||||
|
||||
stateAfterFirstPass, err := aliases.List(context.Background(), testSourceID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
second := first.Add(time.Minute)
|
||||
provider.set(healthySnapshot(second,
|
||||
container.Container{ID: "runtime-1", Name: "pulse-api", State: "exited", Health: "unhealthy", RestartCount: 1}))
|
||||
outcome, err = job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Counts["events"] != 3 {
|
||||
t.Fatalf("second pass counts = %#v, want 3 lifecycle events", outcome.Counts)
|
||||
}
|
||||
types := map[string]bool{}
|
||||
for _, event := range store.Events {
|
||||
types[event.Type] = true
|
||||
if event.EntityID == "" || event.SourceID != testSourceID || event.Severity == "" {
|
||||
t.Fatalf("event is missing identity: %#v", event)
|
||||
}
|
||||
}
|
||||
for _, expected := range []string{"container.state_changed", "container.health_changed", "container.restart"} {
|
||||
if !types[expected] {
|
||||
t.Fatalf("missing event %s in %#v", expected, types)
|
||||
}
|
||||
}
|
||||
emitted := len(store.Events)
|
||||
|
||||
// The same observation replayed emits the same deduplicated events.
|
||||
store2 := discovery.NewMemoryStore()
|
||||
job2 := newDiscoveryJob(provider, &memoryAliasStore{records: stateAfterFirstPass}, store2, newMemoryInventory())
|
||||
if _, err := job2.Run(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(store2.Events) != emitted {
|
||||
t.Fatalf("replayed events = %d, want %d", len(store2.Events), emitted)
|
||||
}
|
||||
for key := range store.Events {
|
||||
if _, ok := store2.Events[key]; !ok {
|
||||
t.Fatalf("replay produced a different dedup key set: %q missing", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveryJobTombstonesMissingContainers(t *testing.T) {
|
||||
first := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
aliases := &memoryAliasStore{records: aliasSnapshotBefore(first)}
|
||||
inv := newMemoryInventory()
|
||||
job := newDiscoveryJob(&fakeContainerProvider{snapshot: healthySnapshot(first.Add(time.Minute))}, aliases, discovery.NewMemoryStore(), inv)
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Counts["tombstoned"] != 1 || len(inv.tombstoned) != 2 {
|
||||
t.Fatalf("counts = %#v tombstoned = %#v", outcome.Counts, inv.tombstoned)
|
||||
}
|
||||
for entityID, relations := range inv.relations {
|
||||
if inv.persisted[entityID].EntityType == "container" && (len(relations) != 1 || relations[0].TombstonedAt == nil) {
|
||||
t.Fatalf("missing container relation was not tombstoned: %+v", relations)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func aliasFixture(runtimeID string, observedAt time.Time) reconciliation.ContainerAlias {
|
||||
return reconciliation.ContainerAlias{
|
||||
EntityID: "22222222-2222-4222-8222-222222222222", SourceID: testSourceID, RuntimeID: runtimeID,
|
||||
Name: "pulse-api", FirstSeenAt: observedAt, LastSeenAt: observedAt, Active: true,
|
||||
}
|
||||
}
|
||||
|
||||
func aliasSnapshotBefore(observedAt time.Time) []ContainerAliasRecord {
|
||||
return []ContainerAliasRecord{{ContainerAlias: aliasFixture("runtime-1", observedAt), State: "running", Health: "healthy"}}
|
||||
}
|
||||
|
||||
// --- probe job -------------------------------------------------------------
|
||||
|
||||
type fakeProbeStore struct {
|
||||
mu sync.Mutex
|
||||
due []probe.Definition
|
||||
saved []probe.Result
|
||||
enabled int
|
||||
listErr error
|
||||
}
|
||||
|
||||
func (s *fakeProbeStore) CountEnabled(context.Context) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.enabled, nil
|
||||
}
|
||||
|
||||
func (s *fakeProbeStore) ListDue(context.Context, time.Time, int) ([]probe.Definition, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]probe.Definition(nil), s.due...), s.listErr
|
||||
}
|
||||
func (s *fakeProbeStore) SaveResults(_ context.Context, results []probe.Result) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.saved = append(s.saved, results...)
|
||||
return len(results), nil
|
||||
}
|
||||
|
||||
type fakeExecutor struct {
|
||||
state string
|
||||
err error
|
||||
}
|
||||
|
||||
func (e fakeExecutor) Execute(_ context.Context, definition probe.Definition) (probe.Result, error) {
|
||||
if e.err != nil {
|
||||
return probe.Result{ProbeID: definition.ID, State: "unknown"}, e.err
|
||||
}
|
||||
return probe.Result{ProbeID: definition.ID, State: e.state, ObservedAt: time.Now().UTC(), CompletedAt: time.Now().UTC()}, nil
|
||||
}
|
||||
|
||||
func probeDefinition(id string, interval, timeout time.Duration) probe.Definition {
|
||||
return probe.Definition{ID: id, ServiceID: "service", Name: id, Type: probe.TypeTCP,
|
||||
Target: probe.Target{Host: "example.internal", Port: 443}, Interval: interval, Timeout: timeout, Enabled: true, Revision: 1}
|
||||
}
|
||||
|
||||
func TestProbeJobExecutesDueProbesAndPersistsResults(t *testing.T) {
|
||||
store := &fakeProbeStore{due: []probe.Definition{
|
||||
probeDefinition("probe-a", time.Minute, 5*time.Second),
|
||||
// An individually invalid probe must not stop the batch.
|
||||
probeDefinition("probe-b", 10*time.Second, 30*time.Second),
|
||||
}}
|
||||
scheduler, err := probe.NewScheduler(fakeExecutor{state: "up"}, probe.SchedulerConfig{MaxConcurrent: 4, MaxAttempts: 1, AttemptTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := &ProbeJob{Store: store, Scheduler: scheduler, Logger: quietLogger()}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Counts["due"] != 2 || outcome.Counts["invalid"] != 1 || outcome.Counts["executed"] != 1 || outcome.Counts["saved"] != 1 {
|
||||
t.Fatalf("counts = %#v", outcome.Counts)
|
||||
}
|
||||
if len(store.saved) != 1 || store.saved[0].ProbeID != "probe-a" || store.saved[0].State != "up" {
|
||||
t.Fatalf("saved = %#v", store.saved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeJobWithoutConfiguredProbesIsDisabled(t *testing.T) {
|
||||
scheduler, err := probe.NewScheduler(fakeExecutor{state: "up"}, probe.SchedulerConfig{MaxConcurrent: 1, MaxAttempts: 1, AttemptTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := &ProbeJob{Store: &fakeProbeStore{}, Scheduler: scheduler, Logger: quietLogger()}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil || !outcome.Disabled || outcome.Reason != "no_probes_configured" {
|
||||
t.Fatalf("outcome = %#v err = %v", outcome, err)
|
||||
}
|
||||
// Probes exist but none are due: the scan itself is the successful unit of work.
|
||||
job = &ProbeJob{Store: &fakeProbeStore{enabled: 3}, Scheduler: scheduler, Logger: quietLogger()}
|
||||
outcome, err = job.Run(context.Background())
|
||||
if err != nil || outcome.Disabled || outcome.Counts["enabled"] != 3 {
|
||||
t.Fatalf("outcome = %#v err = %v", outcome, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeJobReportsStoreFailure(t *testing.T) {
|
||||
store := &fakeProbeStore{listErr: errors.New("probes table unavailable")}
|
||||
scheduler, err := probe.NewScheduler(fakeExecutor{state: "up"}, probe.SchedulerConfig{MaxConcurrent: 1, MaxAttempts: 1, AttemptTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := &ProbeJob{Store: store, Scheduler: scheduler, Logger: quietLogger()}
|
||||
if _, err := job.Run(context.Background()); err == nil {
|
||||
t.Fatal("a store failure must surface as a job failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeJobWithoutAStoreIsDisabled(t *testing.T) {
|
||||
job := &ProbeJob{}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil || !outcome.Disabled {
|
||||
t.Fatalf("outcome = %#v err = %v", outcome, err)
|
||||
}
|
||||
}
|
||||
|
||||
// --- notification drain ----------------------------------------------------
|
||||
|
||||
type fakeOutboxStore struct {
|
||||
mu sync.Mutex
|
||||
items map[string]*notification.Outbox
|
||||
}
|
||||
|
||||
func newFakeOutboxStore(items ...notification.Outbox) *fakeOutboxStore {
|
||||
store := &fakeOutboxStore{items: map[string]*notification.Outbox{}}
|
||||
for index := range items {
|
||||
item := items[index]
|
||||
store.items[item.ID] = &item
|
||||
}
|
||||
return store
|
||||
}
|
||||
func (s *fakeOutboxStore) Enqueue(_ context.Context, item notification.Outbox) (notification.Outbox, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, existing := range s.items {
|
||||
if existing.IdempotencyKey == item.IdempotencyKey {
|
||||
return *existing, true, nil
|
||||
}
|
||||
}
|
||||
if item.ID == "" {
|
||||
item.ID = item.IdempotencyKey
|
||||
}
|
||||
item.Status = notification.StatusPending
|
||||
s.items[item.ID] = &item
|
||||
return item, false, nil
|
||||
}
|
||||
func (s *fakeOutboxStore) GetOutbox(_ context.Context, id string) (notification.Outbox, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
item, ok := s.items[id]
|
||||
if !ok {
|
||||
return notification.Outbox{}, notification.ErrNotFound
|
||||
}
|
||||
return *item, nil
|
||||
}
|
||||
func (s *fakeOutboxStore) ClaimDue(_ context.Context, now time.Time, limit int) ([]notification.Outbox, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
claimed := make([]notification.Outbox, 0, limit)
|
||||
for _, item := range s.items {
|
||||
if len(claimed) >= limit {
|
||||
break
|
||||
}
|
||||
if item.Status != notification.StatusPending && item.Status != notification.StatusRetry {
|
||||
continue
|
||||
}
|
||||
if item.NextAttemptAt.After(now) {
|
||||
continue
|
||||
}
|
||||
item.Status = notification.StatusDelivering
|
||||
item.Attempts++
|
||||
claimed = append(claimed, *item)
|
||||
}
|
||||
return claimed, nil
|
||||
}
|
||||
func (s *fakeOutboxStore) Complete(_ context.Context, id string, attempt int, success bool, _ error, now time.Time) (notification.Outbox, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
item, ok := s.items[id]
|
||||
if !ok {
|
||||
return notification.Outbox{}, notification.ErrNotFound
|
||||
}
|
||||
if item.Status != notification.StatusDelivering || item.Attempts != attempt {
|
||||
return *item, nil
|
||||
}
|
||||
if success {
|
||||
item.Status = notification.StatusDelivered
|
||||
item.DeliveredAt = &now
|
||||
} else {
|
||||
item.Status = notification.StatusRetry
|
||||
item.NextAttemptAt = now.Add(notification.RetryDelay(attempt))
|
||||
}
|
||||
return *item, nil
|
||||
}
|
||||
|
||||
type fakeChannelLister struct{ channels []notification.Channel }
|
||||
|
||||
func (l fakeChannelLister) ListChannels(context.Context, int) ([]notification.Channel, error) {
|
||||
return l.channels, nil
|
||||
}
|
||||
|
||||
func TestNotificationDrainDeliversOnceAndRetriesWithoutDuplicating(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
store := newFakeOutboxStore(notification.Outbox{ID: "out-1", IdempotencyKey: "key-1", ChannelID: "channel-1",
|
||||
EventType: notification.EventFiring, Subject: "firing", Body: "body", Status: notification.StatusPending, NextAttemptAt: now})
|
||||
channel := ¬ification.MemoryChannel{}
|
||||
job := NotificationDrainJob{
|
||||
Store: store,
|
||||
Channels: fakeChannelLister{channels: []notification.Channel{{ID: "channel-1", Type: "memory", Enabled: true}}},
|
||||
Senders: map[string]notification.ChannelSender{"memory": channel},
|
||||
Logger: quietLogger(), Now: func() time.Time { return now },
|
||||
}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Counts["delivered"] != 1 {
|
||||
t.Fatalf("counts = %#v", outcome.Counts)
|
||||
}
|
||||
// A second drain must not deliver the same item again.
|
||||
if _, err := job.Run(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(channel.Deliveries) != 1 {
|
||||
t.Fatalf("deliveries = %d, want 1", len(channel.Deliveries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotificationDrainBuildsChannelSpecificTransport(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
store := newFakeOutboxStore(notification.Outbox{ID: "out-1", IdempotencyKey: "key-1", ChannelID: "channel-1",
|
||||
EventType: notification.EventFiring, Subject: "firing", Body: "body", Status: notification.StatusPending, NextAttemptAt: now})
|
||||
memory := ¬ification.MemoryChannel{}
|
||||
factoryCalls := 0
|
||||
job := NotificationDrainJob{
|
||||
Store: store, Channels: fakeChannelLister{channels: []notification.Channel{{ID: "channel-1", Type: "webhook", Enabled: true}}},
|
||||
Factories: map[string]notification.ChannelSenderFactory{"webhook": notification.ChannelSenderFactoryFunc(func(context.Context, notification.Channel) (notification.ChannelSender, error) {
|
||||
factoryCalls++
|
||||
return memory, nil
|
||||
})}, Logger: quietLogger(), Now: func() time.Time { return now },
|
||||
}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Counts["delivered"] != 1 || factoryCalls != 1 || len(memory.Deliveries) != 1 {
|
||||
t.Fatalf("outcome=%#v factoryCalls=%d deliveries=%d", outcome, factoryCalls, len(memory.Deliveries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotificationDrainStatesWhyItCannotDeliver(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
store := newFakeOutboxStore(notification.Outbox{ID: "out-1", IdempotencyKey: "key-1", ChannelID: "channel-1",
|
||||
EventType: notification.EventFiring, Subject: "firing", Body: "body", Status: notification.StatusPending, NextAttemptAt: now})
|
||||
for name, testCase := range map[string]struct {
|
||||
channels []notification.Channel
|
||||
senders map[string]notification.ChannelSender
|
||||
disabled bool
|
||||
wantErr bool
|
||||
}{
|
||||
"no channels": {channels: nil, disabled: true},
|
||||
"all disabled": {channels: []notification.Channel{{ID: "c", Type: "webhook"}}, disabled: true},
|
||||
"no transport": {channels: []notification.Channel{{ID: "c", Type: "webhook", Enabled: true}}, wantErr: true},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
job := NotificationDrainJob{Store: store, Channels: fakeChannelLister{channels: testCase.channels},
|
||||
Senders: testCase.senders, Logger: quietLogger(), Now: func() time.Time { return now }}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if testCase.wantErr && err == nil {
|
||||
t.Fatal("an undeliverable channel must be reported as a failure")
|
||||
}
|
||||
if !testCase.wantErr && err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Disabled != testCase.disabled {
|
||||
t.Fatalf("outcome = %#v", outcome)
|
||||
}
|
||||
if item, _ := store.GetOutbox(context.Background(), "out-1"); item.Attempts != 0 {
|
||||
t.Fatalf("an undeliverable drain consumed attempt %d", item.Attempts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- alert evaluation ------------------------------------------------------
|
||||
|
||||
type fakeStateWriter struct {
|
||||
mu sync.Mutex
|
||||
observations []alert.Observation
|
||||
to alert.State
|
||||
duplicate bool
|
||||
}
|
||||
|
||||
func (s *fakeStateWriter) ApplyObservation(_ context.Context, input alert.StateInput) (alert.Instance, alert.Occurrence, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.observations = append(s.observations, input.Observation)
|
||||
to := s.to
|
||||
if to == "" {
|
||||
to = alert.StateInactive
|
||||
}
|
||||
instance := alert.Instance{ID: "instance-1", RuleID: input.RuleID, Fingerprint: input.Fingerprint, State: to}
|
||||
occurrence := alert.Occurrence{InstanceID: instance.ID, EvaluationKey: input.Observation.EvaluationKey,
|
||||
From: alert.StateInactive, To: to, ObservedAt: input.Observation.ObservedAt}
|
||||
return instance, occurrence, s.duplicate, nil
|
||||
}
|
||||
func (s *fakeStateWriter) last() alert.Observation {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.observations[len(s.observations)-1]
|
||||
}
|
||||
|
||||
type fakePrior struct{ state PriorAlertState }
|
||||
|
||||
func (p fakePrior) PriorState(context.Context, string, string) (PriorAlertState, error) {
|
||||
return p.state, nil
|
||||
}
|
||||
|
||||
type fakeVersions struct{}
|
||||
|
||||
func (fakeVersions) Versions(context.Context, string, int) ([]alert.Version, error) {
|
||||
return []alert.Version{{ID: "33333333-3333-4333-8333-333333333333", VersionNumber: 1}}, nil
|
||||
}
|
||||
|
||||
type fakeMetricSource struct {
|
||||
value MetricValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (s fakeMetricSource) Value(context.Context, alert.Rule) (MetricValue, error) {
|
||||
return s.value, s.err
|
||||
}
|
||||
|
||||
func ruleFixture() alert.Rule {
|
||||
return alert.Rule{Document: alert.Document{
|
||||
SchemaVersion: 1, ID: "44444444-4444-4444-8444-444444444444", Name: "CPU high", Enabled: true, Severity: alert.SeverityDegraded,
|
||||
Condition: alert.Condition{InputType: "metric", Metric: "host.cpu.utilization", Operator: ">", Threshold: float64(80)},
|
||||
EvaluationIntervalSeconds: 60, UnknownBehavior: alert.UnknownRetain, Message: alert.Message{TitleKey: "t", BodyKey: "b"},
|
||||
}, CurrentVersion: 1}
|
||||
}
|
||||
|
||||
func TestAlertEvaluatorRecordsUnknownWhenTheInputIsUnavailable(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
for name, testCase := range map[string]struct {
|
||||
source MetricSource
|
||||
reason string
|
||||
}{
|
||||
"no source": {source: nil, reason: "metric_source_not_configured"},
|
||||
"query failure": {source: fakeMetricSource{err: errors.New("prometheus is down")}, reason: "metric_query_failed"},
|
||||
"no series": {source: fakeMetricSource{value: MetricValue{Reason: "metric_no_series"}}, reason: "metric_no_series"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
states := &fakeStateWriter{to: alert.StateUnknown}
|
||||
evaluator := &AlertEvaluator{Metrics: testCase.source, States: states, Prior: fakePrior{}, Versions: fakeVersions{},
|
||||
Logger: quietLogger(), Now: func() time.Time { return now }}
|
||||
if err := evaluator.Evaluate(context.Background(), ruleFixture()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
observation := states.last()
|
||||
if !observation.Unknown || observation.Reason != testCase.reason {
|
||||
t.Fatalf("observation = %#v", observation)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertEvaluatorEvaluatesConditionAndQueuesOneNotificationPerChannel(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
states := &fakeStateWriter{to: alert.StateFiring}
|
||||
outbox := newFakeOutboxStore()
|
||||
queue := &fakeNotificationQueue{outbox: outbox, channels: []notification.Channel{
|
||||
{ID: "channel-1", Type: "memory", Enabled: true}, {ID: "channel-2", Type: "memory", Enabled: true}, {ID: "channel-3", Type: "memory"},
|
||||
}}
|
||||
evaluator := &AlertEvaluator{Metrics: fakeMetricSource{value: MetricValue{Value: 91, Known: true, ObservedAt: now}},
|
||||
States: states, Prior: fakePrior{}, Versions: fakeVersions{}, Notifications: queue, Logger: quietLogger(), Now: func() time.Time { return now }}
|
||||
|
||||
if err := evaluator.Evaluate(context.Background(), ruleFixture()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
observation := states.last()
|
||||
if observation.Unknown || !observation.ConditionTrue {
|
||||
t.Fatalf("observation = %#v", observation)
|
||||
}
|
||||
if queued := outbox.count(); queued != 2 {
|
||||
t.Fatalf("queued notifications = %d, want one per enabled channel", queued)
|
||||
}
|
||||
for _, item := range outbox.all() {
|
||||
if strings.Contains(item.Body, "secret") || strings.Contains(item.Body, "token") {
|
||||
t.Fatalf("notification body leaks configuration: %q", item.Body)
|
||||
}
|
||||
}
|
||||
// Re-running the same evaluation window must not queue a second delivery.
|
||||
if err := evaluator.Evaluate(context.Background(), ruleFixture()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queued := outbox.count(); queued != 2 {
|
||||
t.Fatalf("queued notifications after repeat = %d, want 2", queued)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertEvaluatorHonoursCooldownAndDuplicateWindows(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
cooldown := now.Add(5 * time.Minute)
|
||||
for name, testCase := range map[string]struct {
|
||||
prior PriorAlertState
|
||||
duplicate bool
|
||||
want int
|
||||
}{
|
||||
"fresh firing": {prior: PriorAlertState{State: alert.StateInactive}, want: 1},
|
||||
"cooldown active": {prior: PriorAlertState{State: alert.StateResolved, CooldownUntil: &cooldown, Found: true}, want: 0},
|
||||
"already firing": {prior: PriorAlertState{State: alert.StateFiring, Found: true}, want: 0},
|
||||
"duplicate window": {prior: PriorAlertState{State: alert.StateInactive}, duplicate: true, want: 0},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
outbox := newFakeOutboxStore()
|
||||
queue := &fakeNotificationQueue{outbox: outbox, channels: []notification.Channel{{ID: "channel-1", Type: "memory", Enabled: true}}}
|
||||
evaluator := &AlertEvaluator{Metrics: fakeMetricSource{value: MetricValue{Value: 91, Known: true}},
|
||||
States: &fakeStateWriter{to: alert.StateFiring, duplicate: testCase.duplicate}, Prior: fakePrior{state: testCase.prior},
|
||||
Versions: fakeVersions{}, Notifications: queue, Logger: quietLogger(), Now: func() time.Time { return now }}
|
||||
if err := evaluator.Evaluate(context.Background(), ruleFixture()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queued := outbox.count(); queued != testCase.want {
|
||||
t.Fatalf("queued = %d, want %d", queued, testCase.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluationKeyAndFingerprintAreStable(t *testing.T) {
|
||||
rule := ruleFixture()
|
||||
base := time.Date(2026, 8, 4, 12, 0, 30, 0, time.UTC)
|
||||
if EvaluationKey(rule, base) != EvaluationKey(rule, base.Add(20*time.Second)) {
|
||||
t.Fatal("evaluation key changed inside one interval")
|
||||
}
|
||||
if EvaluationKey(rule, base) == EvaluationKey(rule, base.Add(time.Minute)) {
|
||||
t.Fatal("evaluation key did not change between intervals")
|
||||
}
|
||||
scoped := ruleFixture()
|
||||
scoped.Scope = map[string]any{"host": "tower"}
|
||||
if RuleFingerprint(rule) == RuleFingerprint(scoped) {
|
||||
t.Fatal("scope is not part of the fingerprint")
|
||||
}
|
||||
repeated, again := RuleFingerprint(scoped), RuleFingerprint(scoped)
|
||||
if repeated != again || len(repeated) == 0 || len(repeated) > 160 {
|
||||
t.Fatal("fingerprint is unstable or unbounded")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeNotificationQueue struct {
|
||||
outbox *fakeOutboxStore
|
||||
channels []notification.Channel
|
||||
}
|
||||
|
||||
func (q *fakeNotificationQueue) Enqueue(ctx context.Context, item notification.Outbox) (notification.Outbox, bool, error) {
|
||||
return q.outbox.Enqueue(ctx, item)
|
||||
}
|
||||
func (q *fakeNotificationQueue) ListChannels(context.Context, int) ([]notification.Channel, error) {
|
||||
return q.channels, nil
|
||||
}
|
||||
|
||||
func (s *fakeOutboxStore) count() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return len(s.items)
|
||||
}
|
||||
func (s *fakeOutboxStore) all() []notification.Outbox {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
items := make([]notification.Outbox, 0, len(s.items))
|
||||
for _, item := range s.items {
|
||||
items = append(items, *item)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func TestAlertEvaluationJobReportsFailuresAndDisabledState(t *testing.T) {
|
||||
outcome, err := AlertEvaluationJob{}.Run(context.Background())
|
||||
if err != nil || !outcome.Disabled || outcome.Reason != "alert_evaluation_not_configured" {
|
||||
t.Fatalf("outcome = %#v err = %v", outcome, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user