Public source validation / validate (push) Failing after 3m8s
168 lines
7.8 KiB
Go
168 lines
7.8 KiB
Go
package alert
|
|
|
|
import (
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func testPolicy() Policy {
|
|
return Policy{PendingSeconds: 60, ResolveSeconds: 30, UnknownBehavior: UnknownRetain}
|
|
}
|
|
|
|
func observation(at time.Time, key string, fire bool) Observation {
|
|
return Observation{EvaluationKey: key, ObservedAt: at, ConditionTrue: fire, Value: 90, Reason: "condition_evaluated", SourceHealth: map[string]any{"source": "test"}}
|
|
}
|
|
|
|
func TestStateTransitionTable(t *testing.T) {
|
|
start := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
|
tests := []struct {
|
|
name string
|
|
state State
|
|
at time.Time
|
|
fire bool
|
|
want State
|
|
}{
|
|
{name: "inactive enters pending", state: StateInactive, at: start, fire: true, want: StatePending},
|
|
{name: "pending remains pending", state: StatePending, at: start.Add(30 * time.Second), fire: true, want: StatePending},
|
|
{name: "pending fires after duration", state: StatePending, at: start.Add(60 * time.Second), fire: true, want: StateFiring},
|
|
{name: "pending clears", state: StatePending, at: start.Add(30 * time.Second), fire: false, want: StateInactive},
|
|
{name: "firing starts recovery", state: StateFiring, at: start, fire: false, want: StateFiring},
|
|
{name: "firing resolves after duration", state: StateFiring, at: start.Add(30 * time.Second), fire: false, want: StateResolved},
|
|
{name: "resolved reopens", state: StateResolved, at: start, fire: true, want: StatePending},
|
|
{name: "acknowledged remains firing", state: StateAcknowledged, at: start, fire: true, want: StateAcknowledged},
|
|
{name: "acknowledged starts recovery", state: StateAcknowledged, at: start, fire: false, want: StateAcknowledged},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
current := Snapshot{State: test.state, RetainedState: test.state}
|
|
if test.state == StatePending {
|
|
current.ActiveSince = timePtr(start)
|
|
}
|
|
if test.state == StateFiring || test.state == StateAcknowledged {
|
|
current.RecoverySince = timePtr(start)
|
|
}
|
|
result, err := Transition(current, testPolicy(), observation(test.at, "slot-"+test.name, test.fire))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.To != test.want {
|
|
t.Fatalf("state = %s, want %s", result.To, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAcknowledgementIsExplicitAndPreservesFiringContext(t *testing.T) {
|
|
at := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
|
result, err := Acknowledge(Snapshot{State: StateFiring, RetainedState: StateFiring, LastValue: 92}, "operator", at)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.To != StateAcknowledged || result.Snapshot.LastValue != 92 || result.Snapshot.AcknowledgedBy != "operator" {
|
|
t.Fatalf("unexpected acknowledgement: %#v", result)
|
|
}
|
|
later, err := Transition(result.Snapshot, testPolicy(), observation(at.Add(time.Second), "slot-ack", true))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if later.To != StateAcknowledged {
|
|
t.Fatalf("acknowledged alert did not remain acknowledged while firing: %#v", later)
|
|
}
|
|
}
|
|
|
|
func TestUnknownDoesNotFalseResolveAndRecoveryUsesRetainedState(t *testing.T) {
|
|
at := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
|
unknown, err := Transition(Snapshot{State: StateFiring, RetainedState: StateFiring, LastKnownAt: timePtr(at), LastValue: 91}, testPolicy(), Observation{EvaluationKey: "unknown", ObservedAt: at.Add(time.Minute), Unknown: true, Reason: "source_stale", SourceHealth: map[string]any{"state": "unknown"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if unknown.To != StateUnknown || unknown.Snapshot.RetainedState != StateFiring || unknown.Snapshot.LastValue != 91 {
|
|
t.Fatalf("unknown transition lost firing context: %#v", unknown)
|
|
}
|
|
recovered, err := Transition(unknown.Snapshot, testPolicy(), observation(at.Add(2*time.Minute), "recovery", false))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if recovered.To != StateFiring {
|
|
t.Fatalf("unknown recovery falsely resolved immediately: %#v", recovered)
|
|
}
|
|
}
|
|
|
|
func TestStateRejectsOutOfOrderObservation(t *testing.T) {
|
|
at := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
|
_, err := Transition(Snapshot{State: StateFiring, RetainedState: StateFiring, LastEvaluatedAt: at}, testPolicy(), observation(at.Add(-time.Second), "old", false))
|
|
if !errors.Is(err, ErrStaleObservation) {
|
|
t.Fatalf("error = %v, want ErrStaleObservation", err)
|
|
}
|
|
}
|
|
|
|
func TestIgnoreUnknownGapPreservesState(t *testing.T) {
|
|
at := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
|
policy := testPolicy()
|
|
policy.UnknownBehavior = UnknownIgnoreGap
|
|
result, err := Transition(Snapshot{State: StateFiring, RetainedState: StateFiring, LastEvaluatedAt: at, LastValue: 80}, policy, Observation{EvaluationKey: "gap", ObservedAt: at.Add(time.Second), Unknown: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.To != StateFiring || result.Snapshot.LastValue != 80 {
|
|
t.Fatalf("unknown gap changed state: %#v", result)
|
|
}
|
|
}
|
|
|
|
func TestDiskTemperatureScenarioUsesPendingRecoveryAndCooldownDeterministically(t *testing.T) {
|
|
start := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
|
policy := Policy{PendingSeconds: 300, ResolveSeconds: 300, UnknownBehavior: UnknownRetain}
|
|
current := Snapshot{State: StateInactive, RetainedState: StateInactive}
|
|
transition, err := Transition(current, policy, observation(start, "disk-1", true))
|
|
if err != nil || transition.To != StatePending {
|
|
t.Fatalf("initial transition = %#v, %v", transition, err)
|
|
}
|
|
current = transition.Snapshot
|
|
transition, err = Transition(current, policy, observation(start.Add(299*time.Second), "disk-2", true))
|
|
if err != nil || transition.To != StatePending {
|
|
t.Fatalf("boundary pending transition = %#v, %v", transition, err)
|
|
}
|
|
current = transition.Snapshot
|
|
transition, err = Transition(current, policy, observation(start.Add(300*time.Second), "disk-3", true))
|
|
if err != nil || transition.To != StateFiring {
|
|
t.Fatalf("fire transition = %#v, %v", transition, err)
|
|
}
|
|
current = transition.Snapshot
|
|
transition, err = Transition(current, policy, observation(start.Add(330*time.Second), "disk-4", true))
|
|
if err != nil || transition.To != StateFiring {
|
|
t.Fatalf("temperature at recovery threshold changed state: %#v, %v", transition, err)
|
|
}
|
|
current = transition.Snapshot
|
|
transition, err = Transition(current, policy, observation(start.Add(360*time.Second), "disk-5", false))
|
|
if err != nil || transition.To != StateFiring {
|
|
t.Fatalf("recovery start transition = %#v, %v", transition, err)
|
|
}
|
|
current = transition.Snapshot
|
|
transition, err = Transition(current, policy, observation(start.Add(659*time.Second), "disk-6", false))
|
|
if err != nil || transition.To != StateFiring {
|
|
t.Fatalf("pre-recovery boundary transition = %#v, %v", transition, err)
|
|
}
|
|
transition, err = Transition(transition.Snapshot, policy, observation(start.Add(660*time.Second), "disk-7", false))
|
|
if err != nil || transition.To != StateResolved {
|
|
t.Fatalf("recovery boundary transition = %#v, %v", transition, err)
|
|
}
|
|
}
|
|
|
|
func TestCooldownSuppressesRepeatedFiringNotification(t *testing.T) {
|
|
start := time.Date(2026, time.January, 1, 12, 0, 0, 0, time.UTC)
|
|
policy := Policy{PendingSeconds: 0, ResolveSeconds: 0, CooldownSeconds: 60, UnknownBehavior: UnknownRetain}
|
|
first, err := Transition(Snapshot{State: StateInactive, RetainedState: StateInactive}, policy, observation(start, "cool-1", true))
|
|
if err != nil || first.Notification != NotificationFiring {
|
|
t.Fatalf("first notification = %#v, %v", first, err)
|
|
}
|
|
resolved, err := Transition(first.Snapshot, policy, observation(start.Add(time.Second), "cool-2", false))
|
|
if err != nil || resolved.To != StateResolved || resolved.Notification != NotificationRecovery || resolved.Snapshot.CooldownUntil == nil {
|
|
t.Fatalf("resolve notification = %#v, %v", resolved, err)
|
|
}
|
|
suppressed, err := Transition(resolved.Snapshot, policy, observation(start.Add(30*time.Second), "cool-3", true))
|
|
if err != nil || suppressed.To != StateFiring || suppressed.Notification != NotificationNone {
|
|
t.Fatalf("cooldown firing notification = %#v, %v", suppressed, err)
|
|
}
|
|
}
|