This commit is contained in:
@@ -0,0 +1,261 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateInactive State = "inactive"
|
||||
StatePending State = "pending"
|
||||
StateFiring State = "firing"
|
||||
StateAcknowledged State = "acknowledged"
|
||||
StateResolved State = "resolved"
|
||||
StateUnknown State = "unknown"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidObservation = errors.New("invalid alert observation")
|
||||
ErrStaleObservation = errors.New("stale alert observation")
|
||||
ErrInstanceNotFound = errors.New("alert instance not found")
|
||||
ErrStateConflict = errors.New("alert instance state conflict")
|
||||
ErrRevisionConflict = errors.New("alert instance revision conflict")
|
||||
)
|
||||
|
||||
type Policy struct {
|
||||
PendingSeconds int
|
||||
ResolveSeconds int
|
||||
CooldownSeconds int
|
||||
UnknownBehavior string
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
State State
|
||||
RetainedState State
|
||||
ActiveSince *time.Time
|
||||
RecoverySince *time.Time
|
||||
CooldownUntil *time.Time
|
||||
LastEvaluatedAt time.Time
|
||||
LastKnownAt *time.Time
|
||||
LastValue any
|
||||
Reason string
|
||||
SourceHealth map[string]any
|
||||
AcknowledgedBy string
|
||||
AcknowledgedAt *time.Time
|
||||
}
|
||||
|
||||
type Notification string
|
||||
|
||||
const (
|
||||
NotificationNone Notification = ""
|
||||
NotificationFiring Notification = "firing"
|
||||
NotificationRecovery Notification = "recovery"
|
||||
NotificationUnknown Notification = "unknown"
|
||||
)
|
||||
|
||||
type TransitionResult struct {
|
||||
Snapshot Snapshot
|
||||
From State
|
||||
To State
|
||||
EventType string
|
||||
Notification Notification
|
||||
}
|
||||
type Observation struct {
|
||||
EvaluationKey string
|
||||
ObservedAt time.Time
|
||||
ConditionTrue bool
|
||||
Unknown bool
|
||||
Value any
|
||||
Reason string
|
||||
SourceHealth map[string]any
|
||||
}
|
||||
|
||||
func (s Snapshot) normalized() Snapshot {
|
||||
if s.State == "" {
|
||||
s.State = StateInactive
|
||||
}
|
||||
if s.RetainedState == "" {
|
||||
s.RetainedState = s.State
|
||||
}
|
||||
if s.SourceHealth == nil {
|
||||
s.SourceHealth = map[string]any{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func Transition(current Snapshot, policy Policy, observation Observation) (TransitionResult, error) {
|
||||
current = current.normalized()
|
||||
if observation.ObservedAt.IsZero() || observation.EvaluationKey == "" || len(observation.EvaluationKey) > 160 {
|
||||
return TransitionResult{}, ErrInvalidObservation
|
||||
}
|
||||
if !validState(current.State) || !validState(current.RetainedState) {
|
||||
return TransitionResult{}, fmt.Errorf("%w: invalid current state", ErrInvalidObservation)
|
||||
}
|
||||
if policy.PendingSeconds < 0 || policy.ResolveSeconds < 0 || policy.CooldownSeconds < 0 || policy.CooldownSeconds > 2592000 || policy.UnknownBehavior == "" {
|
||||
return TransitionResult{}, fmt.Errorf("%w: invalid policy", ErrInvalidObservation)
|
||||
}
|
||||
at := observation.ObservedAt.UTC()
|
||||
if !current.LastEvaluatedAt.IsZero() && at.Before(current.LastEvaluatedAt.UTC()) {
|
||||
return TransitionResult{}, ErrStaleObservation
|
||||
}
|
||||
result := current
|
||||
result.LastEvaluatedAt = at
|
||||
result.Reason = boundedReason(observation.Reason)
|
||||
result.SourceHealth = cloneMap(observation.SourceHealth)
|
||||
if observation.Unknown {
|
||||
if policy.UnknownBehavior == UnknownIgnoreGap {
|
||||
result.Reason = "unknown_input_ignored_short_gap"
|
||||
return finish(current, result, State(current.State), "evaluation"), nil
|
||||
}
|
||||
result.RetainedState = current.State
|
||||
if current.State == StateUnknown && current.RetainedState != StateUnknown {
|
||||
result.RetainedState = current.RetainedState
|
||||
}
|
||||
result.State = StateUnknown
|
||||
transition := finish(current, result, StateUnknown, "transition")
|
||||
transition.Notification = notificationFor(current, transition, at)
|
||||
return transition, nil
|
||||
}
|
||||
|
||||
base := current.State
|
||||
if base == StateUnknown {
|
||||
base = current.RetainedState
|
||||
if !validState(base) || base == StateUnknown {
|
||||
base = StateInactive
|
||||
}
|
||||
result.State = base
|
||||
}
|
||||
result.RetainedState = base
|
||||
result.LastKnownAt = timePtr(at)
|
||||
result.LastValue = observation.Value
|
||||
if observation.ConditionTrue {
|
||||
result.RecoverySince = nil
|
||||
switch base {
|
||||
case StateInactive, StateResolved:
|
||||
result.ActiveSince = timePtr(at)
|
||||
if policy.PendingSeconds == 0 {
|
||||
result.State = StateFiring
|
||||
} else {
|
||||
result.State = StatePending
|
||||
}
|
||||
case StatePending:
|
||||
if result.ActiveSince == nil {
|
||||
result.ActiveSince = timePtr(at)
|
||||
}
|
||||
if at.Sub(result.ActiveSince.UTC()) >= time.Duration(policy.PendingSeconds)*time.Second {
|
||||
result.State = StateFiring
|
||||
}
|
||||
case StateFiring, StateAcknowledged:
|
||||
result.State = base
|
||||
default:
|
||||
result.State = StateInactive
|
||||
}
|
||||
} else {
|
||||
result.ActiveSince = current.ActiveSince
|
||||
switch base {
|
||||
case StatePending:
|
||||
result.State = StateInactive
|
||||
result.ActiveSince = nil
|
||||
case StateFiring, StateAcknowledged:
|
||||
if result.RecoverySince == nil {
|
||||
result.RecoverySince = timePtr(at)
|
||||
}
|
||||
if at.Sub(result.RecoverySince.UTC()) >= time.Duration(policy.ResolveSeconds)*time.Second {
|
||||
result.State = StateResolved
|
||||
result.CooldownUntil = timePtr(at.Add(time.Duration(policy.CooldownSeconds) * time.Second))
|
||||
result.ActiveSince = nil
|
||||
result.RecoverySince = nil
|
||||
}
|
||||
default:
|
||||
result.State = StateInactive
|
||||
result.ActiveSince = nil
|
||||
result.RecoverySince = nil
|
||||
}
|
||||
}
|
||||
transition := finish(current, result, result.State, stateEvent(current.State, result.State))
|
||||
transition.Notification = notificationFor(current, transition, at)
|
||||
return transition, nil
|
||||
}
|
||||
|
||||
func Acknowledge(current Snapshot, actor string, at time.Time) (TransitionResult, error) {
|
||||
current = current.normalized()
|
||||
if actor == "" || len(actor) > 160 || at.IsZero() {
|
||||
return TransitionResult{}, ErrInvalidObservation
|
||||
}
|
||||
if current.State != StateFiring && current.State != StatePending {
|
||||
return TransitionResult{}, ErrStateConflict
|
||||
}
|
||||
result := current
|
||||
result.State = StateAcknowledged
|
||||
result.RetainedState = StateAcknowledged
|
||||
result.AcknowledgedBy = actor
|
||||
result.AcknowledgedAt = timePtr(at.UTC())
|
||||
result.Reason = "acknowledged"
|
||||
return finish(current, result, StateAcknowledged, "acknowledge"), nil
|
||||
}
|
||||
|
||||
func finish(current, result Snapshot, state State, eventType string) TransitionResult {
|
||||
result.State = state
|
||||
if result.SourceHealth == nil {
|
||||
result.SourceHealth = map[string]any{}
|
||||
}
|
||||
return TransitionResult{Snapshot: result, From: current.State, To: state, EventType: eventType}
|
||||
}
|
||||
|
||||
func notificationFor(current Snapshot, result TransitionResult, at time.Time) Notification {
|
||||
switch {
|
||||
case result.To == StateFiring && current.State != StateFiring && current.State != StateAcknowledged:
|
||||
if current.CooldownUntil != nil && at.Before(current.CooldownUntil.UTC()) {
|
||||
return NotificationNone
|
||||
}
|
||||
return NotificationFiring
|
||||
case result.To == StateResolved && (current.State == StateFiring || current.State == StateAcknowledged):
|
||||
return NotificationRecovery
|
||||
case result.To == StateUnknown && current.State != StateUnknown:
|
||||
return NotificationUnknown
|
||||
default:
|
||||
return NotificationNone
|
||||
}
|
||||
}
|
||||
|
||||
func stateEvent(from, to State) string {
|
||||
if from == to {
|
||||
return "evaluation"
|
||||
}
|
||||
return "transition"
|
||||
}
|
||||
|
||||
func validState(state State) bool {
|
||||
switch state {
|
||||
case StateInactive, StatePending, StateFiring, StateAcknowledged, StateResolved, StateUnknown:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func boundedReason(reason string) string {
|
||||
if len(reason) > 500 {
|
||||
return reason[:500]
|
||||
}
|
||||
return reason
|
||||
}
|
||||
|
||||
func cloneMap(source map[string]any) map[string]any {
|
||||
if source == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
copy := make(map[string]any, len(source))
|
||||
for key, value := range source {
|
||||
copy[key] = value
|
||||
}
|
||||
return copy
|
||||
}
|
||||
|
||||
func timePtr(value time.Time) *time.Time {
|
||||
value = value.UTC()
|
||||
return &value
|
||||
}
|
||||
Reference in New Issue
Block a user