Public source validation / validate (push) Failing after 3m8s
262 lines
7.3 KiB
Go
262 lines
7.3 KiB
Go
package alert
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
)
|
|
|
|
type State string
|
|
|
|
const (
|
|
StateInactive State = "inactive"
|
|
StatePending State = "pending"
|
|
StateFiring State = "firing"
|
|
StateAcknowledged State = "acknowledged"
|
|
StateResolved State = "resolved"
|
|
StateUnknown State = "unknown"
|
|
)
|
|
|
|
var (
|
|
ErrInvalidObservation = errors.New("invalid alert observation")
|
|
ErrStaleObservation = errors.New("stale alert observation")
|
|
ErrInstanceNotFound = errors.New("alert instance not found")
|
|
ErrStateConflict = errors.New("alert instance state conflict")
|
|
ErrRevisionConflict = errors.New("alert instance revision conflict")
|
|
)
|
|
|
|
type Policy struct {
|
|
PendingSeconds int
|
|
ResolveSeconds int
|
|
CooldownSeconds int
|
|
UnknownBehavior string
|
|
}
|
|
|
|
type Snapshot struct {
|
|
State State
|
|
RetainedState State
|
|
ActiveSince *time.Time
|
|
RecoverySince *time.Time
|
|
CooldownUntil *time.Time
|
|
LastEvaluatedAt time.Time
|
|
LastKnownAt *time.Time
|
|
LastValue any
|
|
Reason string
|
|
SourceHealth map[string]any
|
|
AcknowledgedBy string
|
|
AcknowledgedAt *time.Time
|
|
}
|
|
|
|
type Notification string
|
|
|
|
const (
|
|
NotificationNone Notification = ""
|
|
NotificationFiring Notification = "firing"
|
|
NotificationRecovery Notification = "recovery"
|
|
NotificationUnknown Notification = "unknown"
|
|
)
|
|
|
|
type TransitionResult struct {
|
|
Snapshot Snapshot
|
|
From State
|
|
To State
|
|
EventType string
|
|
Notification Notification
|
|
}
|
|
type Observation struct {
|
|
EvaluationKey string
|
|
ObservedAt time.Time
|
|
ConditionTrue bool
|
|
Unknown bool
|
|
Value any
|
|
Reason string
|
|
SourceHealth map[string]any
|
|
}
|
|
|
|
func (s Snapshot) normalized() Snapshot {
|
|
if s.State == "" {
|
|
s.State = StateInactive
|
|
}
|
|
if s.RetainedState == "" {
|
|
s.RetainedState = s.State
|
|
}
|
|
if s.SourceHealth == nil {
|
|
s.SourceHealth = map[string]any{}
|
|
}
|
|
return s
|
|
}
|
|
|
|
func Transition(current Snapshot, policy Policy, observation Observation) (TransitionResult, error) {
|
|
current = current.normalized()
|
|
if observation.ObservedAt.IsZero() || observation.EvaluationKey == "" || len(observation.EvaluationKey) > 160 {
|
|
return TransitionResult{}, ErrInvalidObservation
|
|
}
|
|
if !validState(current.State) || !validState(current.RetainedState) {
|
|
return TransitionResult{}, fmt.Errorf("%w: invalid current state", ErrInvalidObservation)
|
|
}
|
|
if policy.PendingSeconds < 0 || policy.ResolveSeconds < 0 || policy.CooldownSeconds < 0 || policy.CooldownSeconds > 2592000 || policy.UnknownBehavior == "" {
|
|
return TransitionResult{}, fmt.Errorf("%w: invalid policy", ErrInvalidObservation)
|
|
}
|
|
at := observation.ObservedAt.UTC()
|
|
if !current.LastEvaluatedAt.IsZero() && at.Before(current.LastEvaluatedAt.UTC()) {
|
|
return TransitionResult{}, ErrStaleObservation
|
|
}
|
|
result := current
|
|
result.LastEvaluatedAt = at
|
|
result.Reason = boundedReason(observation.Reason)
|
|
result.SourceHealth = cloneMap(observation.SourceHealth)
|
|
if observation.Unknown {
|
|
if policy.UnknownBehavior == UnknownIgnoreGap {
|
|
result.Reason = "unknown_input_ignored_short_gap"
|
|
return finish(current, result, State(current.State), "evaluation"), nil
|
|
}
|
|
result.RetainedState = current.State
|
|
if current.State == StateUnknown && current.RetainedState != StateUnknown {
|
|
result.RetainedState = current.RetainedState
|
|
}
|
|
result.State = StateUnknown
|
|
transition := finish(current, result, StateUnknown, "transition")
|
|
transition.Notification = notificationFor(current, transition, at)
|
|
return transition, nil
|
|
}
|
|
|
|
base := current.State
|
|
if base == StateUnknown {
|
|
base = current.RetainedState
|
|
if !validState(base) || base == StateUnknown {
|
|
base = StateInactive
|
|
}
|
|
result.State = base
|
|
}
|
|
result.RetainedState = base
|
|
result.LastKnownAt = timePtr(at)
|
|
result.LastValue = observation.Value
|
|
if observation.ConditionTrue {
|
|
result.RecoverySince = nil
|
|
switch base {
|
|
case StateInactive, StateResolved:
|
|
result.ActiveSince = timePtr(at)
|
|
if policy.PendingSeconds == 0 {
|
|
result.State = StateFiring
|
|
} else {
|
|
result.State = StatePending
|
|
}
|
|
case StatePending:
|
|
if result.ActiveSince == nil {
|
|
result.ActiveSince = timePtr(at)
|
|
}
|
|
if at.Sub(result.ActiveSince.UTC()) >= time.Duration(policy.PendingSeconds)*time.Second {
|
|
result.State = StateFiring
|
|
}
|
|
case StateFiring, StateAcknowledged:
|
|
result.State = base
|
|
default:
|
|
result.State = StateInactive
|
|
}
|
|
} else {
|
|
result.ActiveSince = current.ActiveSince
|
|
switch base {
|
|
case StatePending:
|
|
result.State = StateInactive
|
|
result.ActiveSince = nil
|
|
case StateFiring, StateAcknowledged:
|
|
if result.RecoverySince == nil {
|
|
result.RecoverySince = timePtr(at)
|
|
}
|
|
if at.Sub(result.RecoverySince.UTC()) >= time.Duration(policy.ResolveSeconds)*time.Second {
|
|
result.State = StateResolved
|
|
result.CooldownUntil = timePtr(at.Add(time.Duration(policy.CooldownSeconds) * time.Second))
|
|
result.ActiveSince = nil
|
|
result.RecoverySince = nil
|
|
}
|
|
default:
|
|
result.State = StateInactive
|
|
result.ActiveSince = nil
|
|
result.RecoverySince = nil
|
|
}
|
|
}
|
|
transition := finish(current, result, result.State, stateEvent(current.State, result.State))
|
|
transition.Notification = notificationFor(current, transition, at)
|
|
return transition, nil
|
|
}
|
|
|
|
func Acknowledge(current Snapshot, actor string, at time.Time) (TransitionResult, error) {
|
|
current = current.normalized()
|
|
if actor == "" || len(actor) > 160 || at.IsZero() {
|
|
return TransitionResult{}, ErrInvalidObservation
|
|
}
|
|
if current.State != StateFiring && current.State != StatePending {
|
|
return TransitionResult{}, ErrStateConflict
|
|
}
|
|
result := current
|
|
result.State = StateAcknowledged
|
|
result.RetainedState = StateAcknowledged
|
|
result.AcknowledgedBy = actor
|
|
result.AcknowledgedAt = timePtr(at.UTC())
|
|
result.Reason = "acknowledged"
|
|
return finish(current, result, StateAcknowledged, "acknowledge"), nil
|
|
}
|
|
|
|
func finish(current, result Snapshot, state State, eventType string) TransitionResult {
|
|
result.State = state
|
|
if result.SourceHealth == nil {
|
|
result.SourceHealth = map[string]any{}
|
|
}
|
|
return TransitionResult{Snapshot: result, From: current.State, To: state, EventType: eventType}
|
|
}
|
|
|
|
func notificationFor(current Snapshot, result TransitionResult, at time.Time) Notification {
|
|
switch {
|
|
case result.To == StateFiring && current.State != StateFiring && current.State != StateAcknowledged:
|
|
if current.CooldownUntil != nil && at.Before(current.CooldownUntil.UTC()) {
|
|
return NotificationNone
|
|
}
|
|
return NotificationFiring
|
|
case result.To == StateResolved && (current.State == StateFiring || current.State == StateAcknowledged):
|
|
return NotificationRecovery
|
|
case result.To == StateUnknown && current.State != StateUnknown:
|
|
return NotificationUnknown
|
|
default:
|
|
return NotificationNone
|
|
}
|
|
}
|
|
|
|
func stateEvent(from, to State) string {
|
|
if from == to {
|
|
return "evaluation"
|
|
}
|
|
return "transition"
|
|
}
|
|
|
|
func validState(state State) bool {
|
|
switch state {
|
|
case StateInactive, StatePending, StateFiring, StateAcknowledged, StateResolved, StateUnknown:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func boundedReason(reason string) string {
|
|
if len(reason) > 500 {
|
|
return reason[:500]
|
|
}
|
|
return reason
|
|
}
|
|
|
|
func cloneMap(source map[string]any) map[string]any {
|
|
if source == nil {
|
|
return map[string]any{}
|
|
}
|
|
copy := make(map[string]any, len(source))
|
|
for key, value := range source {
|
|
copy[key] = value
|
|
}
|
|
return copy
|
|
}
|
|
|
|
func timePtr(value time.Time) *time.Time {
|
|
value = value.UTC()
|
|
return &value
|
|
}
|