Files
ITWorx Pulse release export bd774932d5
Public source validation / validate (push) Failing after 3m8s
Publish ITWorx Pulse source
2026-09-03 02:09:19 +02:00

262 lines
7.3 KiB
Go

package alert
import (
"errors"
"fmt"
"time"
)
type State string
const (
StateInactive State = "inactive"
StatePending State = "pending"
StateFiring State = "firing"
StateAcknowledged State = "acknowledged"
StateResolved State = "resolved"
StateUnknown State = "unknown"
)
var (
ErrInvalidObservation = errors.New("invalid alert observation")
ErrStaleObservation = errors.New("stale alert observation")
ErrInstanceNotFound = errors.New("alert instance not found")
ErrStateConflict = errors.New("alert instance state conflict")
ErrRevisionConflict = errors.New("alert instance revision conflict")
)
type Policy struct {
PendingSeconds int
ResolveSeconds int
CooldownSeconds int
UnknownBehavior string
}
type Snapshot struct {
State State
RetainedState State
ActiveSince *time.Time
RecoverySince *time.Time
CooldownUntil *time.Time
LastEvaluatedAt time.Time
LastKnownAt *time.Time
LastValue any
Reason string
SourceHealth map[string]any
AcknowledgedBy string
AcknowledgedAt *time.Time
}
type Notification string
const (
NotificationNone Notification = ""
NotificationFiring Notification = "firing"
NotificationRecovery Notification = "recovery"
NotificationUnknown Notification = "unknown"
)
type TransitionResult struct {
Snapshot Snapshot
From State
To State
EventType string
Notification Notification
}
type Observation struct {
EvaluationKey string
ObservedAt time.Time
ConditionTrue bool
Unknown bool
Value any
Reason string
SourceHealth map[string]any
}
func (s Snapshot) normalized() Snapshot {
if s.State == "" {
s.State = StateInactive
}
if s.RetainedState == "" {
s.RetainedState = s.State
}
if s.SourceHealth == nil {
s.SourceHealth = map[string]any{}
}
return s
}
func Transition(current Snapshot, policy Policy, observation Observation) (TransitionResult, error) {
current = current.normalized()
if observation.ObservedAt.IsZero() || observation.EvaluationKey == "" || len(observation.EvaluationKey) > 160 {
return TransitionResult{}, ErrInvalidObservation
}
if !validState(current.State) || !validState(current.RetainedState) {
return TransitionResult{}, fmt.Errorf("%w: invalid current state", ErrInvalidObservation)
}
if policy.PendingSeconds < 0 || policy.ResolveSeconds < 0 || policy.CooldownSeconds < 0 || policy.CooldownSeconds > 2592000 || policy.UnknownBehavior == "" {
return TransitionResult{}, fmt.Errorf("%w: invalid policy", ErrInvalidObservation)
}
at := observation.ObservedAt.UTC()
if !current.LastEvaluatedAt.IsZero() && at.Before(current.LastEvaluatedAt.UTC()) {
return TransitionResult{}, ErrStaleObservation
}
result := current
result.LastEvaluatedAt = at
result.Reason = boundedReason(observation.Reason)
result.SourceHealth = cloneMap(observation.SourceHealth)
if observation.Unknown {
if policy.UnknownBehavior == UnknownIgnoreGap {
result.Reason = "unknown_input_ignored_short_gap"
return finish(current, result, State(current.State), "evaluation"), nil
}
result.RetainedState = current.State
if current.State == StateUnknown && current.RetainedState != StateUnknown {
result.RetainedState = current.RetainedState
}
result.State = StateUnknown
transition := finish(current, result, StateUnknown, "transition")
transition.Notification = notificationFor(current, transition, at)
return transition, nil
}
base := current.State
if base == StateUnknown {
base = current.RetainedState
if !validState(base) || base == StateUnknown {
base = StateInactive
}
result.State = base
}
result.RetainedState = base
result.LastKnownAt = timePtr(at)
result.LastValue = observation.Value
if observation.ConditionTrue {
result.RecoverySince = nil
switch base {
case StateInactive, StateResolved:
result.ActiveSince = timePtr(at)
if policy.PendingSeconds == 0 {
result.State = StateFiring
} else {
result.State = StatePending
}
case StatePending:
if result.ActiveSince == nil {
result.ActiveSince = timePtr(at)
}
if at.Sub(result.ActiveSince.UTC()) >= time.Duration(policy.PendingSeconds)*time.Second {
result.State = StateFiring
}
case StateFiring, StateAcknowledged:
result.State = base
default:
result.State = StateInactive
}
} else {
result.ActiveSince = current.ActiveSince
switch base {
case StatePending:
result.State = StateInactive
result.ActiveSince = nil
case StateFiring, StateAcknowledged:
if result.RecoverySince == nil {
result.RecoverySince = timePtr(at)
}
if at.Sub(result.RecoverySince.UTC()) >= time.Duration(policy.ResolveSeconds)*time.Second {
result.State = StateResolved
result.CooldownUntil = timePtr(at.Add(time.Duration(policy.CooldownSeconds) * time.Second))
result.ActiveSince = nil
result.RecoverySince = nil
}
default:
result.State = StateInactive
result.ActiveSince = nil
result.RecoverySince = nil
}
}
transition := finish(current, result, result.State, stateEvent(current.State, result.State))
transition.Notification = notificationFor(current, transition, at)
return transition, nil
}
func Acknowledge(current Snapshot, actor string, at time.Time) (TransitionResult, error) {
current = current.normalized()
if actor == "" || len(actor) > 160 || at.IsZero() {
return TransitionResult{}, ErrInvalidObservation
}
if current.State != StateFiring && current.State != StatePending {
return TransitionResult{}, ErrStateConflict
}
result := current
result.State = StateAcknowledged
result.RetainedState = StateAcknowledged
result.AcknowledgedBy = actor
result.AcknowledgedAt = timePtr(at.UTC())
result.Reason = "acknowledged"
return finish(current, result, StateAcknowledged, "acknowledge"), nil
}
func finish(current, result Snapshot, state State, eventType string) TransitionResult {
result.State = state
if result.SourceHealth == nil {
result.SourceHealth = map[string]any{}
}
return TransitionResult{Snapshot: result, From: current.State, To: state, EventType: eventType}
}
func notificationFor(current Snapshot, result TransitionResult, at time.Time) Notification {
switch {
case result.To == StateFiring && current.State != StateFiring && current.State != StateAcknowledged:
if current.CooldownUntil != nil && at.Before(current.CooldownUntil.UTC()) {
return NotificationNone
}
return NotificationFiring
case result.To == StateResolved && (current.State == StateFiring || current.State == StateAcknowledged):
return NotificationRecovery
case result.To == StateUnknown && current.State != StateUnknown:
return NotificationUnknown
default:
return NotificationNone
}
}
func stateEvent(from, to State) string {
if from == to {
return "evaluation"
}
return "transition"
}
func validState(state State) bool {
switch state {
case StateInactive, StatePending, StateFiring, StateAcknowledged, StateResolved, StateUnknown:
return true
default:
return false
}
}
func boundedReason(reason string) string {
if len(reason) > 500 {
return reason[:500]
}
return reason
}
func cloneMap(source map[string]any) map[string]any {
if source == nil {
return map[string]any{}
}
copy := make(map[string]any, len(source))
for key, value := range source {
copy[key] = value
}
return copy
}
func timePtr(value time.Time) *time.Time {
value = value.UTC()
return &value
}