This commit is contained in:
@@ -0,0 +1,230 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxFingerprintLabels = 10
|
||||
MaxAlertGroups = 1000
|
||||
MaxSignalsPerGroup = 500
|
||||
)
|
||||
|
||||
var ErrInvalidAlertIdentity = errors.New("invalid alert identity")
|
||||
var alertLabelPattern = regexp.MustCompile("^[A-Za-z0-9_.:/-]+$")
|
||||
|
||||
type Signal struct {
|
||||
InstanceID string
|
||||
RuleID string
|
||||
RuleVersionID string
|
||||
EntityID string
|
||||
Severity string
|
||||
State State
|
||||
Fingerprint string
|
||||
EvaluationKey string
|
||||
ObservedAt time.Time
|
||||
Labels map[string]string
|
||||
GroupBy []string
|
||||
SuppressWhen []string
|
||||
}
|
||||
|
||||
type Group struct {
|
||||
Key string
|
||||
Severity string
|
||||
Labels map[string]string
|
||||
Signals []Signal
|
||||
}
|
||||
|
||||
type Cause struct {
|
||||
Key string
|
||||
State State
|
||||
Confirmed bool
|
||||
Confidence float64
|
||||
ObservedAt time.Time
|
||||
}
|
||||
|
||||
type SuppressionDecision struct {
|
||||
Suppressed bool `json:"suppressed"`
|
||||
CauseKey string `json:"causeKey,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
func BuildFingerprint(ruleID, ruleVersionID, entityID string, labels map[string]string) (string, error) {
|
||||
if strings.TrimSpace(ruleID) == "" || strings.TrimSpace(ruleVersionID) == "" || strings.TrimSpace(entityID) == "" || validateLabel(ruleID, 160) != nil || validateLabel(ruleVersionID, 160) != nil || validateLabel(entityID, 160) != nil {
|
||||
return "", ErrInvalidAlertIdentity
|
||||
}
|
||||
canonical, err := canonicalLabels(labels, MaxFingerprintLabels)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
value := "rule=" + ruleID + "\x00version=" + ruleVersionID + "\x00entity=" + entityID + "\x00" + canonical
|
||||
digest := sha256.Sum256([]byte(value))
|
||||
return hex.EncodeToString(digest[:]), nil
|
||||
}
|
||||
|
||||
func GroupSignals(signals []Signal) ([]Group, error) {
|
||||
groups := make(map[string]*Group)
|
||||
for _, signal := range signals {
|
||||
if err := validateSignal(signal); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, labels, err := groupKey(signal)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
group := groups[key]
|
||||
if group == nil {
|
||||
if len(groups) >= MaxAlertGroups {
|
||||
return nil, fmt.Errorf("%w: too many alert groups", ErrInvalidAlertIdentity)
|
||||
}
|
||||
group = &Group{Key: key, Severity: signal.Severity, Labels: labels}
|
||||
groups[key] = group
|
||||
}
|
||||
if len(group.Signals) >= MaxSignalsPerGroup {
|
||||
return nil, fmt.Errorf("%w: too many signals in group", ErrInvalidAlertIdentity)
|
||||
}
|
||||
group.Signals = append(group.Signals, signal)
|
||||
}
|
||||
result := make([]Group, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
sort.SliceStable(group.Signals, func(i, j int) bool { return signalSortKey(group.Signals[i]) < signalSortKey(group.Signals[j]) })
|
||||
result = append(result, *group)
|
||||
}
|
||||
sort.SliceStable(result, func(i, j int) bool { return result[i].Key < result[j].Key })
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func DeduplicateSignals(signals []Signal) ([]Signal, error) {
|
||||
byKey := make(map[string]Signal, len(signals))
|
||||
for _, signal := range signals {
|
||||
if err := validateSignal(signal); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := signal.InstanceID + "\x00" + signal.EvaluationKey
|
||||
if previous, exists := byKey[key]; !exists || signalSortKey(signal) > signalSortKey(previous) {
|
||||
byKey[key] = signal
|
||||
}
|
||||
}
|
||||
result := make([]Signal, 0, len(byKey))
|
||||
for _, signal := range byKey {
|
||||
result = append(result, signal)
|
||||
}
|
||||
sort.SliceStable(result, func(i, j int) bool { return signalSortKey(result[i]) < signalSortKey(result[j]) })
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func EvaluateSuppression(signal Signal, causes []Cause) (SuppressionDecision, error) {
|
||||
if err := validateSignal(signal); err != nil {
|
||||
return SuppressionDecision{}, err
|
||||
}
|
||||
if signal.State != StatePending && signal.State != StateFiring && signal.State != StateAcknowledged && signal.State != StateUnknown {
|
||||
return SuppressionDecision{Reason: "alert_not_active"}, nil
|
||||
}
|
||||
wanted := make(map[string]struct{}, len(signal.SuppressWhen))
|
||||
for _, key := range signal.SuppressWhen {
|
||||
if err := validateLabel(key, 160); err != nil {
|
||||
return SuppressionDecision{}, err
|
||||
}
|
||||
wanted[key] = struct{}{}
|
||||
}
|
||||
ordered := append([]Cause(nil), causes...)
|
||||
sort.SliceStable(ordered, func(i, j int) bool { return causeSortKey(ordered[i]) < causeSortKey(ordered[j]) })
|
||||
for _, cause := range ordered {
|
||||
if _, ok := wanted[cause.Key]; !ok || !causeActive(cause) {
|
||||
continue
|
||||
}
|
||||
if !cause.Confirmed && cause.Confidence < .75 {
|
||||
continue
|
||||
}
|
||||
reason := "dependency_failure"
|
||||
if strings.HasPrefix(cause.Key, "source.") {
|
||||
reason = "source_outage"
|
||||
}
|
||||
return SuppressionDecision{Suppressed: true, CauseKey: cause.Key, Reason: reason}, nil
|
||||
}
|
||||
return SuppressionDecision{Reason: "no_active_suppression_cause"}, nil
|
||||
}
|
||||
|
||||
func groupKey(signal Signal) (string, map[string]string, error) {
|
||||
labels := make(map[string]string, len(signal.GroupBy))
|
||||
for _, key := range signal.GroupBy {
|
||||
if err := validateLabel(key, 80); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if value, ok := signal.Labels[key]; ok {
|
||||
if err := validateLabel(value, 160); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
labels[key] = value
|
||||
}
|
||||
}
|
||||
canonical, err := canonicalLabels(labels, MaxFingerprintLabels)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return signal.RuleID + "|" + signal.Severity + "|" + canonical, labels, nil
|
||||
}
|
||||
|
||||
func validateSignal(signal Signal) error {
|
||||
if signal.InstanceID == "" || signal.RuleID == "" || signal.RuleVersionID == "" || signal.EvaluationKey == "" || signal.Severity == "" || !validState(signal.State) || validateLabel(signal.InstanceID, 160) != nil || validateLabel(signal.RuleID, 160) != nil || validateLabel(signal.RuleVersionID, 160) != nil || validateLabel(signal.EvaluationKey, 160) != nil || validateLabel(signal.Severity, 40) != nil {
|
||||
return ErrInvalidAlertIdentity
|
||||
}
|
||||
if len(signal.GroupBy) > MaxFingerprintLabels || len(signal.Labels) > MaxFingerprintLabels {
|
||||
return fmt.Errorf("%w: label cardinality exceeds limit", ErrInvalidAlertIdentity)
|
||||
}
|
||||
for key, value := range signal.Labels {
|
||||
if err := validateLabel(key, 80); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateLabel(value, 160); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func canonicalLabels(labels map[string]string, max int) (string, error) {
|
||||
if len(labels) > max {
|
||||
return "", fmt.Errorf("%w: too many labels", ErrInvalidAlertIdentity)
|
||||
}
|
||||
keys := make([]string, 0, len(labels))
|
||||
for key, value := range labels {
|
||||
if err := validateLabel(key, 80); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := validateLabel(value, 160); err != nil {
|
||||
return "", err
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
parts = append(parts, key+"="+labels[key])
|
||||
}
|
||||
return strings.Join(parts, "\x00"), nil
|
||||
}
|
||||
|
||||
func validateLabel(value string, max int) error {
|
||||
if value == "" || len(value) > max || strings.ContainsAny(value, "\r\n\x00") || !alertLabelPattern.MatchString(value) {
|
||||
return ErrInvalidAlertIdentity
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func signalSortKey(signal Signal) string {
|
||||
return signal.InstanceID + "|" + signal.EvaluationKey + "|" + signal.ObservedAt.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
func causeSortKey(cause Cause) string {
|
||||
return cause.Key + "|" + string(cause.State) + "|" + cause.ObservedAt.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
func causeActive(cause Cause) bool {
|
||||
return cause.State == StateFiring || cause.State == StateAcknowledged || (cause.State == StateUnknown && cause.Confirmed)
|
||||
}
|
||||
Reference in New Issue
Block a user