Public source validation / validate (push) Failing after 3m8s
231 lines
7.2 KiB
Go
231 lines
7.2 KiB
Go
package alert
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
MaxFingerprintLabels = 10
|
|
MaxAlertGroups = 1000
|
|
MaxSignalsPerGroup = 500
|
|
)
|
|
|
|
var ErrInvalidAlertIdentity = errors.New("invalid alert identity")
|
|
var alertLabelPattern = regexp.MustCompile("^[A-Za-z0-9_.:/-]+$")
|
|
|
|
type Signal struct {
|
|
InstanceID string
|
|
RuleID string
|
|
RuleVersionID string
|
|
EntityID string
|
|
Severity string
|
|
State State
|
|
Fingerprint string
|
|
EvaluationKey string
|
|
ObservedAt time.Time
|
|
Labels map[string]string
|
|
GroupBy []string
|
|
SuppressWhen []string
|
|
}
|
|
|
|
type Group struct {
|
|
Key string
|
|
Severity string
|
|
Labels map[string]string
|
|
Signals []Signal
|
|
}
|
|
|
|
type Cause struct {
|
|
Key string
|
|
State State
|
|
Confirmed bool
|
|
Confidence float64
|
|
ObservedAt time.Time
|
|
}
|
|
|
|
type SuppressionDecision struct {
|
|
Suppressed bool `json:"suppressed"`
|
|
CauseKey string `json:"causeKey,omitempty"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
|
|
func BuildFingerprint(ruleID, ruleVersionID, entityID string, labels map[string]string) (string, error) {
|
|
if strings.TrimSpace(ruleID) == "" || strings.TrimSpace(ruleVersionID) == "" || strings.TrimSpace(entityID) == "" || validateLabel(ruleID, 160) != nil || validateLabel(ruleVersionID, 160) != nil || validateLabel(entityID, 160) != nil {
|
|
return "", ErrInvalidAlertIdentity
|
|
}
|
|
canonical, err := canonicalLabels(labels, MaxFingerprintLabels)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
value := "rule=" + ruleID + "\x00version=" + ruleVersionID + "\x00entity=" + entityID + "\x00" + canonical
|
|
digest := sha256.Sum256([]byte(value))
|
|
return hex.EncodeToString(digest[:]), nil
|
|
}
|
|
|
|
func GroupSignals(signals []Signal) ([]Group, error) {
|
|
groups := make(map[string]*Group)
|
|
for _, signal := range signals {
|
|
if err := validateSignal(signal); err != nil {
|
|
return nil, err
|
|
}
|
|
key, labels, err := groupKey(signal)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
group := groups[key]
|
|
if group == nil {
|
|
if len(groups) >= MaxAlertGroups {
|
|
return nil, fmt.Errorf("%w: too many alert groups", ErrInvalidAlertIdentity)
|
|
}
|
|
group = &Group{Key: key, Severity: signal.Severity, Labels: labels}
|
|
groups[key] = group
|
|
}
|
|
if len(group.Signals) >= MaxSignalsPerGroup {
|
|
return nil, fmt.Errorf("%w: too many signals in group", ErrInvalidAlertIdentity)
|
|
}
|
|
group.Signals = append(group.Signals, signal)
|
|
}
|
|
result := make([]Group, 0, len(groups))
|
|
for _, group := range groups {
|
|
sort.SliceStable(group.Signals, func(i, j int) bool { return signalSortKey(group.Signals[i]) < signalSortKey(group.Signals[j]) })
|
|
result = append(result, *group)
|
|
}
|
|
sort.SliceStable(result, func(i, j int) bool { return result[i].Key < result[j].Key })
|
|
return result, nil
|
|
}
|
|
|
|
func DeduplicateSignals(signals []Signal) ([]Signal, error) {
|
|
byKey := make(map[string]Signal, len(signals))
|
|
for _, signal := range signals {
|
|
if err := validateSignal(signal); err != nil {
|
|
return nil, err
|
|
}
|
|
key := signal.InstanceID + "\x00" + signal.EvaluationKey
|
|
if previous, exists := byKey[key]; !exists || signalSortKey(signal) > signalSortKey(previous) {
|
|
byKey[key] = signal
|
|
}
|
|
}
|
|
result := make([]Signal, 0, len(byKey))
|
|
for _, signal := range byKey {
|
|
result = append(result, signal)
|
|
}
|
|
sort.SliceStable(result, func(i, j int) bool { return signalSortKey(result[i]) < signalSortKey(result[j]) })
|
|
return result, nil
|
|
}
|
|
|
|
func EvaluateSuppression(signal Signal, causes []Cause) (SuppressionDecision, error) {
|
|
if err := validateSignal(signal); err != nil {
|
|
return SuppressionDecision{}, err
|
|
}
|
|
if signal.State != StatePending && signal.State != StateFiring && signal.State != StateAcknowledged && signal.State != StateUnknown {
|
|
return SuppressionDecision{Reason: "alert_not_active"}, nil
|
|
}
|
|
wanted := make(map[string]struct{}, len(signal.SuppressWhen))
|
|
for _, key := range signal.SuppressWhen {
|
|
if err := validateLabel(key, 160); err != nil {
|
|
return SuppressionDecision{}, err
|
|
}
|
|
wanted[key] = struct{}{}
|
|
}
|
|
ordered := append([]Cause(nil), causes...)
|
|
sort.SliceStable(ordered, func(i, j int) bool { return causeSortKey(ordered[i]) < causeSortKey(ordered[j]) })
|
|
for _, cause := range ordered {
|
|
if _, ok := wanted[cause.Key]; !ok || !causeActive(cause) {
|
|
continue
|
|
}
|
|
if !cause.Confirmed && cause.Confidence < .75 {
|
|
continue
|
|
}
|
|
reason := "dependency_failure"
|
|
if strings.HasPrefix(cause.Key, "source.") {
|
|
reason = "source_outage"
|
|
}
|
|
return SuppressionDecision{Suppressed: true, CauseKey: cause.Key, Reason: reason}, nil
|
|
}
|
|
return SuppressionDecision{Reason: "no_active_suppression_cause"}, nil
|
|
}
|
|
|
|
func groupKey(signal Signal) (string, map[string]string, error) {
|
|
labels := make(map[string]string, len(signal.GroupBy))
|
|
for _, key := range signal.GroupBy {
|
|
if err := validateLabel(key, 80); err != nil {
|
|
return "", nil, err
|
|
}
|
|
if value, ok := signal.Labels[key]; ok {
|
|
if err := validateLabel(value, 160); err != nil {
|
|
return "", nil, err
|
|
}
|
|
labels[key] = value
|
|
}
|
|
}
|
|
canonical, err := canonicalLabels(labels, MaxFingerprintLabels)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
return signal.RuleID + "|" + signal.Severity + "|" + canonical, labels, nil
|
|
}
|
|
|
|
func validateSignal(signal Signal) error {
|
|
if signal.InstanceID == "" || signal.RuleID == "" || signal.RuleVersionID == "" || signal.EvaluationKey == "" || signal.Severity == "" || !validState(signal.State) || validateLabel(signal.InstanceID, 160) != nil || validateLabel(signal.RuleID, 160) != nil || validateLabel(signal.RuleVersionID, 160) != nil || validateLabel(signal.EvaluationKey, 160) != nil || validateLabel(signal.Severity, 40) != nil {
|
|
return ErrInvalidAlertIdentity
|
|
}
|
|
if len(signal.GroupBy) > MaxFingerprintLabels || len(signal.Labels) > MaxFingerprintLabels {
|
|
return fmt.Errorf("%w: label cardinality exceeds limit", ErrInvalidAlertIdentity)
|
|
}
|
|
for key, value := range signal.Labels {
|
|
if err := validateLabel(key, 80); err != nil {
|
|
return err
|
|
}
|
|
if err := validateLabel(value, 160); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func canonicalLabels(labels map[string]string, max int) (string, error) {
|
|
if len(labels) > max {
|
|
return "", fmt.Errorf("%w: too many labels", ErrInvalidAlertIdentity)
|
|
}
|
|
keys := make([]string, 0, len(labels))
|
|
for key, value := range labels {
|
|
if err := validateLabel(key, 80); err != nil {
|
|
return "", err
|
|
}
|
|
if err := validateLabel(value, 160); err != nil {
|
|
return "", err
|
|
}
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
parts := make([]string, 0, len(keys))
|
|
for _, key := range keys {
|
|
parts = append(parts, key+"="+labels[key])
|
|
}
|
|
return strings.Join(parts, "\x00"), nil
|
|
}
|
|
|
|
func validateLabel(value string, max int) error {
|
|
if value == "" || len(value) > max || strings.ContainsAny(value, "\r\n\x00") || !alertLabelPattern.MatchString(value) {
|
|
return ErrInvalidAlertIdentity
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func signalSortKey(signal Signal) string {
|
|
return signal.InstanceID + "|" + signal.EvaluationKey + "|" + signal.ObservedAt.UTC().Format(time.RFC3339Nano)
|
|
}
|
|
func causeSortKey(cause Cause) string {
|
|
return cause.Key + "|" + string(cause.State) + "|" + cause.ObservedAt.UTC().Format(time.RFC3339Nano)
|
|
}
|
|
|
|
func causeActive(cause Cause) bool {
|
|
return cause.State == StateFiring || cause.State == StateAcknowledged || (cause.State == StateUnknown && cause.Confirmed)
|
|
}
|