Publish ITWorx Pulse source
Public source validation / validate (push) Failing after 3m8s

This commit is contained in:
ITWorx Pulse release export
2026-09-03 02:09:19 +02:00
commit bd774932d5
614 changed files with 77116 additions and 0 deletions
+79
View File
@@ -0,0 +1,79 @@
package incident
import (
"context"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
)
func (r *Repository) UpdateOwner(ctx context.Context, id, owner string, expectedRevision int64) (Incident, error) {
if r == nil || r.Pool == nil {
return Incident{}, ErrUnavailable
}
if id == "" || expectedRevision < 1 || len(owner) > 160 || (owner != "" && !ownerUUIDPattern.MatchString(owner)) {
return Incident{}, ErrInvalid
}
var item Incident
ownerValue := any(nil)
if owner != "" {
ownerValue = owner
}
err := scanIncident(r.Pool.QueryRow(ctx, `UPDATE incidents SET owner_user_id=$1::uuid,revision=revision+1,updated_at=now() WHERE id=$2 AND revision=$3 RETURNING `+incidentColumns, ownerValue, id, expectedRevision), &item)
if errors.Is(err, pgx.ErrNoRows) {
if _, getErr := r.Get(ctx, id); errors.Is(getErr, ErrNotFound) {
return Incident{}, ErrNotFound
}
return Incident{}, ErrConflict
}
if err != nil {
return Incident{}, mapError("update incident owner", err)
}
return r.Get(ctx, item.ID)
}
func (r *Repository) AddNote(ctx context.Context, incidentID, author, body string) (Note, error) {
if r == nil || r.Pool == nil {
return Note{}, ErrUnavailable
}
if incidentID == "" || author == "" || len(author) > 160 {
return Note{}, ErrInvalid
}
sanitized, err := SanitizeNote(body)
if err != nil {
return Note{}, err
}
var note Note
err = r.Pool.QueryRow(ctx, `INSERT INTO incident_notes (id,incident_id,author,body) VALUES ($1,$2,$3,$4) RETURNING id::text,incident_id::text,author,body,created_at`, newID(), incidentID, author, sanitized).Scan(&note.ID, &note.IncidentID, &note.Author, &note.Body, &note.CreatedAt)
if err != nil {
return Note{}, mapError("add incident note", err)
}
return note, nil
}
func (r *Repository) ListNotes(ctx context.Context, incidentID string, limit int) ([]Note, error) {
if r == nil || r.Pool == nil {
return nil, ErrUnavailable
}
if incidentID == "" || limit < 1 || limit > 100 {
return nil, ErrInvalid
}
rows, err := r.Pool.Query(ctx, `SELECT id::text,incident_id::text,author,body,created_at FROM incident_notes WHERE incident_id=$1 ORDER BY created_at ASC,id ASC LIMIT $2`, incidentID, limit)
if err != nil {
return nil, fmt.Errorf("list incident notes: %w", err)
}
defer rows.Close()
items := make([]Note, 0, limit)
for rows.Next() {
var note Note
if err := rows.Scan(&note.ID, &note.IncidentID, &note.Author, &note.Body, &note.CreatedAt); err != nil {
return nil, fmt.Errorf("scan incident note: %w", err)
}
items = append(items, note)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
+275
View File
@@ -0,0 +1,275 @@
package incident
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
)
type Repository struct{ Pool *pgxpool.Pool }
type Store interface {
List(context.Context, int, Status) ([]Incident, error)
Get(context.Context, string) (Incident, error)
UpsertCandidate(context.Context, Candidate) (Incident, bool, error)
AssociateAlert(context.Context, string, string, string, float64, string) (Association, bool, error)
DisassociateAlert(context.Context, string, string) error
UpdateStatus(context.Context, string, Status, int64, time.Time) (Incident, error)
UpdateOwner(context.Context, string, string, int64) (Incident, error)
AddNote(context.Context, string, string, string) (Note, error)
ListNotes(context.Context, string, int) ([]Note, error)
}
func NewRepository(pool *pgxpool.Pool) (*Repository, error) {
if pool == nil {
return nil, ErrUnavailable
}
return &Repository{Pool: pool}, nil
}
func (r *Repository) UpsertCandidate(ctx context.Context, candidate Candidate) (Incident, bool, error) {
if r == nil || r.Pool == nil {
return Incident{}, false, ErrUnavailable
}
if err := candidate.Validate(); err != nil {
return Incident{}, false, err
}
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return Incident{}, false, fmt.Errorf("begin incident correlation: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtext($1))`, candidate.CorrelationKey); err != nil {
return Incident{}, false, fmt.Errorf("lock incident correlation: %w", err)
}
var incident Incident
err = scanIncident(tx.QueryRow(ctx, incidentSelect+` WHERE correlation_key=$1 AND status <> 'resolved' FOR UPDATE`, candidate.CorrelationKey), &incident)
created := false
if errors.Is(err, pgx.ErrNoRows) {
created = true
err = scanIncident(tx.QueryRow(ctx, `INSERT INTO incidents (id,correlation_key,title,summary,severity,status,started_at,correlation_method,confidence) VALUES ($1,$2,$3,$4,$5,'open',$6,$7,$8) RETURNING `+incidentColumns, newID(), candidate.CorrelationKey, candidate.Title, candidate.Summary, candidate.Severity, candidate.StartedAt.UTC(), candidate.CorrelationMethod, candidate.Confidence), &incident)
}
if err != nil {
return Incident{}, false, mapError("find or create incident", err)
}
if !created {
err = scanIncident(tx.QueryRow(ctx, `UPDATE incidents SET title=$1,summary=$2,severity=$3,started_at=LEAST(started_at,$4),correlation_method=$5,confidence=$6,revision=revision+1,updated_at=now() WHERE id=$7 RETURNING `+incidentColumns, candidate.Title, candidate.Summary, candidate.Severity, candidate.StartedAt.UTC(), candidate.CorrelationMethod, candidate.Confidence, incident.ID), &incident)
if err != nil {
return Incident{}, false, fmt.Errorf("update incident correlation: %w", err)
}
}
for _, signal := range candidate.Signals {
if _, err := tx.Exec(ctx, `INSERT INTO incident_alerts (incident_id,alert_id,rationale,confidence,correlation_method,is_manual,added_by) VALUES ($1,$2,$3,$4,$5,false,'') ON CONFLICT (incident_id,alert_id) DO UPDATE SET rationale=EXCLUDED.rationale,confidence=EXCLUDED.confidence,correlation_method=EXCLUDED.correlation_method WHERE incident_alerts.is_manual=false`, incident.ID, signal.AlertID, signalRationale(signal, candidate.CorrelationMethod), methodConfidence(candidate.CorrelationMethod), candidate.CorrelationMethod); err != nil {
return Incident{}, false, mapError("associate correlated alert", err)
}
if _, err := tx.Exec(ctx, `INSERT INTO incident_entities (incident_id,entity_id,rationale,confidence) VALUES ($1,$2,$3,$4) ON CONFLICT (incident_id,entity_id) DO NOTHING`, incident.ID, signal.EntityID, signalRationale(signal, candidate.CorrelationMethod), methodConfidence(candidate.CorrelationMethod)); err != nil {
return Incident{}, false, mapError("associate incident entity", err)
}
}
if err := tx.Commit(ctx); err != nil {
return Incident{}, false, fmt.Errorf("commit incident correlation: %w", err)
}
item, err := r.Get(ctx, incident.ID)
return item, created, err
}
func (r *Repository) List(ctx context.Context, limit int, status Status) ([]Incident, error) {
if r == nil || r.Pool == nil {
return nil, ErrUnavailable
}
if limit < 1 || limit > 100 {
return nil, fmt.Errorf("%w: incident limit is invalid", ErrInvalid)
}
if status != "" && !validStatus(status) {
return nil, fmt.Errorf("%w: incident status is invalid", ErrInvalid)
}
rows, err := r.Pool.Query(ctx, incidentSelect+` WHERE ($2='' OR status=$2) ORDER BY CASE status WHEN 'open' THEN 1 WHEN 'acknowledged' THEN 2 ELSE 3 END,severity DESC,updated_at DESC,id ASC LIMIT $1`, limit, status)
if err != nil {
return nil, fmt.Errorf("list incidents: %w", err)
}
defer rows.Close()
items := make([]Incident, 0, limit)
for rows.Next() {
var item Incident
if err := scanIncident(rows, &item); err != nil {
return nil, err
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list incident rows: %w", err)
}
return items, nil
}
func (r *Repository) Get(ctx context.Context, id string) (Incident, error) {
if r == nil || r.Pool == nil {
return Incident{}, ErrUnavailable
}
if strings.TrimSpace(id) == "" {
return Incident{}, ErrNotFound
}
var item Incident
if err := scanIncident(r.Pool.QueryRow(ctx, incidentSelect+` WHERE id=$1`, id), &item); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return Incident{}, ErrNotFound
}
return Incident{}, fmt.Errorf("get incident: %w", err)
}
alerts, err := r.Pool.Query(ctx, `SELECT alert_id::text,rationale,confidence,correlation_method,is_manual,added_by,created_at FROM incident_alerts WHERE incident_id=$1 ORDER BY created_at ASC,alert_id ASC`, id)
if err != nil {
return Incident{}, fmt.Errorf("list incident alerts: %w", err)
}
for alerts.Next() {
var association Association
if err := alerts.Scan(&association.AlertID, &association.Rationale, &association.Confidence, &association.CorrelationMethod, &association.Manual, &association.AddedBy, &association.CreatedAt); err != nil {
alerts.Close()
return Incident{}, fmt.Errorf("scan incident alert: %w", err)
}
item.Alerts = append(item.Alerts, association)
}
if err := alerts.Err(); err != nil {
alerts.Close()
return Incident{}, err
}
alerts.Close()
entities, err := r.Pool.Query(ctx, `SELECT entity_id::text,rationale,confidence,created_at FROM incident_entities WHERE incident_id=$1 ORDER BY created_at ASC,entity_id ASC`, id)
if err != nil {
return Incident{}, fmt.Errorf("list incident entities: %w", err)
}
defer entities.Close()
for entities.Next() {
var link EntityLink
if err := entities.Scan(&link.EntityID, &link.Rationale, &link.Confidence, &link.CreatedAt); err != nil {
return Incident{}, fmt.Errorf("scan incident entity: %w", err)
}
item.Entities = append(item.Entities, link)
}
if err := entities.Err(); err != nil {
return Incident{}, err
}
return item, nil
}
func (r *Repository) AssociateAlert(ctx context.Context, incidentID, alertID, rationale string, confidence float64, actor string) (Association, bool, error) {
if r == nil || r.Pool == nil {
return Association{}, false, ErrUnavailable
}
if incidentID == "" || alertID == "" || len(rationale) < 1 || len(rationale) > MaxRationale || confidence < 0 || confidence > 1 || len(actor) > 160 {
return Association{}, false, ErrInvalid
}
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return Association{}, false, fmt.Errorf("begin manual incident association: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
var existing Association
err = tx.QueryRow(ctx, `SELECT alert_id::text,rationale,confidence,correlation_method,is_manual,added_by,created_at FROM incident_alerts WHERE incident_id=$1 AND alert_id=$2 FOR UPDATE`, incidentID, alertID).Scan(&existing.AlertID, &existing.Rationale, &existing.Confidence, &existing.CorrelationMethod, &existing.Manual, &existing.AddedBy, &existing.CreatedAt)
if err == nil && existing.Manual && existing.Rationale == rationale && existing.AddedBy == actor {
if err := tx.Commit(ctx); err != nil {
return Association{}, false, err
}
return existing, true, nil
}
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return Association{}, false, err
}
var association Association
err = tx.QueryRow(ctx, `INSERT INTO incident_alerts (incident_id,alert_id,rationale,confidence,correlation_method,is_manual,added_by) VALUES ($1,$2,$3,$4,'manual',true,$5) ON CONFLICT (incident_id,alert_id) DO UPDATE SET rationale=EXCLUDED.rationale,confidence=EXCLUDED.confidence,correlation_method='manual',is_manual=true,added_by=EXCLUDED.added_by RETURNING alert_id::text,rationale,confidence,correlation_method,is_manual,added_by,created_at`, incidentID, alertID, rationale, confidence, actor).Scan(&association.AlertID, &association.Rationale, &association.Confidence, &association.CorrelationMethod, &association.Manual, &association.AddedBy, &association.CreatedAt)
if err != nil {
return Association{}, false, mapError("associate incident alert", err)
}
if _, err := tx.Exec(ctx, `UPDATE incidents SET revision=revision+1,updated_at=now() WHERE id=$1`, incidentID); err != nil {
return Association{}, false, err
}
if err := tx.Commit(ctx); err != nil {
return Association{}, false, fmt.Errorf("commit manual incident association: %w", err)
}
return association, false, nil
}
func (r *Repository) DisassociateAlert(ctx context.Context, incidentID, alertID string) error {
if r == nil || r.Pool == nil {
return ErrUnavailable
}
command, err := r.Pool.Exec(ctx, `DELETE FROM incident_alerts WHERE incident_id=$1 AND alert_id=$2 AND is_manual=true`, incidentID, alertID)
if err != nil {
return mapError("remove manual incident alert", err)
}
if command.RowsAffected() == 0 {
return ErrNotFound
}
_, err = r.Pool.Exec(ctx, `UPDATE incidents SET revision=revision+1,updated_at=now() WHERE id=$1`, incidentID)
return err
}
func (r *Repository) UpdateStatus(ctx context.Context, id string, status Status, expectedRevision int64, at time.Time) (Incident, error) {
if r == nil || r.Pool == nil {
return Incident{}, ErrUnavailable
}
if !validStatus(status) || expectedRevision < 1 || at.IsZero() {
return Incident{}, ErrInvalid
}
var item Incident
resolved := any(nil)
if status == StatusResolved {
resolved = at.UTC()
}
err := scanIncident(r.Pool.QueryRow(ctx, `UPDATE incidents SET status=$1,resolved_at=$2,revision=revision+1,updated_at=$3 WHERE id=$4 AND revision=$5 RETURNING `+incidentColumns, status, resolved, at.UTC(), id, expectedRevision), &item)
if errors.Is(err, pgx.ErrNoRows) {
if _, getErr := r.Get(ctx, id); errors.Is(getErr, ErrNotFound) {
return Incident{}, ErrNotFound
}
return Incident{}, ErrConflict
}
if err != nil {
return Incident{}, mapError("update incident status", err)
}
return r.Get(ctx, item.ID)
}
const incidentColumns = `id::text,correlation_key,title,summary,severity,status,started_at,resolved_at,COALESCE(owner_user_id::text,''),correlation_method,confidence,revision,created_at,updated_at`
const incidentSelect = `SELECT ` + incidentColumns + ` FROM incidents`
func scanIncident(row interface{ Scan(...any) error }, item *Incident) error {
return row.Scan(&item.ID, &item.CorrelationKey, &item.Title, &item.Summary, &item.Severity, &item.Status, &item.StartedAt, &item.ResolvedAt, &item.OwnerUserID, &item.CorrelationMethod, &item.Confidence, &item.Revision, &item.CreatedAt, &item.UpdatedAt)
}
func signalRationale(signal Signal, method string) string {
if signal.DependencyOutage {
return "Primary dependency outage is the common causal signal."
}
if signal.ParentEntityID != "" {
return "Alert shares a known parent dependency."
}
return "Alert shares a deterministic " + method + " key."
}
func mapError(operation string, err error) error {
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
switch pgErr.Code {
case "23503", "23505":
return fmt.Errorf("%w: %s", ErrConflict, operation)
case "23514", "22P02":
return fmt.Errorf("%w: %s", ErrInvalid, operation)
}
}
return fmt.Errorf("%s: %w", operation, err)
}
func newID() string {
bytes := make([]byte, 16)
if _, err := rand.Read(bytes); err != nil {
return "00000000-0000-4000-8000-000000000000"
}
bytes[6] = (bytes[6] & 15) | 64
bytes[8] = (bytes[8] & 63) | 128
value := hex.EncodeToString(bytes)
return value[:8] + "-" + value[8:12] + "-" + value[12:16] + "-" + value[16:20] + "-" + value[20:]
}
@@ -0,0 +1,86 @@
package incident
import (
"context"
"errors"
"fmt"
"os"
"testing"
"time"
"github.com/itworx/pulse/internal/database"
)
func TestIncidentRepositoryPostgreSQL(t *testing.T) {
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("PULSE_TEST_DATABASE_URL is not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 12, MinConns: 1})
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := database.Migrate(ctx, pool); err != nil {
t.Fatal(err)
}
repository, err := NewRepository(pool)
if err != nil {
t.Fatal(err)
}
run := fmt.Sprintf("%012x", time.Now().UnixNano()&0xffffffffffff)
ruleID := "00000000-0000-4000-8000-" + run
versionID := fmt.Sprintf("00000000-0000-4001-8000-%012x", (time.Now().UnixNano()+1)&0xffffffffffff)
alertID := fmt.Sprintf("00000000-0000-4002-8000-%012x", (time.Now().UnixNano()+2)&0xffffffffffff)
entityID := fmt.Sprintf("00000000-0000-4003-8000-%012x", (time.Now().UnixNano()+3)&0xffffffffffff)
incidentID := fmt.Sprintf("00000000-0000-4004-8000-%012x", (time.Now().UnixNano()+4)&0xffffffffffff)
_, _ = pool.Exec(ctx, `DELETE FROM incidents WHERE id=$1; DELETE FROM alert_instances WHERE id=$2; DELETE FROM alert_rule_versions WHERE id=$3; DELETE FROM alert_rules WHERE id=$4; DELETE FROM entities WHERE id=$5`, incidentID, alertID, versionID, ruleID, entityID)
if _, err := pool.Exec(ctx, `INSERT INTO entities (id,entity_type,canonical_name,display_name,first_seen_at) VALUES ($1,'host',$2,$3,now())`, entityID, "m8-09-host-"+run, "M8-09 host "+run); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO alert_rules
(id,schema_version,name,severity,condition,evaluation_interval_seconds,pending_seconds,resolve_seconds,unknown_behavior,message)
VALUES ($1,1,'M8-09 rule','critical','{}',60,0,0,'become-unknown','{}')`, ruleID); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO alert_rule_versions (id,rule_id,version_number,document) VALUES ($1,$2,1,'{}')`, versionID, ruleID); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `UPDATE alert_rules SET current_version_id=$1 WHERE id=$2`, versionID, ruleID); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO alert_instances (id,rule_id,rule_version_id,fingerprint,entity_id,current_state,last_evaluated_at) VALUES ($1,$2,$3,$4,$5,'firing',now())`, alertID, ruleID, versionID, "m8-09-fingerprint-"+run, entityID); err != nil {
t.Fatal(err)
}
defer func() {
_, _ = pool.Exec(ctx, `DELETE FROM incidents WHERE id=$1; DELETE FROM alert_instances WHERE id=$2; DELETE FROM alert_rule_versions WHERE id=$3; DELETE FROM alert_rules WHERE id=$4; DELETE FROM entities WHERE id=$5`, incidentID, alertID, versionID, ruleID, entityID)
}()
now := time.Now().UTC().Truncate(time.Microsecond)
candidate := Candidate{CorrelationKey: "dependency:" + entityID, Title: "Host unreachable", Summary: "primary dependency outage", Severity: SeverityCritical, StartedAt: now, Confidence: 1, CorrelationMethod: "primary_dependency", Signals: []Signal{{AlertID: alertID, EntityID: entityID, RuleName: "Host unreachable", Severity: SeverityCritical, State: "firing", DependencyOutage: true, ObservedAt: now}}}
created, first, err := repository.UpsertCandidate(ctx, candidate)
if err != nil || !first || len(created.Alerts) != 1 || len(created.Entities) != 1 {
t.Fatalf("created=%+v first=%v err=%v", created, first, err)
}
repeated, first, err := repository.UpsertCandidate(ctx, candidate)
if err != nil || first || repeated.ID != created.ID || len(repeated.Alerts) != 1 {
t.Fatalf("repeated=%+v first=%v err=%v", repeated, first, err)
}
association, duplicate, err := repository.AssociateAlert(ctx, created.ID, alertID, "Operator confirmed common dependency.", 1, "operator-1")
if err != nil || duplicate || !association.Manual {
t.Fatalf("association=%+v duplicate=%v err=%v", association, duplicate, err)
}
if _, duplicate, err := repository.AssociateAlert(ctx, created.ID, alertID, "Operator confirmed common dependency.", 1, "operator-1"); err != nil || !duplicate {
t.Fatalf("idempotent association duplicate=%v err=%v", duplicate, err)
}
if _, err := repository.UpdateStatus(ctx, created.ID, StatusResolved, 2, now.Add(time.Minute)); !errors.Is(err, ErrConflict) {
t.Fatalf("stale status error=%v", err)
}
item, err := repository.Get(ctx, created.ID)
if err != nil || len(item.Alerts) != 1 || !item.Alerts[0].Manual {
t.Fatalf("get incident=%+v err=%v", item, err)
}
}
+264
View File
@@ -0,0 +1,264 @@
package incident
import (
"errors"
"fmt"
"html"
"regexp"
"sort"
"strings"
"time"
)
const (
MaxTitle = 240
MaxSummary = 2000
MaxRationale = 500
MaxSignals = 100
MaxNote = 2000
)
var (
ErrInvalid = errors.New("invalid incident")
ErrNotFound = errors.New("incident not found")
ErrConflict = errors.New("incident conflict")
ErrUnavailable = errors.New("incident repository unavailable")
noteTags = regexp.MustCompile(`<[^>]{0,200}>`)
ownerUUIDPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
)
type Severity string
const (
SeverityAttention Severity = "attention"
SeverityDegraded Severity = "degraded"
SeverityCritical Severity = "critical"
)
type Status string
const (
StatusOpen Status = "open"
StatusAcknowledged Status = "acknowledged"
StatusResolved Status = "resolved"
)
type Signal struct {
AlertID string
EntityID string
EntityType string
RuleName string
Severity Severity
State string
Summary string
ObservedAt time.Time
ParentEntityID string
DependencyOutage bool
Labels map[string]string
}
type Association struct {
AlertID string `json:"alertId"`
Rationale string `json:"rationale"`
Confidence float64 `json:"confidence"`
CorrelationMethod string `json:"correlationMethod"`
Manual bool `json:"manual"`
AddedBy string `json:"addedBy,omitempty"`
CreatedAt time.Time `json:"createdAt"`
}
type EntityLink struct {
EntityID string `json:"entityId"`
Rationale string `json:"rationale"`
Confidence float64 `json:"confidence"`
CreatedAt time.Time `json:"createdAt"`
}
type Note struct {
ID string `json:"id"`
IncidentID string `json:"incidentId"`
Author string `json:"author"`
Body string `json:"body"`
CreatedAt time.Time `json:"createdAt"`
}
type Incident struct {
ID string `json:"id"`
CorrelationKey string `json:"correlationKey"`
Title string `json:"title"`
Summary string `json:"summary"`
Severity Severity `json:"severity"`
Status Status `json:"status"`
StartedAt time.Time `json:"startedAt"`
ResolvedAt *time.Time `json:"resolvedAt,omitempty"`
OwnerUserID string `json:"ownerUserId,omitempty"`
CorrelationMethod string `json:"correlationMethod"`
Confidence float64 `json:"confidence"`
Revision int64 `json:"revision"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
Alerts []Association `json:"alerts,omitempty"`
Entities []EntityLink `json:"entities,omitempty"`
Notes []Note `json:"notes,omitempty"`
}
type Candidate struct {
CorrelationKey string
Title string
Summary string
Severity Severity
StartedAt time.Time
Confidence float64
CorrelationMethod string
Signals []Signal
}
func SanitizeNote(value string) (string, error) {
value = noteTags.ReplaceAllString(html.UnescapeString(value), "")
value = strings.TrimSpace(strings.ReplaceAll(strings.ReplaceAll(value, "\r\n", "\n"), "\r", "\n"))
if value == "" || len(value) > MaxNote {
return "", fmt.Errorf("%w: note body is empty or too large", ErrInvalid)
}
return value, nil
}
func (signal Signal) Validate() error {
if signal.AlertID == "" || signal.EntityID == "" || signal.RuleName == "" || len(signal.RuleName) > 160 || signal.Severity == "" || !validSeverity(signal.Severity) || signal.ObservedAt.IsZero() || len(signal.State) > 32 || len(signal.Summary) > MaxSummary {
return fmt.Errorf("%w: signal fields are invalid", ErrInvalid)
}
if signal.State != "firing" && signal.State != "acknowledged" && signal.State != "unknown" && signal.State != "pending" {
return fmt.Errorf("%w: signal state is not active", ErrInvalid)
}
if len(signal.Labels) > 20 {
return fmt.Errorf("%w: signal label count is bounded", ErrInvalid)
}
return nil
}
func (candidate Candidate) Validate() error {
if candidate.CorrelationKey == "" || len(candidate.CorrelationKey) > 255 || candidate.Title == "" || len(candidate.Title) > MaxTitle || len(candidate.Summary) > MaxSummary || !validSeverity(candidate.Severity) || candidate.StartedAt.IsZero() || candidate.Confidence < 0 || candidate.Confidence > 1 || candidate.CorrelationMethod == "" || len(candidate.CorrelationMethod) > 80 || len(candidate.Signals) < 1 || len(candidate.Signals) > MaxSignals {
return fmt.Errorf("%w: candidate fields are invalid", ErrInvalid)
}
for _, signal := range candidate.Signals {
if err := signal.Validate(); err != nil {
return err
}
}
return nil
}
func (incident Incident) Validate() error {
if incident.ID == "" || incident.CorrelationKey == "" || len(incident.Title) < 1 || len(incident.Title) > MaxTitle || len(incident.Summary) > MaxSummary || !validSeverity(incident.Severity) || !validStatus(incident.Status) || incident.StartedAt.IsZero() || incident.CorrelationMethod == "" || len(incident.CorrelationMethod) > 80 || incident.Confidence < 0 || incident.Confidence > 1 || incident.Revision < 1 {
return fmt.Errorf("%w: incident fields are invalid", ErrInvalid)
}
if incident.Status == StatusResolved && incident.ResolvedAt == nil {
return fmt.Errorf("%w: resolved incident needs resolvedAt", ErrInvalid)
}
return nil
}
func validSeverity(value Severity) bool {
return value == SeverityAttention || value == SeverityDegraded || value == SeverityCritical
}
func validStatus(value Status) bool {
return value == StatusOpen || value == StatusAcknowledged || value == StatusResolved
}
func Correlate(signals []Signal, now time.Time) ([]Candidate, error) {
if len(signals) < 1 || len(signals) > MaxSignals {
return nil, fmt.Errorf("%w: signal count is bounded", ErrInvalid)
}
ordered := append([]Signal(nil), signals...)
sort.SliceStable(ordered, func(i, j int) bool { return ordered[i].AlertID < ordered[j].AlertID })
groups := map[string][]Signal{}
methods := map[string]string{}
for _, signal := range ordered {
if err := signal.Validate(); err != nil {
return nil, err
}
key, method := groupKey(signal)
groups[key] = append(groups[key], signal)
if _, ok := methods[key]; !ok || methodConfidence(method) > methodConfidence(methods[key]) {
methods[key] = method
}
}
keys := make([]string, 0, len(groups))
for key := range groups {
keys = append(keys, key)
}
sort.Strings(keys)
result := make([]Candidate, 0, len(keys))
for _, key := range keys {
items := groups[key]
sort.SliceStable(items, func(i, j int) bool { return items[i].AlertID < items[j].AlertID })
candidate := Candidate{CorrelationKey: key, CorrelationMethod: methods[key], Severity: SeverityAttention, Confidence: 1, StartedAt: items[0].ObservedAt, Signals: items}
for _, signal := range items {
if signal.ObservedAt.Before(candidate.StartedAt) {
candidate.StartedAt = signal.ObservedAt
}
if severityRank(signal.Severity) > severityRank(candidate.Severity) {
candidate.Severity = signal.Severity
}
confidence := methodConfidence(methods[key])
if confidence < candidate.Confidence {
candidate.Confidence = confidence
}
}
candidate.Title, candidate.Summary = buildText(candidate, now)
if err := candidate.Validate(); err != nil {
return nil, err
}
result = append(result, candidate)
}
return result, nil
}
func groupKey(signal Signal) (string, string) {
if signal.DependencyOutage {
return "dependency:" + signal.EntityID, "primary_dependency"
}
if signal.ParentEntityID != "" {
return "dependency:" + signal.ParentEntityID, "dependency_child"
}
if application := signal.Labels["application"]; application != "" {
return "application:" + application, "same_application"
}
if host := signal.Labels["host"]; host != "" {
return "host:" + host, "same_host"
}
return "entity:" + signal.EntityID, "same_entity"
}
func methodConfidence(method string) float64 {
switch method {
case "primary_dependency":
return 1
case "dependency_child":
return .95
case "same_entity":
return .85
case "same_application":
return .75
case "same_host":
return .7
default:
return .5
}
}
func severityRank(value Severity) int {
switch value {
case SeverityCritical:
return 3
case SeverityDegraded:
return 2
default:
return 1
}
}
func buildText(candidate Candidate, now time.Time) (string, string) {
name := strings.TrimSpace(candidate.Signals[0].RuleName)
if name == "" {
name = "Incident"
}
return name, fmt.Sprintf("%d related alert signals correlated by %s at %s.", len(candidate.Signals), candidate.CorrelationMethod, now.UTC().Format(time.RFC3339))
}
+71
View File
@@ -0,0 +1,71 @@
package incident
import (
"strings"
"testing"
"time"
)
func TestCorrelatePrimaryDependencyWithDownstreamAlerts(t *testing.T) {
now := time.Date(2026, 8, 2, 7, 0, 0, 0, time.UTC)
signals := []Signal{
{AlertID: "alert-downstream", EntityID: "service-1", ParentEntityID: "host-1", RuleName: "Service unreachable", Severity: SeverityDegraded, State: "firing", ObservedAt: now.Add(time.Second)},
{AlertID: "alert-primary", EntityID: "host-1", RuleName: "Host unreachable", Severity: SeverityCritical, State: "firing", DependencyOutage: true, ObservedAt: now},
}
groups, err := Correlate(signals, now)
if err != nil {
t.Fatal(err)
}
if len(groups) != 1 || groups[0].CorrelationKey != "dependency:host-1" || groups[0].Severity != SeverityCritical {
t.Fatalf("groups=%+v", groups)
}
if groups[0].CorrelationMethod != "primary_dependency" || groups[0].Confidence != 1 {
t.Fatalf("correlation=%+v", groups[0])
}
if groups[0].Signals[0].AlertID != "alert-downstream" || groups[0].Signals[1].AlertID != "alert-primary" {
t.Fatalf("signals are not deterministic: %+v", groups[0].Signals)
}
}
func TestCorrelateIsDeterministicAndSeparatesUnrelatedSignals(t *testing.T) {
now := time.Unix(10, 0).UTC()
input := []Signal{
{AlertID: "b", EntityID: "entity-2", RuleName: "App B", Severity: SeverityAttention, State: "firing", ObservedAt: now, Labels: map[string]string{"application": "app"}},
{AlertID: "a", EntityID: "entity-1", RuleName: "App A", Severity: SeverityAttention, State: "firing", ObservedAt: now, Labels: map[string]string{"application": "app"}},
{AlertID: "c", EntityID: "entity-3", RuleName: "Other", Severity: SeverityAttention, State: "unknown", ObservedAt: now},
}
groups, err := Correlate(input, now)
if err != nil || len(groups) != 2 {
t.Fatalf("groups=%+v err=%v", groups, err)
}
if groups[0].CorrelationKey != "application:app" || len(groups[0].Signals) != 2 || groups[0].Signals[0].AlertID != "a" {
t.Fatalf("groups=%+v", groups)
}
if groups[1].CorrelationKey != "entity:entity-3" {
t.Fatalf("groups=%+v", groups)
}
}
func TestIncidentValidationAndStatus(t *testing.T) {
now := time.Now().UTC()
item := Incident{ID: "incident-1", CorrelationKey: "entity:1", Title: "title", Severity: SeverityCritical, Status: StatusOpen, StartedAt: now, CorrelationMethod: "same_entity", Confidence: .85, Revision: 1}
if err := item.Validate(); err != nil {
t.Fatal(err)
}
item.Status = StatusResolved
if err := item.Validate(); err == nil {
t.Fatal("resolved incident without timestamp was accepted")
}
}
func TestSanitizeNote(t *testing.T) {
sanitized, err := SanitizeNote(" <b>operator</b>\r\nconfirmed &amp; bounded ")
if err != nil || sanitized != "operator\nconfirmed & bounded" {
t.Fatalf("sanitized=%q err=%v", sanitized, err)
}
if _, err := SanitizeNote("<b></b>"); err == nil {
t.Fatal("empty sanitized note was accepted")
}
if _, err := SanitizeNote(strings.Repeat("x", MaxNote+1)); err == nil {
t.Fatal("oversized note was accepted")
}
}