This commit is contained in:
@@ -0,0 +1,79 @@
|
||||
package incident
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func (r *Repository) UpdateOwner(ctx context.Context, id, owner string, expectedRevision int64) (Incident, error) {
|
||||
if r == nil || r.Pool == nil {
|
||||
return Incident{}, ErrUnavailable
|
||||
}
|
||||
if id == "" || expectedRevision < 1 || len(owner) > 160 || (owner != "" && !ownerUUIDPattern.MatchString(owner)) {
|
||||
return Incident{}, ErrInvalid
|
||||
}
|
||||
var item Incident
|
||||
ownerValue := any(nil)
|
||||
if owner != "" {
|
||||
ownerValue = owner
|
||||
}
|
||||
err := scanIncident(r.Pool.QueryRow(ctx, `UPDATE incidents SET owner_user_id=$1::uuid,revision=revision+1,updated_at=now() WHERE id=$2 AND revision=$3 RETURNING `+incidentColumns, ownerValue, id, expectedRevision), &item)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
if _, getErr := r.Get(ctx, id); errors.Is(getErr, ErrNotFound) {
|
||||
return Incident{}, ErrNotFound
|
||||
}
|
||||
return Incident{}, ErrConflict
|
||||
}
|
||||
if err != nil {
|
||||
return Incident{}, mapError("update incident owner", err)
|
||||
}
|
||||
return r.Get(ctx, item.ID)
|
||||
}
|
||||
|
||||
func (r *Repository) AddNote(ctx context.Context, incidentID, author, body string) (Note, error) {
|
||||
if r == nil || r.Pool == nil {
|
||||
return Note{}, ErrUnavailable
|
||||
}
|
||||
if incidentID == "" || author == "" || len(author) > 160 {
|
||||
return Note{}, ErrInvalid
|
||||
}
|
||||
sanitized, err := SanitizeNote(body)
|
||||
if err != nil {
|
||||
return Note{}, err
|
||||
}
|
||||
var note Note
|
||||
err = r.Pool.QueryRow(ctx, `INSERT INTO incident_notes (id,incident_id,author,body) VALUES ($1,$2,$3,$4) RETURNING id::text,incident_id::text,author,body,created_at`, newID(), incidentID, author, sanitized).Scan(¬e.ID, ¬e.IncidentID, ¬e.Author, ¬e.Body, ¬e.CreatedAt)
|
||||
if err != nil {
|
||||
return Note{}, mapError("add incident note", err)
|
||||
}
|
||||
return note, nil
|
||||
}
|
||||
|
||||
func (r *Repository) ListNotes(ctx context.Context, incidentID string, limit int) ([]Note, error) {
|
||||
if r == nil || r.Pool == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if incidentID == "" || limit < 1 || limit > 100 {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, `SELECT id::text,incident_id::text,author,body,created_at FROM incident_notes WHERE incident_id=$1 ORDER BY created_at ASC,id ASC LIMIT $2`, incidentID, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list incident notes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]Note, 0, limit)
|
||||
for rows.Next() {
|
||||
var note Note
|
||||
if err := rows.Scan(¬e.ID, ¬e.IncidentID, ¬e.Author, ¬e.Body, ¬e.CreatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan incident note: %w", err)
|
||||
}
|
||||
items = append(items, note)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
package incident
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type Repository struct{ Pool *pgxpool.Pool }
|
||||
|
||||
type Store interface {
|
||||
List(context.Context, int, Status) ([]Incident, error)
|
||||
Get(context.Context, string) (Incident, error)
|
||||
UpsertCandidate(context.Context, Candidate) (Incident, bool, error)
|
||||
AssociateAlert(context.Context, string, string, string, float64, string) (Association, bool, error)
|
||||
DisassociateAlert(context.Context, string, string) error
|
||||
UpdateStatus(context.Context, string, Status, int64, time.Time) (Incident, error)
|
||||
UpdateOwner(context.Context, string, string, int64) (Incident, error)
|
||||
AddNote(context.Context, string, string, string) (Note, error)
|
||||
ListNotes(context.Context, string, int) ([]Note, error)
|
||||
}
|
||||
|
||||
func NewRepository(pool *pgxpool.Pool) (*Repository, error) {
|
||||
if pool == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
return &Repository{Pool: pool}, nil
|
||||
}
|
||||
|
||||
func (r *Repository) UpsertCandidate(ctx context.Context, candidate Candidate) (Incident, bool, error) {
|
||||
if r == nil || r.Pool == nil {
|
||||
return Incident{}, false, ErrUnavailable
|
||||
}
|
||||
if err := candidate.Validate(); err != nil {
|
||||
return Incident{}, false, err
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Incident{}, false, fmt.Errorf("begin incident correlation: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtext($1))`, candidate.CorrelationKey); err != nil {
|
||||
return Incident{}, false, fmt.Errorf("lock incident correlation: %w", err)
|
||||
}
|
||||
var incident Incident
|
||||
err = scanIncident(tx.QueryRow(ctx, incidentSelect+` WHERE correlation_key=$1 AND status <> 'resolved' FOR UPDATE`, candidate.CorrelationKey), &incident)
|
||||
created := false
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
created = true
|
||||
err = scanIncident(tx.QueryRow(ctx, `INSERT INTO incidents (id,correlation_key,title,summary,severity,status,started_at,correlation_method,confidence) VALUES ($1,$2,$3,$4,$5,'open',$6,$7,$8) RETURNING `+incidentColumns, newID(), candidate.CorrelationKey, candidate.Title, candidate.Summary, candidate.Severity, candidate.StartedAt.UTC(), candidate.CorrelationMethod, candidate.Confidence), &incident)
|
||||
}
|
||||
if err != nil {
|
||||
return Incident{}, false, mapError("find or create incident", err)
|
||||
}
|
||||
if !created {
|
||||
err = scanIncident(tx.QueryRow(ctx, `UPDATE incidents SET title=$1,summary=$2,severity=$3,started_at=LEAST(started_at,$4),correlation_method=$5,confidence=$6,revision=revision+1,updated_at=now() WHERE id=$7 RETURNING `+incidentColumns, candidate.Title, candidate.Summary, candidate.Severity, candidate.StartedAt.UTC(), candidate.CorrelationMethod, candidate.Confidence, incident.ID), &incident)
|
||||
if err != nil {
|
||||
return Incident{}, false, fmt.Errorf("update incident correlation: %w", err)
|
||||
}
|
||||
}
|
||||
for _, signal := range candidate.Signals {
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO incident_alerts (incident_id,alert_id,rationale,confidence,correlation_method,is_manual,added_by) VALUES ($1,$2,$3,$4,$5,false,'') ON CONFLICT (incident_id,alert_id) DO UPDATE SET rationale=EXCLUDED.rationale,confidence=EXCLUDED.confidence,correlation_method=EXCLUDED.correlation_method WHERE incident_alerts.is_manual=false`, incident.ID, signal.AlertID, signalRationale(signal, candidate.CorrelationMethod), methodConfidence(candidate.CorrelationMethod), candidate.CorrelationMethod); err != nil {
|
||||
return Incident{}, false, mapError("associate correlated alert", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO incident_entities (incident_id,entity_id,rationale,confidence) VALUES ($1,$2,$3,$4) ON CONFLICT (incident_id,entity_id) DO NOTHING`, incident.ID, signal.EntityID, signalRationale(signal, candidate.CorrelationMethod), methodConfidence(candidate.CorrelationMethod)); err != nil {
|
||||
return Incident{}, false, mapError("associate incident entity", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Incident{}, false, fmt.Errorf("commit incident correlation: %w", err)
|
||||
}
|
||||
item, err := r.Get(ctx, incident.ID)
|
||||
return item, created, err
|
||||
}
|
||||
|
||||
func (r *Repository) List(ctx context.Context, limit int, status Status) ([]Incident, error) {
|
||||
if r == nil || r.Pool == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if limit < 1 || limit > 100 {
|
||||
return nil, fmt.Errorf("%w: incident limit is invalid", ErrInvalid)
|
||||
}
|
||||
if status != "" && !validStatus(status) {
|
||||
return nil, fmt.Errorf("%w: incident status is invalid", ErrInvalid)
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, incidentSelect+` WHERE ($2='' OR status=$2) ORDER BY CASE status WHEN 'open' THEN 1 WHEN 'acknowledged' THEN 2 ELSE 3 END,severity DESC,updated_at DESC,id ASC LIMIT $1`, limit, status)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list incidents: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]Incident, 0, limit)
|
||||
for rows.Next() {
|
||||
var item Incident
|
||||
if err := scanIncident(rows, &item); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("list incident rows: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (r *Repository) Get(ctx context.Context, id string) (Incident, error) {
|
||||
if r == nil || r.Pool == nil {
|
||||
return Incident{}, ErrUnavailable
|
||||
}
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return Incident{}, ErrNotFound
|
||||
}
|
||||
var item Incident
|
||||
if err := scanIncident(r.Pool.QueryRow(ctx, incidentSelect+` WHERE id=$1`, id), &item); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Incident{}, ErrNotFound
|
||||
}
|
||||
return Incident{}, fmt.Errorf("get incident: %w", err)
|
||||
}
|
||||
alerts, err := r.Pool.Query(ctx, `SELECT alert_id::text,rationale,confidence,correlation_method,is_manual,added_by,created_at FROM incident_alerts WHERE incident_id=$1 ORDER BY created_at ASC,alert_id ASC`, id)
|
||||
if err != nil {
|
||||
return Incident{}, fmt.Errorf("list incident alerts: %w", err)
|
||||
}
|
||||
for alerts.Next() {
|
||||
var association Association
|
||||
if err := alerts.Scan(&association.AlertID, &association.Rationale, &association.Confidence, &association.CorrelationMethod, &association.Manual, &association.AddedBy, &association.CreatedAt); err != nil {
|
||||
alerts.Close()
|
||||
return Incident{}, fmt.Errorf("scan incident alert: %w", err)
|
||||
}
|
||||
item.Alerts = append(item.Alerts, association)
|
||||
}
|
||||
if err := alerts.Err(); err != nil {
|
||||
alerts.Close()
|
||||
return Incident{}, err
|
||||
}
|
||||
alerts.Close()
|
||||
entities, err := r.Pool.Query(ctx, `SELECT entity_id::text,rationale,confidence,created_at FROM incident_entities WHERE incident_id=$1 ORDER BY created_at ASC,entity_id ASC`, id)
|
||||
if err != nil {
|
||||
return Incident{}, fmt.Errorf("list incident entities: %w", err)
|
||||
}
|
||||
defer entities.Close()
|
||||
for entities.Next() {
|
||||
var link EntityLink
|
||||
if err := entities.Scan(&link.EntityID, &link.Rationale, &link.Confidence, &link.CreatedAt); err != nil {
|
||||
return Incident{}, fmt.Errorf("scan incident entity: %w", err)
|
||||
}
|
||||
item.Entities = append(item.Entities, link)
|
||||
}
|
||||
if err := entities.Err(); err != nil {
|
||||
return Incident{}, err
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (r *Repository) AssociateAlert(ctx context.Context, incidentID, alertID, rationale string, confidence float64, actor string) (Association, bool, error) {
|
||||
if r == nil || r.Pool == nil {
|
||||
return Association{}, false, ErrUnavailable
|
||||
}
|
||||
if incidentID == "" || alertID == "" || len(rationale) < 1 || len(rationale) > MaxRationale || confidence < 0 || confidence > 1 || len(actor) > 160 {
|
||||
return Association{}, false, ErrInvalid
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Association{}, false, fmt.Errorf("begin manual incident association: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var existing Association
|
||||
err = tx.QueryRow(ctx, `SELECT alert_id::text,rationale,confidence,correlation_method,is_manual,added_by,created_at FROM incident_alerts WHERE incident_id=$1 AND alert_id=$2 FOR UPDATE`, incidentID, alertID).Scan(&existing.AlertID, &existing.Rationale, &existing.Confidence, &existing.CorrelationMethod, &existing.Manual, &existing.AddedBy, &existing.CreatedAt)
|
||||
if err == nil && existing.Manual && existing.Rationale == rationale && existing.AddedBy == actor {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Association{}, false, err
|
||||
}
|
||||
return existing, true, nil
|
||||
}
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Association{}, false, err
|
||||
}
|
||||
var association Association
|
||||
err = tx.QueryRow(ctx, `INSERT INTO incident_alerts (incident_id,alert_id,rationale,confidence,correlation_method,is_manual,added_by) VALUES ($1,$2,$3,$4,'manual',true,$5) ON CONFLICT (incident_id,alert_id) DO UPDATE SET rationale=EXCLUDED.rationale,confidence=EXCLUDED.confidence,correlation_method='manual',is_manual=true,added_by=EXCLUDED.added_by RETURNING alert_id::text,rationale,confidence,correlation_method,is_manual,added_by,created_at`, incidentID, alertID, rationale, confidence, actor).Scan(&association.AlertID, &association.Rationale, &association.Confidence, &association.CorrelationMethod, &association.Manual, &association.AddedBy, &association.CreatedAt)
|
||||
if err != nil {
|
||||
return Association{}, false, mapError("associate incident alert", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE incidents SET revision=revision+1,updated_at=now() WHERE id=$1`, incidentID); err != nil {
|
||||
return Association{}, false, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Association{}, false, fmt.Errorf("commit manual incident association: %w", err)
|
||||
}
|
||||
return association, false, nil
|
||||
}
|
||||
|
||||
func (r *Repository) DisassociateAlert(ctx context.Context, incidentID, alertID string) error {
|
||||
if r == nil || r.Pool == nil {
|
||||
return ErrUnavailable
|
||||
}
|
||||
command, err := r.Pool.Exec(ctx, `DELETE FROM incident_alerts WHERE incident_id=$1 AND alert_id=$2 AND is_manual=true`, incidentID, alertID)
|
||||
if err != nil {
|
||||
return mapError("remove manual incident alert", err)
|
||||
}
|
||||
if command.RowsAffected() == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
_, err = r.Pool.Exec(ctx, `UPDATE incidents SET revision=revision+1,updated_at=now() WHERE id=$1`, incidentID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *Repository) UpdateStatus(ctx context.Context, id string, status Status, expectedRevision int64, at time.Time) (Incident, error) {
|
||||
if r == nil || r.Pool == nil {
|
||||
return Incident{}, ErrUnavailable
|
||||
}
|
||||
if !validStatus(status) || expectedRevision < 1 || at.IsZero() {
|
||||
return Incident{}, ErrInvalid
|
||||
}
|
||||
var item Incident
|
||||
resolved := any(nil)
|
||||
if status == StatusResolved {
|
||||
resolved = at.UTC()
|
||||
}
|
||||
err := scanIncident(r.Pool.QueryRow(ctx, `UPDATE incidents SET status=$1,resolved_at=$2,revision=revision+1,updated_at=$3 WHERE id=$4 AND revision=$5 RETURNING `+incidentColumns, status, resolved, at.UTC(), id, expectedRevision), &item)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
if _, getErr := r.Get(ctx, id); errors.Is(getErr, ErrNotFound) {
|
||||
return Incident{}, ErrNotFound
|
||||
}
|
||||
return Incident{}, ErrConflict
|
||||
}
|
||||
if err != nil {
|
||||
return Incident{}, mapError("update incident status", err)
|
||||
}
|
||||
return r.Get(ctx, item.ID)
|
||||
}
|
||||
|
||||
const incidentColumns = `id::text,correlation_key,title,summary,severity,status,started_at,resolved_at,COALESCE(owner_user_id::text,''),correlation_method,confidence,revision,created_at,updated_at`
|
||||
const incidentSelect = `SELECT ` + incidentColumns + ` FROM incidents`
|
||||
|
||||
func scanIncident(row interface{ Scan(...any) error }, item *Incident) error {
|
||||
return row.Scan(&item.ID, &item.CorrelationKey, &item.Title, &item.Summary, &item.Severity, &item.Status, &item.StartedAt, &item.ResolvedAt, &item.OwnerUserID, &item.CorrelationMethod, &item.Confidence, &item.Revision, &item.CreatedAt, &item.UpdatedAt)
|
||||
}
|
||||
func signalRationale(signal Signal, method string) string {
|
||||
if signal.DependencyOutage {
|
||||
return "Primary dependency outage is the common causal signal."
|
||||
}
|
||||
if signal.ParentEntityID != "" {
|
||||
return "Alert shares a known parent dependency."
|
||||
}
|
||||
return "Alert shares a deterministic " + method + " key."
|
||||
}
|
||||
func mapError(operation string, err error) error {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
switch pgErr.Code {
|
||||
case "23503", "23505":
|
||||
return fmt.Errorf("%w: %s", ErrConflict, operation)
|
||||
case "23514", "22P02":
|
||||
return fmt.Errorf("%w: %s", ErrInvalid, operation)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("%s: %w", operation, err)
|
||||
}
|
||||
func newID() string {
|
||||
bytes := make([]byte, 16)
|
||||
if _, err := rand.Read(bytes); err != nil {
|
||||
return "00000000-0000-4000-8000-000000000000"
|
||||
}
|
||||
bytes[6] = (bytes[6] & 15) | 64
|
||||
bytes[8] = (bytes[8] & 63) | 128
|
||||
value := hex.EncodeToString(bytes)
|
||||
return value[:8] + "-" + value[8:12] + "-" + value[12:16] + "-" + value[16:20] + "-" + value[20:]
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package incident
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
)
|
||||
|
||||
func TestIncidentRepositoryPostgreSQL(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 12, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repository, err := NewRepository(pool)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run := fmt.Sprintf("%012x", time.Now().UnixNano()&0xffffffffffff)
|
||||
ruleID := "00000000-0000-4000-8000-" + run
|
||||
versionID := fmt.Sprintf("00000000-0000-4001-8000-%012x", (time.Now().UnixNano()+1)&0xffffffffffff)
|
||||
alertID := fmt.Sprintf("00000000-0000-4002-8000-%012x", (time.Now().UnixNano()+2)&0xffffffffffff)
|
||||
entityID := fmt.Sprintf("00000000-0000-4003-8000-%012x", (time.Now().UnixNano()+3)&0xffffffffffff)
|
||||
incidentID := fmt.Sprintf("00000000-0000-4004-8000-%012x", (time.Now().UnixNano()+4)&0xffffffffffff)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM incidents WHERE id=$1; DELETE FROM alert_instances WHERE id=$2; DELETE FROM alert_rule_versions WHERE id=$3; DELETE FROM alert_rules WHERE id=$4; DELETE FROM entities WHERE id=$5`, incidentID, alertID, versionID, ruleID, entityID)
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO entities (id,entity_type,canonical_name,display_name,first_seen_at) VALUES ($1,'host',$2,$3,now())`, entityID, "m8-09-host-"+run, "M8-09 host "+run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO alert_rules
|
||||
(id,schema_version,name,severity,condition,evaluation_interval_seconds,pending_seconds,resolve_seconds,unknown_behavior,message)
|
||||
VALUES ($1,1,'M8-09 rule','critical','{}',60,0,0,'become-unknown','{}')`, ruleID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO alert_rule_versions (id,rule_id,version_number,document) VALUES ($1,$2,1,'{}')`, versionID, ruleID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE alert_rules SET current_version_id=$1 WHERE id=$2`, versionID, ruleID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO alert_instances (id,rule_id,rule_version_id,fingerprint,entity_id,current_state,last_evaluated_at) VALUES ($1,$2,$3,$4,$5,'firing',now())`, alertID, ruleID, versionID, "m8-09-fingerprint-"+run, entityID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM incidents WHERE id=$1; DELETE FROM alert_instances WHERE id=$2; DELETE FROM alert_rule_versions WHERE id=$3; DELETE FROM alert_rules WHERE id=$4; DELETE FROM entities WHERE id=$5`, incidentID, alertID, versionID, ruleID, entityID)
|
||||
}()
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
candidate := Candidate{CorrelationKey: "dependency:" + entityID, Title: "Host unreachable", Summary: "primary dependency outage", Severity: SeverityCritical, StartedAt: now, Confidence: 1, CorrelationMethod: "primary_dependency", Signals: []Signal{{AlertID: alertID, EntityID: entityID, RuleName: "Host unreachable", Severity: SeverityCritical, State: "firing", DependencyOutage: true, ObservedAt: now}}}
|
||||
created, first, err := repository.UpsertCandidate(ctx, candidate)
|
||||
if err != nil || !first || len(created.Alerts) != 1 || len(created.Entities) != 1 {
|
||||
t.Fatalf("created=%+v first=%v err=%v", created, first, err)
|
||||
}
|
||||
repeated, first, err := repository.UpsertCandidate(ctx, candidate)
|
||||
if err != nil || first || repeated.ID != created.ID || len(repeated.Alerts) != 1 {
|
||||
t.Fatalf("repeated=%+v first=%v err=%v", repeated, first, err)
|
||||
}
|
||||
association, duplicate, err := repository.AssociateAlert(ctx, created.ID, alertID, "Operator confirmed common dependency.", 1, "operator-1")
|
||||
if err != nil || duplicate || !association.Manual {
|
||||
t.Fatalf("association=%+v duplicate=%v err=%v", association, duplicate, err)
|
||||
}
|
||||
if _, duplicate, err := repository.AssociateAlert(ctx, created.ID, alertID, "Operator confirmed common dependency.", 1, "operator-1"); err != nil || !duplicate {
|
||||
t.Fatalf("idempotent association duplicate=%v err=%v", duplicate, err)
|
||||
}
|
||||
if _, err := repository.UpdateStatus(ctx, created.ID, StatusResolved, 2, now.Add(time.Minute)); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("stale status error=%v", err)
|
||||
}
|
||||
item, err := repository.Get(ctx, created.ID)
|
||||
if err != nil || len(item.Alerts) != 1 || !item.Alerts[0].Manual {
|
||||
t.Fatalf("get incident=%+v err=%v", item, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package incident
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxTitle = 240
|
||||
MaxSummary = 2000
|
||||
MaxRationale = 500
|
||||
MaxSignals = 100
|
||||
MaxNote = 2000
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalid = errors.New("invalid incident")
|
||||
ErrNotFound = errors.New("incident not found")
|
||||
ErrConflict = errors.New("incident conflict")
|
||||
ErrUnavailable = errors.New("incident repository unavailable")
|
||||
noteTags = regexp.MustCompile(`<[^>]{0,200}>`)
|
||||
ownerUUIDPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
||||
)
|
||||
|
||||
type Severity string
|
||||
|
||||
const (
|
||||
SeverityAttention Severity = "attention"
|
||||
SeverityDegraded Severity = "degraded"
|
||||
SeverityCritical Severity = "critical"
|
||||
)
|
||||
|
||||
type Status string
|
||||
|
||||
const (
|
||||
StatusOpen Status = "open"
|
||||
StatusAcknowledged Status = "acknowledged"
|
||||
StatusResolved Status = "resolved"
|
||||
)
|
||||
|
||||
type Signal struct {
|
||||
AlertID string
|
||||
EntityID string
|
||||
EntityType string
|
||||
RuleName string
|
||||
Severity Severity
|
||||
State string
|
||||
Summary string
|
||||
ObservedAt time.Time
|
||||
ParentEntityID string
|
||||
DependencyOutage bool
|
||||
Labels map[string]string
|
||||
}
|
||||
|
||||
type Association struct {
|
||||
AlertID string `json:"alertId"`
|
||||
Rationale string `json:"rationale"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
CorrelationMethod string `json:"correlationMethod"`
|
||||
Manual bool `json:"manual"`
|
||||
AddedBy string `json:"addedBy,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
type EntityLink struct {
|
||||
EntityID string `json:"entityId"`
|
||||
Rationale string `json:"rationale"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
type Note struct {
|
||||
ID string `json:"id"`
|
||||
IncidentID string `json:"incidentId"`
|
||||
Author string `json:"author"`
|
||||
Body string `json:"body"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
type Incident struct {
|
||||
ID string `json:"id"`
|
||||
CorrelationKey string `json:"correlationKey"`
|
||||
Title string `json:"title"`
|
||||
Summary string `json:"summary"`
|
||||
Severity Severity `json:"severity"`
|
||||
Status Status `json:"status"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
ResolvedAt *time.Time `json:"resolvedAt,omitempty"`
|
||||
OwnerUserID string `json:"ownerUserId,omitempty"`
|
||||
CorrelationMethod string `json:"correlationMethod"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Revision int64 `json:"revision"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
Alerts []Association `json:"alerts,omitempty"`
|
||||
Entities []EntityLink `json:"entities,omitempty"`
|
||||
Notes []Note `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
type Candidate struct {
|
||||
CorrelationKey string
|
||||
Title string
|
||||
Summary string
|
||||
Severity Severity
|
||||
StartedAt time.Time
|
||||
Confidence float64
|
||||
CorrelationMethod string
|
||||
Signals []Signal
|
||||
}
|
||||
|
||||
func SanitizeNote(value string) (string, error) {
|
||||
value = noteTags.ReplaceAllString(html.UnescapeString(value), "")
|
||||
value = strings.TrimSpace(strings.ReplaceAll(strings.ReplaceAll(value, "\r\n", "\n"), "\r", "\n"))
|
||||
if value == "" || len(value) > MaxNote {
|
||||
return "", fmt.Errorf("%w: note body is empty or too large", ErrInvalid)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func (signal Signal) Validate() error {
|
||||
if signal.AlertID == "" || signal.EntityID == "" || signal.RuleName == "" || len(signal.RuleName) > 160 || signal.Severity == "" || !validSeverity(signal.Severity) || signal.ObservedAt.IsZero() || len(signal.State) > 32 || len(signal.Summary) > MaxSummary {
|
||||
return fmt.Errorf("%w: signal fields are invalid", ErrInvalid)
|
||||
}
|
||||
if signal.State != "firing" && signal.State != "acknowledged" && signal.State != "unknown" && signal.State != "pending" {
|
||||
return fmt.Errorf("%w: signal state is not active", ErrInvalid)
|
||||
}
|
||||
if len(signal.Labels) > 20 {
|
||||
return fmt.Errorf("%w: signal label count is bounded", ErrInvalid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (candidate Candidate) Validate() error {
|
||||
if candidate.CorrelationKey == "" || len(candidate.CorrelationKey) > 255 || candidate.Title == "" || len(candidate.Title) > MaxTitle || len(candidate.Summary) > MaxSummary || !validSeverity(candidate.Severity) || candidate.StartedAt.IsZero() || candidate.Confidence < 0 || candidate.Confidence > 1 || candidate.CorrelationMethod == "" || len(candidate.CorrelationMethod) > 80 || len(candidate.Signals) < 1 || len(candidate.Signals) > MaxSignals {
|
||||
return fmt.Errorf("%w: candidate fields are invalid", ErrInvalid)
|
||||
}
|
||||
for _, signal := range candidate.Signals {
|
||||
if err := signal.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (incident Incident) Validate() error {
|
||||
if incident.ID == "" || incident.CorrelationKey == "" || len(incident.Title) < 1 || len(incident.Title) > MaxTitle || len(incident.Summary) > MaxSummary || !validSeverity(incident.Severity) || !validStatus(incident.Status) || incident.StartedAt.IsZero() || incident.CorrelationMethod == "" || len(incident.CorrelationMethod) > 80 || incident.Confidence < 0 || incident.Confidence > 1 || incident.Revision < 1 {
|
||||
return fmt.Errorf("%w: incident fields are invalid", ErrInvalid)
|
||||
}
|
||||
if incident.Status == StatusResolved && incident.ResolvedAt == nil {
|
||||
return fmt.Errorf("%w: resolved incident needs resolvedAt", ErrInvalid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validSeverity(value Severity) bool {
|
||||
return value == SeverityAttention || value == SeverityDegraded || value == SeverityCritical
|
||||
}
|
||||
func validStatus(value Status) bool {
|
||||
return value == StatusOpen || value == StatusAcknowledged || value == StatusResolved
|
||||
}
|
||||
|
||||
func Correlate(signals []Signal, now time.Time) ([]Candidate, error) {
|
||||
if len(signals) < 1 || len(signals) > MaxSignals {
|
||||
return nil, fmt.Errorf("%w: signal count is bounded", ErrInvalid)
|
||||
}
|
||||
ordered := append([]Signal(nil), signals...)
|
||||
sort.SliceStable(ordered, func(i, j int) bool { return ordered[i].AlertID < ordered[j].AlertID })
|
||||
groups := map[string][]Signal{}
|
||||
methods := map[string]string{}
|
||||
for _, signal := range ordered {
|
||||
if err := signal.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, method := groupKey(signal)
|
||||
groups[key] = append(groups[key], signal)
|
||||
if _, ok := methods[key]; !ok || methodConfidence(method) > methodConfidence(methods[key]) {
|
||||
methods[key] = method
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
result := make([]Candidate, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
items := groups[key]
|
||||
sort.SliceStable(items, func(i, j int) bool { return items[i].AlertID < items[j].AlertID })
|
||||
candidate := Candidate{CorrelationKey: key, CorrelationMethod: methods[key], Severity: SeverityAttention, Confidence: 1, StartedAt: items[0].ObservedAt, Signals: items}
|
||||
for _, signal := range items {
|
||||
if signal.ObservedAt.Before(candidate.StartedAt) {
|
||||
candidate.StartedAt = signal.ObservedAt
|
||||
}
|
||||
if severityRank(signal.Severity) > severityRank(candidate.Severity) {
|
||||
candidate.Severity = signal.Severity
|
||||
}
|
||||
confidence := methodConfidence(methods[key])
|
||||
if confidence < candidate.Confidence {
|
||||
candidate.Confidence = confidence
|
||||
}
|
||||
}
|
||||
candidate.Title, candidate.Summary = buildText(candidate, now)
|
||||
if err := candidate.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, candidate)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func groupKey(signal Signal) (string, string) {
|
||||
if signal.DependencyOutage {
|
||||
return "dependency:" + signal.EntityID, "primary_dependency"
|
||||
}
|
||||
if signal.ParentEntityID != "" {
|
||||
return "dependency:" + signal.ParentEntityID, "dependency_child"
|
||||
}
|
||||
if application := signal.Labels["application"]; application != "" {
|
||||
return "application:" + application, "same_application"
|
||||
}
|
||||
if host := signal.Labels["host"]; host != "" {
|
||||
return "host:" + host, "same_host"
|
||||
}
|
||||
return "entity:" + signal.EntityID, "same_entity"
|
||||
}
|
||||
|
||||
func methodConfidence(method string) float64 {
|
||||
switch method {
|
||||
case "primary_dependency":
|
||||
return 1
|
||||
case "dependency_child":
|
||||
return .95
|
||||
case "same_entity":
|
||||
return .85
|
||||
case "same_application":
|
||||
return .75
|
||||
case "same_host":
|
||||
return .7
|
||||
default:
|
||||
return .5
|
||||
}
|
||||
}
|
||||
func severityRank(value Severity) int {
|
||||
switch value {
|
||||
case SeverityCritical:
|
||||
return 3
|
||||
case SeverityDegraded:
|
||||
return 2
|
||||
default:
|
||||
return 1
|
||||
}
|
||||
}
|
||||
func buildText(candidate Candidate, now time.Time) (string, string) {
|
||||
name := strings.TrimSpace(candidate.Signals[0].RuleName)
|
||||
if name == "" {
|
||||
name = "Incident"
|
||||
}
|
||||
return name, fmt.Sprintf("%d related alert signals correlated by %s at %s.", len(candidate.Signals), candidate.CorrelationMethod, now.UTC().Format(time.RFC3339))
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package incident
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCorrelatePrimaryDependencyWithDownstreamAlerts(t *testing.T) {
|
||||
now := time.Date(2026, 8, 2, 7, 0, 0, 0, time.UTC)
|
||||
signals := []Signal{
|
||||
{AlertID: "alert-downstream", EntityID: "service-1", ParentEntityID: "host-1", RuleName: "Service unreachable", Severity: SeverityDegraded, State: "firing", ObservedAt: now.Add(time.Second)},
|
||||
{AlertID: "alert-primary", EntityID: "host-1", RuleName: "Host unreachable", Severity: SeverityCritical, State: "firing", DependencyOutage: true, ObservedAt: now},
|
||||
}
|
||||
groups, err := Correlate(signals, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(groups) != 1 || groups[0].CorrelationKey != "dependency:host-1" || groups[0].Severity != SeverityCritical {
|
||||
t.Fatalf("groups=%+v", groups)
|
||||
}
|
||||
if groups[0].CorrelationMethod != "primary_dependency" || groups[0].Confidence != 1 {
|
||||
t.Fatalf("correlation=%+v", groups[0])
|
||||
}
|
||||
if groups[0].Signals[0].AlertID != "alert-downstream" || groups[0].Signals[1].AlertID != "alert-primary" {
|
||||
t.Fatalf("signals are not deterministic: %+v", groups[0].Signals)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrelateIsDeterministicAndSeparatesUnrelatedSignals(t *testing.T) {
|
||||
now := time.Unix(10, 0).UTC()
|
||||
input := []Signal{
|
||||
{AlertID: "b", EntityID: "entity-2", RuleName: "App B", Severity: SeverityAttention, State: "firing", ObservedAt: now, Labels: map[string]string{"application": "app"}},
|
||||
{AlertID: "a", EntityID: "entity-1", RuleName: "App A", Severity: SeverityAttention, State: "firing", ObservedAt: now, Labels: map[string]string{"application": "app"}},
|
||||
{AlertID: "c", EntityID: "entity-3", RuleName: "Other", Severity: SeverityAttention, State: "unknown", ObservedAt: now},
|
||||
}
|
||||
groups, err := Correlate(input, now)
|
||||
if err != nil || len(groups) != 2 {
|
||||
t.Fatalf("groups=%+v err=%v", groups, err)
|
||||
}
|
||||
if groups[0].CorrelationKey != "application:app" || len(groups[0].Signals) != 2 || groups[0].Signals[0].AlertID != "a" {
|
||||
t.Fatalf("groups=%+v", groups)
|
||||
}
|
||||
if groups[1].CorrelationKey != "entity:entity-3" {
|
||||
t.Fatalf("groups=%+v", groups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncidentValidationAndStatus(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
item := Incident{ID: "incident-1", CorrelationKey: "entity:1", Title: "title", Severity: SeverityCritical, Status: StatusOpen, StartedAt: now, CorrelationMethod: "same_entity", Confidence: .85, Revision: 1}
|
||||
if err := item.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item.Status = StatusResolved
|
||||
if err := item.Validate(); err == nil {
|
||||
t.Fatal("resolved incident without timestamp was accepted")
|
||||
}
|
||||
}
|
||||
func TestSanitizeNote(t *testing.T) {
|
||||
sanitized, err := SanitizeNote(" <b>operator</b>\r\nconfirmed & bounded ")
|
||||
if err != nil || sanitized != "operator\nconfirmed & bounded" {
|
||||
t.Fatalf("sanitized=%q err=%v", sanitized, err)
|
||||
}
|
||||
if _, err := SanitizeNote("<b></b>"); err == nil {
|
||||
t.Fatal("empty sanitized note was accepted")
|
||||
}
|
||||
if _, err := SanitizeNote(strings.Repeat("x", MaxNote+1)); err == nil {
|
||||
t.Fatal("oversized note was accepted")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user