ITWorx Pulse release export
bd774932d5
Public source validation / validate (push) Failing after 3m8s
276 lines
12 KiB
Go
276 lines
12 KiB
Go
package incident
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgconn"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
type Repository struct{ Pool *pgxpool.Pool }
|
|
|
|
type Store interface {
|
|
List(context.Context, int, Status) ([]Incident, error)
|
|
Get(context.Context, string) (Incident, error)
|
|
UpsertCandidate(context.Context, Candidate) (Incident, bool, error)
|
|
AssociateAlert(context.Context, string, string, string, float64, string) (Association, bool, error)
|
|
DisassociateAlert(context.Context, string, string) error
|
|
UpdateStatus(context.Context, string, Status, int64, time.Time) (Incident, error)
|
|
UpdateOwner(context.Context, string, string, int64) (Incident, error)
|
|
AddNote(context.Context, string, string, string) (Note, error)
|
|
ListNotes(context.Context, string, int) ([]Note, error)
|
|
}
|
|
|
|
func NewRepository(pool *pgxpool.Pool) (*Repository, error) {
|
|
if pool == nil {
|
|
return nil, ErrUnavailable
|
|
}
|
|
return &Repository{Pool: pool}, nil
|
|
}
|
|
|
|
func (r *Repository) UpsertCandidate(ctx context.Context, candidate Candidate) (Incident, bool, error) {
|
|
if r == nil || r.Pool == nil {
|
|
return Incident{}, false, ErrUnavailable
|
|
}
|
|
if err := candidate.Validate(); err != nil {
|
|
return Incident{}, false, err
|
|
}
|
|
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
|
if err != nil {
|
|
return Incident{}, false, fmt.Errorf("begin incident correlation: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback(ctx) }()
|
|
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtext($1))`, candidate.CorrelationKey); err != nil {
|
|
return Incident{}, false, fmt.Errorf("lock incident correlation: %w", err)
|
|
}
|
|
var incident Incident
|
|
err = scanIncident(tx.QueryRow(ctx, incidentSelect+` WHERE correlation_key=$1 AND status <> 'resolved' FOR UPDATE`, candidate.CorrelationKey), &incident)
|
|
created := false
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
created = true
|
|
err = scanIncident(tx.QueryRow(ctx, `INSERT INTO incidents (id,correlation_key,title,summary,severity,status,started_at,correlation_method,confidence) VALUES ($1,$2,$3,$4,$5,'open',$6,$7,$8) RETURNING `+incidentColumns, newID(), candidate.CorrelationKey, candidate.Title, candidate.Summary, candidate.Severity, candidate.StartedAt.UTC(), candidate.CorrelationMethod, candidate.Confidence), &incident)
|
|
}
|
|
if err != nil {
|
|
return Incident{}, false, mapError("find or create incident", err)
|
|
}
|
|
if !created {
|
|
err = scanIncident(tx.QueryRow(ctx, `UPDATE incidents SET title=$1,summary=$2,severity=$3,started_at=LEAST(started_at,$4),correlation_method=$5,confidence=$6,revision=revision+1,updated_at=now() WHERE id=$7 RETURNING `+incidentColumns, candidate.Title, candidate.Summary, candidate.Severity, candidate.StartedAt.UTC(), candidate.CorrelationMethod, candidate.Confidence, incident.ID), &incident)
|
|
if err != nil {
|
|
return Incident{}, false, fmt.Errorf("update incident correlation: %w", err)
|
|
}
|
|
}
|
|
for _, signal := range candidate.Signals {
|
|
if _, err := tx.Exec(ctx, `INSERT INTO incident_alerts (incident_id,alert_id,rationale,confidence,correlation_method,is_manual,added_by) VALUES ($1,$2,$3,$4,$5,false,'') ON CONFLICT (incident_id,alert_id) DO UPDATE SET rationale=EXCLUDED.rationale,confidence=EXCLUDED.confidence,correlation_method=EXCLUDED.correlation_method WHERE incident_alerts.is_manual=false`, incident.ID, signal.AlertID, signalRationale(signal, candidate.CorrelationMethod), methodConfidence(candidate.CorrelationMethod), candidate.CorrelationMethod); err != nil {
|
|
return Incident{}, false, mapError("associate correlated alert", err)
|
|
}
|
|
if _, err := tx.Exec(ctx, `INSERT INTO incident_entities (incident_id,entity_id,rationale,confidence) VALUES ($1,$2,$3,$4) ON CONFLICT (incident_id,entity_id) DO NOTHING`, incident.ID, signal.EntityID, signalRationale(signal, candidate.CorrelationMethod), methodConfidence(candidate.CorrelationMethod)); err != nil {
|
|
return Incident{}, false, mapError("associate incident entity", err)
|
|
}
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return Incident{}, false, fmt.Errorf("commit incident correlation: %w", err)
|
|
}
|
|
item, err := r.Get(ctx, incident.ID)
|
|
return item, created, err
|
|
}
|
|
|
|
func (r *Repository) List(ctx context.Context, limit int, status Status) ([]Incident, error) {
|
|
if r == nil || r.Pool == nil {
|
|
return nil, ErrUnavailable
|
|
}
|
|
if limit < 1 || limit > 100 {
|
|
return nil, fmt.Errorf("%w: incident limit is invalid", ErrInvalid)
|
|
}
|
|
if status != "" && !validStatus(status) {
|
|
return nil, fmt.Errorf("%w: incident status is invalid", ErrInvalid)
|
|
}
|
|
rows, err := r.Pool.Query(ctx, incidentSelect+` WHERE ($2='' OR status=$2) ORDER BY CASE status WHEN 'open' THEN 1 WHEN 'acknowledged' THEN 2 ELSE 3 END,severity DESC,updated_at DESC,id ASC LIMIT $1`, limit, status)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list incidents: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
items := make([]Incident, 0, limit)
|
|
for rows.Next() {
|
|
var item Incident
|
|
if err := scanIncident(rows, &item); err != nil {
|
|
return nil, err
|
|
}
|
|
items = append(items, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("list incident rows: %w", err)
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func (r *Repository) Get(ctx context.Context, id string) (Incident, error) {
|
|
if r == nil || r.Pool == nil {
|
|
return Incident{}, ErrUnavailable
|
|
}
|
|
if strings.TrimSpace(id) == "" {
|
|
return Incident{}, ErrNotFound
|
|
}
|
|
var item Incident
|
|
if err := scanIncident(r.Pool.QueryRow(ctx, incidentSelect+` WHERE id=$1`, id), &item); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return Incident{}, ErrNotFound
|
|
}
|
|
return Incident{}, fmt.Errorf("get incident: %w", err)
|
|
}
|
|
alerts, err := r.Pool.Query(ctx, `SELECT alert_id::text,rationale,confidence,correlation_method,is_manual,added_by,created_at FROM incident_alerts WHERE incident_id=$1 ORDER BY created_at ASC,alert_id ASC`, id)
|
|
if err != nil {
|
|
return Incident{}, fmt.Errorf("list incident alerts: %w", err)
|
|
}
|
|
for alerts.Next() {
|
|
var association Association
|
|
if err := alerts.Scan(&association.AlertID, &association.Rationale, &association.Confidence, &association.CorrelationMethod, &association.Manual, &association.AddedBy, &association.CreatedAt); err != nil {
|
|
alerts.Close()
|
|
return Incident{}, fmt.Errorf("scan incident alert: %w", err)
|
|
}
|
|
item.Alerts = append(item.Alerts, association)
|
|
}
|
|
if err := alerts.Err(); err != nil {
|
|
alerts.Close()
|
|
return Incident{}, err
|
|
}
|
|
alerts.Close()
|
|
entities, err := r.Pool.Query(ctx, `SELECT entity_id::text,rationale,confidence,created_at FROM incident_entities WHERE incident_id=$1 ORDER BY created_at ASC,entity_id ASC`, id)
|
|
if err != nil {
|
|
return Incident{}, fmt.Errorf("list incident entities: %w", err)
|
|
}
|
|
defer entities.Close()
|
|
for entities.Next() {
|
|
var link EntityLink
|
|
if err := entities.Scan(&link.EntityID, &link.Rationale, &link.Confidence, &link.CreatedAt); err != nil {
|
|
return Incident{}, fmt.Errorf("scan incident entity: %w", err)
|
|
}
|
|
item.Entities = append(item.Entities, link)
|
|
}
|
|
if err := entities.Err(); err != nil {
|
|
return Incident{}, err
|
|
}
|
|
return item, nil
|
|
}
|
|
|
|
func (r *Repository) AssociateAlert(ctx context.Context, incidentID, alertID, rationale string, confidence float64, actor string) (Association, bool, error) {
|
|
if r == nil || r.Pool == nil {
|
|
return Association{}, false, ErrUnavailable
|
|
}
|
|
if incidentID == "" || alertID == "" || len(rationale) < 1 || len(rationale) > MaxRationale || confidence < 0 || confidence > 1 || len(actor) > 160 {
|
|
return Association{}, false, ErrInvalid
|
|
}
|
|
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
|
if err != nil {
|
|
return Association{}, false, fmt.Errorf("begin manual incident association: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback(ctx) }()
|
|
var existing Association
|
|
err = tx.QueryRow(ctx, `SELECT alert_id::text,rationale,confidence,correlation_method,is_manual,added_by,created_at FROM incident_alerts WHERE incident_id=$1 AND alert_id=$2 FOR UPDATE`, incidentID, alertID).Scan(&existing.AlertID, &existing.Rationale, &existing.Confidence, &existing.CorrelationMethod, &existing.Manual, &existing.AddedBy, &existing.CreatedAt)
|
|
if err == nil && existing.Manual && existing.Rationale == rationale && existing.AddedBy == actor {
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return Association{}, false, err
|
|
}
|
|
return existing, true, nil
|
|
}
|
|
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
|
return Association{}, false, err
|
|
}
|
|
var association Association
|
|
err = tx.QueryRow(ctx, `INSERT INTO incident_alerts (incident_id,alert_id,rationale,confidence,correlation_method,is_manual,added_by) VALUES ($1,$2,$3,$4,'manual',true,$5) ON CONFLICT (incident_id,alert_id) DO UPDATE SET rationale=EXCLUDED.rationale,confidence=EXCLUDED.confidence,correlation_method='manual',is_manual=true,added_by=EXCLUDED.added_by RETURNING alert_id::text,rationale,confidence,correlation_method,is_manual,added_by,created_at`, incidentID, alertID, rationale, confidence, actor).Scan(&association.AlertID, &association.Rationale, &association.Confidence, &association.CorrelationMethod, &association.Manual, &association.AddedBy, &association.CreatedAt)
|
|
if err != nil {
|
|
return Association{}, false, mapError("associate incident alert", err)
|
|
}
|
|
if _, err := tx.Exec(ctx, `UPDATE incidents SET revision=revision+1,updated_at=now() WHERE id=$1`, incidentID); err != nil {
|
|
return Association{}, false, err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return Association{}, false, fmt.Errorf("commit manual incident association: %w", err)
|
|
}
|
|
return association, false, nil
|
|
}
|
|
|
|
func (r *Repository) DisassociateAlert(ctx context.Context, incidentID, alertID string) error {
|
|
if r == nil || r.Pool == nil {
|
|
return ErrUnavailable
|
|
}
|
|
command, err := r.Pool.Exec(ctx, `DELETE FROM incident_alerts WHERE incident_id=$1 AND alert_id=$2 AND is_manual=true`, incidentID, alertID)
|
|
if err != nil {
|
|
return mapError("remove manual incident alert", err)
|
|
}
|
|
if command.RowsAffected() == 0 {
|
|
return ErrNotFound
|
|
}
|
|
_, err = r.Pool.Exec(ctx, `UPDATE incidents SET revision=revision+1,updated_at=now() WHERE id=$1`, incidentID)
|
|
return err
|
|
}
|
|
|
|
func (r *Repository) UpdateStatus(ctx context.Context, id string, status Status, expectedRevision int64, at time.Time) (Incident, error) {
|
|
if r == nil || r.Pool == nil {
|
|
return Incident{}, ErrUnavailable
|
|
}
|
|
if !validStatus(status) || expectedRevision < 1 || at.IsZero() {
|
|
return Incident{}, ErrInvalid
|
|
}
|
|
var item Incident
|
|
resolved := any(nil)
|
|
if status == StatusResolved {
|
|
resolved = at.UTC()
|
|
}
|
|
err := scanIncident(r.Pool.QueryRow(ctx, `UPDATE incidents SET status=$1,resolved_at=$2,revision=revision+1,updated_at=$3 WHERE id=$4 AND revision=$5 RETURNING `+incidentColumns, status, resolved, at.UTC(), id, expectedRevision), &item)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
if _, getErr := r.Get(ctx, id); errors.Is(getErr, ErrNotFound) {
|
|
return Incident{}, ErrNotFound
|
|
}
|
|
return Incident{}, ErrConflict
|
|
}
|
|
if err != nil {
|
|
return Incident{}, mapError("update incident status", err)
|
|
}
|
|
return r.Get(ctx, item.ID)
|
|
}
|
|
|
|
const incidentColumns = `id::text,correlation_key,title,summary,severity,status,started_at,resolved_at,COALESCE(owner_user_id::text,''),correlation_method,confidence,revision,created_at,updated_at`
|
|
const incidentSelect = `SELECT ` + incidentColumns + ` FROM incidents`
|
|
|
|
func scanIncident(row interface{ Scan(...any) error }, item *Incident) error {
|
|
return row.Scan(&item.ID, &item.CorrelationKey, &item.Title, &item.Summary, &item.Severity, &item.Status, &item.StartedAt, &item.ResolvedAt, &item.OwnerUserID, &item.CorrelationMethod, &item.Confidence, &item.Revision, &item.CreatedAt, &item.UpdatedAt)
|
|
}
|
|
func signalRationale(signal Signal, method string) string {
|
|
if signal.DependencyOutage {
|
|
return "Primary dependency outage is the common causal signal."
|
|
}
|
|
if signal.ParentEntityID != "" {
|
|
return "Alert shares a known parent dependency."
|
|
}
|
|
return "Alert shares a deterministic " + method + " key."
|
|
}
|
|
func mapError(operation string, err error) error {
|
|
var pgErr *pgconn.PgError
|
|
if errors.As(err, &pgErr) {
|
|
switch pgErr.Code {
|
|
case "23503", "23505":
|
|
return fmt.Errorf("%w: %s", ErrConflict, operation)
|
|
case "23514", "22P02":
|
|
return fmt.Errorf("%w: %s", ErrInvalid, operation)
|
|
}
|
|
}
|
|
return fmt.Errorf("%s: %w", operation, err)
|
|
}
|
|
func newID() string {
|
|
bytes := make([]byte, 16)
|
|
if _, err := rand.Read(bytes); err != nil {
|
|
return "00000000-0000-4000-8000-000000000000"
|
|
}
|
|
bytes[6] = (bytes[6] & 15) | 64
|
|
bytes[8] = (bytes[8] & 63) | 128
|
|
value := hex.EncodeToString(bytes)
|
|
return value[:8] + "-" + value[8:12] + "-" + value[12:16] + "-" + value[16:20] + "-" + value[20:]
|
|
}
|