package alert import ( "context" "encoding/json" "errors" "fmt" "strings" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" ) type StateInput struct { RuleID string RuleVersionID string Fingerprint string EntityID string Policy Policy Observation Observation } type Instance struct { ID string `json:"id"` RuleID string `json:"ruleId"` RuleVersionID string `json:"ruleVersionId"` Fingerprint string `json:"fingerprint"` EntityID string `json:"entityId,omitempty"` State State `json:"state"` RetainedState State `json:"retainedState"` ActiveSince *time.Time `json:"activeSince,omitempty"` RecoverySince *time.Time `json:"recoverySince,omitempty"` LastEvaluatedAt time.Time `json:"lastEvaluatedAt"` LastKnownAt *time.Time `json:"lastKnownAt,omitempty"` LastValue any `json:"lastValue,omitempty"` Reason string `json:"reason"` SourceHealth map[string]any `json:"sourceHealth"` AcknowledgedBy string `json:"acknowledgedBy,omitempty"` AcknowledgedAt *time.Time `json:"acknowledgedAt,omitempty"` CooldownUntil *time.Time `json:"cooldownUntil,omitempty"` Revision int64 `json:"revision"` CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` } type Occurrence struct { ID string `json:"id"` InstanceID string `json:"instanceId"` EvaluationKey string `json:"evaluationKey"` EventType string `json:"eventType"` From State `json:"from"` To State `json:"to"` ObservedAt time.Time `json:"observedAt"` Value any `json:"value,omitempty"` Reason string `json:"reason"` SourceHealth map[string]any `json:"sourceHealth"` CreatedAt time.Time `json:"createdAt"` } type StateStore interface { ApplyObservation(context.Context, StateInput) (Instance, Occurrence, bool, error) GetInstance(context.Context, string) (Instance, error) ListOccurrences(context.Context, string, int) ([]Occurrence, error) Acknowledge(context.Context, string, string, string, time.Time) (Instance, Occurrence, bool, error) } type StateRepository struct { Pool *pgxpool.Pool } func (r StateRepository) ApplyObservation(ctx context.Context, input StateInput) (Instance, Occurrence, bool, error) { if r.Pool == nil { return Instance{}, Occurrence{}, false, ErrUnavailable } if err := validateStateInput(input); err != nil { return Instance{}, Occurrence{}, false, err } healthJSON, err := boundedJSON(nonNilMap(input.Observation.SourceHealth), 64<<10) if err != nil { return Instance{}, Occurrence{}, false, err } tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{}) if err != nil { return Instance{}, Occurrence{}, false, fmt.Errorf("begin alert state transition: %w", err) } defer func() { _ = tx.Rollback(ctx) }() instanceID := NewID() if err := tx.QueryRow(ctx, `INSERT INTO alert_instances (id,rule_id,rule_version_id,fingerprint,entity_id,last_evaluated_at) VALUES ($1,$2,$3,$4,$5,$6) ON CONFLICT (rule_id,fingerprint) DO NOTHING RETURNING id`, instanceID, input.RuleID, input.RuleVersionID, input.Fingerprint, nullableID(input.EntityID), input.Observation.ObservedAt.UTC()).Scan(&instanceID); err != nil && !errors.Is(err, pgx.ErrNoRows) { return Instance{}, Occurrence{}, false, mapStateError(fmt.Errorf("create alert instance: %w", err)) } var current Instance if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE rule_id=$1 AND fingerprint=$2 FOR UPDATE`, input.RuleID, input.Fingerprint), ¤t); err != nil { return Instance{}, Occurrence{}, false, err } if current.EntityID != input.EntityID { return Instance{}, Occurrence{}, false, ErrStateConflict } if occurrence, err := scanOccurrence(tx.QueryRow(ctx, `SELECT id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at FROM alert_occurrences WHERE instance_id=$1 AND evaluation_key=$2`, current.ID, input.Observation.EvaluationKey)); err == nil { if err := tx.Commit(ctx); err != nil { return Instance{}, Occurrence{}, false, fmt.Errorf("commit idempotent alert evaluation: %w", err) } return current, occurrence, true, nil } else if !errors.Is(err, pgx.ErrNoRows) { return Instance{}, Occurrence{}, false, err } transition, err := Transition(snapshotFromInstance(current), input.Policy, input.Observation) if err != nil { return Instance{}, Occurrence{}, false, err } resultValue := transition.Snapshot.LastValue if input.Observation.Unknown || input.Policy.UnknownBehavior == UnknownIgnoreGap && transition.Snapshot.LastKnownAt == nil { resultValue = current.LastValue } resultValueJSON, err := boundedJSON(resultValue, 128<<10) if err != nil { return Instance{}, Occurrence{}, false, err } if _, err := tx.Exec(ctx, `UPDATE alert_instances SET rule_version_id=$1,current_state=$2,retained_state=$3,active_since=$4,recovery_since=$5,cooldown_until=$6,last_evaluated_at=$7,last_known_at=$8,last_value=$9::jsonb,reason=$10,source_health=$11::jsonb,acknowledged_by=$12,acknowledged_at=$13,revision=revision+1,updated_at=now() WHERE id=$14`, input.RuleVersionID, transition.Snapshot.State, transition.Snapshot.RetainedState, transition.Snapshot.ActiveSince, transition.Snapshot.RecoverySince, transition.Snapshot.CooldownUntil, transition.Snapshot.LastEvaluatedAt.UTC(), transition.Snapshot.LastKnownAt, resultValueJSON, transition.Snapshot.Reason, healthJSON, nullableText(transition.Snapshot.AcknowledgedBy), transition.Snapshot.AcknowledgedAt, current.ID); err != nil { return Instance{}, Occurrence{}, false, fmt.Errorf("update alert instance: %w", err) } occurrence, err := insertOccurrence(ctx, tx, current.ID, input.Observation.EvaluationKey, transition, input.Observation, resultValueJSON, healthJSON) if err != nil { return Instance{}, Occurrence{}, false, err } if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1`, current.ID), ¤t); err != nil { return Instance{}, Occurrence{}, false, err } if err := tx.Commit(ctx); err != nil { return Instance{}, Occurrence{}, false, fmt.Errorf("commit alert state transition: %w", err) } return current, occurrence, false, nil } func (r StateRepository) GetInstance(ctx context.Context, id string) (Instance, error) { if r.Pool == nil { return Instance{}, ErrUnavailable } var instance Instance err := scanInstance(r.Pool.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1`, id), &instance) if errors.Is(err, pgx.ErrNoRows) { return Instance{}, ErrInstanceNotFound } return instance, err } func (r StateRepository) ListOccurrences(ctx context.Context, instanceID string, limit int) ([]Occurrence, error) { if r.Pool == nil { return nil, ErrUnavailable } if limit < 1 || limit > 500 { return nil, errors.New("alert occurrence limit is invalid") } rows, err := r.Pool.Query(ctx, `SELECT id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at FROM alert_occurrences WHERE instance_id=$1 ORDER BY observed_at DESC,id ASC LIMIT $2`, instanceID, limit) if err != nil { return nil, fmt.Errorf("list alert occurrences: %w", err) } defer rows.Close() result := make([]Occurrence, 0, limit) for rows.Next() { occurrence, err := scanOccurrence(rows) if err != nil { return nil, err } result = append(result, occurrence) } if err := rows.Err(); err != nil { return nil, err } if len(result) == 0 { if _, err := r.GetInstance(ctx, instanceID); errors.Is(err, ErrInstanceNotFound) { return nil, ErrInstanceNotFound } } return result, nil } func (r StateRepository) Acknowledge(ctx context.Context, instanceID, actor, evaluationKey string, at time.Time) (Instance, Occurrence, bool, error) { if r.Pool == nil { return Instance{}, Occurrence{}, false, ErrUnavailable } if instanceID == "" || actor == "" || len(actor) > 160 || evaluationKey == "" || len(evaluationKey) > 160 || at.IsZero() { return Instance{}, Occurrence{}, false, ErrInvalidObservation } tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{}) if err != nil { return Instance{}, Occurrence{}, false, fmt.Errorf("begin alert acknowledgement: %w", err) } defer func() { _ = tx.Rollback(ctx) }() var current Instance if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1 FOR UPDATE`, instanceID), ¤t); errors.Is(err, pgx.ErrNoRows) { return Instance{}, Occurrence{}, false, ErrInstanceNotFound } else if err != nil { return Instance{}, Occurrence{}, false, err } if occurrence, err := scanOccurrence(tx.QueryRow(ctx, `SELECT id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at FROM alert_occurrences WHERE instance_id=$1 AND evaluation_key=$2`, instanceID, evaluationKey)); err == nil { if err := tx.Commit(ctx); err != nil { return Instance{}, Occurrence{}, false, err } return current, occurrence, true, nil } else if !errors.Is(err, pgx.ErrNoRows) { return Instance{}, Occurrence{}, false, err } transition, err := Acknowledge(snapshotFromInstance(current), actor, at.UTC()) if err != nil { return Instance{}, Occurrence{}, false, err } valueJSON, err := boundedJSON(current.LastValue, 128<<10) if err != nil { return Instance{}, Occurrence{}, false, err } healthJSON, err := boundedJSON(nonNilMap(current.SourceHealth), 64<<10) if err != nil { return Instance{}, Occurrence{}, false, err } if _, err := tx.Exec(ctx, `UPDATE alert_instances SET current_state=$1,retained_state=$2,reason=$3,acknowledged_by=$4,acknowledged_at=$5,revision=revision+1,updated_at=now() WHERE id=$6`, transition.Snapshot.State, transition.Snapshot.RetainedState, transition.Snapshot.Reason, actor, transition.Snapshot.AcknowledgedAt, instanceID); err != nil { return Instance{}, Occurrence{}, false, err } occurrence, err := insertOccurrence(ctx, tx, instanceID, evaluationKey, transition, Observation{ObservedAt: at.UTC(), Reason: "acknowledged", Value: current.LastValue}, valueJSON, healthJSON) if err != nil { return Instance{}, Occurrence{}, false, err } if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1`, instanceID), ¤t); err != nil { return Instance{}, Occurrence{}, false, err } if err := tx.Commit(ctx); err != nil { return Instance{}, Occurrence{}, false, err } return current, occurrence, false, nil } func scanInstance(row pgx.Row, instance *Instance) error { var valueJSON, healthJSON []byte err := row.Scan(&instance.ID, &instance.RuleID, &instance.RuleVersionID, &instance.Fingerprint, &instance.EntityID, &instance.State, &instance.RetainedState, &instance.ActiveSince, &instance.RecoverySince, &instance.CooldownUntil, &instance.LastEvaluatedAt, &instance.LastKnownAt, &valueJSON, &instance.Reason, &healthJSON, &instance.AcknowledgedBy, &instance.AcknowledgedAt, &instance.Revision, &instance.CreatedAt, &instance.UpdatedAt) if errors.Is(err, pgx.ErrNoRows) { return ErrInstanceNotFound } if err != nil { return fmt.Errorf("scan alert instance: %w", err) } instance.LastValue, err = decodeJSON(valueJSON) if err != nil { return fmt.Errorf("decode alert instance value: %w", err) } instance.SourceHealth, err = decodeMap(healthJSON) if err != nil { return fmt.Errorf("decode alert instance source health: %w", err) } return nil } func scanOccurrence(row interface{ Scan(...any) error }) (Occurrence, error) { var occurrence Occurrence var valueJSON, healthJSON []byte err := row.Scan(&occurrence.ID, &occurrence.InstanceID, &occurrence.EvaluationKey, &occurrence.EventType, &occurrence.From, &occurrence.To, &occurrence.ObservedAt, &valueJSON, &occurrence.Reason, &healthJSON, &occurrence.CreatedAt) if err != nil { return Occurrence{}, err } occurrence.Value, err = decodeJSON(valueJSON) if err != nil { return Occurrence{}, fmt.Errorf("decode alert occurrence value: %w", err) } occurrence.SourceHealth, err = decodeMap(healthJSON) if err != nil { return Occurrence{}, fmt.Errorf("decode alert occurrence source health: %w", err) } return occurrence, nil } func insertOccurrence(ctx context.Context, tx pgx.Tx, instanceID, key string, transition TransitionResult, observation Observation, valueJSON, healthJSON []byte) (Occurrence, error) { var occurrence Occurrence err := tx.QueryRow(ctx, `INSERT INTO alert_occurrences (id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health) VALUES ($1,$2,$3,$4,$5,$6,$7,$8::jsonb,$9,$10::jsonb) RETURNING id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at`, NewID(), instanceID, key, transition.EventType, transition.From, transition.To, observation.ObservedAt.UTC(), valueJSON, transition.Snapshot.Reason, healthJSON).Scan(&occurrence.ID, &occurrence.InstanceID, &occurrence.EvaluationKey, &occurrence.EventType, &occurrence.From, &occurrence.To, &occurrence.ObservedAt, &valueJSON, &occurrence.Reason, &healthJSON, &occurrence.CreatedAt) if err != nil { return Occurrence{}, mapStateError(fmt.Errorf("insert alert occurrence: %w", err)) } occurrence.Value, err = decodeJSON(valueJSON) if err != nil { return Occurrence{}, err } occurrence.SourceHealth, err = decodeMap(healthJSON) if err != nil { return Occurrence{}, err } return occurrence, nil } func snapshotFromInstance(instance Instance) Snapshot { return Snapshot{State: instance.State, RetainedState: instance.RetainedState, ActiveSince: instance.ActiveSince, RecoverySince: instance.RecoverySince, CooldownUntil: instance.CooldownUntil, LastEvaluatedAt: instance.LastEvaluatedAt, LastKnownAt: instance.LastKnownAt, LastValue: instance.LastValue, Reason: instance.Reason, SourceHealth: instance.SourceHealth, AcknowledgedBy: instance.AcknowledgedBy, AcknowledgedAt: instance.AcknowledgedAt} } func validateStateInput(input StateInput) error { if strings.TrimSpace(input.RuleID) == "" || strings.TrimSpace(input.RuleVersionID) == "" || input.Fingerprint == "" || len(input.Fingerprint) > 160 { return ErrInvalidObservation } if input.Policy.UnknownBehavior != UnknownRetain && input.Policy.UnknownBehavior != UnknownBecome && input.Policy.UnknownBehavior != UnknownIgnoreGap { return ErrInvalidObservation } if input.Policy.PendingSeconds < 0 || input.Policy.ResolveSeconds < 0 { return ErrInvalidObservation } return nil } func boundedJSON(value any, max int) ([]byte, error) { encoded, err := json.Marshal(value) if err != nil { return nil, fmt.Errorf("encode alert state JSON: %w", err) } if len(encoded) > max { return nil, ErrInvalidObservation } return encoded, nil } func decodeJSON(raw []byte) (any, error) { if len(raw) == 0 || string(raw) == "null" { return nil, nil } var value any if err := json.Unmarshal(raw, &value); err != nil { return nil, err } return value, nil } func decodeMap(raw []byte) (map[string]any, error) { if len(raw) == 0 || string(raw) == "null" { return map[string]any{}, nil } var value map[string]any if err := json.Unmarshal(raw, &value); err != nil { return nil, err } if value == nil { return map[string]any{}, nil } return value, nil } func nullableID(value string) any { if value == "" { return nil } return value } func nullableText(value string) any { if value == "" { return nil } return value } func mapStateError(err error) error { var pgErr interface{ SQLState() string } if errors.As(err, &pgErr) && pgErr.SQLState() == "23505" { return ErrStateConflict } return err }