ITWorx Pulse release export
bd774932d5
Public source validation / validate (push) Failing after 3m8s
361 lines
17 KiB
Go
361 lines
17 KiB
Go
package alert
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
type StateInput struct {
|
|
RuleID string
|
|
RuleVersionID string
|
|
Fingerprint string
|
|
EntityID string
|
|
Policy Policy
|
|
Observation Observation
|
|
}
|
|
|
|
type Instance struct {
|
|
ID string `json:"id"`
|
|
RuleID string `json:"ruleId"`
|
|
RuleVersionID string `json:"ruleVersionId"`
|
|
Fingerprint string `json:"fingerprint"`
|
|
EntityID string `json:"entityId,omitempty"`
|
|
State State `json:"state"`
|
|
RetainedState State `json:"retainedState"`
|
|
ActiveSince *time.Time `json:"activeSince,omitempty"`
|
|
RecoverySince *time.Time `json:"recoverySince,omitempty"`
|
|
LastEvaluatedAt time.Time `json:"lastEvaluatedAt"`
|
|
LastKnownAt *time.Time `json:"lastKnownAt,omitempty"`
|
|
LastValue any `json:"lastValue,omitempty"`
|
|
Reason string `json:"reason"`
|
|
SourceHealth map[string]any `json:"sourceHealth"`
|
|
AcknowledgedBy string `json:"acknowledgedBy,omitempty"`
|
|
AcknowledgedAt *time.Time `json:"acknowledgedAt,omitempty"`
|
|
CooldownUntil *time.Time `json:"cooldownUntil,omitempty"`
|
|
Revision int64 `json:"revision"`
|
|
CreatedAt time.Time `json:"createdAt"`
|
|
UpdatedAt time.Time `json:"updatedAt"`
|
|
}
|
|
|
|
type Occurrence struct {
|
|
ID string `json:"id"`
|
|
InstanceID string `json:"instanceId"`
|
|
EvaluationKey string `json:"evaluationKey"`
|
|
EventType string `json:"eventType"`
|
|
From State `json:"from"`
|
|
To State `json:"to"`
|
|
ObservedAt time.Time `json:"observedAt"`
|
|
Value any `json:"value,omitempty"`
|
|
Reason string `json:"reason"`
|
|
SourceHealth map[string]any `json:"sourceHealth"`
|
|
CreatedAt time.Time `json:"createdAt"`
|
|
}
|
|
|
|
type StateStore interface {
|
|
ApplyObservation(context.Context, StateInput) (Instance, Occurrence, bool, error)
|
|
GetInstance(context.Context, string) (Instance, error)
|
|
ListOccurrences(context.Context, string, int) ([]Occurrence, error)
|
|
Acknowledge(context.Context, string, string, string, time.Time) (Instance, Occurrence, bool, error)
|
|
}
|
|
|
|
type StateRepository struct {
|
|
Pool *pgxpool.Pool
|
|
}
|
|
|
|
func (r StateRepository) ApplyObservation(ctx context.Context, input StateInput) (Instance, Occurrence, bool, error) {
|
|
if r.Pool == nil {
|
|
return Instance{}, Occurrence{}, false, ErrUnavailable
|
|
}
|
|
if err := validateStateInput(input); err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
healthJSON, err := boundedJSON(nonNilMap(input.Observation.SourceHealth), 64<<10)
|
|
if err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
|
if err != nil {
|
|
return Instance{}, Occurrence{}, false, fmt.Errorf("begin alert state transition: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback(ctx) }()
|
|
instanceID := NewID()
|
|
if err := tx.QueryRow(ctx, `INSERT INTO alert_instances (id,rule_id,rule_version_id,fingerprint,entity_id,last_evaluated_at) VALUES ($1,$2,$3,$4,$5,$6) ON CONFLICT (rule_id,fingerprint) DO NOTHING RETURNING id`, instanceID, input.RuleID, input.RuleVersionID, input.Fingerprint, nullableID(input.EntityID), input.Observation.ObservedAt.UTC()).Scan(&instanceID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
|
return Instance{}, Occurrence{}, false, mapStateError(fmt.Errorf("create alert instance: %w", err))
|
|
}
|
|
var current Instance
|
|
if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE rule_id=$1 AND fingerprint=$2 FOR UPDATE`, input.RuleID, input.Fingerprint), ¤t); err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
if current.EntityID != input.EntityID {
|
|
return Instance{}, Occurrence{}, false, ErrStateConflict
|
|
}
|
|
if occurrence, err := scanOccurrence(tx.QueryRow(ctx, `SELECT id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at FROM alert_occurrences WHERE instance_id=$1 AND evaluation_key=$2`, current.ID, input.Observation.EvaluationKey)); err == nil {
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return Instance{}, Occurrence{}, false, fmt.Errorf("commit idempotent alert evaluation: %w", err)
|
|
}
|
|
return current, occurrence, true, nil
|
|
} else if !errors.Is(err, pgx.ErrNoRows) {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
transition, err := Transition(snapshotFromInstance(current), input.Policy, input.Observation)
|
|
if err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
resultValue := transition.Snapshot.LastValue
|
|
if input.Observation.Unknown || input.Policy.UnknownBehavior == UnknownIgnoreGap && transition.Snapshot.LastKnownAt == nil {
|
|
resultValue = current.LastValue
|
|
}
|
|
resultValueJSON, err := boundedJSON(resultValue, 128<<10)
|
|
if err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
if _, err := tx.Exec(ctx, `UPDATE alert_instances SET rule_version_id=$1,current_state=$2,retained_state=$3,active_since=$4,recovery_since=$5,cooldown_until=$6,last_evaluated_at=$7,last_known_at=$8,last_value=$9::jsonb,reason=$10,source_health=$11::jsonb,acknowledged_by=$12,acknowledged_at=$13,revision=revision+1,updated_at=now() WHERE id=$14`, input.RuleVersionID, transition.Snapshot.State, transition.Snapshot.RetainedState, transition.Snapshot.ActiveSince, transition.Snapshot.RecoverySince, transition.Snapshot.CooldownUntil, transition.Snapshot.LastEvaluatedAt.UTC(), transition.Snapshot.LastKnownAt, resultValueJSON, transition.Snapshot.Reason, healthJSON, nullableText(transition.Snapshot.AcknowledgedBy), transition.Snapshot.AcknowledgedAt, current.ID); err != nil {
|
|
return Instance{}, Occurrence{}, false, fmt.Errorf("update alert instance: %w", err)
|
|
}
|
|
occurrence, err := insertOccurrence(ctx, tx, current.ID, input.Observation.EvaluationKey, transition, input.Observation, resultValueJSON, healthJSON)
|
|
if err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1`, current.ID), ¤t); err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return Instance{}, Occurrence{}, false, fmt.Errorf("commit alert state transition: %w", err)
|
|
}
|
|
return current, occurrence, false, nil
|
|
}
|
|
|
|
func (r StateRepository) GetInstance(ctx context.Context, id string) (Instance, error) {
|
|
if r.Pool == nil {
|
|
return Instance{}, ErrUnavailable
|
|
}
|
|
var instance Instance
|
|
err := scanInstance(r.Pool.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1`, id), &instance)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return Instance{}, ErrInstanceNotFound
|
|
}
|
|
return instance, err
|
|
}
|
|
|
|
func (r StateRepository) ListOccurrences(ctx context.Context, instanceID string, limit int) ([]Occurrence, error) {
|
|
if r.Pool == nil {
|
|
return nil, ErrUnavailable
|
|
}
|
|
if limit < 1 || limit > 500 {
|
|
return nil, errors.New("alert occurrence limit is invalid")
|
|
}
|
|
rows, err := r.Pool.Query(ctx, `SELECT id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at FROM alert_occurrences WHERE instance_id=$1 ORDER BY observed_at DESC,id ASC LIMIT $2`, instanceID, limit)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list alert occurrences: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
result := make([]Occurrence, 0, limit)
|
|
for rows.Next() {
|
|
occurrence, err := scanOccurrence(rows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result = append(result, occurrence)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(result) == 0 {
|
|
if _, err := r.GetInstance(ctx, instanceID); errors.Is(err, ErrInstanceNotFound) {
|
|
return nil, ErrInstanceNotFound
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (r StateRepository) Acknowledge(ctx context.Context, instanceID, actor, evaluationKey string, at time.Time) (Instance, Occurrence, bool, error) {
|
|
if r.Pool == nil {
|
|
return Instance{}, Occurrence{}, false, ErrUnavailable
|
|
}
|
|
if instanceID == "" || actor == "" || len(actor) > 160 || evaluationKey == "" || len(evaluationKey) > 160 || at.IsZero() {
|
|
return Instance{}, Occurrence{}, false, ErrInvalidObservation
|
|
}
|
|
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
|
if err != nil {
|
|
return Instance{}, Occurrence{}, false, fmt.Errorf("begin alert acknowledgement: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback(ctx) }()
|
|
var current Instance
|
|
if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1 FOR UPDATE`, instanceID), ¤t); errors.Is(err, pgx.ErrNoRows) {
|
|
return Instance{}, Occurrence{}, false, ErrInstanceNotFound
|
|
} else if err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
if occurrence, err := scanOccurrence(tx.QueryRow(ctx, `SELECT id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at FROM alert_occurrences WHERE instance_id=$1 AND evaluation_key=$2`, instanceID, evaluationKey)); err == nil {
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
return current, occurrence, true, nil
|
|
} else if !errors.Is(err, pgx.ErrNoRows) {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
transition, err := Acknowledge(snapshotFromInstance(current), actor, at.UTC())
|
|
if err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
valueJSON, err := boundedJSON(current.LastValue, 128<<10)
|
|
if err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
healthJSON, err := boundedJSON(nonNilMap(current.SourceHealth), 64<<10)
|
|
if err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
if _, err := tx.Exec(ctx, `UPDATE alert_instances SET current_state=$1,retained_state=$2,reason=$3,acknowledged_by=$4,acknowledged_at=$5,revision=revision+1,updated_at=now() WHERE id=$6`, transition.Snapshot.State, transition.Snapshot.RetainedState, transition.Snapshot.Reason, actor, transition.Snapshot.AcknowledgedAt, instanceID); err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
occurrence, err := insertOccurrence(ctx, tx, instanceID, evaluationKey, transition, Observation{ObservedAt: at.UTC(), Reason: "acknowledged", Value: current.LastValue}, valueJSON, healthJSON)
|
|
if err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
if err := scanInstance(tx.QueryRow(ctx, `SELECT id,rule_id,rule_version_id,fingerprint,COALESCE(entity_id::text,''),current_state,retained_state,active_since,recovery_since,cooldown_until,last_evaluated_at,last_known_at,last_value,reason,source_health,COALESCE(acknowledged_by,''),acknowledged_at,revision,created_at,updated_at FROM alert_instances WHERE id=$1`, instanceID), ¤t); err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return Instance{}, Occurrence{}, false, err
|
|
}
|
|
return current, occurrence, false, nil
|
|
}
|
|
|
|
func scanInstance(row pgx.Row, instance *Instance) error {
|
|
var valueJSON, healthJSON []byte
|
|
err := row.Scan(&instance.ID, &instance.RuleID, &instance.RuleVersionID, &instance.Fingerprint, &instance.EntityID, &instance.State, &instance.RetainedState, &instance.ActiveSince, &instance.RecoverySince, &instance.CooldownUntil, &instance.LastEvaluatedAt, &instance.LastKnownAt, &valueJSON, &instance.Reason, &healthJSON, &instance.AcknowledgedBy, &instance.AcknowledgedAt, &instance.Revision, &instance.CreatedAt, &instance.UpdatedAt)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return ErrInstanceNotFound
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("scan alert instance: %w", err)
|
|
}
|
|
instance.LastValue, err = decodeJSON(valueJSON)
|
|
if err != nil {
|
|
return fmt.Errorf("decode alert instance value: %w", err)
|
|
}
|
|
instance.SourceHealth, err = decodeMap(healthJSON)
|
|
if err != nil {
|
|
return fmt.Errorf("decode alert instance source health: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func scanOccurrence(row interface{ Scan(...any) error }) (Occurrence, error) {
|
|
var occurrence Occurrence
|
|
var valueJSON, healthJSON []byte
|
|
err := row.Scan(&occurrence.ID, &occurrence.InstanceID, &occurrence.EvaluationKey, &occurrence.EventType, &occurrence.From, &occurrence.To, &occurrence.ObservedAt, &valueJSON, &occurrence.Reason, &healthJSON, &occurrence.CreatedAt)
|
|
if err != nil {
|
|
return Occurrence{}, err
|
|
}
|
|
occurrence.Value, err = decodeJSON(valueJSON)
|
|
if err != nil {
|
|
return Occurrence{}, fmt.Errorf("decode alert occurrence value: %w", err)
|
|
}
|
|
occurrence.SourceHealth, err = decodeMap(healthJSON)
|
|
if err != nil {
|
|
return Occurrence{}, fmt.Errorf("decode alert occurrence source health: %w", err)
|
|
}
|
|
return occurrence, nil
|
|
}
|
|
|
|
func insertOccurrence(ctx context.Context, tx pgx.Tx, instanceID, key string, transition TransitionResult, observation Observation, valueJSON, healthJSON []byte) (Occurrence, error) {
|
|
var occurrence Occurrence
|
|
err := tx.QueryRow(ctx, `INSERT INTO alert_occurrences (id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health) VALUES ($1,$2,$3,$4,$5,$6,$7,$8::jsonb,$9,$10::jsonb) RETURNING id,instance_id,evaluation_key,event_type,from_state,to_state,observed_at,value,reason,source_health,created_at`, NewID(), instanceID, key, transition.EventType, transition.From, transition.To, observation.ObservedAt.UTC(), valueJSON, transition.Snapshot.Reason, healthJSON).Scan(&occurrence.ID, &occurrence.InstanceID, &occurrence.EvaluationKey, &occurrence.EventType, &occurrence.From, &occurrence.To, &occurrence.ObservedAt, &valueJSON, &occurrence.Reason, &healthJSON, &occurrence.CreatedAt)
|
|
if err != nil {
|
|
return Occurrence{}, mapStateError(fmt.Errorf("insert alert occurrence: %w", err))
|
|
}
|
|
occurrence.Value, err = decodeJSON(valueJSON)
|
|
if err != nil {
|
|
return Occurrence{}, err
|
|
}
|
|
occurrence.SourceHealth, err = decodeMap(healthJSON)
|
|
if err != nil {
|
|
return Occurrence{}, err
|
|
}
|
|
return occurrence, nil
|
|
}
|
|
|
|
func snapshotFromInstance(instance Instance) Snapshot {
|
|
return Snapshot{State: instance.State, RetainedState: instance.RetainedState, ActiveSince: instance.ActiveSince, RecoverySince: instance.RecoverySince, CooldownUntil: instance.CooldownUntil, LastEvaluatedAt: instance.LastEvaluatedAt, LastKnownAt: instance.LastKnownAt, LastValue: instance.LastValue, Reason: instance.Reason, SourceHealth: instance.SourceHealth, AcknowledgedBy: instance.AcknowledgedBy, AcknowledgedAt: instance.AcknowledgedAt}
|
|
}
|
|
|
|
func validateStateInput(input StateInput) error {
|
|
if strings.TrimSpace(input.RuleID) == "" || strings.TrimSpace(input.RuleVersionID) == "" || input.Fingerprint == "" || len(input.Fingerprint) > 160 {
|
|
return ErrInvalidObservation
|
|
}
|
|
if input.Policy.UnknownBehavior != UnknownRetain && input.Policy.UnknownBehavior != UnknownBecome && input.Policy.UnknownBehavior != UnknownIgnoreGap {
|
|
return ErrInvalidObservation
|
|
}
|
|
if input.Policy.PendingSeconds < 0 || input.Policy.ResolveSeconds < 0 {
|
|
return ErrInvalidObservation
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func boundedJSON(value any, max int) ([]byte, error) {
|
|
encoded, err := json.Marshal(value)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("encode alert state JSON: %w", err)
|
|
}
|
|
if len(encoded) > max {
|
|
return nil, ErrInvalidObservation
|
|
}
|
|
return encoded, nil
|
|
}
|
|
|
|
func decodeJSON(raw []byte) (any, error) {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return nil, nil
|
|
}
|
|
var value any
|
|
if err := json.Unmarshal(raw, &value); err != nil {
|
|
return nil, err
|
|
}
|
|
return value, nil
|
|
}
|
|
|
|
func decodeMap(raw []byte) (map[string]any, error) {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return map[string]any{}, nil
|
|
}
|
|
var value map[string]any
|
|
if err := json.Unmarshal(raw, &value); err != nil {
|
|
return nil, err
|
|
}
|
|
if value == nil {
|
|
return map[string]any{}, nil
|
|
}
|
|
return value, nil
|
|
}
|
|
|
|
func nullableID(value string) any {
|
|
if value == "" {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
|
|
func nullableText(value string) any {
|
|
if value == "" {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
|
|
func mapStateError(err error) error {
|
|
var pgErr interface{ SQLState() string }
|
|
if errors.As(err, &pgErr) && pgErr.SQLState() == "23505" {
|
|
return ErrStateConflict
|
|
}
|
|
return err
|
|
}
|