package workerruntime import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "log/slog" "sort" "strconv" "strings" "sync" "time" "github.com/itworx/pulse/internal/alert" "github.com/itworx/pulse/internal/alertworker" "github.com/itworx/pulse/internal/auth" "github.com/itworx/pulse/internal/metricquery" "github.com/itworx/pulse/internal/notification" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" ) // MaxAlertRules bounds one evaluation pass. const MaxAlertRules = 100 // MetricValue is the resolved input of one alert condition. type MetricValue struct { Value float64 ObservedAt time.Time // Known is false when the source could not supply a usable, fresh value. // The evaluation then records Unknown rather than assuming healthy // (ADR-0008). Known bool // Reason is a stable snake_case code explaining an unknown value. Reason string } // MetricSource resolves an alert rule condition to a current value. type MetricSource interface { Value(ctx context.Context, rule alert.Rule) (MetricValue, error) } // AlertStateWriter applies one observation to the alert state machine. // alert.StateRepository satisfies it. type AlertStateWriter interface { ApplyObservation(ctx context.Context, input alert.StateInput) (alert.Instance, alert.Occurrence, bool, error) } // PriorAlertState is the state of an alert instance before an observation is // applied. It is required to decide whether a transition should notify, because // notification suppression during cooldown depends on the previous state. type PriorAlertState struct { State alert.State CooldownUntil *time.Time Found bool } // AlertStateReader reads the pre-transition state of an alert instance. type AlertStateReader interface { PriorState(ctx context.Context, ruleID, fingerprint string) (PriorAlertState, error) } // RuleVersionSource resolves the immutable rule version an evaluation is // recorded against. alert.Repository satisfies it. type RuleVersionSource interface { Versions(ctx context.Context, id string, limit int) ([]alert.Version, error) } // NotificationQueue is the outbox surface the evaluator writes to. // *notification.Repository satisfies it. type NotificationQueue interface { Enqueue(ctx context.Context, item notification.Outbox) (notification.Outbox, bool, error) ListChannels(ctx context.Context, limit int) ([]notification.Channel, error) } // AlertEvaluator evaluates one rule and records the resulting state transition. // // It implements alertworker.Evaluator, so per-rule leasing, bounded // concurrency and attempt timeouts all come from internal/alertworker rather // than being reimplemented here. type AlertEvaluator struct { Metrics MetricSource States AlertStateWriter Prior AlertStateReader Versions RuleVersionSource Notifications NotificationQueue Logger *slog.Logger Now func() time.Time mu sync.Mutex versions map[string]cachedVersion channels []notification.Channel loadedAt time.Time } type cachedVersion struct { id string number int } func (e *AlertEvaluator) now() time.Time { if e.Now != nil { return e.Now().UTC() } return time.Now().UTC() } func (e *AlertEvaluator) logger() *slog.Logger { if e.Logger != nil { return e.Logger } return slog.Default() } // Evaluate resolves the rule input, applies the state machine transition and // queues any resulting notification. func (e *AlertEvaluator) Evaluate(ctx context.Context, rule alert.Rule) error { if e.States == nil { return errors.New("alert evaluator requires a state store") } now := e.now() versionID, err := e.versionID(ctx, rule) if err != nil { return err } fingerprint := RuleFingerprint(rule) prior := PriorAlertState{State: alert.StateInactive} if e.Prior != nil { prior, err = e.Prior.PriorState(ctx, rule.ID, fingerprint) if err != nil { return fmt.Errorf("read prior alert state: %w", err) } if !prior.Found { prior.State = alert.StateInactive } } observation := e.observe(ctx, rule, prior, now) instance, occurrence, duplicate, err := e.States.ApplyObservation(ctx, alert.StateInput{ RuleID: rule.ID, RuleVersionID: versionID, Fingerprint: fingerprint, Policy: alert.Policy{PendingSeconds: rule.PendingSeconds, ResolveSeconds: rule.ResolveSeconds, CooldownSeconds: rule.CooldownSeconds, UnknownBehavior: rule.UnknownBehavior}, Observation: observation, }) if err != nil { if errors.Is(err, alert.ErrStaleObservation) { // An out-of-order observation is a normal race between two // evaluation windows, not a failure. return nil } return fmt.Errorf("apply alert observation: %w", err) } if duplicate { // The same evaluation window was already recorded, so the notification // for it was queued too. Repeating must not deliver twice. return nil } return e.notify(ctx, rule, instance, occurrence, prior, now) } // observe resolves the rule condition into an observation. Any input that is // missing, stale or unsupported becomes an explicit Unknown observation with a // reason instead of a silent healthy result. func (e *AlertEvaluator) observe(ctx context.Context, rule alert.Rule, prior PriorAlertState, now time.Time) alert.Observation { observation := alert.Observation{ EvaluationKey: EvaluationKey(rule, now), ObservedAt: now, Unknown: true, Reason: "input_not_available", SourceHealth: map[string]any{"inputType": rule.Condition.InputType}, } if rule.Condition.InputType != "metric" { observation.Reason = "input_type_not_supported" return observation } if e.Metrics == nil { observation.Reason = "metric_source_not_configured" return observation } value, err := e.Metrics.Value(ctx, rule) if err != nil { observation.Reason = "metric_query_failed" e.logger().Warn("alert metric query failed", "rule", rule.ID, "error", boundedError(err)) return observation } if !value.Known { observation.Reason = defaultReason(value.Reason, "metric_unavailable") return observation } active := prior.State == alert.StateFiring || prior.State == alert.StateAcknowledged || prior.State == alert.StateUnknown fires, err := rule.Condition.Evaluate(value.Value, active) if err != nil { observation.Reason = "condition_not_evaluable" e.logger().Warn("alert condition could not be evaluated", "rule", rule.ID, "error", boundedError(err)) return observation } observation.Unknown = false observation.ConditionTrue = fires observation.Value = value.Value observation.Reason = "condition_evaluated" if !value.ObservedAt.IsZero() { observation.SourceHealth["observedAt"] = value.ObservedAt.UTC().Format(time.RFC3339) } return observation } // notify queues one outbox item per enabled channel for a transition that the // state machine considers notifiable. The idempotency key is derived from the // instance, the evaluation window and the channel, so a repeated evaluation or // a retried enqueue can never produce a second delivery. func (e *AlertEvaluator) notify(ctx context.Context, rule alert.Rule, instance alert.Instance, occurrence alert.Occurrence, prior PriorAlertState, now time.Time) error { if e.Notifications == nil { return nil } eventType := NotificationFor(prior, occurrence, now) if eventType == "" { return nil } channels, err := e.enabledChannels(ctx, now) if err != nil { return err } for _, channel := range channels { key := instance.ID + ":" + occurrence.EvaluationKey + ":" + channel.ID if len(key) > 255 { key = key[:255] } item := notification.Outbox{ IdempotencyKey: key, ChannelID: channel.ID, EventType: eventType, Subject: notificationSubject(rule, eventType), Body: notificationBody(rule, occurrence), Status: notification.StatusPending, NextAttemptAt: now, } // A duplicate enqueue is the outbox recognizing the idempotency key and // returning the existing row, which is exactly the no-duplicate-delivery // guarantee this key exists for. if _, _, err := e.Notifications.Enqueue(ctx, item); err != nil { return fmt.Errorf("enqueue alert notification: %w", err) } } return nil } // enabledChannels caches the channel list briefly. Channel configuration // changes rarely and re-reading it for every rule would multiply database load // by the number of rules. func (e *AlertEvaluator) enabledChannels(ctx context.Context, now time.Time) ([]notification.Channel, error) { e.mu.Lock() if !e.loadedAt.IsZero() && now.Sub(e.loadedAt) < 30*time.Second { channels := e.channels e.mu.Unlock() return channels, nil } e.mu.Unlock() all, err := e.Notifications.ListChannels(ctx, 100) if err != nil { return nil, fmt.Errorf("list notification channels: %w", err) } enabled := make([]notification.Channel, 0, len(all)) for _, channel := range all { if channel.Enabled { enabled = append(enabled, channel) } } e.mu.Lock() e.channels, e.loadedAt = enabled, now e.mu.Unlock() return enabled, nil } func (e *AlertEvaluator) versionID(ctx context.Context, rule alert.Rule) (string, error) { e.mu.Lock() cached, ok := e.versions[rule.ID] e.mu.Unlock() if ok && cached.number == rule.CurrentVersion { return cached.id, nil } if e.Versions == nil { return "", errors.New("alert evaluator requires a rule version source") } versions, err := e.Versions.Versions(ctx, rule.ID, 1) if err != nil { return "", fmt.Errorf("read alert rule version: %w", err) } if len(versions) == 0 { return "", fmt.Errorf("alert rule %s has no version", rule.ID) } e.mu.Lock() if e.versions == nil { e.versions = make(map[string]cachedVersion, MaxAlertRules) } if len(e.versions) > MaxAlertRules*2 { e.versions = make(map[string]cachedVersion, MaxAlertRules) } e.versions[rule.ID] = cachedVersion{id: versions[0].ID, number: versions[0].VersionNumber} e.mu.Unlock() return versions[0].ID, nil } // NotificationFor mirrors the notification decision the alert state machine // makes internally (internal/alert/state.go notificationFor), which is not // exported: a fresh firing transition notifies unless the instance is still in // its cooldown window, a resolution from firing or acknowledged notifies, and a // first transition into unknown notifies. func NotificationFor(prior PriorAlertState, occurrence alert.Occurrence, now time.Time) notification.EventType { from := prior.State if !prior.Found { from = alert.StateInactive } switch { case occurrence.To == alert.StateFiring && from != alert.StateFiring && from != alert.StateAcknowledged: if prior.CooldownUntil != nil && now.Before(prior.CooldownUntil.UTC()) { return "" } return notification.EventFiring case occurrence.To == alert.StateResolved && (from == alert.StateFiring || from == alert.StateAcknowledged): return notification.EventRecovery case occurrence.To == alert.StateUnknown && from != alert.StateUnknown: return notification.EventUnknown default: return "" } } // EvaluationKey identifies one evaluation window of a rule. Deriving it from // the rule's own interval is what makes a repeated evaluation idempotent: the // state store recognizes the key and returns the existing occurrence. func EvaluationKey(rule alert.Rule, now time.Time) string { interval := time.Duration(rule.EvaluationIntervalSeconds) * time.Second if interval <= 0 { interval = time.Minute } return "eval:" + strconv.FormatInt(now.UTC().Truncate(interval).Unix(), 10) } // RuleFingerprint is the stable alert instance identity for a rule scope. func RuleFingerprint(rule alert.Rule) string { keys := make([]string, 0, len(rule.Scope)) for key := range rule.Scope { keys = append(keys, key) } sort.Strings(keys) builder := strings.Builder{} builder.WriteString(rule.ID) for _, key := range keys { builder.WriteString("\x00" + key + "=" + fmt.Sprint(rule.Scope[key])) } sum := sha256.Sum256([]byte("itworx-pulse/alert-fingerprint/v1/" + builder.String())) return hex.EncodeToString(sum[:16]) } func notificationSubject(rule alert.Rule, eventType notification.EventType) string { subject := string(eventType) + ": " + rule.Name if len(subject) > notification.MaxSubject { subject = subject[:notification.MaxSubject] } return subject } // notificationBody carries stable identifiers and localization keys only. It // deliberately contains no configuration values, so a notification can never // leak a secret. func notificationBody(rule alert.Rule, occurrence alert.Occurrence) string { body := strings.Join([]string{ "ruleId=" + rule.ID, "severity=" + rule.Severity, "titleKey=" + rule.Message.TitleKey, "bodyKey=" + rule.Message.BodyKey, "from=" + string(occurrence.From), "to=" + string(occurrence.To), "observedAt=" + occurrence.ObservedAt.UTC().Format(time.RFC3339), "reason=" + occurrence.Reason, }, "\n") if len(body) > notification.MaxBody { body = body[:notification.MaxBody] } return body } // PostgresAlertStateReader reads pre-transition alert state. type PostgresAlertStateReader struct { Pool *pgxpool.Pool } // PriorState returns the current state and cooldown of an alert instance. func (r PostgresAlertStateReader) PriorState(ctx context.Context, ruleID, fingerprint string) (PriorAlertState, error) { if r.Pool == nil { return PriorAlertState{}, fmt.Errorf("%w: alert state reader has no database pool", ErrInvalidConfig) } var state string var cooldown *time.Time err := r.Pool.QueryRow(ctx, `SELECT current_state,cooldown_until FROM alert_instances WHERE rule_id=$1 AND fingerprint=$2`, ruleID, fingerprint).Scan(&state, &cooldown) if errors.Is(err, pgx.ErrNoRows) { return PriorAlertState{State: alert.StateInactive}, nil } if err != nil { return PriorAlertState{}, fmt.Errorf("read alert instance state: %w", err) } return PriorAlertState{State: alert.State(state), CooldownUntil: cooldown, Found: true}, nil } // AlertEvaluationJob runs one alert evaluation pass through internal/alertworker. type AlertEvaluationJob struct { Worker *alertworker.Worker Enabled bool Reason string } // Run evaluates every due rule. func (j AlertEvaluationJob) Run(ctx context.Context) (Outcome, error) { if !j.Enabled || j.Worker == nil { return Outcome{Disabled: true, Reason: defaultReason(j.Reason, "alert_evaluation_not_configured")}, nil } report, err := j.Worker.RunOnce(ctx) counts := map[string]int64{ "scheduled": int64(report.Scheduled), "started": int64(report.Started), "completed": int64(report.Completed), "skipped": int64(report.Skipped), "failed": int64(report.Failed), "canceled": int64(report.Canceled), } if err != nil { return Outcome{Counts: counts}, err } if report.Failed > 0 { // A rule that could not be evaluated must not be hidden behind an // otherwise successful pass. return Outcome{Counts: counts}, fmt.Errorf("%d of %d alert rule evaluations failed", report.Failed, report.Scheduled) } return Outcome{Counts: counts}, nil } // PrometheusMetricSource resolves rule conditions through the shared metric // query service, which enforces the semantic catalog, plan limits and // server-side query bounds. type PrometheusMetricSource struct { Service interface { ExecuteInstant(context.Context, metricquery.InstantRequest) (metricquery.Response, error) } } // Value executes a bounded instant query for the rule's semantic metric. func (s PrometheusMetricSource) Value(ctx context.Context, rule alert.Rule) (MetricValue, error) { if s.Service == nil { return MetricValue{Reason: "metric_source_not_configured"}, nil } request := metricquery.InstantRequest{Metric: rule.Condition.Metric, Aggregation: rule.Condition.Aggregation, Scope: metricQueryScope(rule.Scope), MaxSeries: 1} // The planner intentionally requires an authenticated read principal. Worker // jobs have no HTTP session, so identify this narrow internal caller rather // than bypassing the planner, its catalog, or any query limits. queryContext := ctx if _, ok := auth.PrincipalFromContext(ctx); !ok { queryContext = auth.WithPrincipal(ctx, auth.Principal{Subject: "pulse-worker", Role: auth.RoleViewer}) } response, err := s.Service.ExecuteInstant(queryContext, request) if err != nil { return MetricValue{Reason: "metric_query_failed"}, err } if response.Freshness != metricquery.FreshnessFresh { return MetricValue{Reason: "metric_" + defaultReason(response.Freshness, "unknown"), ObservedAt: response.SourceObservedAt}, nil } value, ok := firstInstantValue(response.Data) if !ok { return MetricValue{Reason: "metric_no_series", ObservedAt: response.SourceObservedAt}, nil } return MetricValue{Value: value, ObservedAt: response.SourceObservedAt, Known: true}, nil } // firstInstantValue extracts the single sample of a Prometheus instant vector. func firstInstantValue(payload []byte) (float64, bool) { if len(payload) == 0 || len(payload) > 1<<20 { return 0, false } var document struct { ResultType string `json:"resultType"` Result []struct { Value []json.RawMessage `json:"value"` } `json:"result"` } if err := json.Unmarshal(payload, &document); err != nil { return 0, false } if len(document.Result) == 0 || len(document.Result[0].Value) != 2 { return 0, false } var text string if err := json.Unmarshal(document.Result[0].Value[1], &text); err != nil { return 0, false } value, err := strconv.ParseFloat(text, 64) if err != nil { return 0, false } return value, true } // metricQueryScope separates alert-target metadata (`entityType`, `critical`, // and similar inventory selectors) from the query planner's explicit metric // label aliases. Forwarding the whole alert scope made implementation-owned // defaults fail before reaching Prometheus and could accidentally treat product // metadata as a raw label selector. func metricQueryScope(scope map[string]any) map[string]string { if len(scope) == 0 { return nil } allowed := map[string]struct{}{ "entityId": {}, "serverId": {}, "containerId": {}, "diskId": {}, "poolId": {}, "serviceId": {}, "probeId": {}, } result := make(map[string]string, len(allowed)) for key, value := range scope { if _, ok := allowed[key]; !ok { continue } result[key] = fmt.Sprint(value) } if len(result) == 0 { return nil } return result }