ITWorx-Pulse-Public/internal/workerruntime/alertjob.go

package workerruntime

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"log/slog"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/itworx/pulse/internal/alert"
	"github.com/itworx/pulse/internal/alertworker"
	"github.com/itworx/pulse/internal/auth"
	"github.com/itworx/pulse/internal/metricquery"
	"github.com/itworx/pulse/internal/notification"
	"github.com/jackc/pgx/v5"
	"github.com/jackc/pgx/v5/pgxpool"
)

// MaxAlertRules bounds one evaluation pass.
const MaxAlertRules = 100

// MetricValue is the resolved input of one alert condition.
type MetricValue struct {
	Value      float64
	ObservedAt time.Time
	// Known is false when the source could not supply a usable, fresh value.
	// The evaluation then records Unknown rather than assuming healthy
	// (ADR-0008).
	Known bool
	// Reason is a stable snake_case code explaining an unknown value.
	Reason string
}

// MetricSource resolves an alert rule condition to a current value.
type MetricSource interface {
	Value(ctx context.Context, rule alert.Rule) (MetricValue, error)
}

// AlertStateWriter applies one observation to the alert state machine.
// alert.StateRepository satisfies it.
type AlertStateWriter interface {
	ApplyObservation(ctx context.Context, input alert.StateInput) (alert.Instance, alert.Occurrence, bool, error)
}

// PriorAlertState is the state of an alert instance before an observation is
// applied. It is required to decide whether a transition should notify, because
// notification suppression during cooldown depends on the previous state.
type PriorAlertState struct {
	State         alert.State
	CooldownUntil *time.Time
	Found         bool
}

// AlertStateReader reads the pre-transition state of an alert instance.
type AlertStateReader interface {
	PriorState(ctx context.Context, ruleID, fingerprint string) (PriorAlertState, error)
}

// RuleVersionSource resolves the immutable rule version an evaluation is
// recorded against. alert.Repository satisfies it.
type RuleVersionSource interface {
	Versions(ctx context.Context, id string, limit int) ([]alert.Version, error)
}

// NotificationQueue is the outbox surface the evaluator writes to.
// *notification.Repository satisfies it.
type NotificationQueue interface {
	Enqueue(ctx context.Context, item notification.Outbox) (notification.Outbox, bool, error)
	ListChannels(ctx context.Context, limit int) ([]notification.Channel, error)
}

// AlertEvaluator evaluates one rule and records the resulting state transition.
//
// It implements alertworker.Evaluator, so per-rule leasing, bounded
// concurrency and attempt timeouts all come from internal/alertworker rather
// than being reimplemented here.
type AlertEvaluator struct {
	Metrics       MetricSource
	States        AlertStateWriter
	Prior         AlertStateReader
	Versions      RuleVersionSource
	Notifications NotificationQueue
	Logger        *slog.Logger
	Now           func() time.Time

	mu       sync.Mutex
	versions map[string]cachedVersion
	channels []notification.Channel
	loadedAt time.Time
}

type cachedVersion struct {
	id     string
	number int
}

func (e *AlertEvaluator) now() time.Time {
	if e.Now != nil {
		return e.Now().UTC()
	}
	return time.Now().UTC()
}

func (e *AlertEvaluator) logger() *slog.Logger {
	if e.Logger != nil {
		return e.Logger
	}
	return slog.Default()
}

// Evaluate resolves the rule input, applies the state machine transition and
// queues any resulting notification.
func (e *AlertEvaluator) Evaluate(ctx context.Context, rule alert.Rule) error {
	if e.States == nil {
		return errors.New("alert evaluator requires a state store")
	}
	now := e.now()
	versionID, err := e.versionID(ctx, rule)
	if err != nil {
		return err
	}
	fingerprint := RuleFingerprint(rule)
	prior := PriorAlertState{State: alert.StateInactive}
	if e.Prior != nil {
		prior, err = e.Prior.PriorState(ctx, rule.ID, fingerprint)
		if err != nil {
			return fmt.Errorf("read prior alert state: %w", err)
		}
		if !prior.Found {
			prior.State = alert.StateInactive
		}
	}
	observation := e.observe(ctx, rule, prior, now)
	instance, occurrence, duplicate, err := e.States.ApplyObservation(ctx, alert.StateInput{
		RuleID: rule.ID, RuleVersionID: versionID, Fingerprint: fingerprint,
		Policy:      alert.Policy{PendingSeconds: rule.PendingSeconds, ResolveSeconds: rule.ResolveSeconds, CooldownSeconds: rule.CooldownSeconds, UnknownBehavior: rule.UnknownBehavior},
		Observation: observation,
	})
	if err != nil {
		if errors.Is(err, alert.ErrStaleObservation) {
			// An out-of-order observation is a normal race between two
			// evaluation windows, not a failure.
			return nil
		}
		return fmt.Errorf("apply alert observation: %w", err)
	}
	if duplicate {
		// The same evaluation window was already recorded, so the notification
		// for it was queued too. Repeating must not deliver twice.
		return nil
	}
	return e.notify(ctx, rule, instance, occurrence, prior, now)
}

// observe resolves the rule condition into an observation. Any input that is
// missing, stale or unsupported becomes an explicit Unknown observation with a
// reason instead of a silent healthy result.
func (e *AlertEvaluator) observe(ctx context.Context, rule alert.Rule, prior PriorAlertState, now time.Time) alert.Observation {
	observation := alert.Observation{
		EvaluationKey: EvaluationKey(rule, now), ObservedAt: now, Unknown: true, Reason: "input_not_available",
		SourceHealth: map[string]any{"inputType": rule.Condition.InputType},
	}
	if rule.Condition.InputType != "metric" {
		observation.Reason = "input_type_not_supported"
		return observation
	}
	if e.Metrics == nil {
		observation.Reason = "metric_source_not_configured"
		return observation
	}
	value, err := e.Metrics.Value(ctx, rule)
	if err != nil {
		observation.Reason = "metric_query_failed"
		e.logger().Warn("alert metric query failed", "rule", rule.ID, "error", boundedError(err))
		return observation
	}
	if !value.Known {
		observation.Reason = defaultReason(value.Reason, "metric_unavailable")
		return observation
	}
	active := prior.State == alert.StateFiring || prior.State == alert.StateAcknowledged || prior.State == alert.StateUnknown
	fires, err := rule.Condition.Evaluate(value.Value, active)
	if err != nil {
		observation.Reason = "condition_not_evaluable"
		e.logger().Warn("alert condition could not be evaluated", "rule", rule.ID, "error", boundedError(err))
		return observation
	}
	observation.Unknown = false
	observation.ConditionTrue = fires
	observation.Value = value.Value
	observation.Reason = "condition_evaluated"
	if !value.ObservedAt.IsZero() {
		observation.SourceHealth["observedAt"] = value.ObservedAt.UTC().Format(time.RFC3339)
	}
	return observation
}

// notify queues one outbox item per enabled channel for a transition that the
// state machine considers notifiable. The idempotency key is derived from the
// instance, the evaluation window and the channel, so a repeated evaluation or
// a retried enqueue can never produce a second delivery.
func (e *AlertEvaluator) notify(ctx context.Context, rule alert.Rule, instance alert.Instance, occurrence alert.Occurrence, prior PriorAlertState, now time.Time) error {
	if e.Notifications == nil {
		return nil
	}
	eventType := NotificationFor(prior, occurrence, now)
	if eventType == "" {
		return nil
	}
	channels, err := e.enabledChannels(ctx, now)
	if err != nil {
		return err
	}
	for _, channel := range channels {
		key := instance.ID + ":" + occurrence.EvaluationKey + ":" + channel.ID
		if len(key) > 255 {
			key = key[:255]
		}
		item := notification.Outbox{
			IdempotencyKey: key, ChannelID: channel.ID, EventType: eventType,
			Subject: notificationSubject(rule, eventType), Body: notificationBody(rule, occurrence),
			Status: notification.StatusPending, NextAttemptAt: now,
		}
		// A duplicate enqueue is the outbox recognizing the idempotency key and
		// returning the existing row, which is exactly the no-duplicate-delivery
		// guarantee this key exists for.
		if _, _, err := e.Notifications.Enqueue(ctx, item); err != nil {
			return fmt.Errorf("enqueue alert notification: %w", err)
		}
	}
	return nil
}

// enabledChannels caches the channel list briefly. Channel configuration
// changes rarely and re-reading it for every rule would multiply database load
// by the number of rules.
func (e *AlertEvaluator) enabledChannels(ctx context.Context, now time.Time) ([]notification.Channel, error) {
	e.mu.Lock()
	if !e.loadedAt.IsZero() && now.Sub(e.loadedAt) < 30*time.Second {
		channels := e.channels
		e.mu.Unlock()
		return channels, nil
	}
	e.mu.Unlock()
	all, err := e.Notifications.ListChannels(ctx, 100)
	if err != nil {
		return nil, fmt.Errorf("list notification channels: %w", err)
	}
	enabled := make([]notification.Channel, 0, len(all))
	for _, channel := range all {
		if channel.Enabled {
			enabled = append(enabled, channel)
		}
	}
	e.mu.Lock()
	e.channels, e.loadedAt = enabled, now
	e.mu.Unlock()
	return enabled, nil
}

func (e *AlertEvaluator) versionID(ctx context.Context, rule alert.Rule) (string, error) {
	e.mu.Lock()
	cached, ok := e.versions[rule.ID]
	e.mu.Unlock()
	if ok && cached.number == rule.CurrentVersion {
		return cached.id, nil
	}
	if e.Versions == nil {
		return "", errors.New("alert evaluator requires a rule version source")
	}
	versions, err := e.Versions.Versions(ctx, rule.ID, 1)
	if err != nil {
		return "", fmt.Errorf("read alert rule version: %w", err)
	}
	if len(versions) == 0 {
		return "", fmt.Errorf("alert rule %s has no version", rule.ID)
	}
	e.mu.Lock()
	if e.versions == nil {
		e.versions = make(map[string]cachedVersion, MaxAlertRules)
	}
	if len(e.versions) > MaxAlertRules*2 {
		e.versions = make(map[string]cachedVersion, MaxAlertRules)
	}
	e.versions[rule.ID] = cachedVersion{id: versions[0].ID, number: versions[0].VersionNumber}
	e.mu.Unlock()
	return versions[0].ID, nil
}

// NotificationFor mirrors the notification decision the alert state machine
// makes internally (internal/alert/state.go notificationFor), which is not
// exported: a fresh firing transition notifies unless the instance is still in
// its cooldown window, a resolution from firing or acknowledged notifies, and a
// first transition into unknown notifies.
func NotificationFor(prior PriorAlertState, occurrence alert.Occurrence, now time.Time) notification.EventType {
	from := prior.State
	if !prior.Found {
		from = alert.StateInactive
	}
	switch {
	case occurrence.To == alert.StateFiring && from != alert.StateFiring && from != alert.StateAcknowledged:
		if prior.CooldownUntil != nil && now.Before(prior.CooldownUntil.UTC()) {
			return ""
		}
		return notification.EventFiring
	case occurrence.To == alert.StateResolved && (from == alert.StateFiring || from == alert.StateAcknowledged):
		return notification.EventRecovery
	case occurrence.To == alert.StateUnknown && from != alert.StateUnknown:
		return notification.EventUnknown
	default:
		return ""
	}
}

// EvaluationKey identifies one evaluation window of a rule. Deriving it from
// the rule's own interval is what makes a repeated evaluation idempotent: the
// state store recognizes the key and returns the existing occurrence.
func EvaluationKey(rule alert.Rule, now time.Time) string {
	interval := time.Duration(rule.EvaluationIntervalSeconds) * time.Second
	if interval <= 0 {
		interval = time.Minute
	}
	return "eval:" + strconv.FormatInt(now.UTC().Truncate(interval).Unix(), 10)
}

// RuleFingerprint is the stable alert instance identity for a rule scope.
func RuleFingerprint(rule alert.Rule) string {
	keys := make([]string, 0, len(rule.Scope))
	for key := range rule.Scope {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	builder := strings.Builder{}
	builder.WriteString(rule.ID)
	for _, key := range keys {
		builder.WriteString("\x00" + key + "=" + fmt.Sprint(rule.Scope[key]))
	}
	sum := sha256.Sum256([]byte("itworx-pulse/alert-fingerprint/v1/" + builder.String()))
	return hex.EncodeToString(sum[:16])
}

func notificationSubject(rule alert.Rule, eventType notification.EventType) string {
	subject := string(eventType) + ": " + rule.Name
	if len(subject) > notification.MaxSubject {
		subject = subject[:notification.MaxSubject]
	}
	return subject
}

// notificationBody carries stable identifiers and localization keys only. It
// deliberately contains no configuration values, so a notification can never
// leak a secret.
func notificationBody(rule alert.Rule, occurrence alert.Occurrence) string {
	body := strings.Join([]string{
		"ruleId=" + rule.ID,
		"severity=" + rule.Severity,
		"titleKey=" + rule.Message.TitleKey,
		"bodyKey=" + rule.Message.BodyKey,
		"from=" + string(occurrence.From),
		"to=" + string(occurrence.To),
		"observedAt=" + occurrence.ObservedAt.UTC().Format(time.RFC3339),
		"reason=" + occurrence.Reason,
	}, "\n")
	if len(body) > notification.MaxBody {
		body = body[:notification.MaxBody]
	}
	return body
}

// PostgresAlertStateReader reads pre-transition alert state.
type PostgresAlertStateReader struct {
	Pool *pgxpool.Pool
}

// PriorState returns the current state and cooldown of an alert instance.
func (r PostgresAlertStateReader) PriorState(ctx context.Context, ruleID, fingerprint string) (PriorAlertState, error) {
	if r.Pool == nil {
		return PriorAlertState{}, fmt.Errorf("%w: alert state reader has no database pool", ErrInvalidConfig)
	}
	var state string
	var cooldown *time.Time
	err := r.Pool.QueryRow(ctx, `SELECT current_state,cooldown_until FROM alert_instances WHERE rule_id=$1 AND fingerprint=$2`, ruleID, fingerprint).Scan(&state, &cooldown)
	if errors.Is(err, pgx.ErrNoRows) {
		return PriorAlertState{State: alert.StateInactive}, nil
	}
	if err != nil {
		return PriorAlertState{}, fmt.Errorf("read alert instance state: %w", err)
	}
	return PriorAlertState{State: alert.State(state), CooldownUntil: cooldown, Found: true}, nil
}

// AlertEvaluationJob runs one alert evaluation pass through internal/alertworker.
type AlertEvaluationJob struct {
	Worker  *alertworker.Worker
	Enabled bool
	Reason  string
}

// Run evaluates every due rule.
func (j AlertEvaluationJob) Run(ctx context.Context) (Outcome, error) {
	if !j.Enabled || j.Worker == nil {
		return Outcome{Disabled: true, Reason: defaultReason(j.Reason, "alert_evaluation_not_configured")}, nil
	}
	report, err := j.Worker.RunOnce(ctx)
	counts := map[string]int64{
		"scheduled": int64(report.Scheduled), "started": int64(report.Started), "completed": int64(report.Completed),
		"skipped": int64(report.Skipped), "failed": int64(report.Failed), "canceled": int64(report.Canceled),
	}
	if err != nil {
		return Outcome{Counts: counts}, err
	}
	if report.Failed > 0 {
		// A rule that could not be evaluated must not be hidden behind an
		// otherwise successful pass.
		return Outcome{Counts: counts}, fmt.Errorf("%d of %d alert rule evaluations failed", report.Failed, report.Scheduled)
	}
	return Outcome{Counts: counts}, nil
}

// PrometheusMetricSource resolves rule conditions through the shared metric
// query service, which enforces the semantic catalog, plan limits and
// server-side query bounds.
type PrometheusMetricSource struct {
	Service interface {
		ExecuteInstant(context.Context, metricquery.InstantRequest) (metricquery.Response, error)
	}
}

// Value executes a bounded instant query for the rule's semantic metric.
func (s PrometheusMetricSource) Value(ctx context.Context, rule alert.Rule) (MetricValue, error) {
	if s.Service == nil {
		return MetricValue{Reason: "metric_source_not_configured"}, nil
	}
	request := metricquery.InstantRequest{Metric: rule.Condition.Metric, Aggregation: rule.Condition.Aggregation, Scope: metricQueryScope(rule.Scope), MaxSeries: 1}
	// The planner intentionally requires an authenticated read principal. Worker
	// jobs have no HTTP session, so identify this narrow internal caller rather
	// than bypassing the planner, its catalog, or any query limits.
	queryContext := ctx
	if _, ok := auth.PrincipalFromContext(ctx); !ok {
		queryContext = auth.WithPrincipal(ctx, auth.Principal{Subject: "pulse-worker", Role: auth.RoleViewer})
	}
	response, err := s.Service.ExecuteInstant(queryContext, request)
	if err != nil {
		return MetricValue{Reason: "metric_query_failed"}, err
	}
	if response.Freshness != metricquery.FreshnessFresh {
		return MetricValue{Reason: "metric_" + defaultReason(response.Freshness, "unknown"), ObservedAt: response.SourceObservedAt}, nil
	}
	value, ok := firstInstantValue(response.Data)
	if !ok {
		return MetricValue{Reason: "metric_no_series", ObservedAt: response.SourceObservedAt}, nil
	}
	return MetricValue{Value: value, ObservedAt: response.SourceObservedAt, Known: true}, nil
}

// firstInstantValue extracts the single sample of a Prometheus instant vector.
func firstInstantValue(payload []byte) (float64, bool) {
	if len(payload) == 0 || len(payload) > 1<<20 {
		return 0, false
	}
	var document struct {
		ResultType string `json:"resultType"`
		Result     []struct {
			Value []json.RawMessage `json:"value"`
		} `json:"result"`
	}
	if err := json.Unmarshal(payload, &document); err != nil {
		return 0, false
	}
	if len(document.Result) == 0 || len(document.Result[0].Value) != 2 {
		return 0, false
	}
	var text string
	if err := json.Unmarshal(document.Result[0].Value[1], &text); err != nil {
		return 0, false
	}
	value, err := strconv.ParseFloat(text, 64)
	if err != nil {
		return 0, false
	}
	return value, true
}

// metricQueryScope separates alert-target metadata (`entityType`, `critical`,
// and similar inventory selectors) from the query planner's explicit metric
// label aliases. Forwarding the whole alert scope made implementation-owned
// defaults fail before reaching Prometheus and could accidentally treat product
// metadata as a raw label selector.
func metricQueryScope(scope map[string]any) map[string]string {
	if len(scope) == 0 {
		return nil
	}
	allowed := map[string]struct{}{
		"entityId": {}, "serverId": {}, "containerId": {}, "diskId": {},
		"poolId": {}, "serviceId": {}, "probeId": {},
	}
	result := make(map[string]string, len(allowed))
	for key, value := range scope {
		if _, ok := allowed[key]; !ok {
			continue
		}
		result[key] = fmt.Sprint(value)
	}
	if len(result) == 0 {
		return nil
	}
	return result
}