This commit is contained in:
ITWorx Pulse release export
@@ -0,0 +1,515 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/alertworker"
|
||||
"github.com/itworx/pulse/internal/auth"
|
||||
"github.com/itworx/pulse/internal/metricquery"
|
||||
"github.com/itworx/pulse/internal/notification"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// MaxAlertRules bounds one evaluation pass.
|
||||
const MaxAlertRules = 100
|
||||
|
||||
// MetricValue is the resolved input of one alert condition.
|
||||
type MetricValue struct {
|
||||
Value float64
|
||||
ObservedAt time.Time
|
||||
// Known is false when the source could not supply a usable, fresh value.
|
||||
// The evaluation then records Unknown rather than assuming healthy
|
||||
// (ADR-0008).
|
||||
Known bool
|
||||
// Reason is a stable snake_case code explaining an unknown value.
|
||||
Reason string
|
||||
}
|
||||
|
||||
// MetricSource resolves an alert rule condition to a current value.
|
||||
type MetricSource interface {
|
||||
Value(ctx context.Context, rule alert.Rule) (MetricValue, error)
|
||||
}
|
||||
|
||||
// AlertStateWriter applies one observation to the alert state machine.
|
||||
// alert.StateRepository satisfies it.
|
||||
type AlertStateWriter interface {
|
||||
ApplyObservation(ctx context.Context, input alert.StateInput) (alert.Instance, alert.Occurrence, bool, error)
|
||||
}
|
||||
|
||||
// PriorAlertState is the state of an alert instance before an observation is
|
||||
// applied. It is required to decide whether a transition should notify, because
|
||||
// notification suppression during cooldown depends on the previous state.
|
||||
type PriorAlertState struct {
|
||||
State alert.State
|
||||
CooldownUntil *time.Time
|
||||
Found bool
|
||||
}
|
||||
|
||||
// AlertStateReader reads the pre-transition state of an alert instance.
|
||||
type AlertStateReader interface {
|
||||
PriorState(ctx context.Context, ruleID, fingerprint string) (PriorAlertState, error)
|
||||
}
|
||||
|
||||
// RuleVersionSource resolves the immutable rule version an evaluation is
|
||||
// recorded against. alert.Repository satisfies it.
|
||||
type RuleVersionSource interface {
|
||||
Versions(ctx context.Context, id string, limit int) ([]alert.Version, error)
|
||||
}
|
||||
|
||||
// NotificationQueue is the outbox surface the evaluator writes to.
|
||||
// *notification.Repository satisfies it.
|
||||
type NotificationQueue interface {
|
||||
Enqueue(ctx context.Context, item notification.Outbox) (notification.Outbox, bool, error)
|
||||
ListChannels(ctx context.Context, limit int) ([]notification.Channel, error)
|
||||
}
|
||||
|
||||
// AlertEvaluator evaluates one rule and records the resulting state transition.
|
||||
//
|
||||
// It implements alertworker.Evaluator, so per-rule leasing, bounded
|
||||
// concurrency and attempt timeouts all come from internal/alertworker rather
|
||||
// than being reimplemented here.
|
||||
type AlertEvaluator struct {
|
||||
Metrics MetricSource
|
||||
States AlertStateWriter
|
||||
Prior AlertStateReader
|
||||
Versions RuleVersionSource
|
||||
Notifications NotificationQueue
|
||||
Logger *slog.Logger
|
||||
Now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
versions map[string]cachedVersion
|
||||
channels []notification.Channel
|
||||
loadedAt time.Time
|
||||
}
|
||||
|
||||
type cachedVersion struct {
|
||||
id string
|
||||
number int
|
||||
}
|
||||
|
||||
func (e *AlertEvaluator) now() time.Time {
|
||||
if e.Now != nil {
|
||||
return e.Now().UTC()
|
||||
}
|
||||
return time.Now().UTC()
|
||||
}
|
||||
|
||||
func (e *AlertEvaluator) logger() *slog.Logger {
|
||||
if e.Logger != nil {
|
||||
return e.Logger
|
||||
}
|
||||
return slog.Default()
|
||||
}
|
||||
|
||||
// Evaluate resolves the rule input, applies the state machine transition and
|
||||
// queues any resulting notification.
|
||||
func (e *AlertEvaluator) Evaluate(ctx context.Context, rule alert.Rule) error {
|
||||
if e.States == nil {
|
||||
return errors.New("alert evaluator requires a state store")
|
||||
}
|
||||
now := e.now()
|
||||
versionID, err := e.versionID(ctx, rule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fingerprint := RuleFingerprint(rule)
|
||||
prior := PriorAlertState{State: alert.StateInactive}
|
||||
if e.Prior != nil {
|
||||
prior, err = e.Prior.PriorState(ctx, rule.ID, fingerprint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read prior alert state: %w", err)
|
||||
}
|
||||
if !prior.Found {
|
||||
prior.State = alert.StateInactive
|
||||
}
|
||||
}
|
||||
observation := e.observe(ctx, rule, prior, now)
|
||||
instance, occurrence, duplicate, err := e.States.ApplyObservation(ctx, alert.StateInput{
|
||||
RuleID: rule.ID, RuleVersionID: versionID, Fingerprint: fingerprint,
|
||||
Policy: alert.Policy{PendingSeconds: rule.PendingSeconds, ResolveSeconds: rule.ResolveSeconds, CooldownSeconds: rule.CooldownSeconds, UnknownBehavior: rule.UnknownBehavior},
|
||||
Observation: observation,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, alert.ErrStaleObservation) {
|
||||
// An out-of-order observation is a normal race between two
|
||||
// evaluation windows, not a failure.
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("apply alert observation: %w", err)
|
||||
}
|
||||
if duplicate {
|
||||
// The same evaluation window was already recorded, so the notification
|
||||
// for it was queued too. Repeating must not deliver twice.
|
||||
return nil
|
||||
}
|
||||
return e.notify(ctx, rule, instance, occurrence, prior, now)
|
||||
}
|
||||
|
||||
// observe resolves the rule condition into an observation. Any input that is
|
||||
// missing, stale or unsupported becomes an explicit Unknown observation with a
|
||||
// reason instead of a silent healthy result.
|
||||
func (e *AlertEvaluator) observe(ctx context.Context, rule alert.Rule, prior PriorAlertState, now time.Time) alert.Observation {
|
||||
observation := alert.Observation{
|
||||
EvaluationKey: EvaluationKey(rule, now), ObservedAt: now, Unknown: true, Reason: "input_not_available",
|
||||
SourceHealth: map[string]any{"inputType": rule.Condition.InputType},
|
||||
}
|
||||
if rule.Condition.InputType != "metric" {
|
||||
observation.Reason = "input_type_not_supported"
|
||||
return observation
|
||||
}
|
||||
if e.Metrics == nil {
|
||||
observation.Reason = "metric_source_not_configured"
|
||||
return observation
|
||||
}
|
||||
value, err := e.Metrics.Value(ctx, rule)
|
||||
if err != nil {
|
||||
observation.Reason = "metric_query_failed"
|
||||
e.logger().Warn("alert metric query failed", "rule", rule.ID, "error", boundedError(err))
|
||||
return observation
|
||||
}
|
||||
if !value.Known {
|
||||
observation.Reason = defaultReason(value.Reason, "metric_unavailable")
|
||||
return observation
|
||||
}
|
||||
active := prior.State == alert.StateFiring || prior.State == alert.StateAcknowledged || prior.State == alert.StateUnknown
|
||||
fires, err := rule.Condition.Evaluate(value.Value, active)
|
||||
if err != nil {
|
||||
observation.Reason = "condition_not_evaluable"
|
||||
e.logger().Warn("alert condition could not be evaluated", "rule", rule.ID, "error", boundedError(err))
|
||||
return observation
|
||||
}
|
||||
observation.Unknown = false
|
||||
observation.ConditionTrue = fires
|
||||
observation.Value = value.Value
|
||||
observation.Reason = "condition_evaluated"
|
||||
if !value.ObservedAt.IsZero() {
|
||||
observation.SourceHealth["observedAt"] = value.ObservedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return observation
|
||||
}
|
||||
|
||||
// notify queues one outbox item per enabled channel for a transition that the
|
||||
// state machine considers notifiable. The idempotency key is derived from the
|
||||
// instance, the evaluation window and the channel, so a repeated evaluation or
|
||||
// a retried enqueue can never produce a second delivery.
|
||||
func (e *AlertEvaluator) notify(ctx context.Context, rule alert.Rule, instance alert.Instance, occurrence alert.Occurrence, prior PriorAlertState, now time.Time) error {
|
||||
if e.Notifications == nil {
|
||||
return nil
|
||||
}
|
||||
eventType := NotificationFor(prior, occurrence, now)
|
||||
if eventType == "" {
|
||||
return nil
|
||||
}
|
||||
channels, err := e.enabledChannels(ctx, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, channel := range channels {
|
||||
key := instance.ID + ":" + occurrence.EvaluationKey + ":" + channel.ID
|
||||
if len(key) > 255 {
|
||||
key = key[:255]
|
||||
}
|
||||
item := notification.Outbox{
|
||||
IdempotencyKey: key, ChannelID: channel.ID, EventType: eventType,
|
||||
Subject: notificationSubject(rule, eventType), Body: notificationBody(rule, occurrence),
|
||||
Status: notification.StatusPending, NextAttemptAt: now,
|
||||
}
|
||||
// A duplicate enqueue is the outbox recognizing the idempotency key and
|
||||
// returning the existing row, which is exactly the no-duplicate-delivery
|
||||
// guarantee this key exists for.
|
||||
if _, _, err := e.Notifications.Enqueue(ctx, item); err != nil {
|
||||
return fmt.Errorf("enqueue alert notification: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// enabledChannels caches the channel list briefly. Channel configuration
|
||||
// changes rarely and re-reading it for every rule would multiply database load
|
||||
// by the number of rules.
|
||||
func (e *AlertEvaluator) enabledChannels(ctx context.Context, now time.Time) ([]notification.Channel, error) {
|
||||
e.mu.Lock()
|
||||
if !e.loadedAt.IsZero() && now.Sub(e.loadedAt) < 30*time.Second {
|
||||
channels := e.channels
|
||||
e.mu.Unlock()
|
||||
return channels, nil
|
||||
}
|
||||
e.mu.Unlock()
|
||||
all, err := e.Notifications.ListChannels(ctx, 100)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list notification channels: %w", err)
|
||||
}
|
||||
enabled := make([]notification.Channel, 0, len(all))
|
||||
for _, channel := range all {
|
||||
if channel.Enabled {
|
||||
enabled = append(enabled, channel)
|
||||
}
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.channels, e.loadedAt = enabled, now
|
||||
e.mu.Unlock()
|
||||
return enabled, nil
|
||||
}
|
||||
|
||||
func (e *AlertEvaluator) versionID(ctx context.Context, rule alert.Rule) (string, error) {
|
||||
e.mu.Lock()
|
||||
cached, ok := e.versions[rule.ID]
|
||||
e.mu.Unlock()
|
||||
if ok && cached.number == rule.CurrentVersion {
|
||||
return cached.id, nil
|
||||
}
|
||||
if e.Versions == nil {
|
||||
return "", errors.New("alert evaluator requires a rule version source")
|
||||
}
|
||||
versions, err := e.Versions.Versions(ctx, rule.ID, 1)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read alert rule version: %w", err)
|
||||
}
|
||||
if len(versions) == 0 {
|
||||
return "", fmt.Errorf("alert rule %s has no version", rule.ID)
|
||||
}
|
||||
e.mu.Lock()
|
||||
if e.versions == nil {
|
||||
e.versions = make(map[string]cachedVersion, MaxAlertRules)
|
||||
}
|
||||
if len(e.versions) > MaxAlertRules*2 {
|
||||
e.versions = make(map[string]cachedVersion, MaxAlertRules)
|
||||
}
|
||||
e.versions[rule.ID] = cachedVersion{id: versions[0].ID, number: versions[0].VersionNumber}
|
||||
e.mu.Unlock()
|
||||
return versions[0].ID, nil
|
||||
}
|
||||
|
||||
// NotificationFor mirrors the notification decision the alert state machine
|
||||
// makes internally (internal/alert/state.go notificationFor), which is not
|
||||
// exported: a fresh firing transition notifies unless the instance is still in
|
||||
// its cooldown window, a resolution from firing or acknowledged notifies, and a
|
||||
// first transition into unknown notifies.
|
||||
func NotificationFor(prior PriorAlertState, occurrence alert.Occurrence, now time.Time) notification.EventType {
|
||||
from := prior.State
|
||||
if !prior.Found {
|
||||
from = alert.StateInactive
|
||||
}
|
||||
switch {
|
||||
case occurrence.To == alert.StateFiring && from != alert.StateFiring && from != alert.StateAcknowledged:
|
||||
if prior.CooldownUntil != nil && now.Before(prior.CooldownUntil.UTC()) {
|
||||
return ""
|
||||
}
|
||||
return notification.EventFiring
|
||||
case occurrence.To == alert.StateResolved && (from == alert.StateFiring || from == alert.StateAcknowledged):
|
||||
return notification.EventRecovery
|
||||
case occurrence.To == alert.StateUnknown && from != alert.StateUnknown:
|
||||
return notification.EventUnknown
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// EvaluationKey identifies one evaluation window of a rule. Deriving it from
|
||||
// the rule's own interval is what makes a repeated evaluation idempotent: the
|
||||
// state store recognizes the key and returns the existing occurrence.
|
||||
func EvaluationKey(rule alert.Rule, now time.Time) string {
|
||||
interval := time.Duration(rule.EvaluationIntervalSeconds) * time.Second
|
||||
if interval <= 0 {
|
||||
interval = time.Minute
|
||||
}
|
||||
return "eval:" + strconv.FormatInt(now.UTC().Truncate(interval).Unix(), 10)
|
||||
}
|
||||
|
||||
// RuleFingerprint is the stable alert instance identity for a rule scope.
|
||||
func RuleFingerprint(rule alert.Rule) string {
|
||||
keys := make([]string, 0, len(rule.Scope))
|
||||
for key := range rule.Scope {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
builder := strings.Builder{}
|
||||
builder.WriteString(rule.ID)
|
||||
for _, key := range keys {
|
||||
builder.WriteString("\x00" + key + "=" + fmt.Sprint(rule.Scope[key]))
|
||||
}
|
||||
sum := sha256.Sum256([]byte("itworx-pulse/alert-fingerprint/v1/" + builder.String()))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
func notificationSubject(rule alert.Rule, eventType notification.EventType) string {
|
||||
subject := string(eventType) + ": " + rule.Name
|
||||
if len(subject) > notification.MaxSubject {
|
||||
subject = subject[:notification.MaxSubject]
|
||||
}
|
||||
return subject
|
||||
}
|
||||
|
||||
// notificationBody carries stable identifiers and localization keys only. It
|
||||
// deliberately contains no configuration values, so a notification can never
|
||||
// leak a secret.
|
||||
func notificationBody(rule alert.Rule, occurrence alert.Occurrence) string {
|
||||
body := strings.Join([]string{
|
||||
"ruleId=" + rule.ID,
|
||||
"severity=" + rule.Severity,
|
||||
"titleKey=" + rule.Message.TitleKey,
|
||||
"bodyKey=" + rule.Message.BodyKey,
|
||||
"from=" + string(occurrence.From),
|
||||
"to=" + string(occurrence.To),
|
||||
"observedAt=" + occurrence.ObservedAt.UTC().Format(time.RFC3339),
|
||||
"reason=" + occurrence.Reason,
|
||||
}, "\n")
|
||||
if len(body) > notification.MaxBody {
|
||||
body = body[:notification.MaxBody]
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// PostgresAlertStateReader reads pre-transition alert state.
|
||||
type PostgresAlertStateReader struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
// PriorState returns the current state and cooldown of an alert instance.
|
||||
func (r PostgresAlertStateReader) PriorState(ctx context.Context, ruleID, fingerprint string) (PriorAlertState, error) {
|
||||
if r.Pool == nil {
|
||||
return PriorAlertState{}, fmt.Errorf("%w: alert state reader has no database pool", ErrInvalidConfig)
|
||||
}
|
||||
var state string
|
||||
var cooldown *time.Time
|
||||
err := r.Pool.QueryRow(ctx, `SELECT current_state,cooldown_until FROM alert_instances WHERE rule_id=$1 AND fingerprint=$2`, ruleID, fingerprint).Scan(&state, &cooldown)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return PriorAlertState{State: alert.StateInactive}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return PriorAlertState{}, fmt.Errorf("read alert instance state: %w", err)
|
||||
}
|
||||
return PriorAlertState{State: alert.State(state), CooldownUntil: cooldown, Found: true}, nil
|
||||
}
|
||||
|
||||
// AlertEvaluationJob runs one alert evaluation pass through internal/alertworker.
|
||||
type AlertEvaluationJob struct {
|
||||
Worker *alertworker.Worker
|
||||
Enabled bool
|
||||
Reason string
|
||||
}
|
||||
|
||||
// Run evaluates every due rule.
|
||||
func (j AlertEvaluationJob) Run(ctx context.Context) (Outcome, error) {
|
||||
if !j.Enabled || j.Worker == nil {
|
||||
return Outcome{Disabled: true, Reason: defaultReason(j.Reason, "alert_evaluation_not_configured")}, nil
|
||||
}
|
||||
report, err := j.Worker.RunOnce(ctx)
|
||||
counts := map[string]int64{
|
||||
"scheduled": int64(report.Scheduled), "started": int64(report.Started), "completed": int64(report.Completed),
|
||||
"skipped": int64(report.Skipped), "failed": int64(report.Failed), "canceled": int64(report.Canceled),
|
||||
}
|
||||
if err != nil {
|
||||
return Outcome{Counts: counts}, err
|
||||
}
|
||||
if report.Failed > 0 {
|
||||
// A rule that could not be evaluated must not be hidden behind an
|
||||
// otherwise successful pass.
|
||||
return Outcome{Counts: counts}, fmt.Errorf("%d of %d alert rule evaluations failed", report.Failed, report.Scheduled)
|
||||
}
|
||||
return Outcome{Counts: counts}, nil
|
||||
}
|
||||
|
||||
// PrometheusMetricSource resolves rule conditions through the shared metric
|
||||
// query service, which enforces the semantic catalog, plan limits and
|
||||
// server-side query bounds.
|
||||
type PrometheusMetricSource struct {
|
||||
Service interface {
|
||||
ExecuteInstant(context.Context, metricquery.InstantRequest) (metricquery.Response, error)
|
||||
}
|
||||
}
|
||||
|
||||
// Value executes a bounded instant query for the rule's semantic metric.
|
||||
func (s PrometheusMetricSource) Value(ctx context.Context, rule alert.Rule) (MetricValue, error) {
|
||||
if s.Service == nil {
|
||||
return MetricValue{Reason: "metric_source_not_configured"}, nil
|
||||
}
|
||||
request := metricquery.InstantRequest{Metric: rule.Condition.Metric, Aggregation: rule.Condition.Aggregation, Scope: metricQueryScope(rule.Scope), MaxSeries: 1}
|
||||
// The planner intentionally requires an authenticated read principal. Worker
|
||||
// jobs have no HTTP session, so identify this narrow internal caller rather
|
||||
// than bypassing the planner, its catalog, or any query limits.
|
||||
queryContext := ctx
|
||||
if _, ok := auth.PrincipalFromContext(ctx); !ok {
|
||||
queryContext = auth.WithPrincipal(ctx, auth.Principal{Subject: "pulse-worker", Role: auth.RoleViewer})
|
||||
}
|
||||
response, err := s.Service.ExecuteInstant(queryContext, request)
|
||||
if err != nil {
|
||||
return MetricValue{Reason: "metric_query_failed"}, err
|
||||
}
|
||||
if response.Freshness != metricquery.FreshnessFresh {
|
||||
return MetricValue{Reason: "metric_" + defaultReason(response.Freshness, "unknown"), ObservedAt: response.SourceObservedAt}, nil
|
||||
}
|
||||
value, ok := firstInstantValue(response.Data)
|
||||
if !ok {
|
||||
return MetricValue{Reason: "metric_no_series", ObservedAt: response.SourceObservedAt}, nil
|
||||
}
|
||||
return MetricValue{Value: value, ObservedAt: response.SourceObservedAt, Known: true}, nil
|
||||
}
|
||||
|
||||
// firstInstantValue extracts the single sample of a Prometheus instant vector.
|
||||
func firstInstantValue(payload []byte) (float64, bool) {
|
||||
if len(payload) == 0 || len(payload) > 1<<20 {
|
||||
return 0, false
|
||||
}
|
||||
var document struct {
|
||||
ResultType string `json:"resultType"`
|
||||
Result []struct {
|
||||
Value []json.RawMessage `json:"value"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := json.Unmarshal(payload, &document); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
if len(document.Result) == 0 || len(document.Result[0].Value) != 2 {
|
||||
return 0, false
|
||||
}
|
||||
var text string
|
||||
if err := json.Unmarshal(document.Result[0].Value[1], &text); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
value, err := strconv.ParseFloat(text, 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return value, true
|
||||
}
|
||||
|
||||
// metricQueryScope separates alert-target metadata (`entityType`, `critical`,
|
||||
// and similar inventory selectors) from the query planner's explicit metric
|
||||
// label aliases. Forwarding the whole alert scope made implementation-owned
|
||||
// defaults fail before reaching Prometheus and could accidentally treat product
|
||||
// metadata as a raw label selector.
|
||||
func metricQueryScope(scope map[string]any) map[string]string {
|
||||
if len(scope) == 0 {
|
||||
return nil
|
||||
}
|
||||
allowed := map[string]struct{}{
|
||||
"entityId": {}, "serverId": {}, "containerId": {}, "diskId": {},
|
||||
"poolId": {}, "serviceId": {}, "probeId": {},
|
||||
}
|
||||
result := make(map[string]string, len(allowed))
|
||||
for key, value := range scope {
|
||||
if _, ok := allowed[key]; !ok {
|
||||
continue
|
||||
}
|
||||
result[key] = fmt.Sprint(value)
|
||||
}
|
||||
if len(result) == 0 {
|
||||
return nil
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,620 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/application"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/discovery"
|
||||
"github.com/itworx/pulse/internal/inventory"
|
||||
"github.com/itworx/pulse/internal/lifecycle"
|
||||
"github.com/itworx/pulse/internal/reconciliation"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// MaxDiscoveredContainers bounds one discovery pass. It sits above the
|
||||
// documented scale target of 150 containers (SYSTEM_ARCHITECTURE section 7) so
|
||||
// a normal host is never truncated while a runaway source still cannot flood
|
||||
// the database.
|
||||
const MaxDiscoveredContainers = 400
|
||||
|
||||
// ContainerAliasRecord is one runtime container as last observed. It carries the
|
||||
// reconciliation identity plus the runtime facts lifecycle event derivation
|
||||
// needs to decide whether anything actually changed.
|
||||
type ContainerAliasRecord struct {
|
||||
reconciliation.ContainerAlias
|
||||
State string
|
||||
Health string
|
||||
RestartCount int
|
||||
IntentionalStop bool
|
||||
}
|
||||
|
||||
// ContainerAliasStore persists the previous container observation per source.
|
||||
type ContainerAliasStore interface {
|
||||
List(ctx context.Context, sourceID string) ([]ContainerAliasRecord, error)
|
||||
Save(ctx context.Context, sourceID string, records []ContainerAliasRecord) error
|
||||
}
|
||||
|
||||
// InventoryStore is the narrow inventory surface the discovery job writes to.
|
||||
// *inventory.Repository satisfies it.
|
||||
type InventoryStore interface {
|
||||
PersistDiscovery(ctx context.Context, entity inventory.Entity, alias inventory.Alias, facts []inventory.Fact, relations []inventory.Relation) error
|
||||
}
|
||||
|
||||
// DiscoveryJob observes containers, reconciles them against the previous
|
||||
// snapshot and emits the resulting lifecycle events.
|
||||
//
|
||||
// It is strictly observational (ADR-0001): it reads a snapshot and writes Pulse
|
||||
// state only. Nothing here starts, stops or otherwise mutates a container, the
|
||||
// array, a volume or the host.
|
||||
type DiscoveryJob struct {
|
||||
// SourceID is the data_sources UUID the observations belong to.
|
||||
SourceID string
|
||||
// Provider supplies the container snapshot.
|
||||
Provider container.Provider
|
||||
// Aliases stores the previous observation used for identity and diffing.
|
||||
Aliases ContainerAliasStore
|
||||
// Inventory persists reconciled entities. It may be nil, in which case the
|
||||
// job only derives and emits events.
|
||||
Inventory InventoryStore
|
||||
// Runner claims the discovery window and emits events idempotently.
|
||||
Runner discovery.Runner
|
||||
// Now is injectable for tests.
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (j DiscoveryJob) now() time.Time {
|
||||
if j.Now != nil {
|
||||
return j.Now().UTC()
|
||||
}
|
||||
return time.Now().UTC()
|
||||
}
|
||||
|
||||
// Run performs one discovery and reconciliation pass.
|
||||
func (j DiscoveryJob) Run(ctx context.Context) (Outcome, error) {
|
||||
if j.Provider == nil || j.Aliases == nil || j.Runner.Store == nil {
|
||||
return Outcome{Disabled: true, Reason: "container_source_not_configured"}, nil
|
||||
}
|
||||
if j.SourceID == "" {
|
||||
return Outcome{Disabled: true, Reason: "container_source_not_registered"}, nil
|
||||
}
|
||||
snapshot, err := j.Provider.Snapshot(ctx)
|
||||
if err != nil {
|
||||
return Outcome{}, fmt.Errorf("read container snapshot: %w", err)
|
||||
}
|
||||
// A source that is not healthy and fresh must not drive reconciliation:
|
||||
// tombstoning every container because a collector is down would be exactly
|
||||
// the "source failure tombstones all inventory" failure the standards
|
||||
// forbid, and reporting success would be a silent fallback to green.
|
||||
if snapshot.Source.State != "healthy" || snapshot.Source.Freshness != "fresh" {
|
||||
reason := strings.TrimSpace(snapshot.Source.Reason)
|
||||
if reason == "" {
|
||||
reason = "source_" + defaultReason(snapshot.Source.State, "unknown")
|
||||
}
|
||||
return Outcome{Skipped: true, Reason: boundedCode(reason)}, nil
|
||||
}
|
||||
if len(snapshot.Containers) > MaxDiscoveredContainers {
|
||||
return Outcome{}, fmt.Errorf("container snapshot of %d exceeds the %d bound", len(snapshot.Containers), MaxDiscoveredContainers)
|
||||
}
|
||||
counts := map[string]int64{}
|
||||
observedAt := snapshot.Source.ObservedAt.UTC()
|
||||
if observedAt.IsZero() {
|
||||
observedAt = j.now()
|
||||
}
|
||||
// The job key contains the observation window, so re-running the same
|
||||
// window is claimed once and a retry of a partially applied pass repeats
|
||||
// safely: entity upserts and event inserts are both keyed by identity.
|
||||
jobKey := "container:" + j.SourceID + ":" + observedAt.Truncate(time.Minute).UTC().Format(time.RFC3339)
|
||||
runErr := j.Runner.Run(ctx, jobKey, func(runCtx context.Context) ([]discovery.Event, error) {
|
||||
events, applied, err := j.reconcile(runCtx, snapshot, observedAt)
|
||||
for key, value := range applied {
|
||||
counts[key] = value
|
||||
}
|
||||
return events, err
|
||||
})
|
||||
if runErr != nil {
|
||||
return Outcome{Counts: counts}, runErr
|
||||
}
|
||||
counts["containers"] = int64(len(snapshot.Containers))
|
||||
return Outcome{Counts: counts}, nil
|
||||
}
|
||||
|
||||
func (j DiscoveryJob) reconcile(ctx context.Context, snapshot container.Snapshot, observedAt time.Time) ([]discovery.Event, map[string]int64, error) {
|
||||
counts := map[string]int64{}
|
||||
previous, err := j.Aliases.List(ctx, j.SourceID)
|
||||
if err != nil {
|
||||
return nil, counts, fmt.Errorf("list container aliases: %w", err)
|
||||
}
|
||||
previousByRuntime := make(map[string]ContainerAliasRecord, len(previous))
|
||||
priorAliases := make([]reconciliation.ContainerAlias, 0, len(previous))
|
||||
for _, record := range previous {
|
||||
previousByRuntime[record.RuntimeID] = record
|
||||
priorAliases = append(priorAliases, record.ContainerAlias)
|
||||
}
|
||||
observations := make([]reconciliation.ContainerObservation, 0, len(snapshot.Containers))
|
||||
byRuntime := make(map[string]container.Container, len(snapshot.Containers))
|
||||
for _, item := range snapshot.Containers {
|
||||
project, service := composeIdentity(item)
|
||||
observations = append(observations, reconciliation.ContainerObservation{
|
||||
SourceID: j.SourceID, RuntimeID: item.ID, Name: item.Name, Project: project,
|
||||
Service: service, ImageDigest: item.ImageDigest, ObservedAt: observedAt,
|
||||
})
|
||||
byRuntime[item.ID] = item
|
||||
}
|
||||
result, err := reconciliation.ReconcileContainers(observations, priorAliases, observedAt)
|
||||
if err != nil {
|
||||
return nil, counts, fmt.Errorf("reconcile containers: %w", err)
|
||||
}
|
||||
counts["added"] = int64(len(result.Added))
|
||||
counts["updated"] = int64(len(result.Updated))
|
||||
counts["recreated"] = int64(len(result.Recreated))
|
||||
events := make([]lifecycle.Event, 0, len(result.Aliases))
|
||||
records := make([]ContainerAliasRecord, 0, len(result.Aliases))
|
||||
tombstoned := 0
|
||||
for _, alias := range result.Aliases {
|
||||
record := ContainerAliasRecord{ContainerAlias: alias}
|
||||
current, observed := byRuntime[alias.RuntimeID]
|
||||
if observed {
|
||||
record.State, record.Health = current.State, current.Health
|
||||
record.RestartCount, record.IntentionalStop = current.RestartCount, current.IntentionalStop
|
||||
after := runtimeState{State: current.State, Health: current.Health, RestartCount: current.RestartCount, IntentionalStop: current.IntentionalStop}
|
||||
// A container observed for the first time has no "before", so it
|
||||
// produces an addition rather than a stream of change events.
|
||||
if prior, ok := previousByRuntime[alias.RuntimeID]; ok {
|
||||
before := runtimeState{State: prior.State, Health: prior.Health, RestartCount: prior.RestartCount, IntentionalStop: prior.IntentionalStop}
|
||||
if before != after {
|
||||
events = append(events, lifecycle.ContainerTransitionEvents(j.SourceID, alias.EntityID, before.container(), after.container(), observedAt, observedAt)...)
|
||||
}
|
||||
}
|
||||
} else if prior, ok := previousByRuntime[alias.RuntimeID]; ok {
|
||||
record.State, record.Health = prior.State, prior.Health
|
||||
record.RestartCount, record.IntentionalStop = prior.RestartCount, prior.IntentionalStop
|
||||
}
|
||||
if !alias.TombstonedAt.IsZero() {
|
||||
tombstoned++
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
counts["tombstoned"] = int64(tombstoned)
|
||||
for _, recreated := range result.Recreated {
|
||||
dedup := recreated.EntityID + ":recreated:" + recreated.CurrentRuntimeID
|
||||
events = append(events, lifecycle.Event{
|
||||
ID: dedup, EventType: recreated.EventType, Severity: lifecycle.SeverityAttention, EntityID: recreated.EntityID,
|
||||
SourceID: j.SourceID, OccurredAt: recreated.OccurredAt, ReceivedAt: observedAt, DedupKey: dedup,
|
||||
Summary: "Container was recreated.",
|
||||
})
|
||||
}
|
||||
counts["warnings"] = int64(len(result.Warnings))
|
||||
if err := j.persist(ctx, result, byRuntime, records); err != nil {
|
||||
return nil, counts, err
|
||||
}
|
||||
normalized, err := lifecycle.Normalize(events, observedAt)
|
||||
if err != nil {
|
||||
return nil, counts, fmt.Errorf("normalize lifecycle events: %w", err)
|
||||
}
|
||||
counts["events"] = int64(len(normalized))
|
||||
discovered := make([]discovery.Event, 0, len(normalized))
|
||||
for _, event := range normalized {
|
||||
discovered = append(discovered, discovery.Event{
|
||||
SourceID: j.SourceID, DedupKey: event.DedupKey, Type: event.EventType, Summary: event.Summary,
|
||||
EntityID: event.EntityID, Severity: string(event.Severity), OccurredAt: event.OccurredAt,
|
||||
})
|
||||
}
|
||||
return discovered, counts, nil
|
||||
}
|
||||
|
||||
// persist writes reconciled inventory before events are emitted, so an event
|
||||
// can always reference an entity that exists. Every write is an upsert keyed by
|
||||
// identity, which is what makes a repeated pass safe.
|
||||
func (j DiscoveryJob) persist(ctx context.Context, result reconciliation.ContainerIdentityResult, byRuntime map[string]container.Container, records []ContainerAliasRecord) error {
|
||||
if j.Inventory != nil {
|
||||
groups, err := applicationGroups(result.Current, byRuntime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Applications are persisted before their container relations so every
|
||||
// foreign-key target exists even on the first discovery pass.
|
||||
for _, group := range groups {
|
||||
firstSeen, lastSeen := group.FirstSeenAt, group.LastSeenAt
|
||||
entity := inventory.Entity{
|
||||
ID: group.ID, EntityType: "application", CanonicalName: group.Name,
|
||||
DisplayName: group.Name, Status: group.Status, FirstSeenAt: &firstSeen, LastSeenAt: &lastSeen,
|
||||
}
|
||||
alias := inventory.Alias{EntityID: group.ID, SourceID: j.SourceID, ExternalType: group.ExternalType, ExternalID: group.Name}
|
||||
facts, factErr := applicationFacts(group, j.SourceID)
|
||||
if factErr != nil {
|
||||
return factErr
|
||||
}
|
||||
if err := j.Inventory.PersistDiscovery(ctx, entity, alias, facts, nil); err != nil {
|
||||
return fmt.Errorf("persist application entity: %w", err)
|
||||
}
|
||||
}
|
||||
activeEntities := make(map[string]struct{}, len(result.Current))
|
||||
activeGroups := make(map[string]struct{}, len(groups))
|
||||
for _, group := range groups {
|
||||
activeGroups[group.Name] = struct{}{}
|
||||
}
|
||||
// Missing application groups are materialized before missing-container
|
||||
// relations are tombstoned, preserving the relation foreign key even
|
||||
// when upgrading a legacy database that did not yet persist apps.
|
||||
for _, group := range tombstonedApplicationGroups(result.Aliases, activeGroups) {
|
||||
firstSeen, lastSeen, tombstoned := group.FirstSeenAt, group.LastSeenAt, group.TombstonedAt
|
||||
entity := inventory.Entity{ID: group.ID, EntityType: "application", CanonicalName: group.Name, DisplayName: group.Name, Status: "unknown", FirstSeenAt: &firstSeen, LastSeenAt: &lastSeen, TombstonedAt: &tombstoned}
|
||||
alias := inventory.Alias{EntityID: group.ID, SourceID: j.SourceID, ExternalType: group.ExternalType, ExternalID: group.Name}
|
||||
if err := j.Inventory.PersistDiscovery(ctx, entity, alias, nil, nil); err != nil {
|
||||
return fmt.Errorf("tombstone application entity: %w", err)
|
||||
}
|
||||
}
|
||||
for _, alias := range result.Current {
|
||||
activeEntities[alias.EntityID] = struct{}{}
|
||||
current := byRuntime[alias.RuntimeID]
|
||||
firstSeen := alias.FirstSeenAt
|
||||
lastSeen := alias.LastSeenAt
|
||||
entity := inventory.Entity{
|
||||
ID: alias.EntityID, EntityType: "container", CanonicalName: canonicalContainerName(alias),
|
||||
DisplayName: alias.Name, Status: containerStatus(current), FirstSeenAt: &firstSeen, LastSeenAt: &lastSeen,
|
||||
}
|
||||
externalType, externalID := containerExternalIdentity(alias)
|
||||
aliasRow := inventory.Alias{EntityID: alias.EntityID, SourceID: j.SourceID, ExternalType: externalType, ExternalID: externalID}
|
||||
facts, factErr := containerFacts(alias.EntityID, j.SourceID, current, lastSeen)
|
||||
if factErr != nil {
|
||||
return factErr
|
||||
}
|
||||
groupKey, _ := applicationGroupKey(current)
|
||||
applicationID := application.StableApplicationID(application.SourceID, groupKey)
|
||||
relationFirst, relationLast := firstSeen, lastSeen
|
||||
relation := inventory.Relation{
|
||||
ID: reconciliation.StableEntityID(j.SourceID, "relation", alias.EntityID+"|member_of|"+applicationID),
|
||||
SourceEntityID: alias.EntityID, RelationType: "member_of", TargetEntityID: applicationID,
|
||||
SourceID: j.SourceID, Confidence: 1, Confirmed: true,
|
||||
FirstSeenAt: &relationFirst, LastSeenAt: &relationLast,
|
||||
}
|
||||
if err := j.Inventory.PersistDiscovery(ctx, entity, aliasRow, facts, []inventory.Relation{relation}); err != nil {
|
||||
return fmt.Errorf("persist container entity: %w", err)
|
||||
}
|
||||
}
|
||||
for _, alias := range result.Aliases {
|
||||
if alias.Active || alias.TombstonedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
// A recreation may retire an old runtime alias while reusing the
|
||||
// same logical entity. The active observation wins and must not be
|
||||
// tombstoned by the historical alias later in this loop.
|
||||
if _, active := activeEntities[alias.EntityID]; active {
|
||||
continue
|
||||
}
|
||||
firstSeen := alias.FirstSeenAt
|
||||
lastSeen := alias.LastSeenAt
|
||||
tombstoned := alias.TombstonedAt
|
||||
entity := inventory.Entity{
|
||||
ID: alias.EntityID, EntityType: "container", CanonicalName: canonicalContainerName(alias),
|
||||
DisplayName: alias.Name, Status: "unknown", FirstSeenAt: &firstSeen, LastSeenAt: &lastSeen, TombstonedAt: &tombstoned,
|
||||
}
|
||||
externalType, externalID := containerExternalIdentity(alias)
|
||||
groupKey, _ := applicationGroupKeyFromAlias(alias)
|
||||
applicationID := application.StableApplicationID(application.SourceID, groupKey)
|
||||
relation := inventory.Relation{
|
||||
ID: reconciliation.StableEntityID(j.SourceID, "relation", alias.EntityID+"|member_of|"+applicationID),
|
||||
SourceEntityID: alias.EntityID, RelationType: "member_of", TargetEntityID: applicationID,
|
||||
SourceID: j.SourceID, Confidence: 1, Confirmed: true,
|
||||
FirstSeenAt: &firstSeen, LastSeenAt: &lastSeen, TombstonedAt: &tombstoned,
|
||||
}
|
||||
if err := j.Inventory.PersistDiscovery(ctx, entity, inventory.Alias{EntityID: alias.EntityID, SourceID: j.SourceID, ExternalType: externalType, ExternalID: externalID}, nil, []inventory.Relation{relation}); err != nil {
|
||||
return fmt.Errorf("tombstone container entity: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := j.Aliases.Save(ctx, j.SourceID, records); err != nil {
|
||||
return fmt.Errorf("save container aliases: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type discoveredApplicationGroup struct {
|
||||
ID, Name, Status, ExternalType string
|
||||
ComponentCount int
|
||||
FirstSeenAt, LastSeenAt time.Time
|
||||
TombstonedAt time.Time
|
||||
}
|
||||
|
||||
func applicationGroups(aliases []reconciliation.ContainerAlias, containers map[string]container.Container) ([]discoveredApplicationGroup, error) {
|
||||
byName := make(map[string]*discoveredApplicationGroup, len(aliases))
|
||||
order := make([]string, 0, len(aliases))
|
||||
for _, alias := range aliases {
|
||||
item, ok := containers[alias.RuntimeID]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key, compose := applicationGroupKey(item)
|
||||
if key == "" {
|
||||
return nil, errors.New("container application identity is incomplete")
|
||||
}
|
||||
group := byName[key]
|
||||
if group == nil {
|
||||
externalType := "application-instance"
|
||||
if compose {
|
||||
externalType = "application-project"
|
||||
}
|
||||
group = &discoveredApplicationGroup{ID: application.StableApplicationID(application.SourceID, key), Name: key, Status: "healthy", ExternalType: externalType, FirstSeenAt: alias.FirstSeenAt, LastSeenAt: alias.LastSeenAt}
|
||||
byName[key] = group
|
||||
order = append(order, key)
|
||||
}
|
||||
group.ComponentCount++
|
||||
if alias.FirstSeenAt.Before(group.FirstSeenAt) {
|
||||
group.FirstSeenAt = alias.FirstSeenAt
|
||||
}
|
||||
if alias.LastSeenAt.After(group.LastSeenAt) {
|
||||
group.LastSeenAt = alias.LastSeenAt
|
||||
}
|
||||
group.Status = worstApplicationStatus(group.Status, containerStatus(item))
|
||||
}
|
||||
sort.Strings(order)
|
||||
result := make([]discoveredApplicationGroup, 0, len(order))
|
||||
for _, key := range order {
|
||||
result = append(result, *byName[key])
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func applicationGroupKey(item container.Container) (string, bool) {
|
||||
project := strings.TrimSpace(item.Project)
|
||||
if project == "" {
|
||||
project = strings.TrimSpace(item.Labels["com.docker.compose.project"])
|
||||
}
|
||||
if project != "" {
|
||||
return project, true
|
||||
}
|
||||
return strings.TrimPrefix(strings.TrimSpace(item.Name), "/"), false
|
||||
}
|
||||
|
||||
func applicationGroupKeyFromAlias(alias reconciliation.ContainerAlias) (string, bool) {
|
||||
if project := strings.TrimSpace(alias.Project); project != "" {
|
||||
return project, true
|
||||
}
|
||||
return strings.TrimPrefix(strings.TrimSpace(alias.Name), "/"), false
|
||||
}
|
||||
|
||||
func tombstonedApplicationGroups(aliases []reconciliation.ContainerAlias, active map[string]struct{}) []discoveredApplicationGroup {
|
||||
groups := make(map[string]*discoveredApplicationGroup)
|
||||
for _, alias := range aliases {
|
||||
if alias.Active || alias.TombstonedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
key, compose := applicationGroupKeyFromAlias(alias)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := active[key]; exists {
|
||||
continue
|
||||
}
|
||||
group := groups[key]
|
||||
if group == nil {
|
||||
externalType := "application-instance"
|
||||
if compose {
|
||||
externalType = "application-project"
|
||||
}
|
||||
group = &discoveredApplicationGroup{ID: application.StableApplicationID(application.SourceID, key), Name: key, ExternalType: externalType, FirstSeenAt: alias.FirstSeenAt, LastSeenAt: alias.LastSeenAt, TombstonedAt: alias.TombstonedAt}
|
||||
groups[key] = group
|
||||
}
|
||||
if alias.FirstSeenAt.Before(group.FirstSeenAt) {
|
||||
group.FirstSeenAt = alias.FirstSeenAt
|
||||
}
|
||||
if alias.LastSeenAt.After(group.LastSeenAt) {
|
||||
group.LastSeenAt = alias.LastSeenAt
|
||||
}
|
||||
if alias.TombstonedAt.After(group.TombstonedAt) {
|
||||
group.TombstonedAt = alias.TombstonedAt
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
result := make([]discoveredApplicationGroup, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
result = append(result, *groups[key])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func worstApplicationStatus(current, candidate string) string {
|
||||
rank := map[string]int{"healthy": 0, "up": 0, "unknown": 1, "degraded": 2, "down": 2}
|
||||
if rank[candidate] > rank[current] {
|
||||
if candidate == "down" {
|
||||
return "degraded"
|
||||
}
|
||||
return candidate
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
func applicationFacts(group discoveredApplicationGroup, sourceID string) ([]inventory.Fact, error) {
|
||||
mode := "standalone"
|
||||
if group.ExternalType == "application-project" {
|
||||
mode = "compose"
|
||||
}
|
||||
return inventoryFacts(group.ID, sourceID, group.LastSeenAt, map[string]any{
|
||||
"groupingMode": mode, "componentCount": group.ComponentCount,
|
||||
})
|
||||
}
|
||||
|
||||
func containerFacts(entityID, sourceID string, item container.Container, observedAt time.Time) ([]inventory.Fact, error) {
|
||||
values := map[string]any{
|
||||
"runtimeState": item.State, "health": item.Health, "restartCount": item.RestartCount,
|
||||
"intentionalStop": item.IntentionalStop, "metricsAvailable": item.MetricsAvailable,
|
||||
"lifecycleAvailable": item.LifecycleAvailable,
|
||||
}
|
||||
if value := strings.TrimSpace(item.Image); value != "" {
|
||||
values["image"] = value
|
||||
}
|
||||
if value := strings.TrimSpace(item.Project); value != "" {
|
||||
values["project"] = value
|
||||
}
|
||||
if value := strings.TrimSpace(item.Labels["com.docker.compose.service"]); value != "" {
|
||||
values["composeService"] = value
|
||||
}
|
||||
return inventoryFacts(entityID, sourceID, observedAt, values)
|
||||
}
|
||||
|
||||
func inventoryFacts(entityID, sourceID string, observedAt time.Time, values map[string]any) ([]inventory.Fact, error) {
|
||||
fields := make([]string, 0, len(values))
|
||||
for field := range values {
|
||||
fields = append(fields, field)
|
||||
}
|
||||
sort.Strings(fields)
|
||||
validUntil := observedAt.Add(2 * time.Minute)
|
||||
facts := make([]inventory.Fact, 0, len(fields))
|
||||
for _, field := range fields {
|
||||
value, err := inventory.MarshalValue(values[field])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal %s fact: %w", field, err)
|
||||
}
|
||||
facts = append(facts, inventory.Fact{EntityID: entityID, FieldName: field, SourceID: sourceID, Value: value, ObservedAt: observedAt, Confidence: 1, ValidUntil: &validUntil})
|
||||
}
|
||||
return facts, nil
|
||||
}
|
||||
|
||||
// runtimeState is the comparable subset of a container observation that decides
|
||||
// whether a lifecycle event is warranted. container.Container itself holds
|
||||
// slices and maps and cannot be compared directly.
|
||||
type runtimeState struct {
|
||||
State string
|
||||
Health string
|
||||
RestartCount int
|
||||
IntentionalStop bool
|
||||
}
|
||||
|
||||
func (s runtimeState) container() container.Container {
|
||||
return container.Container{State: s.State, Health: s.Health, RestartCount: s.RestartCount, IntentionalStop: s.IntentionalStop}
|
||||
}
|
||||
|
||||
func composeIdentity(item container.Container) (string, string) {
|
||||
project := strings.TrimSpace(item.Project)
|
||||
if project == "" {
|
||||
project = strings.TrimSpace(item.Labels["com.docker.compose.project"])
|
||||
}
|
||||
service := strings.TrimSpace(item.Labels["com.docker.compose.service"])
|
||||
if project == "" || service == "" {
|
||||
return "", ""
|
||||
}
|
||||
return project, service
|
||||
}
|
||||
|
||||
func canonicalContainerName(alias reconciliation.ContainerAlias) string {
|
||||
if alias.Project != "" && alias.Service != "" {
|
||||
return alias.Project + "/" + alias.Service
|
||||
}
|
||||
return alias.Name
|
||||
}
|
||||
|
||||
func containerExternalIdentity(alias reconciliation.ContainerAlias) (string, string) {
|
||||
if alias.Project != "" && alias.Service != "" {
|
||||
return "container-service", alias.Project + "/" + alias.Service
|
||||
}
|
||||
return "container-instance", alias.RuntimeID
|
||||
}
|
||||
|
||||
// containerStatus maps a runtime state onto the shared status vocabulary. An
|
||||
// unrecognized state is unknown, never healthy (ADR-0008).
|
||||
func containerStatus(item container.Container) string {
|
||||
switch strings.ToLower(strings.TrimSpace(item.State)) {
|
||||
case "running":
|
||||
if strings.EqualFold(item.Health, "unhealthy") {
|
||||
return "degraded"
|
||||
}
|
||||
return "up"
|
||||
case "exited", "dead", "removing":
|
||||
return "down"
|
||||
case "":
|
||||
return "unknown"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// PostgresContainerAliasStore persists container aliases in container_aliases.
|
||||
type PostgresContainerAliasStore struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
// List returns every recorded alias for one source.
|
||||
func (s PostgresContainerAliasStore) List(ctx context.Context, sourceID string) ([]ContainerAliasRecord, error) {
|
||||
if s.Pool == nil {
|
||||
return nil, fmt.Errorf("%w: container alias store has no database pool", ErrInvalidConfig)
|
||||
}
|
||||
rows, err := s.Pool.Query(ctx, `SELECT entity_id::text,runtime_id,name,project,service,image_digest,observed_state,observed_health,restart_count,intentional_stop,first_seen_at,last_seen_at,tombstoned_at FROM container_aliases WHERE source_id=$1 ORDER BY runtime_id ASC LIMIT $2`, sourceID, MaxDiscoveredContainers*2)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list container aliases: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
records := make([]ContainerAliasRecord, 0, 32)
|
||||
for rows.Next() {
|
||||
var record ContainerAliasRecord
|
||||
var tombstoned *time.Time
|
||||
if err := rows.Scan(&record.EntityID, &record.RuntimeID, &record.Name, &record.Project, &record.Service, &record.ImageDigest,
|
||||
&record.State, &record.Health, &record.RestartCount, &record.IntentionalStop, &record.FirstSeenAt, &record.LastSeenAt, &tombstoned); err != nil {
|
||||
return nil, fmt.Errorf("scan container alias: %w", err)
|
||||
}
|
||||
record.SourceID = sourceID
|
||||
if tombstoned != nil {
|
||||
record.TombstonedAt = tombstoned.UTC()
|
||||
}
|
||||
record.Active = tombstoned == nil
|
||||
records = append(records, record)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("read container alias rows: %w", err)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// Save replaces the recorded aliases for one source in a single transaction.
|
||||
// Aliases that disappeared from the reconciliation result are removed only
|
||||
// after reconciliation already decided they are gone; a live alias is upserted,
|
||||
// never deleted and recreated, so its first_seen_at survives.
|
||||
func (s PostgresContainerAliasStore) Save(ctx context.Context, sourceID string, records []ContainerAliasRecord) error {
|
||||
if s.Pool == nil {
|
||||
return fmt.Errorf("%w: container alias store has no database pool", ErrInvalidConfig)
|
||||
}
|
||||
if len(records) > MaxDiscoveredContainers*2 {
|
||||
return fmt.Errorf("container alias count %d exceeds bounds", len(records))
|
||||
}
|
||||
tx, err := s.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin container alias save: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
keep := make([]string, 0, len(records))
|
||||
for _, record := range records {
|
||||
if strings.TrimSpace(record.RuntimeID) == "" || strings.TrimSpace(record.EntityID) == "" || strings.TrimSpace(record.Name) == "" {
|
||||
return errors.New("container alias identity is incomplete")
|
||||
}
|
||||
var tombstoned any
|
||||
if !record.TombstonedAt.IsZero() {
|
||||
tombstoned = record.TombstonedAt.UTC()
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO container_aliases (source_id,runtime_id,entity_id,name,project,service,image_digest,observed_state,observed_health,restart_count,intentional_stop,first_seen_at,last_seen_at,tombstoned_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)
|
||||
ON CONFLICT (source_id,runtime_id) DO UPDATE SET entity_id=EXCLUDED.entity_id,name=EXCLUDED.name,project=EXCLUDED.project,service=EXCLUDED.service,image_digest=EXCLUDED.image_digest,observed_state=EXCLUDED.observed_state,observed_health=EXCLUDED.observed_health,restart_count=EXCLUDED.restart_count,intentional_stop=EXCLUDED.intentional_stop,last_seen_at=EXCLUDED.last_seen_at,tombstoned_at=EXCLUDED.tombstoned_at`,
|
||||
sourceID, record.RuntimeID, record.EntityID, record.Name, record.Project, record.Service, record.ImageDigest,
|
||||
record.State, record.Health, record.RestartCount, record.IntentionalStop, record.FirstSeenAt.UTC(), record.LastSeenAt.UTC(), tombstoned); err != nil {
|
||||
return fmt.Errorf("save container alias: %w", err)
|
||||
}
|
||||
keep = append(keep, record.RuntimeID)
|
||||
}
|
||||
sort.Strings(keep)
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM container_aliases WHERE source_id=$1 AND NOT (runtime_id = ANY($2::text[]))`, sourceID, keep); err != nil {
|
||||
return fmt.Errorf("prune container aliases: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit container alias save: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
"github.com/itworx/pulse/internal/discovery"
|
||||
"github.com/itworx/pulse/internal/inventory"
|
||||
)
|
||||
|
||||
// TestPostgreSQLDiscoveryPersistsInventoryProvenanceAtTargetScale proves the
|
||||
// complete worker-to-inventory persistence path against real PostgreSQL. The
|
||||
// v1 target of 150 active containers is replayed in a later observation window
|
||||
// to verify stable application IDs, fact upserts and relation uniqueness.
|
||||
func TestPostgreSQLDiscoveryPersistsInventoryProvenanceAtTargetScale(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 8, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceID := newID()
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO data_sources (id,type,name,configuration_ref) VALUES ($1,'agent',$2,'integration')`, sourceID, "m12-discovery-"+sourceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cleanupCancel()
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM entity_relations WHERE source_id=$1`, sourceID)
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM entity_facts WHERE source_id=$1`, sourceID)
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM container_aliases WHERE source_id=$1`, sourceID)
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM entities WHERE id IN (SELECT entity_id FROM entity_aliases WHERE source_id=$1)`, sourceID)
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM data_sources WHERE id=$1`, sourceID)
|
||||
})
|
||||
|
||||
repository, err := inventory.NewRepository(pool)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
items := make([]container.Container, 0, 150)
|
||||
for index := 0; index < 150; index++ {
|
||||
name := fmt.Sprintf("m12-scale-%03d", index)
|
||||
items = append(items, container.Container{ID: "runtime-" + name, Name: name, State: "running", Health: "healthy", MetricsAvailable: true, LifecycleAvailable: true})
|
||||
}
|
||||
provider := &fakeContainerProvider{snapshot: healthySnapshot(now, items...)}
|
||||
job := DiscoveryJob{
|
||||
SourceID: sourceID, Provider: provider, Aliases: PostgresContainerAliasStore{Pool: pool}, Inventory: repository,
|
||||
Runner: discovery.Runner{Store: discovery.NewMemoryStore(), MaxAttempts: 1},
|
||||
}
|
||||
if _, err := job.Run(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertInventoryScale := func() {
|
||||
t.Helper()
|
||||
var entities, facts, relations int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(DISTINCT entity_id) FROM entity_aliases WHERE source_id=$1`, sourceID).Scan(&entities); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM entity_facts WHERE source_id=$1`, sourceID).Scan(&facts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM entity_relations WHERE source_id=$1`, sourceID).Scan(&relations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if entities != 300 || facts < 1_200 || relations != 150 {
|
||||
t.Fatalf("inventory scale entities=%d facts=%d relations=%d", entities, facts, relations)
|
||||
}
|
||||
}
|
||||
assertInventoryScale()
|
||||
|
||||
provider.set(healthySnapshot(now.Add(time.Minute), items...))
|
||||
if _, err := job.Run(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertInventoryScale()
|
||||
|
||||
provider.set(healthySnapshot(now.Add(2 * time.Minute)))
|
||||
if _, err := job.Run(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var activeEntities, tombstonedRelations int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM entities e JOIN entity_aliases a ON a.entity_id=e.id WHERE a.source_id=$1 AND e.tombstoned_at IS NULL`, sourceID).Scan(&activeEntities); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM entity_relations WHERE source_id=$1 AND tombstoned_at IS NOT NULL`, sourceID).Scan(&tombstonedRelations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if activeEntities != 0 || tombstonedRelations != 150 {
|
||||
t.Fatalf("disappearance reconciliation active=%d tombstonedRelations=%d", activeEntities, tombstonedRelations)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,704 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
"github.com/itworx/pulse/internal/discovery"
|
||||
"github.com/itworx/pulse/internal/inventory"
|
||||
"github.com/itworx/pulse/internal/notification"
|
||||
"github.com/itworx/pulse/internal/probe"
|
||||
"github.com/itworx/pulse/internal/reconciliation"
|
||||
)
|
||||
|
||||
const testSourceID = "11111111-1111-4111-8111-111111111111"
|
||||
|
||||
// --- discovery job ---------------------------------------------------------
|
||||
|
||||
type fakeContainerProvider struct {
|
||||
mu sync.Mutex
|
||||
snapshot container.Snapshot
|
||||
err error
|
||||
}
|
||||
|
||||
func (p *fakeContainerProvider) Snapshot(context.Context) (container.Snapshot, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.snapshot, p.err
|
||||
}
|
||||
func (p *fakeContainerProvider) set(snapshot container.Snapshot) {
|
||||
p.mu.Lock()
|
||||
p.snapshot = snapshot
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
type memoryAliasStore struct {
|
||||
mu sync.Mutex
|
||||
records []ContainerAliasRecord
|
||||
saves int
|
||||
}
|
||||
|
||||
func (s *memoryAliasStore) List(context.Context, string) ([]ContainerAliasRecord, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]ContainerAliasRecord(nil), s.records...), nil
|
||||
}
|
||||
func (s *memoryAliasStore) Save(_ context.Context, _ string, records []ContainerAliasRecord) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.records = append([]ContainerAliasRecord(nil), records...)
|
||||
s.saves++
|
||||
return nil
|
||||
}
|
||||
|
||||
type memoryInventory struct {
|
||||
mu sync.Mutex
|
||||
persisted map[string]inventory.Entity
|
||||
facts map[string][]inventory.Fact
|
||||
relations map[string][]inventory.Relation
|
||||
tombstoned map[string]inventory.Entity
|
||||
}
|
||||
|
||||
func newMemoryInventory() *memoryInventory {
|
||||
return &memoryInventory{persisted: map[string]inventory.Entity{}, facts: map[string][]inventory.Fact{}, relations: map[string][]inventory.Relation{}, tombstoned: map[string]inventory.Entity{}}
|
||||
}
|
||||
func (s *memoryInventory) PersistDiscovery(_ context.Context, entity inventory.Entity, _ inventory.Alias, facts []inventory.Fact, relations []inventory.Relation) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.persisted[entity.ID] = entity
|
||||
s.facts[entity.ID] = append([]inventory.Fact(nil), facts...)
|
||||
s.relations[entity.ID] = append([]inventory.Relation(nil), relations...)
|
||||
if entity.TombstonedAt != nil {
|
||||
s.tombstoned[entity.ID] = entity
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *memoryInventory) UpsertEntity(_ context.Context, entity inventory.Entity) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.tombstoned[entity.ID] = entity
|
||||
return nil
|
||||
}
|
||||
|
||||
func healthySnapshot(observedAt time.Time, containers ...container.Container) container.Snapshot {
|
||||
return container.Snapshot{
|
||||
ContractVersion: container.ContractVersion,
|
||||
Source: container.Source{ID: "container", Type: "agent", ObservedAt: observedAt, ReceivedAt: observedAt, Freshness: "fresh", State: "healthy"},
|
||||
Containers: containers, Total: len(containers),
|
||||
}
|
||||
}
|
||||
|
||||
func newDiscoveryJob(provider container.Provider, aliases ContainerAliasStore, store discovery.Store, inv InventoryStore) DiscoveryJob {
|
||||
return DiscoveryJob{SourceID: testSourceID, Provider: provider, Aliases: aliases, Inventory: inv,
|
||||
Runner: discovery.Runner{Store: store, MaxAttempts: 1}}
|
||||
}
|
||||
|
||||
func TestDiscoveryJobSkipsAnUnavailableSourceWithoutTombstoning(t *testing.T) {
|
||||
observedAt := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
aliases := &memoryAliasStore{records: []ContainerAliasRecord{{ContainerAlias: aliasFixture("runtime-1", observedAt)}}}
|
||||
inv := newMemoryInventory()
|
||||
store := discovery.NewMemoryStore()
|
||||
provider := &fakeContainerProvider{snapshot: container.UnknownSnapshot(observedAt, "container", "agent", "source_unavailable")}
|
||||
job := newDiscoveryJob(provider, aliases, store, inv)
|
||||
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !outcome.Skipped || outcome.Reason != "source_unavailable" {
|
||||
t.Fatalf("outcome = %#v", outcome)
|
||||
}
|
||||
if aliases.saves != 0 || len(inv.tombstoned) != 0 || len(store.Events) != 0 {
|
||||
t.Fatalf("an unavailable source changed inventory: saves=%d tombstoned=%d events=%d", aliases.saves, len(inv.tombstoned), len(store.Events))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveryJobDerivesLifecycleEventsAndIsIdempotent(t *testing.T) {
|
||||
first := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
aliases := &memoryAliasStore{}
|
||||
inv := newMemoryInventory()
|
||||
store := discovery.NewMemoryStore()
|
||||
provider := &fakeContainerProvider{snapshot: healthySnapshot(first,
|
||||
container.Container{ID: "runtime-1", Name: "pulse-api", State: "running", Health: "healthy"})}
|
||||
job := newDiscoveryJob(provider, aliases, store, inv)
|
||||
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Counts["added"] != 1 || outcome.Counts["events"] != 0 {
|
||||
t.Fatalf("first pass counts = %#v", outcome.Counts)
|
||||
}
|
||||
if len(inv.persisted) != 2 || aliases.saves != 1 {
|
||||
t.Fatalf("first pass persisted=%d saves=%d", len(inv.persisted), aliases.saves)
|
||||
}
|
||||
containerID := aliases.records[0].EntityID
|
||||
if len(inv.facts[containerID]) < 6 || len(inv.relations[containerID]) != 1 {
|
||||
t.Fatalf("container provenance facts=%d relations=%d", len(inv.facts[containerID]), len(inv.relations[containerID]))
|
||||
}
|
||||
relation := inv.relations[containerID][0]
|
||||
if relation.RelationType != "member_of" || inv.persisted[relation.TargetEntityID].EntityType != "application" {
|
||||
t.Fatalf("application relation = %+v target = %+v", relation, inv.persisted[relation.TargetEntityID])
|
||||
}
|
||||
|
||||
// A repeat of the same window is claimed once, so nothing runs twice.
|
||||
if _, err := job.Run(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if aliases.saves != 1 {
|
||||
t.Fatalf("repeated window saves = %d, want 1", aliases.saves)
|
||||
}
|
||||
|
||||
stateAfterFirstPass, err := aliases.List(context.Background(), testSourceID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
second := first.Add(time.Minute)
|
||||
provider.set(healthySnapshot(second,
|
||||
container.Container{ID: "runtime-1", Name: "pulse-api", State: "exited", Health: "unhealthy", RestartCount: 1}))
|
||||
outcome, err = job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Counts["events"] != 3 {
|
||||
t.Fatalf("second pass counts = %#v, want 3 lifecycle events", outcome.Counts)
|
||||
}
|
||||
types := map[string]bool{}
|
||||
for _, event := range store.Events {
|
||||
types[event.Type] = true
|
||||
if event.EntityID == "" || event.SourceID != testSourceID || event.Severity == "" {
|
||||
t.Fatalf("event is missing identity: %#v", event)
|
||||
}
|
||||
}
|
||||
for _, expected := range []string{"container.state_changed", "container.health_changed", "container.restart"} {
|
||||
if !types[expected] {
|
||||
t.Fatalf("missing event %s in %#v", expected, types)
|
||||
}
|
||||
}
|
||||
emitted := len(store.Events)
|
||||
|
||||
// The same observation replayed emits the same deduplicated events.
|
||||
store2 := discovery.NewMemoryStore()
|
||||
job2 := newDiscoveryJob(provider, &memoryAliasStore{records: stateAfterFirstPass}, store2, newMemoryInventory())
|
||||
if _, err := job2.Run(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(store2.Events) != emitted {
|
||||
t.Fatalf("replayed events = %d, want %d", len(store2.Events), emitted)
|
||||
}
|
||||
for key := range store.Events {
|
||||
if _, ok := store2.Events[key]; !ok {
|
||||
t.Fatalf("replay produced a different dedup key set: %q missing", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveryJobTombstonesMissingContainers(t *testing.T) {
|
||||
first := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
aliases := &memoryAliasStore{records: aliasSnapshotBefore(first)}
|
||||
inv := newMemoryInventory()
|
||||
job := newDiscoveryJob(&fakeContainerProvider{snapshot: healthySnapshot(first.Add(time.Minute))}, aliases, discovery.NewMemoryStore(), inv)
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Counts["tombstoned"] != 1 || len(inv.tombstoned) != 2 {
|
||||
t.Fatalf("counts = %#v tombstoned = %#v", outcome.Counts, inv.tombstoned)
|
||||
}
|
||||
for entityID, relations := range inv.relations {
|
||||
if inv.persisted[entityID].EntityType == "container" && (len(relations) != 1 || relations[0].TombstonedAt == nil) {
|
||||
t.Fatalf("missing container relation was not tombstoned: %+v", relations)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func aliasFixture(runtimeID string, observedAt time.Time) reconciliation.ContainerAlias {
|
||||
return reconciliation.ContainerAlias{
|
||||
EntityID: "22222222-2222-4222-8222-222222222222", SourceID: testSourceID, RuntimeID: runtimeID,
|
||||
Name: "pulse-api", FirstSeenAt: observedAt, LastSeenAt: observedAt, Active: true,
|
||||
}
|
||||
}
|
||||
|
||||
func aliasSnapshotBefore(observedAt time.Time) []ContainerAliasRecord {
|
||||
return []ContainerAliasRecord{{ContainerAlias: aliasFixture("runtime-1", observedAt), State: "running", Health: "healthy"}}
|
||||
}
|
||||
|
||||
// --- probe job -------------------------------------------------------------
|
||||
|
||||
type fakeProbeStore struct {
|
||||
mu sync.Mutex
|
||||
due []probe.Definition
|
||||
saved []probe.Result
|
||||
enabled int
|
||||
listErr error
|
||||
}
|
||||
|
||||
func (s *fakeProbeStore) CountEnabled(context.Context) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.enabled, nil
|
||||
}
|
||||
|
||||
func (s *fakeProbeStore) ListDue(context.Context, time.Time, int) ([]probe.Definition, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]probe.Definition(nil), s.due...), s.listErr
|
||||
}
|
||||
func (s *fakeProbeStore) SaveResults(_ context.Context, results []probe.Result) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.saved = append(s.saved, results...)
|
||||
return len(results), nil
|
||||
}
|
||||
|
||||
type fakeExecutor struct {
|
||||
state string
|
||||
err error
|
||||
}
|
||||
|
||||
func (e fakeExecutor) Execute(_ context.Context, definition probe.Definition) (probe.Result, error) {
|
||||
if e.err != nil {
|
||||
return probe.Result{ProbeID: definition.ID, State: "unknown"}, e.err
|
||||
}
|
||||
return probe.Result{ProbeID: definition.ID, State: e.state, ObservedAt: time.Now().UTC(), CompletedAt: time.Now().UTC()}, nil
|
||||
}
|
||||
|
||||
func probeDefinition(id string, interval, timeout time.Duration) probe.Definition {
|
||||
return probe.Definition{ID: id, ServiceID: "service", Name: id, Type: probe.TypeTCP,
|
||||
Target: probe.Target{Host: "example.internal", Port: 443}, Interval: interval, Timeout: timeout, Enabled: true, Revision: 1}
|
||||
}
|
||||
|
||||
func TestProbeJobExecutesDueProbesAndPersistsResults(t *testing.T) {
|
||||
store := &fakeProbeStore{due: []probe.Definition{
|
||||
probeDefinition("probe-a", time.Minute, 5*time.Second),
|
||||
// An individually invalid probe must not stop the batch.
|
||||
probeDefinition("probe-b", 10*time.Second, 30*time.Second),
|
||||
}}
|
||||
scheduler, err := probe.NewScheduler(fakeExecutor{state: "up"}, probe.SchedulerConfig{MaxConcurrent: 4, MaxAttempts: 1, AttemptTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := &ProbeJob{Store: store, Scheduler: scheduler, Logger: quietLogger()}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Counts["due"] != 2 || outcome.Counts["invalid"] != 1 || outcome.Counts["executed"] != 1 || outcome.Counts["saved"] != 1 {
|
||||
t.Fatalf("counts = %#v", outcome.Counts)
|
||||
}
|
||||
if len(store.saved) != 1 || store.saved[0].ProbeID != "probe-a" || store.saved[0].State != "up" {
|
||||
t.Fatalf("saved = %#v", store.saved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeJobWithoutConfiguredProbesIsDisabled(t *testing.T) {
|
||||
scheduler, err := probe.NewScheduler(fakeExecutor{state: "up"}, probe.SchedulerConfig{MaxConcurrent: 1, MaxAttempts: 1, AttemptTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := &ProbeJob{Store: &fakeProbeStore{}, Scheduler: scheduler, Logger: quietLogger()}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil || !outcome.Disabled || outcome.Reason != "no_probes_configured" {
|
||||
t.Fatalf("outcome = %#v err = %v", outcome, err)
|
||||
}
|
||||
// Probes exist but none are due: the scan itself is the successful unit of work.
|
||||
job = &ProbeJob{Store: &fakeProbeStore{enabled: 3}, Scheduler: scheduler, Logger: quietLogger()}
|
||||
outcome, err = job.Run(context.Background())
|
||||
if err != nil || outcome.Disabled || outcome.Counts["enabled"] != 3 {
|
||||
t.Fatalf("outcome = %#v err = %v", outcome, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeJobReportsStoreFailure(t *testing.T) {
|
||||
store := &fakeProbeStore{listErr: errors.New("probes table unavailable")}
|
||||
scheduler, err := probe.NewScheduler(fakeExecutor{state: "up"}, probe.SchedulerConfig{MaxConcurrent: 1, MaxAttempts: 1, AttemptTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := &ProbeJob{Store: store, Scheduler: scheduler, Logger: quietLogger()}
|
||||
if _, err := job.Run(context.Background()); err == nil {
|
||||
t.Fatal("a store failure must surface as a job failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeJobWithoutAStoreIsDisabled(t *testing.T) {
|
||||
job := &ProbeJob{}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil || !outcome.Disabled {
|
||||
t.Fatalf("outcome = %#v err = %v", outcome, err)
|
||||
}
|
||||
}
|
||||
|
||||
// --- notification drain ----------------------------------------------------
|
||||
|
||||
type fakeOutboxStore struct {
|
||||
mu sync.Mutex
|
||||
items map[string]*notification.Outbox
|
||||
}
|
||||
|
||||
func newFakeOutboxStore(items ...notification.Outbox) *fakeOutboxStore {
|
||||
store := &fakeOutboxStore{items: map[string]*notification.Outbox{}}
|
||||
for index := range items {
|
||||
item := items[index]
|
||||
store.items[item.ID] = &item
|
||||
}
|
||||
return store
|
||||
}
|
||||
func (s *fakeOutboxStore) Enqueue(_ context.Context, item notification.Outbox) (notification.Outbox, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, existing := range s.items {
|
||||
if existing.IdempotencyKey == item.IdempotencyKey {
|
||||
return *existing, true, nil
|
||||
}
|
||||
}
|
||||
if item.ID == "" {
|
||||
item.ID = item.IdempotencyKey
|
||||
}
|
||||
item.Status = notification.StatusPending
|
||||
s.items[item.ID] = &item
|
||||
return item, false, nil
|
||||
}
|
||||
func (s *fakeOutboxStore) GetOutbox(_ context.Context, id string) (notification.Outbox, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
item, ok := s.items[id]
|
||||
if !ok {
|
||||
return notification.Outbox{}, notification.ErrNotFound
|
||||
}
|
||||
return *item, nil
|
||||
}
|
||||
func (s *fakeOutboxStore) ClaimDue(_ context.Context, now time.Time, limit int) ([]notification.Outbox, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
claimed := make([]notification.Outbox, 0, limit)
|
||||
for _, item := range s.items {
|
||||
if len(claimed) >= limit {
|
||||
break
|
||||
}
|
||||
if item.Status != notification.StatusPending && item.Status != notification.StatusRetry {
|
||||
continue
|
||||
}
|
||||
if item.NextAttemptAt.After(now) {
|
||||
continue
|
||||
}
|
||||
item.Status = notification.StatusDelivering
|
||||
item.Attempts++
|
||||
claimed = append(claimed, *item)
|
||||
}
|
||||
return claimed, nil
|
||||
}
|
||||
func (s *fakeOutboxStore) Complete(_ context.Context, id string, attempt int, success bool, _ error, now time.Time) (notification.Outbox, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
item, ok := s.items[id]
|
||||
if !ok {
|
||||
return notification.Outbox{}, notification.ErrNotFound
|
||||
}
|
||||
if item.Status != notification.StatusDelivering || item.Attempts != attempt {
|
||||
return *item, nil
|
||||
}
|
||||
if success {
|
||||
item.Status = notification.StatusDelivered
|
||||
item.DeliveredAt = &now
|
||||
} else {
|
||||
item.Status = notification.StatusRetry
|
||||
item.NextAttemptAt = now.Add(notification.RetryDelay(attempt))
|
||||
}
|
||||
return *item, nil
|
||||
}
|
||||
|
||||
type fakeChannelLister struct{ channels []notification.Channel }
|
||||
|
||||
func (l fakeChannelLister) ListChannels(context.Context, int) ([]notification.Channel, error) {
|
||||
return l.channels, nil
|
||||
}
|
||||
|
||||
func TestNotificationDrainDeliversOnceAndRetriesWithoutDuplicating(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
store := newFakeOutboxStore(notification.Outbox{ID: "out-1", IdempotencyKey: "key-1", ChannelID: "channel-1",
|
||||
EventType: notification.EventFiring, Subject: "firing", Body: "body", Status: notification.StatusPending, NextAttemptAt: now})
|
||||
channel := ¬ification.MemoryChannel{}
|
||||
job := NotificationDrainJob{
|
||||
Store: store,
|
||||
Channels: fakeChannelLister{channels: []notification.Channel{{ID: "channel-1", Type: "memory", Enabled: true}}},
|
||||
Senders: map[string]notification.ChannelSender{"memory": channel},
|
||||
Logger: quietLogger(), Now: func() time.Time { return now },
|
||||
}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Counts["delivered"] != 1 {
|
||||
t.Fatalf("counts = %#v", outcome.Counts)
|
||||
}
|
||||
// A second drain must not deliver the same item again.
|
||||
if _, err := job.Run(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(channel.Deliveries) != 1 {
|
||||
t.Fatalf("deliveries = %d, want 1", len(channel.Deliveries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotificationDrainBuildsChannelSpecificTransport(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
store := newFakeOutboxStore(notification.Outbox{ID: "out-1", IdempotencyKey: "key-1", ChannelID: "channel-1",
|
||||
EventType: notification.EventFiring, Subject: "firing", Body: "body", Status: notification.StatusPending, NextAttemptAt: now})
|
||||
memory := ¬ification.MemoryChannel{}
|
||||
factoryCalls := 0
|
||||
job := NotificationDrainJob{
|
||||
Store: store, Channels: fakeChannelLister{channels: []notification.Channel{{ID: "channel-1", Type: "webhook", Enabled: true}}},
|
||||
Factories: map[string]notification.ChannelSenderFactory{"webhook": notification.ChannelSenderFactoryFunc(func(context.Context, notification.Channel) (notification.ChannelSender, error) {
|
||||
factoryCalls++
|
||||
return memory, nil
|
||||
})}, Logger: quietLogger(), Now: func() time.Time { return now },
|
||||
}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Counts["delivered"] != 1 || factoryCalls != 1 || len(memory.Deliveries) != 1 {
|
||||
t.Fatalf("outcome=%#v factoryCalls=%d deliveries=%d", outcome, factoryCalls, len(memory.Deliveries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotificationDrainStatesWhyItCannotDeliver(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
store := newFakeOutboxStore(notification.Outbox{ID: "out-1", IdempotencyKey: "key-1", ChannelID: "channel-1",
|
||||
EventType: notification.EventFiring, Subject: "firing", Body: "body", Status: notification.StatusPending, NextAttemptAt: now})
|
||||
for name, testCase := range map[string]struct {
|
||||
channels []notification.Channel
|
||||
senders map[string]notification.ChannelSender
|
||||
disabled bool
|
||||
wantErr bool
|
||||
}{
|
||||
"no channels": {channels: nil, disabled: true},
|
||||
"all disabled": {channels: []notification.Channel{{ID: "c", Type: "webhook"}}, disabled: true},
|
||||
"no transport": {channels: []notification.Channel{{ID: "c", Type: "webhook", Enabled: true}}, wantErr: true},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
job := NotificationDrainJob{Store: store, Channels: fakeChannelLister{channels: testCase.channels},
|
||||
Senders: testCase.senders, Logger: quietLogger(), Now: func() time.Time { return now }}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if testCase.wantErr && err == nil {
|
||||
t.Fatal("an undeliverable channel must be reported as a failure")
|
||||
}
|
||||
if !testCase.wantErr && err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Disabled != testCase.disabled {
|
||||
t.Fatalf("outcome = %#v", outcome)
|
||||
}
|
||||
if item, _ := store.GetOutbox(context.Background(), "out-1"); item.Attempts != 0 {
|
||||
t.Fatalf("an undeliverable drain consumed attempt %d", item.Attempts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- alert evaluation ------------------------------------------------------
|
||||
|
||||
type fakeStateWriter struct {
|
||||
mu sync.Mutex
|
||||
observations []alert.Observation
|
||||
to alert.State
|
||||
duplicate bool
|
||||
}
|
||||
|
||||
func (s *fakeStateWriter) ApplyObservation(_ context.Context, input alert.StateInput) (alert.Instance, alert.Occurrence, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.observations = append(s.observations, input.Observation)
|
||||
to := s.to
|
||||
if to == "" {
|
||||
to = alert.StateInactive
|
||||
}
|
||||
instance := alert.Instance{ID: "instance-1", RuleID: input.RuleID, Fingerprint: input.Fingerprint, State: to}
|
||||
occurrence := alert.Occurrence{InstanceID: instance.ID, EvaluationKey: input.Observation.EvaluationKey,
|
||||
From: alert.StateInactive, To: to, ObservedAt: input.Observation.ObservedAt}
|
||||
return instance, occurrence, s.duplicate, nil
|
||||
}
|
||||
func (s *fakeStateWriter) last() alert.Observation {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.observations[len(s.observations)-1]
|
||||
}
|
||||
|
||||
type fakePrior struct{ state PriorAlertState }
|
||||
|
||||
func (p fakePrior) PriorState(context.Context, string, string) (PriorAlertState, error) {
|
||||
return p.state, nil
|
||||
}
|
||||
|
||||
type fakeVersions struct{}
|
||||
|
||||
func (fakeVersions) Versions(context.Context, string, int) ([]alert.Version, error) {
|
||||
return []alert.Version{{ID: "33333333-3333-4333-8333-333333333333", VersionNumber: 1}}, nil
|
||||
}
|
||||
|
||||
type fakeMetricSource struct {
|
||||
value MetricValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (s fakeMetricSource) Value(context.Context, alert.Rule) (MetricValue, error) {
|
||||
return s.value, s.err
|
||||
}
|
||||
|
||||
func ruleFixture() alert.Rule {
|
||||
return alert.Rule{Document: alert.Document{
|
||||
SchemaVersion: 1, ID: "44444444-4444-4444-8444-444444444444", Name: "CPU high", Enabled: true, Severity: alert.SeverityDegraded,
|
||||
Condition: alert.Condition{InputType: "metric", Metric: "host.cpu.utilization", Operator: ">", Threshold: float64(80)},
|
||||
EvaluationIntervalSeconds: 60, UnknownBehavior: alert.UnknownRetain, Message: alert.Message{TitleKey: "t", BodyKey: "b"},
|
||||
}, CurrentVersion: 1}
|
||||
}
|
||||
|
||||
func TestAlertEvaluatorRecordsUnknownWhenTheInputIsUnavailable(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
for name, testCase := range map[string]struct {
|
||||
source MetricSource
|
||||
reason string
|
||||
}{
|
||||
"no source": {source: nil, reason: "metric_source_not_configured"},
|
||||
"query failure": {source: fakeMetricSource{err: errors.New("prometheus is down")}, reason: "metric_query_failed"},
|
||||
"no series": {source: fakeMetricSource{value: MetricValue{Reason: "metric_no_series"}}, reason: "metric_no_series"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
states := &fakeStateWriter{to: alert.StateUnknown}
|
||||
evaluator := &AlertEvaluator{Metrics: testCase.source, States: states, Prior: fakePrior{}, Versions: fakeVersions{},
|
||||
Logger: quietLogger(), Now: func() time.Time { return now }}
|
||||
if err := evaluator.Evaluate(context.Background(), ruleFixture()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
observation := states.last()
|
||||
if !observation.Unknown || observation.Reason != testCase.reason {
|
||||
t.Fatalf("observation = %#v", observation)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertEvaluatorEvaluatesConditionAndQueuesOneNotificationPerChannel(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
states := &fakeStateWriter{to: alert.StateFiring}
|
||||
outbox := newFakeOutboxStore()
|
||||
queue := &fakeNotificationQueue{outbox: outbox, channels: []notification.Channel{
|
||||
{ID: "channel-1", Type: "memory", Enabled: true}, {ID: "channel-2", Type: "memory", Enabled: true}, {ID: "channel-3", Type: "memory"},
|
||||
}}
|
||||
evaluator := &AlertEvaluator{Metrics: fakeMetricSource{value: MetricValue{Value: 91, Known: true, ObservedAt: now}},
|
||||
States: states, Prior: fakePrior{}, Versions: fakeVersions{}, Notifications: queue, Logger: quietLogger(), Now: func() time.Time { return now }}
|
||||
|
||||
if err := evaluator.Evaluate(context.Background(), ruleFixture()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
observation := states.last()
|
||||
if observation.Unknown || !observation.ConditionTrue {
|
||||
t.Fatalf("observation = %#v", observation)
|
||||
}
|
||||
if queued := outbox.count(); queued != 2 {
|
||||
t.Fatalf("queued notifications = %d, want one per enabled channel", queued)
|
||||
}
|
||||
for _, item := range outbox.all() {
|
||||
if strings.Contains(item.Body, "secret") || strings.Contains(item.Body, "token") {
|
||||
t.Fatalf("notification body leaks configuration: %q", item.Body)
|
||||
}
|
||||
}
|
||||
// Re-running the same evaluation window must not queue a second delivery.
|
||||
if err := evaluator.Evaluate(context.Background(), ruleFixture()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queued := outbox.count(); queued != 2 {
|
||||
t.Fatalf("queued notifications after repeat = %d, want 2", queued)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertEvaluatorHonoursCooldownAndDuplicateWindows(t *testing.T) {
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
cooldown := now.Add(5 * time.Minute)
|
||||
for name, testCase := range map[string]struct {
|
||||
prior PriorAlertState
|
||||
duplicate bool
|
||||
want int
|
||||
}{
|
||||
"fresh firing": {prior: PriorAlertState{State: alert.StateInactive}, want: 1},
|
||||
"cooldown active": {prior: PriorAlertState{State: alert.StateResolved, CooldownUntil: &cooldown, Found: true}, want: 0},
|
||||
"already firing": {prior: PriorAlertState{State: alert.StateFiring, Found: true}, want: 0},
|
||||
"duplicate window": {prior: PriorAlertState{State: alert.StateInactive}, duplicate: true, want: 0},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
outbox := newFakeOutboxStore()
|
||||
queue := &fakeNotificationQueue{outbox: outbox, channels: []notification.Channel{{ID: "channel-1", Type: "memory", Enabled: true}}}
|
||||
evaluator := &AlertEvaluator{Metrics: fakeMetricSource{value: MetricValue{Value: 91, Known: true}},
|
||||
States: &fakeStateWriter{to: alert.StateFiring, duplicate: testCase.duplicate}, Prior: fakePrior{state: testCase.prior},
|
||||
Versions: fakeVersions{}, Notifications: queue, Logger: quietLogger(), Now: func() time.Time { return now }}
|
||||
if err := evaluator.Evaluate(context.Background(), ruleFixture()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queued := outbox.count(); queued != testCase.want {
|
||||
t.Fatalf("queued = %d, want %d", queued, testCase.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluationKeyAndFingerprintAreStable(t *testing.T) {
|
||||
rule := ruleFixture()
|
||||
base := time.Date(2026, 8, 4, 12, 0, 30, 0, time.UTC)
|
||||
if EvaluationKey(rule, base) != EvaluationKey(rule, base.Add(20*time.Second)) {
|
||||
t.Fatal("evaluation key changed inside one interval")
|
||||
}
|
||||
if EvaluationKey(rule, base) == EvaluationKey(rule, base.Add(time.Minute)) {
|
||||
t.Fatal("evaluation key did not change between intervals")
|
||||
}
|
||||
scoped := ruleFixture()
|
||||
scoped.Scope = map[string]any{"host": "tower"}
|
||||
if RuleFingerprint(rule) == RuleFingerprint(scoped) {
|
||||
t.Fatal("scope is not part of the fingerprint")
|
||||
}
|
||||
repeated, again := RuleFingerprint(scoped), RuleFingerprint(scoped)
|
||||
if repeated != again || len(repeated) == 0 || len(repeated) > 160 {
|
||||
t.Fatal("fingerprint is unstable or unbounded")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeNotificationQueue struct {
|
||||
outbox *fakeOutboxStore
|
||||
channels []notification.Channel
|
||||
}
|
||||
|
||||
func (q *fakeNotificationQueue) Enqueue(ctx context.Context, item notification.Outbox) (notification.Outbox, bool, error) {
|
||||
return q.outbox.Enqueue(ctx, item)
|
||||
}
|
||||
func (q *fakeNotificationQueue) ListChannels(context.Context, int) ([]notification.Channel, error) {
|
||||
return q.channels, nil
|
||||
}
|
||||
|
||||
func (s *fakeOutboxStore) count() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return len(s.items)
|
||||
}
|
||||
func (s *fakeOutboxStore) all() []notification.Outbox {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
items := make([]notification.Outbox, 0, len(s.items))
|
||||
for _, item := range s.items {
|
||||
items = append(items, *item)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func TestAlertEvaluationJobReportsFailuresAndDisabledState(t *testing.T) {
|
||||
outcome, err := AlertEvaluationJob{}.Run(context.Background())
|
||||
if err != nil || !outcome.Disabled || outcome.Reason != "alert_evaluation_not_configured" {
|
||||
t.Fatalf("outcome = %#v err = %v", outcome, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// ErrLeaseLost reports that the job run row this worker owned was completed or
|
||||
// reclaimed elsewhere, so this worker's result must not overwrite it.
|
||||
var ErrLeaseLost = errors.New("worker job lease was lost")
|
||||
|
||||
// Lease identifies one claimed scheduling window.
|
||||
type Lease struct {
|
||||
ID string
|
||||
JobType string
|
||||
JobKey string
|
||||
ScheduledAt time.Time
|
||||
Owner string
|
||||
}
|
||||
|
||||
// LeaseStore coordinates duplicate workers through the database.
|
||||
//
|
||||
// Acquire must be atomic per (jobType, jobKey, scheduledAt): exactly one caller
|
||||
// may hold a window at a time, an already completed window must never be handed
|
||||
// out again, and a window whose lease has expired must be reclaimable so a
|
||||
// crashed worker cannot block a job forever.
|
||||
type LeaseStore interface {
|
||||
Acquire(ctx context.Context, jobType, jobKey string, scheduledAt time.Time, owner string, now time.Time, ttl time.Duration) (Lease, bool, error)
|
||||
Complete(ctx context.Context, lease Lease, status, errorCode string, counts map[string]int64) error
|
||||
}
|
||||
|
||||
// PostgresLeaseStore implements LeaseStore on the existing job_runs table.
|
||||
//
|
||||
// It follows the same pattern as alertworker.PostgresLeaseStore, which is
|
||||
// hardwired to the alert evaluation job type; this one is parameterized by job
|
||||
// type and additionally records result counts, so every worker job shares one
|
||||
// coordination and reporting mechanism instead of inventing its own.
|
||||
type PostgresLeaseStore struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
// Acquire claims one scheduling window, reclaiming an expired lease if needed.
|
||||
func (s PostgresLeaseStore) Acquire(ctx context.Context, jobType, jobKey string, scheduledAt time.Time, owner string, now time.Time, ttl time.Duration) (Lease, bool, error) {
|
||||
if s.Pool == nil {
|
||||
return Lease{}, false, fmt.Errorf("%w: lease store has no database pool", ErrInvalidConfig)
|
||||
}
|
||||
if strings.TrimSpace(jobType) == "" || strings.TrimSpace(jobKey) == "" || strings.TrimSpace(owner) == "" || ttl <= 0 {
|
||||
return Lease{}, false, fmt.Errorf("%w: lease identity is invalid", ErrInvalidConfig)
|
||||
}
|
||||
scheduled := scheduledAt.UTC()
|
||||
moment := now.UTC()
|
||||
leaseUntil := moment.Add(ttl)
|
||||
tx, err := s.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Lease{}, false, fmt.Errorf("begin worker lease: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
leaseID := newID()
|
||||
var insertedID string
|
||||
err = tx.QueryRow(ctx, `INSERT INTO job_runs (id,job_type,job_key,scheduled_at,started_at,status,lease_owner,lease_until) VALUES ($1,$2,$3,$4,$5,'running',$6,$7) ON CONFLICT (job_type,job_key,scheduled_at) DO NOTHING RETURNING id`,
|
||||
leaseID, jobType, jobKey, scheduled, moment, owner, leaseUntil).Scan(&insertedID)
|
||||
if err == nil {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Lease{}, false, fmt.Errorf("commit worker lease: %w", err)
|
||||
}
|
||||
return Lease{ID: insertedID, JobType: jobType, JobKey: jobKey, ScheduledAt: scheduled, Owner: owner}, true, nil
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Lease{}, false, fmt.Errorf("insert worker lease: %w", err)
|
||||
}
|
||||
var status string
|
||||
var existingUntil *time.Time
|
||||
if err := tx.QueryRow(ctx, `SELECT status,lease_until FROM job_runs WHERE job_type=$1 AND job_key=$2 AND scheduled_at=$3 FOR UPDATE`, jobType, jobKey, scheduled).Scan(&status, &existingUntil); err != nil {
|
||||
return Lease{}, false, fmt.Errorf("read worker lease: %w", err)
|
||||
}
|
||||
if status != "running" || (existingUntil != nil && existingUntil.After(moment)) {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Lease{}, false, fmt.Errorf("commit worker lease contention: %w", err)
|
||||
}
|
||||
return Lease{}, false, nil
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `UPDATE job_runs SET started_at=$1,completed_at=NULL,error_code=NULL,lease_owner=$2,lease_until=$3 WHERE job_type=$4 AND job_key=$5 AND scheduled_at=$6 AND status='running' AND (lease_until IS NULL OR lease_until <= $7)`,
|
||||
moment, owner, leaseUntil, jobType, jobKey, scheduled, moment)
|
||||
if err != nil {
|
||||
return Lease{}, false, fmt.Errorf("reclaim worker lease: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Lease{}, false, fmt.Errorf("commit worker lease reclaim contention: %w", err)
|
||||
}
|
||||
return Lease{}, false, nil
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Lease{}, false, fmt.Errorf("commit worker lease reclaim: %w", err)
|
||||
}
|
||||
return Lease{ID: leaseID, JobType: jobType, JobKey: jobKey, ScheduledAt: scheduled, Owner: owner}, true, nil
|
||||
}
|
||||
|
||||
// Complete records the outcome of a claimed window. The update is scoped to the
|
||||
// owning worker, so a run whose lease already expired cannot overwrite the
|
||||
// result of the worker that took the window over.
|
||||
func (s PostgresLeaseStore) Complete(ctx context.Context, lease Lease, status, errorCode string, counts map[string]int64) error {
|
||||
if s.Pool == nil {
|
||||
return fmt.Errorf("%w: lease store has no database pool", ErrInvalidConfig)
|
||||
}
|
||||
if !recordedStatus(status) {
|
||||
return fmt.Errorf("%w: worker job status %q is invalid", ErrInvalidConfig, status)
|
||||
}
|
||||
payload, err := encodeCounts(status, errorCode, counts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// error_code describes a failure; the reason for a skipped or disabled run
|
||||
// lives in counts so a non-failure is never read back as an error.
|
||||
failureCode := ""
|
||||
if status == StatusFailed {
|
||||
failureCode = boundedCode(errorCode)
|
||||
}
|
||||
tag, err := s.Pool.Exec(ctx, `UPDATE job_runs SET status=$1,completed_at=now(),counts=$2::jsonb,error_code=NULLIF($3,''),lease_owner=NULL,lease_until=NULL WHERE job_type=$4 AND job_key=$5 AND scheduled_at=$6 AND lease_owner=$7 AND status='running'`,
|
||||
status, payload, failureCode, lease.JobType, lease.JobKey, lease.ScheduledAt.UTC(), lease.Owner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete worker job: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return ErrLeaseLost
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// recordedStatus bounds what may be written to job_runs.status. Skipped and
|
||||
// disabled are kept distinct from completed on purpose: a run that did nothing
|
||||
// must never be read back as a successful observation.
|
||||
func recordedStatus(status string) bool {
|
||||
switch status {
|
||||
case StatusCompleted, StatusFailed, StatusSkipped, StatusDisabled:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func encodeCounts(status, reason string, counts map[string]int64) ([]byte, error) {
|
||||
document := make(map[string]any, len(counts)+2)
|
||||
for key, value := range counts {
|
||||
if len(document) >= 20 {
|
||||
break
|
||||
}
|
||||
document[boundedCode(key)] = value
|
||||
}
|
||||
document["status"] = status
|
||||
if reason != "" {
|
||||
document["reason"] = boundedCode(reason)
|
||||
}
|
||||
payload, err := json.Marshal(document)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode worker job counts: %w", err)
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func boundedCode(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if len(value) > 160 {
|
||||
return value[:160]
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// MemoryLeaseStore is an in-memory LeaseStore with the same semantics, used by
|
||||
// tests and by a worker running without a database.
|
||||
type MemoryLeaseStore struct {
|
||||
mu sync.Mutex
|
||||
records map[string]memoryLease
|
||||
}
|
||||
|
||||
type memoryLease struct {
|
||||
lease Lease
|
||||
status string
|
||||
leaseUntil time.Time
|
||||
counts map[string]int64
|
||||
reason string
|
||||
}
|
||||
|
||||
// NewMemoryLeaseStore returns an empty in-memory lease store.
|
||||
func NewMemoryLeaseStore() *MemoryLeaseStore {
|
||||
return &MemoryLeaseStore{records: make(map[string]memoryLease)}
|
||||
}
|
||||
|
||||
// Acquire claims a window unless it is completed or still leased.
|
||||
func (s *MemoryLeaseStore) Acquire(_ context.Context, jobType, jobKey string, scheduledAt time.Time, owner string, now time.Time, ttl time.Duration) (Lease, bool, error) {
|
||||
if strings.TrimSpace(jobType) == "" || strings.TrimSpace(jobKey) == "" || strings.TrimSpace(owner) == "" || ttl <= 0 {
|
||||
return Lease{}, false, fmt.Errorf("%w: lease identity is invalid", ErrInvalidConfig)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.records == nil {
|
||||
s.records = make(map[string]memoryLease)
|
||||
}
|
||||
key := memoryLeaseKey(jobType, jobKey, scheduledAt)
|
||||
if record, exists := s.records[key]; exists {
|
||||
if record.status != StatusRunning || record.leaseUntil.After(now.UTC()) {
|
||||
return Lease{}, false, nil
|
||||
}
|
||||
}
|
||||
lease := Lease{ID: newID(), JobType: jobType, JobKey: jobKey, ScheduledAt: scheduledAt.UTC(), Owner: owner}
|
||||
s.records[key] = memoryLease{lease: lease, status: StatusRunning, leaseUntil: now.UTC().Add(ttl)}
|
||||
return lease, true, nil
|
||||
}
|
||||
|
||||
// Complete records the outcome for a window this owner still holds.
|
||||
func (s *MemoryLeaseStore) Complete(_ context.Context, lease Lease, status, errorCode string, counts map[string]int64) error {
|
||||
if !recordedStatus(status) {
|
||||
return fmt.Errorf("%w: worker job status %q is invalid", ErrInvalidConfig, status)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
key := memoryLeaseKey(lease.JobType, lease.JobKey, lease.ScheduledAt)
|
||||
record, exists := s.records[key]
|
||||
if !exists || record.lease.ID != lease.ID || record.lease.Owner != lease.Owner || record.status != StatusRunning {
|
||||
return ErrLeaseLost
|
||||
}
|
||||
record.status = status
|
||||
record.reason = errorCode
|
||||
record.counts = counts
|
||||
record.leaseUntil = time.Time{}
|
||||
s.records[key] = record
|
||||
return nil
|
||||
}
|
||||
|
||||
// Status reports the recorded status of one window, or an empty string when the
|
||||
// window was never claimed.
|
||||
func (s *MemoryLeaseStore) Status(jobType, jobKey string, scheduledAt time.Time) string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
record, exists := s.records[memoryLeaseKey(jobType, jobKey, scheduledAt)]
|
||||
if !exists {
|
||||
return ""
|
||||
}
|
||||
return record.status
|
||||
}
|
||||
|
||||
// Windows reports how many distinct scheduling windows were claimed.
|
||||
func (s *MemoryLeaseStore) Windows() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return len(s.records)
|
||||
}
|
||||
|
||||
func memoryLeaseKey(jobType, jobKey string, scheduledAt time.Time) string {
|
||||
return jobType + "|" + jobKey + "|" + scheduledAt.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
func newID() string {
|
||||
raw := make([]byte, 16)
|
||||
if _, err := rand.Read(raw); err != nil {
|
||||
return fmt.Sprintf("00000000-0000-4000-8000-%012d", time.Now().UnixNano()%1_000_000_000_000)
|
||||
}
|
||||
raw[6] = (raw[6] & 0x0f) | 0x40
|
||||
raw[8] = (raw[8] & 0x3f) | 0x80
|
||||
encoded := hex.EncodeToString(raw)
|
||||
return encoded[:8] + "-" + encoded[8:12] + "-" + encoded[12:16] + "-" + encoded[16:20] + "-" + encoded[20:]
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/alert"
|
||||
"github.com/itworx/pulse/internal/metriccatalog"
|
||||
"github.com/itworx/pulse/internal/metricquery"
|
||||
"github.com/itworx/pulse/internal/prometheus"
|
||||
"github.com/itworx/pulse/internal/queryplan"
|
||||
)
|
||||
|
||||
type workerPrometheusSource struct{ queries int }
|
||||
|
||||
func (source *workerPrometheusSource) Query(context.Context, string, *time.Time) (prometheus.QueryResult, error) {
|
||||
source.queries++
|
||||
return prometheus.QueryResult{Status: "success", Data: []byte(`{"resultType":"vector","result":[{"metric":{"instance":"smoke-host"},"value":[1770000000,"95"]}]}`)}, nil
|
||||
}
|
||||
func (*workerPrometheusSource) QueryRange(context.Context, string, time.Time, time.Time, time.Duration) (prometheus.QueryResult, error) {
|
||||
return prometheus.QueryResult{}, nil
|
||||
}
|
||||
|
||||
func TestPrometheusMetricSourceUsesReadOnlyWorkerPrincipal(t *testing.T) {
|
||||
registry, err := metriccatalog.DefaultRegistry()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upstream := &workerPrometheusSource{}
|
||||
service := metricquery.NewService(queryplan.NewPlanner(registry, queryplan.Limits{}), upstream, nil)
|
||||
source := PrometheusMetricSource{Service: service}
|
||||
value, err := source.Value(context.Background(), alert.Rule{Document: alert.Document{
|
||||
Condition: alert.Condition{InputType: "metric", Metric: "host.cpu.utilization", Aggregation: "avg"},
|
||||
Scope: map[string]any{"serverId": "smoke-host"},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !value.Known || value.Value != 95 || upstream.queries != 1 {
|
||||
t.Fatalf("value=%+v queries=%d", value, upstream.queries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrometheusMetricSourceDoesNotForwardAlertTargetMetadata(t *testing.T) {
|
||||
registry, err := metriccatalog.DefaultRegistry()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upstream := &workerPrometheusSource{}
|
||||
service := metricquery.NewService(queryplan.NewPlanner(registry, queryplan.Limits{}), upstream, nil)
|
||||
source := PrometheusMetricSource{Service: service}
|
||||
rules := []alert.Rule{
|
||||
{Document: alert.Document{Condition: alert.Condition{InputType: "metric", Metric: "storage.disk.temperature.maximum", Aggregation: "max"}, Scope: map[string]any{"entityType": "disk"}}},
|
||||
{Document: alert.Document{Condition: alert.Condition{InputType: "metric", Metric: "service.availability.minimum", Aggregation: "min"}, Scope: map[string]any{"entityType": "service", "critical": true}}},
|
||||
}
|
||||
for _, rule := range rules {
|
||||
value, valueErr := source.Value(context.Background(), rule)
|
||||
if valueErr != nil || !value.Known {
|
||||
t.Fatalf("metric=%s value=%+v err=%v", rule.Condition.Metric, value, valueErr)
|
||||
}
|
||||
}
|
||||
if upstream.queries != len(rules) {
|
||||
t.Fatalf("queries=%d want=%d", upstream.queries, len(rules))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/notification"
|
||||
)
|
||||
|
||||
// MaxNotificationBatch bounds one outbox drain.
|
||||
const MaxNotificationBatch = 50
|
||||
|
||||
// ChannelLister lists configured notification channels.
|
||||
type ChannelLister interface {
|
||||
ListChannels(ctx context.Context, limit int) ([]notification.Channel, error)
|
||||
}
|
||||
|
||||
// NotificationDrainJob delivers due outbox items.
|
||||
//
|
||||
// Delivery semantics come entirely from internal/notification: ClaimDue moves a
|
||||
// due item to "delivering", increments its attempt counter and takes a bounded
|
||||
// lock in one transaction (FOR UPDATE SKIP LOCKED), and Complete only applies
|
||||
// to the exact attempt it claimed. That is what makes retry safe: a second
|
||||
// worker cannot claim a locked item, a crashed attempt is retried only after
|
||||
// its lock expires, and a completion for a stale attempt is ignored instead of
|
||||
// delivering twice.
|
||||
type NotificationDrainJob struct {
|
||||
Store notification.Store
|
||||
Channels ChannelLister
|
||||
// Senders maps a channel type onto its transport. A configured channel
|
||||
// whose type has no registered transport is reported, never silently
|
||||
// dropped and never retried into failure.
|
||||
Senders map[string]notification.ChannelSender
|
||||
Factories map[string]notification.ChannelSenderFactory
|
||||
BatchSize int
|
||||
Logger *slog.Logger
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (j NotificationDrainJob) now() time.Time {
|
||||
if j.Now != nil {
|
||||
return j.Now().UTC()
|
||||
}
|
||||
return time.Now().UTC()
|
||||
}
|
||||
|
||||
func (j NotificationDrainJob) logger() *slog.Logger {
|
||||
if j.Logger != nil {
|
||||
return j.Logger
|
||||
}
|
||||
return slog.Default()
|
||||
}
|
||||
|
||||
// Run drains one bounded batch of due notifications.
|
||||
func (j NotificationDrainJob) Run(ctx context.Context) (Outcome, error) {
|
||||
if j.Store == nil || j.Channels == nil {
|
||||
return Outcome{Disabled: true, Reason: "notifications_not_configured"}, nil
|
||||
}
|
||||
channels, err := j.Channels.ListChannels(ctx, 100)
|
||||
if err != nil {
|
||||
return Outcome{}, fmt.Errorf("list notification channels: %w", err)
|
||||
}
|
||||
senders := make(map[string]notification.ChannelSender, len(channels))
|
||||
counts := map[string]int64{}
|
||||
unsupported := 0
|
||||
for _, channel := range channels {
|
||||
if !channel.Enabled {
|
||||
continue
|
||||
}
|
||||
counts["channels"]++
|
||||
sender, ok := j.Senders[channel.Type]
|
||||
if (!ok || sender == nil) && j.Factories[channel.Type] != nil {
|
||||
sender, err = j.Factories[channel.Type].Sender(ctx, channel)
|
||||
ok = err == nil && sender != nil
|
||||
if err != nil {
|
||||
j.logger().Error("notification channel transport configuration failed", "channel", channel.ID, "type", channel.Type, "error", notification.RedactError(err))
|
||||
}
|
||||
}
|
||||
if !ok || sender == nil {
|
||||
unsupported++
|
||||
j.logger().Error("notification channel has no delivery transport", "channel", channel.ID, "type", channel.Type)
|
||||
continue
|
||||
}
|
||||
senders[channel.ID] = sender
|
||||
}
|
||||
counts["unsupported_channels"] = int64(unsupported)
|
||||
if counts["channels"] == 0 {
|
||||
return Outcome{Disabled: true, Reason: "no_enabled_notification_channels", Counts: counts}, nil
|
||||
}
|
||||
if len(senders) == 0 {
|
||||
// Claiming items we cannot deliver would burn their bounded attempts,
|
||||
// so the drain stops and the failure stays visible instead.
|
||||
return Outcome{Counts: counts}, fmt.Errorf("no delivery transport is registered for %d enabled notification channel(s)", unsupported)
|
||||
}
|
||||
batch := j.BatchSize
|
||||
if batch < 1 || batch > MaxNotificationBatch {
|
||||
batch = MaxNotificationBatch
|
||||
}
|
||||
dispatcher := notification.Dispatcher{Store: j.Store, Channels: senders, Limiter: ¬ification.RateLimiter{}}
|
||||
delivered, err := dispatcher.DispatchDue(ctx, j.now(), batch)
|
||||
counts["delivered"] = int64(delivered)
|
||||
if err != nil {
|
||||
return Outcome{Counts: counts}, fmt.Errorf("drain notification outbox: %w", err)
|
||||
}
|
||||
if unsupported > 0 {
|
||||
return Outcome{Counts: counts}, fmt.Errorf("%d enabled notification channel(s) have no delivery transport", unsupported)
|
||||
}
|
||||
return Outcome{Counts: counts}, nil
|
||||
}
|
||||
|
||||
// DrainDeadline is the longest a single drain may take. It is exported so the
|
||||
// job schedule and the outbox lock window stay visibly related: the outbox lock
|
||||
// is five minutes, comfortably above this bound.
|
||||
const DrainDeadline = 30 * time.Second
|
||||
@@ -0,0 +1,196 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
"github.com/itworx/pulse/internal/probe"
|
||||
"github.com/itworx/pulse/internal/systemstatus"
|
||||
)
|
||||
|
||||
// TestPostgreSQLLeaseStoreAndJobHealth exercises the real job_runs coordination
|
||||
// and the status projection the API reads back. It is skipped unless
|
||||
// PULSE_TEST_DATABASE_URL points at a disposable PostgreSQL instance.
|
||||
func TestPostgreSQLLeaseStoreAndJobHealth(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 4, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := PostgresLeaseStore{Pool: pool}
|
||||
job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
|
||||
Run: func(context.Context) (Outcome, error) { return Outcome{}, nil }}
|
||||
jobKey := "integration-" + newID()
|
||||
now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
|
||||
t.Cleanup(func() {
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cleanupCancel()
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM job_runs WHERE job_key=$1`, jobKey)
|
||||
})
|
||||
|
||||
lease, acquired, err := store.Acquire(ctx, job.JobType(), jobKey, now, "worker-a", now, time.Minute)
|
||||
if err != nil || !acquired {
|
||||
t.Fatalf("first acquire = %v err = %v", acquired, err)
|
||||
}
|
||||
if _, acquired, err := store.Acquire(ctx, job.JobType(), jobKey, now, "worker-b", now, time.Minute); err != nil || acquired {
|
||||
t.Fatalf("a held window was acquired twice: %v err = %v", acquired, err)
|
||||
}
|
||||
if err := store.Complete(ctx, lease, StatusCompleted, "", map[string]int64{"items": 3}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Complete(ctx, lease, StatusCompleted, "", nil); err != ErrLeaseLost {
|
||||
t.Fatalf("completing twice = %v, want ErrLeaseLost", err)
|
||||
}
|
||||
if _, acquired, err := store.Acquire(ctx, job.JobType(), jobKey, now, "worker-b", now, time.Minute); err != nil || acquired {
|
||||
t.Fatalf("a completed window was reacquired: %v err = %v", acquired, err)
|
||||
}
|
||||
// An expired lease is reclaimable so a crashed worker cannot block the job.
|
||||
expired, acquired, err := store.Acquire(ctx, job.JobType(), jobKey, now.Add(time.Minute), "worker-a", now, time.Second)
|
||||
if err != nil || !acquired {
|
||||
t.Fatalf("expired-window acquire = %v err = %v", acquired, err)
|
||||
}
|
||||
if _, acquired, err := store.Acquire(ctx, job.JobType(), jobKey, expired.ScheduledAt, "worker-b", now.Add(10*time.Second), time.Minute); err != nil || !acquired {
|
||||
t.Fatalf("expired lease was not reclaimed: %v err = %v", acquired, err)
|
||||
}
|
||||
|
||||
health, err := ReadJobHealth(ctx, pool, []Job{job})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(health) != 1 || health[0].Component != systemstatus.ComponentWorker || health[0].LastRunAt.IsZero() {
|
||||
t.Fatalf("job health = %#v", health)
|
||||
}
|
||||
// A job type that never ran is absent, so its component stays Unknown.
|
||||
absent, err := ReadJobHealth(ctx, pool, []Job{{Name: "never-scheduled", Component: systemstatus.ComponentProbes, Interval: time.Minute, Timeout: time.Second, Run: job.Run}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(absent) != 0 {
|
||||
t.Fatalf("a job that never ran reported health: %#v", absent)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPostgreSQLProbeAndAliasStoresAreIdempotent verifies that replaying a
|
||||
// probe batch and a container alias snapshot creates no duplicate rows.
|
||||
func TestPostgreSQLProbeAndAliasStoresAreIdempotent(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 4, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceID, serviceID, probeID, entityID := newID(), newID(), newID(), newID()
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
seed := []struct {
|
||||
query string
|
||||
args []any
|
||||
}{
|
||||
{`INSERT INTO data_sources (id,type,name,configuration_ref) VALUES ($1,'agent','worker-test','test')`, []any{sourceID}},
|
||||
{`INSERT INTO entities (id,entity_type,canonical_name,display_name,first_seen_at) VALUES ($1,'container','worker/test','worker-test',$2)`, []any{entityID, now}},
|
||||
{`INSERT INTO services (id,name) VALUES ($1,'worker-test-service')`, []any{serviceID}},
|
||||
{`INSERT INTO probes (id,service_id,name,probe_type,target,interval_seconds,timeout_seconds) VALUES ($1,$2,'worker-test-probe','tcp','{"host":"example.internal","port":443}'::jsonb,60,5)`, []any{probeID, serviceID}},
|
||||
}
|
||||
for _, statement := range seed {
|
||||
if _, err := pool.Exec(ctx, statement.query, statement.args...); err != nil {
|
||||
t.Fatalf("seed %q: %v", statement.query, err)
|
||||
}
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cleanupCancel()
|
||||
for _, statement := range []string{
|
||||
`DELETE FROM probe_results WHERE probe_id=$1`, `DELETE FROM probes WHERE id=$1`,
|
||||
} {
|
||||
_, _ = pool.Exec(cleanupCtx, statement, probeID)
|
||||
}
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM services WHERE id=$1`, serviceID)
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM container_aliases WHERE source_id=$1`, sourceID)
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM entities WHERE id=$1`, entityID)
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM data_sources WHERE id=$1`, sourceID)
|
||||
})
|
||||
|
||||
probeStore := PostgresProbeStore{Pool: pool, SourceID: sourceID}
|
||||
due, err := probeStore.ListDue(ctx, now, MaxProbeBatch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, definition := range due {
|
||||
if definition.ID == probeID {
|
||||
found = true
|
||||
if definition.Interval != time.Minute || definition.Timeout != 5*time.Second || definition.Target.Host != "example.internal" {
|
||||
t.Fatalf("decoded probe = %#v", definition)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("a probe without results is not due")
|
||||
}
|
||||
batch := []probe.Result{{ProbeID: probeID, ObservedAt: now, CompletedAt: now, State: "up", Attempts: 1}}
|
||||
saved, err := probeStore.SaveResults(ctx, batch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved != 1 {
|
||||
t.Fatalf("saved = %d, want 1", saved)
|
||||
}
|
||||
saved, err = probeStore.SaveResults(ctx, batch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved != 0 {
|
||||
t.Fatalf("replayed save = %d, want 0", saved)
|
||||
}
|
||||
if due, err := probeStore.ListDue(ctx, now, MaxProbeBatch); err == nil {
|
||||
for _, definition := range due {
|
||||
if definition.ID == probeID {
|
||||
t.Fatal("a probe with a fresh result is still due")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
aliasStore := PostgresContainerAliasStore{Pool: pool}
|
||||
record := ContainerAliasRecord{State: "running", Health: "healthy"}
|
||||
record.EntityID, record.SourceID, record.RuntimeID, record.Name = entityID, sourceID, "runtime-1", "worker-test"
|
||||
record.FirstSeenAt, record.LastSeenAt, record.Active = now, now, true
|
||||
for range 2 {
|
||||
if err := aliasStore.Save(ctx, sourceID, []ContainerAliasRecord{record}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
stored, err := aliasStore.List(ctx, sourceID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(stored) != 1 || stored[0].RuntimeID != "runtime-1" || stored[0].State != "running" || !stored[0].Active {
|
||||
t.Fatalf("stored aliases = %#v", stored)
|
||||
}
|
||||
if err := aliasStore.Save(ctx, sourceID, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if remaining, err := aliasStore.List(ctx, sourceID); err != nil || len(remaining) != 0 {
|
||||
t.Fatalf("aliases after prune = %#v err = %v", remaining, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/probe"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const (
|
||||
// MaxProbeBatch bounds one probe scan. It matches the documented scale
|
||||
// target of 300 service probes (SYSTEM_ARCHITECTURE section 7).
|
||||
MaxProbeBatch = 300
|
||||
// DefaultProbeConcurrency bounds simultaneous probe executions.
|
||||
DefaultProbeConcurrency = 16
|
||||
)
|
||||
|
||||
// ProbeStore is the persistence surface of the probe execution job.
|
||||
type ProbeStore interface {
|
||||
// ListDue returns enabled, unarchived probes whose interval has elapsed.
|
||||
ListDue(ctx context.Context, now time.Time, limit int) ([]probe.Definition, error)
|
||||
// CountEnabled reports how many probes are configured and enabled. It
|
||||
// separates "nothing was due" from "nothing is configured", so an empty
|
||||
// installation reports Disabled instead of a green probe subsystem.
|
||||
CountEnabled(ctx context.Context) (int, error)
|
||||
// SaveResults records probe results and reports how many were new.
|
||||
SaveResults(ctx context.Context, results []probe.Result) (int, error)
|
||||
}
|
||||
|
||||
// ProbeJob executes due probes and records their results.
|
||||
//
|
||||
// Execution goes through probe.Scheduler and probe.ProbeExecutor with the
|
||||
// configured network policy. HTTP transport and dialing are deliberately left
|
||||
// to the executor's own safe client, so the SSRF protections in
|
||||
// internal/probe/policy.go (blocked link-local/metadata/loopback ranges,
|
||||
// redirect re-validation at dial time, response caps) stay fully in force.
|
||||
type ProbeJob struct {
|
||||
Store ProbeStore
|
||||
Scheduler *probe.Scheduler
|
||||
Logger *slog.Logger
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// NewProbeJob builds a probe job with a bounded scheduler around the policy.
|
||||
func NewProbeJob(store ProbeStore, policy probe.NetworkPolicy, logger *slog.Logger) (*ProbeJob, error) {
|
||||
if store == nil {
|
||||
return nil, fmt.Errorf("%w: probe job requires a store", ErrInvalidConfig)
|
||||
}
|
||||
if err := policy.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scheduler, err := probe.NewScheduler(probe.ProbeExecutor{Policy: policy}, probe.SchedulerConfig{
|
||||
MaxConcurrent: DefaultProbeConcurrency, MaxAttempts: 2, AttemptTimeout: 10 * time.Second, RetryBackoff: 250 * time.Millisecond,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ProbeJob{Store: store, Scheduler: scheduler, Logger: logger}, nil
|
||||
}
|
||||
|
||||
func (j *ProbeJob) now() time.Time {
|
||||
if j.Now != nil {
|
||||
return j.Now().UTC()
|
||||
}
|
||||
return time.Now().UTC()
|
||||
}
|
||||
|
||||
func (j *ProbeJob) logger() *slog.Logger {
|
||||
if j.Logger != nil {
|
||||
return j.Logger
|
||||
}
|
||||
return slog.Default()
|
||||
}
|
||||
|
||||
// Run executes every due probe once and persists the results.
|
||||
func (j *ProbeJob) Run(ctx context.Context) (Outcome, error) {
|
||||
if j == nil || j.Store == nil || j.Scheduler == nil {
|
||||
return Outcome{Disabled: true, Reason: "probe_execution_not_configured"}, nil
|
||||
}
|
||||
now := j.now()
|
||||
definitions, err := j.Store.ListDue(ctx, now, MaxProbeBatch)
|
||||
if err != nil {
|
||||
return Outcome{}, fmt.Errorf("list due probes: %w", err)
|
||||
}
|
||||
counts := map[string]int64{"due": int64(len(definitions))}
|
||||
if len(definitions) == 0 {
|
||||
configured, err := j.Store.CountEnabled(ctx)
|
||||
if err != nil {
|
||||
return Outcome{Counts: counts}, fmt.Errorf("count enabled probes: %w", err)
|
||||
}
|
||||
counts["enabled"] = int64(configured)
|
||||
if configured == 0 {
|
||||
return Outcome{Disabled: true, Reason: "no_probes_configured", Counts: counts}, nil
|
||||
}
|
||||
return Outcome{Counts: counts}, nil
|
||||
}
|
||||
// An individually invalid definition is skipped and counted rather than
|
||||
// failing the whole batch, so one bad probe cannot stop the other 299.
|
||||
valid := make([]probe.Definition, 0, len(definitions))
|
||||
for _, definition := range definitions {
|
||||
if err := definition.Validate(); err != nil {
|
||||
counts["invalid"]++
|
||||
j.logger().Error("probe definition is invalid", "probe", definition.ID, "error", boundedError(err))
|
||||
continue
|
||||
}
|
||||
valid = append(valid, definition)
|
||||
}
|
||||
if len(valid) == 0 {
|
||||
return Outcome{Counts: counts}, fmt.Errorf("all %d due probes are invalid", len(definitions))
|
||||
}
|
||||
report, err := j.Scheduler.Run(ctx, valid)
|
||||
if err != nil {
|
||||
return Outcome{Counts: counts}, fmt.Errorf("execute probes: %w", err)
|
||||
}
|
||||
for _, result := range report.Results {
|
||||
counts["state_"+result.State]++
|
||||
}
|
||||
counts["executed"] = int64(len(report.Results))
|
||||
// A batch that ran into the job timeout or a shutdown still produced real
|
||||
// observations. They are persisted under a short detached deadline so the
|
||||
// work is not silently thrown away, and the job still reports the failure.
|
||||
saveCtx := ctx
|
||||
if ctx.Err() != nil {
|
||||
detached, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer cancel()
|
||||
saveCtx = detached
|
||||
}
|
||||
saved, err := j.Store.SaveResults(saveCtx, report.Results)
|
||||
if err != nil {
|
||||
return Outcome{Counts: counts}, fmt.Errorf("save probe results: %w", err)
|
||||
}
|
||||
counts["saved"] = int64(saved)
|
||||
return Outcome{Counts: counts}, nil
|
||||
}
|
||||
|
||||
// Shutdown cancels in-flight probes and waits for them, bounded by ctx.
|
||||
func (j *ProbeJob) Shutdown(ctx context.Context) error {
|
||||
if j == nil || j.Scheduler == nil {
|
||||
return nil
|
||||
}
|
||||
return j.Scheduler.Shutdown(ctx)
|
||||
}
|
||||
|
||||
// PostgresProbeStore reads probe definitions and writes probe_results.
|
||||
type PostgresProbeStore struct {
|
||||
Pool *pgxpool.Pool
|
||||
// SourceID optionally attributes results to a data source.
|
||||
SourceID string
|
||||
}
|
||||
|
||||
// ListDue returns probes whose interval has elapsed since their last result.
|
||||
func (s PostgresProbeStore) ListDue(ctx context.Context, now time.Time, limit int) ([]probe.Definition, error) {
|
||||
if s.Pool == nil {
|
||||
return nil, fmt.Errorf("%w: probe store has no database pool", ErrInvalidConfig)
|
||||
}
|
||||
if limit < 1 || limit > MaxProbeBatch {
|
||||
return nil, fmt.Errorf("%w: probe batch must be between 1 and %d", ErrInvalidConfig, MaxProbeBatch)
|
||||
}
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT p.id::text,p.service_id::text,COALESCE(p.endpoint_id::text,''),p.name,p.probe_type,p.target,p.interval_seconds,p.timeout_seconds,
|
||||
p.expected_status_codes,p.follow_redirects,p.verify_tls,p.content_assertion,COALESCE(p.secret_reference,''),
|
||||
COALESCE(p.network_policy_id::text,''),p.revision,p.created_at,p.updated_at
|
||||
FROM probes p
|
||||
LEFT JOIN LATERAL (SELECT max(r.observed_at) AS last_observed_at FROM probe_results r WHERE r.probe_id = p.id) latest ON true
|
||||
WHERE p.archived_at IS NULL AND p.enabled = true
|
||||
AND (latest.last_observed_at IS NULL OR latest.last_observed_at <= $1::timestamptz - make_interval(secs => p.interval_seconds))
|
||||
ORDER BY p.id ASC
|
||||
LIMIT $2`, now.UTC(), limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query due probes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
definitions := make([]probe.Definition, 0, limit)
|
||||
for rows.Next() {
|
||||
var definition probe.Definition
|
||||
var target, expected, assertion []byte
|
||||
var intervalSeconds, timeoutSeconds int
|
||||
if err := rows.Scan(&definition.ID, &definition.ServiceID, &definition.EndpointID, &definition.Name, &definition.Type, &target,
|
||||
&intervalSeconds, &timeoutSeconds, &expected, &definition.FollowRedirects, &definition.VerifyTLS, &assertion,
|
||||
&definition.SecretReference, &definition.NetworkPolicyID, &definition.Revision, &definition.CreatedAt, &definition.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan probe definition: %w", err)
|
||||
}
|
||||
definition.Enabled = true
|
||||
definition.Interval = time.Duration(intervalSeconds) * time.Second
|
||||
definition.Timeout = time.Duration(timeoutSeconds) * time.Second
|
||||
if err := json.Unmarshal(target, &definition.Target); err != nil {
|
||||
return nil, fmt.Errorf("decode probe target: %w", err)
|
||||
}
|
||||
if len(expected) > 0 {
|
||||
if err := json.Unmarshal(expected, &definition.ExpectedStatusCodes); err != nil {
|
||||
return nil, fmt.Errorf("decode probe expected status codes: %w", err)
|
||||
}
|
||||
}
|
||||
if len(assertion) > 0 {
|
||||
if err := json.Unmarshal(assertion, &definition.ContentAssertion); err != nil {
|
||||
return nil, fmt.Errorf("decode probe content assertion: %w", err)
|
||||
}
|
||||
}
|
||||
definitions = append(definitions, definition)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("read probe definition rows: %w", err)
|
||||
}
|
||||
return definitions, nil
|
||||
}
|
||||
|
||||
// CountEnabled counts configured, enabled, unarchived probes.
|
||||
func (s PostgresProbeStore) CountEnabled(ctx context.Context) (int, error) {
|
||||
if s.Pool == nil {
|
||||
return 0, fmt.Errorf("%w: probe store has no database pool", ErrInvalidConfig)
|
||||
}
|
||||
var count int
|
||||
if err := s.Pool.QueryRow(ctx, `SELECT count(*) FROM probes WHERE archived_at IS NULL AND enabled = true`).Scan(&count); err != nil {
|
||||
return 0, fmt.Errorf("count enabled probes: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// SaveResults writes probe results and any observed certificate. Both inserts
|
||||
// use their natural key with ON CONFLICT DO NOTHING, so replaying a batch after
|
||||
// a crash cannot create duplicate history.
|
||||
func (s PostgresProbeStore) SaveResults(ctx context.Context, results []probe.Result) (int, error) {
|
||||
if s.Pool == nil {
|
||||
return 0, fmt.Errorf("%w: probe store has no database pool", ErrInvalidConfig)
|
||||
}
|
||||
if len(results) > MaxProbeBatch {
|
||||
return 0, fmt.Errorf("probe result batch of %d exceeds the %d bound", len(results), MaxProbeBatch)
|
||||
}
|
||||
if len(results) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
tx, err := s.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("begin probe result save: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
saved := 0
|
||||
for _, result := range results {
|
||||
attributes, err := json.Marshal(nonNilAttributes(result.Attributes))
|
||||
if err != nil {
|
||||
return saved, fmt.Errorf("encode probe attributes: %w", err)
|
||||
}
|
||||
observedAt := result.ObservedAt.UTC()
|
||||
completedAt := result.CompletedAt.UTC()
|
||||
if completedAt.IsZero() {
|
||||
completedAt = observedAt
|
||||
}
|
||||
var inserted string
|
||||
err = tx.QueryRow(ctx, `INSERT INTO probe_results (id,probe_id,source_id,observed_at,completed_at,state,response_time_ms,status_code,error_class,error_message,attributes)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,NULLIF($9,''),NULLIF($10,''),$11::jsonb)
|
||||
ON CONFLICT (probe_id,observed_at) DO NOTHING RETURNING id`,
|
||||
newID(), result.ProbeID, nullableUUID(s.SourceID), observedAt, completedAt, result.State,
|
||||
result.ResponseTimeMS, result.StatusCode, boundedCode(result.ErrorClass), boundedText(result.ErrorMessage, 500), attributes).Scan(&inserted)
|
||||
switch {
|
||||
case err == nil:
|
||||
saved++
|
||||
case errors.Is(err, pgx.ErrNoRows):
|
||||
// The result for this probe and observation time already exists.
|
||||
default:
|
||||
return saved, fmt.Errorf("insert probe result: %w", err)
|
||||
}
|
||||
if result.Certificate == nil {
|
||||
continue
|
||||
}
|
||||
certificate := result.Certificate
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO service_certificates (id,service_id,endpoint_id,source_id,observed_at,expires_at,issuer,subject,hostname_valid,verification_state)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,NULLIF($7,''),NULLIF($8,''),$9,$10) ON CONFLICT DO NOTHING`,
|
||||
newID(), certificate.ServiceID, nullableUUID(certificate.EndpointID), nullableUUID(s.SourceID), certificate.ObservedAt.UTC(),
|
||||
certificate.ExpiresAt, boundedText(certificate.Issuer, 500), boundedText(certificate.Subject, 500), certificate.HostnameValid, certificate.VerificationState); err != nil {
|
||||
return saved, fmt.Errorf("insert service certificate: %w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return saved, fmt.Errorf("commit probe results: %w", err)
|
||||
}
|
||||
return saved, nil
|
||||
}
|
||||
|
||||
func nonNilAttributes(value map[string]any) map[string]any {
|
||||
if value == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func nullableUUID(value string) any {
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func boundedText(value string, max int) string {
|
||||
if len(value) > max {
|
||||
return value[:max]
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,562 @@
|
||||
// Package workerruntime is the supervised background runtime of pulse-worker.
|
||||
//
|
||||
// It owns one scheduling loop that starts a small, fixed set of jobs on
|
||||
// independent intervals. Every job is:
|
||||
//
|
||||
// - database-coordinated: a job run holds a lease row in job_runs keyed by
|
||||
// (job_type, job_key, scheduled_at), so two workers never run the same unit
|
||||
// of work twice and a worker that dies mid-run has its lease expire instead
|
||||
// of blocking the job forever;
|
||||
// - idempotent: the lease key is derived from the schedule, and each job's
|
||||
// writes use natural-key conflict handling, so repeating a run is safe;
|
||||
// - bounded: every run executes under a timeout and there is at most one
|
||||
// in-flight run per job, which caps concurrency at the number of jobs;
|
||||
// - isolated: a failing or panicking job is recorded, logged and counted, and
|
||||
// never stops the other jobs or the process;
|
||||
// - observable: per-job last run, duration and outcome are exposed through
|
||||
// Status and persisted in job_runs, which is what internal/systemstatus
|
||||
// reads instead of hardcoding "not recorded".
|
||||
//
|
||||
// The scheduling loop itself performs no blocking I/O: jobs run in their own
|
||||
// goroutines, so the loop keeps completing iterations and writing the heartbeat
|
||||
// file required by docs/operations/WORKER_AGENT_HEALTHCHECK_CONTRACT.md even
|
||||
// while a job is stuck against a slow database or network target.
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/systemstatus"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultTick is the scheduling loop period. The healthcheck contract
|
||||
// requires a completed iteration at least every 10 seconds; 5 seconds
|
||||
// leaves room for a slow heartbeat write without going stale.
|
||||
DefaultTick = 5 * time.Second
|
||||
// MaxTick is the largest loop period the healthcheck contract tolerates.
|
||||
MaxTick = 10 * time.Second
|
||||
// DefaultDrainTimeout bounds how long shutdown waits for in-flight jobs.
|
||||
DefaultDrainTimeout = 20 * time.Second
|
||||
// DefaultHeartbeatFile matches the healthcheck script's default.
|
||||
DefaultHeartbeatFile = "/tmp/healthy"
|
||||
|
||||
maxJobs = 16
|
||||
maxErrorLength = 200
|
||||
)
|
||||
|
||||
// Job outcome statuses recorded per run.
|
||||
const (
|
||||
StatusCompleted = "completed"
|
||||
StatusFailed = "failed"
|
||||
StatusSkipped = "skipped"
|
||||
StatusDisabled = "disabled"
|
||||
StatusRunning = "running"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrInvalidConfig reports a runtime or job definition outside safe bounds.
|
||||
ErrInvalidConfig = errors.New("worker runtime configuration is invalid")
|
||||
|
||||
jobNamePattern = regexp.MustCompile(`^[a-z][a-z0-9-]{1,39}$`)
|
||||
)
|
||||
|
||||
// Outcome is the bounded, structured result of one job run. Counts are recorded
|
||||
// in job_runs so an operator can see what a run actually did.
|
||||
type Outcome struct {
|
||||
// Reason is a stable snake_case code explaining a skipped or disabled run.
|
||||
Reason string
|
||||
// Counts are small integer result counters, for example processed items.
|
||||
Counts map[string]int64
|
||||
// Skipped marks a run that intentionally did nothing, for example because
|
||||
// another worker held the inner lease.
|
||||
Skipped bool
|
||||
// Disabled marks a run whose feature is not configured. It is reported as
|
||||
// Disabled rather than Healthy so an unconfigured feature never looks green.
|
||||
Disabled bool
|
||||
}
|
||||
|
||||
// RunFunc executes one unit of work. It must respect the supplied context.
|
||||
type RunFunc func(context.Context) (Outcome, error)
|
||||
|
||||
// Job is a scheduled unit of background work with a stable key.
|
||||
type Job struct {
|
||||
// Name is the stable job key. It becomes job_runs.job_key and, prefixed,
|
||||
// job_runs.job_type, so it must never change casually.
|
||||
Name string
|
||||
// Component is the internal/systemstatus component this job reports to.
|
||||
Component string
|
||||
// Interval is how often the job becomes due. It also defines the lease
|
||||
// window: two runs inside one interval are the same unit of work.
|
||||
Interval time.Duration
|
||||
// Timeout bounds a single run.
|
||||
Timeout time.Duration
|
||||
// Run performs the work.
|
||||
Run RunFunc
|
||||
}
|
||||
|
||||
func (j Job) validate() error {
|
||||
if !jobNamePattern.MatchString(j.Name) {
|
||||
return fmt.Errorf("%w: job name %q must be a bounded lowercase key", ErrInvalidConfig, j.Name)
|
||||
}
|
||||
switch j.Component {
|
||||
case systemstatus.ComponentWorker, systemstatus.ComponentProbes, systemstatus.ComponentNotifications:
|
||||
default:
|
||||
return fmt.Errorf("%w: job %s has unknown component %q", ErrInvalidConfig, j.Name, j.Component)
|
||||
}
|
||||
if j.Interval < time.Second || j.Interval > time.Hour {
|
||||
return fmt.Errorf("%w: job %s interval must be between 1s and 1h", ErrInvalidConfig, j.Name)
|
||||
}
|
||||
if j.Timeout < time.Second || j.Timeout > 5*time.Minute {
|
||||
return fmt.Errorf("%w: job %s timeout must be between 1s and 5m", ErrInvalidConfig, j.Name)
|
||||
}
|
||||
if j.Run == nil {
|
||||
return fmt.Errorf("%w: job %s has no run function", ErrInvalidConfig, j.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// JobType is the job_runs.job_type value used for a job's scheduling lease.
|
||||
func (j Job) JobType() string { return "worker-" + j.Name }
|
||||
|
||||
// MetricSink receives in-process counters and gauges. *observability.Registry
|
||||
// satisfies it. The worker binds no network port by design, so these values are
|
||||
// a process-local aid; the durable, operator-visible record is job_runs.
|
||||
type MetricSink interface {
|
||||
IncCounter(name string)
|
||||
SetGauge(name string, value float64)
|
||||
}
|
||||
|
||||
// Config configures the runtime loop.
|
||||
type Config struct {
|
||||
// Owner identifies this worker in job_runs.lease_owner.
|
||||
Owner string
|
||||
// Tick is the scheduling loop period; DefaultTick is used when zero.
|
||||
Tick time.Duration
|
||||
// HeartbeatFile is the path written after every completed loop iteration.
|
||||
// DefaultHeartbeatFile is used when empty.
|
||||
HeartbeatFile string
|
||||
// DrainTimeout bounds graceful shutdown; DefaultDrainTimeout when zero.
|
||||
DrainTimeout time.Duration
|
||||
// Leases coordinates duplicate workers.
|
||||
Leases LeaseStore
|
||||
// Logger receives structured, secret-free job events.
|
||||
Logger *slog.Logger
|
||||
// Metrics optionally receives counters and gauges.
|
||||
Metrics MetricSink
|
||||
// Now is injectable for tests; time.Now is used when nil.
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// JobStatus is the observable state of one job.
|
||||
type JobStatus struct {
|
||||
Name string
|
||||
Component string
|
||||
Interval time.Duration
|
||||
LastStartedAt time.Time
|
||||
LastFinishedAt time.Time
|
||||
LastSuccessAt time.Time
|
||||
LastDuration time.Duration
|
||||
LastStatus string
|
||||
LastReason string
|
||||
LastError string
|
||||
Runs uint64
|
||||
Successes uint64
|
||||
Failures uint64
|
||||
Skips uint64
|
||||
Running bool
|
||||
}
|
||||
|
||||
type jobState struct {
|
||||
job Job
|
||||
running bool
|
||||
// lastAttemptAt is when the job last became due, which drives scheduling
|
||||
// independently of how long the run itself took.
|
||||
lastAttemptAt time.Time
|
||||
status JobStatus
|
||||
}
|
||||
|
||||
// Runtime supervises the scheduled jobs.
|
||||
type Runtime struct {
|
||||
config Config
|
||||
mu sync.Mutex
|
||||
states []*jobState
|
||||
wg sync.WaitGroup
|
||||
|
||||
heartbeatMu sync.Mutex
|
||||
heartbeatCount uint64
|
||||
heartbeatAt time.Time
|
||||
}
|
||||
|
||||
// New validates the configuration and job set and returns a runtime.
|
||||
func New(config Config, jobs ...Job) (*Runtime, error) {
|
||||
if strings.TrimSpace(config.Owner) == "" || len(config.Owner) > 120 {
|
||||
return nil, fmt.Errorf("%w: owner must be 1-120 characters", ErrInvalidConfig)
|
||||
}
|
||||
if config.Leases == nil {
|
||||
return nil, fmt.Errorf("%w: a lease store is required", ErrInvalidConfig)
|
||||
}
|
||||
if len(jobs) == 0 || len(jobs) > maxJobs {
|
||||
return nil, fmt.Errorf("%w: between 1 and %d jobs are required", ErrInvalidConfig, maxJobs)
|
||||
}
|
||||
if config.Tick == 0 {
|
||||
config.Tick = DefaultTick
|
||||
}
|
||||
// The floor only guards against a busy loop; the ceiling is what the
|
||||
// healthcheck contract requires.
|
||||
if config.Tick < 10*time.Millisecond || config.Tick > MaxTick {
|
||||
return nil, fmt.Errorf("%w: tick must be between 10ms and %s", ErrInvalidConfig, MaxTick)
|
||||
}
|
||||
if config.DrainTimeout == 0 {
|
||||
config.DrainTimeout = DefaultDrainTimeout
|
||||
}
|
||||
if config.DrainTimeout < time.Second || config.DrainTimeout > 2*time.Minute {
|
||||
return nil, fmt.Errorf("%w: drain timeout must be between 1s and 2m", ErrInvalidConfig)
|
||||
}
|
||||
if config.HeartbeatFile == "" {
|
||||
config.HeartbeatFile = DefaultHeartbeatFile
|
||||
}
|
||||
if config.Logger == nil {
|
||||
config.Logger = slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
seen := make(map[string]struct{}, len(jobs))
|
||||
states := make([]*jobState, 0, len(jobs))
|
||||
for _, job := range jobs {
|
||||
if err := job.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, duplicate := seen[job.Name]; duplicate {
|
||||
return nil, fmt.Errorf("%w: duplicate job name %q", ErrInvalidConfig, job.Name)
|
||||
}
|
||||
seen[job.Name] = struct{}{}
|
||||
states = append(states, &jobState{job: job, status: JobStatus{Name: job.Name, Component: job.Component, Interval: job.Interval}})
|
||||
}
|
||||
sort.Slice(states, func(i, j int) bool { return states[i].job.Name < states[j].job.Name })
|
||||
return &Runtime{config: config, states: states}, nil
|
||||
}
|
||||
|
||||
// Run drives the scheduling loop until ctx is cancelled, then drains in-flight
|
||||
// jobs within the configured budget. It returns nil on a clean shutdown and an
|
||||
// error only when shutdown could not complete within that budget.
|
||||
func (r *Runtime) Run(ctx context.Context) error {
|
||||
// The first heartbeat is written before the first blocking call so a slow
|
||||
// but healthy cold start is not mistaken for a hang.
|
||||
r.writeHeartbeat()
|
||||
// In-flight jobs keep a context that outlives cancellation of ctx, so
|
||||
// shutdown can drain them instead of tearing them down mid-transaction.
|
||||
jobsCtx, cancelJobs := context.WithCancel(context.WithoutCancel(ctx))
|
||||
defer cancelJobs()
|
||||
timer := time.NewTimer(r.config.Tick)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
r.schedule(jobsCtx)
|
||||
// The heartbeat marks a completed loop iteration, including an
|
||||
// iteration in which nothing was due.
|
||||
r.writeHeartbeat()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return r.drain(cancelJobs)
|
||||
case <-timer.C:
|
||||
timer.Reset(r.config.Tick)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// schedule starts every job that is due and not already running.
|
||||
func (r *Runtime) schedule(jobsCtx context.Context) {
|
||||
now := r.now()
|
||||
r.mu.Lock()
|
||||
due := make([]*jobState, 0, len(r.states))
|
||||
for _, state := range r.states {
|
||||
if state.running {
|
||||
continue
|
||||
}
|
||||
if !state.lastAttemptAt.IsZero() && now.Sub(state.lastAttemptAt) < state.job.Interval {
|
||||
continue
|
||||
}
|
||||
state.running = true
|
||||
state.lastAttemptAt = now
|
||||
state.status.Running = true
|
||||
due = append(due, state)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
for _, state := range due {
|
||||
r.wg.Add(1)
|
||||
go func(state *jobState) {
|
||||
defer r.wg.Done()
|
||||
r.execute(jobsCtx, state)
|
||||
}(state)
|
||||
}
|
||||
}
|
||||
|
||||
// drain stops scheduling and waits for in-flight jobs. Jobs that outlast the
|
||||
// drain budget are cancelled so shutdown always terminates.
|
||||
func (r *Runtime) drain(cancelJobs context.CancelFunc) error {
|
||||
done := make(chan struct{})
|
||||
go func() { r.wg.Wait(); close(done) }()
|
||||
timer := time.NewTimer(r.config.DrainTimeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-done:
|
||||
r.config.Logger.Info("worker runtime drained", "in_flight", 0)
|
||||
return nil
|
||||
case <-timer.C:
|
||||
}
|
||||
r.config.Logger.Warn("worker runtime drain timeout, cancelling in-flight jobs", "drain_timeout", r.config.DrainTimeout.String())
|
||||
cancelJobs()
|
||||
grace := time.NewTimer(5 * time.Second)
|
||||
defer grace.Stop()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-grace.C:
|
||||
return errors.New("worker runtime shutdown left cancelled jobs running")
|
||||
}
|
||||
}
|
||||
|
||||
// execute runs one job under its lease and timeout. It never returns an error:
|
||||
// a failure is recorded, logged and counted so it stays visible without
|
||||
// affecting any other job.
|
||||
func (r *Runtime) execute(parent context.Context, state *jobState) {
|
||||
job := state.job
|
||||
started := r.now()
|
||||
runCtx, cancel := context.WithTimeout(parent, job.Timeout)
|
||||
defer cancel()
|
||||
scheduledAt := started.Truncate(job.Interval)
|
||||
lease, acquired, err := r.config.Leases.Acquire(runCtx, job.JobType(), job.Name, scheduledAt, r.config.Owner, started, r.leaseTTL(job))
|
||||
if err != nil {
|
||||
r.finish(state, started, StatusFailed, "lease_unavailable", err)
|
||||
return
|
||||
}
|
||||
if !acquired {
|
||||
// Another worker owns this window. That is coordination working, not a
|
||||
// failure, so the previous successful outcome is retained.
|
||||
r.finish(state, started, StatusSkipped, "lease_held_elsewhere", nil)
|
||||
return
|
||||
}
|
||||
outcome, runErr := safeRun(runCtx, job)
|
||||
status, reason := StatusCompleted, outcome.Reason
|
||||
switch {
|
||||
case runErr != nil:
|
||||
status = StatusFailed
|
||||
reason = classifyError(runCtx, runErr)
|
||||
case outcome.Disabled:
|
||||
status = StatusDisabled
|
||||
reason = defaultReason(reason, "not_configured")
|
||||
case outcome.Skipped:
|
||||
status = StatusSkipped
|
||||
reason = defaultReason(reason, "nothing_due")
|
||||
}
|
||||
// Lease completion must not be skipped because the run context expired, or
|
||||
// the row would stay "running" until its lease expires.
|
||||
completeCtx, completeCancel := context.WithTimeout(context.WithoutCancel(parent), 5*time.Second)
|
||||
completeErr := r.config.Leases.Complete(completeCtx, lease, status, reason, outcome.Counts)
|
||||
completeCancel()
|
||||
if completeErr != nil && runErr == nil {
|
||||
status = StatusFailed
|
||||
reason = "lease_completion_failed"
|
||||
runErr = completeErr
|
||||
} else if completeErr != nil {
|
||||
r.config.Logger.Error("worker job lease completion failed", "job", job.Name, "error", boundedError(completeErr))
|
||||
}
|
||||
r.finish(state, started, status, reason, runErr)
|
||||
}
|
||||
|
||||
// safeRun converts a panicking job into an ordinary error so one broken job
|
||||
// cannot take the process down.
|
||||
func safeRun(ctx context.Context, job Job) (outcome Outcome, err error) {
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
outcome = Outcome{}
|
||||
err = fmt.Errorf("job %s panicked: %v", job.Name, recovered)
|
||||
}
|
||||
}()
|
||||
return job.Run(ctx)
|
||||
}
|
||||
|
||||
func (r *Runtime) finish(state *jobState, started time.Time, status, reason string, runErr error) {
|
||||
finished := r.now()
|
||||
duration := finished.Sub(started)
|
||||
message := boundedError(runErr)
|
||||
r.mu.Lock()
|
||||
state.running = false
|
||||
state.status.Running = false
|
||||
state.status.LastStartedAt = started
|
||||
state.status.LastFinishedAt = finished
|
||||
state.status.LastDuration = duration
|
||||
state.status.LastStatus = status
|
||||
state.status.LastReason = reason
|
||||
state.status.LastError = message
|
||||
state.status.Runs++
|
||||
switch status {
|
||||
case StatusCompleted:
|
||||
state.status.Successes++
|
||||
state.status.LastSuccessAt = finished
|
||||
case StatusFailed:
|
||||
state.status.Failures++
|
||||
case StatusSkipped, StatusDisabled:
|
||||
state.status.Skips++
|
||||
}
|
||||
name := state.job.Name
|
||||
component := state.job.Component
|
||||
r.mu.Unlock()
|
||||
r.record(name, status, duration)
|
||||
attributes := []any{"job", name, "component", component, "status", status, "duration_ms", duration.Milliseconds()}
|
||||
if reason != "" {
|
||||
attributes = append(attributes, "reason", reason)
|
||||
}
|
||||
if status == StatusFailed {
|
||||
r.config.Logger.Error("worker job failed", append(attributes, "error", message)...)
|
||||
return
|
||||
}
|
||||
r.config.Logger.Info("worker job finished", attributes...)
|
||||
}
|
||||
|
||||
func (r *Runtime) record(name, status string, duration time.Duration) {
|
||||
if r.config.Metrics == nil {
|
||||
return
|
||||
}
|
||||
r.config.Metrics.IncCounter("pulse_worker_job_runs_total_" + metricSuffix(name) + "_" + status)
|
||||
r.config.Metrics.SetGauge("pulse_worker_job_duration_seconds_"+metricSuffix(name), duration.Seconds())
|
||||
}
|
||||
|
||||
// Status returns a copy of every job's observable state.
|
||||
func (r *Runtime) Status() []JobStatus {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
result := make([]JobStatus, 0, len(r.states))
|
||||
for _, state := range r.states {
|
||||
result = append(result, state.status)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// JobHealth projects the in-process job status onto the system status model.
|
||||
// A job that has never reported is deliberately omitted so the component keeps
|
||||
// its "not recorded" Unknown state instead of being invented as healthy.
|
||||
func (r *Runtime) JobHealth() []systemstatus.JobHealth {
|
||||
statuses := r.Status()
|
||||
health := make([]systemstatus.JobHealth, 0, len(statuses))
|
||||
for _, status := range statuses {
|
||||
if status.LastStatus == "" {
|
||||
continue
|
||||
}
|
||||
health = append(health, systemstatus.JobHealth{
|
||||
Component: status.Component, JobKey: status.Name, Status: mapStatus(status.LastStatus),
|
||||
Reason: status.LastReason, ErrorCode: errorCodeFor(status), LastRunAt: status.LastFinishedAt,
|
||||
LastSuccessAt: status.LastSuccessAt, Duration: status.LastDuration,
|
||||
})
|
||||
}
|
||||
return health
|
||||
}
|
||||
|
||||
// HeartbeatCount reports how many loop iterations completed. Tests use it to
|
||||
// assert the heartbeat is driven by the loop rather than by a free-running
|
||||
// timer.
|
||||
func (r *Runtime) HeartbeatCount() uint64 {
|
||||
r.heartbeatMu.Lock()
|
||||
defer r.heartbeatMu.Unlock()
|
||||
return r.heartbeatCount
|
||||
}
|
||||
|
||||
// LastHeartbeat reports when the loop last wrote the heartbeat file.
|
||||
func (r *Runtime) LastHeartbeat() time.Time {
|
||||
r.heartbeatMu.Lock()
|
||||
defer r.heartbeatMu.Unlock()
|
||||
return r.heartbeatAt
|
||||
}
|
||||
|
||||
// writeHeartbeat records loop liveness. A write failure is logged and the loop
|
||||
// continues: the healthcheck notices sustained staleness on its own, and
|
||||
// crashing on a tmpfs hiccup would be worse than reporting it.
|
||||
func (r *Runtime) writeHeartbeat() {
|
||||
now := r.now()
|
||||
content := now.Format(time.RFC3339) + "\n"
|
||||
if err := os.WriteFile(r.config.HeartbeatFile, []byte(content), 0o600); err != nil {
|
||||
r.config.Logger.Error("worker heartbeat write failed", "path", r.config.HeartbeatFile, "error", boundedError(err))
|
||||
return
|
||||
}
|
||||
r.heartbeatMu.Lock()
|
||||
r.heartbeatCount++
|
||||
r.heartbeatAt = now
|
||||
r.heartbeatMu.Unlock()
|
||||
}
|
||||
|
||||
func (r *Runtime) leaseTTL(job Job) time.Duration {
|
||||
ttl := 2 * job.Timeout
|
||||
if ttl < job.Interval {
|
||||
ttl = job.Interval + job.Timeout
|
||||
}
|
||||
if ttl > 10*time.Minute {
|
||||
ttl = 10 * time.Minute
|
||||
}
|
||||
return ttl
|
||||
}
|
||||
|
||||
func (r *Runtime) now() time.Time { return r.config.Now().UTC() }
|
||||
|
||||
// mapStatus projects a recorded run status onto the system status vocabulary.
|
||||
// A skipped run is reported as completed but never updates the last success, so
|
||||
// a job that only ever skips can reach Unknown but never Healthy.
|
||||
func mapStatus(status string) string {
|
||||
switch status {
|
||||
case StatusFailed:
|
||||
return systemstatus.JobFailed
|
||||
case StatusDisabled:
|
||||
return systemstatus.JobDisabled
|
||||
case StatusRunning:
|
||||
return systemstatus.JobRunning
|
||||
default:
|
||||
return systemstatus.JobCompleted
|
||||
}
|
||||
}
|
||||
|
||||
func errorCodeFor(status JobStatus) string {
|
||||
if status.LastStatus != StatusFailed {
|
||||
return ""
|
||||
}
|
||||
return defaultReason(status.LastReason, "last_run_failed")
|
||||
}
|
||||
|
||||
func defaultReason(reason, fallback string) string {
|
||||
if strings.TrimSpace(reason) == "" {
|
||||
return fallback
|
||||
}
|
||||
return reason
|
||||
}
|
||||
|
||||
func classifyError(ctx context.Context, err error) string {
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded):
|
||||
return "timeout"
|
||||
case errors.Is(err, context.Canceled) || errors.Is(ctx.Err(), context.Canceled):
|
||||
return "canceled"
|
||||
default:
|
||||
return "run_failed"
|
||||
}
|
||||
}
|
||||
|
||||
func boundedError(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
message := strings.ReplaceAll(strings.ReplaceAll(err.Error(), "\n", " "), "\r", " ")
|
||||
if len(message) > maxErrorLength {
|
||||
return message[:maxErrorLength]
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func metricSuffix(name string) string { return strings.ReplaceAll(name, "-", "_") }
|
||||
@@ -0,0 +1,525 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/config"
|
||||
"github.com/itworx/pulse/internal/systemstatus"
|
||||
)
|
||||
|
||||
// configFixture is a minimal valid application configuration for status tests.
|
||||
func configFixture() config.Config { return config.Config{AuthMode: "mock"} }
|
||||
|
||||
// fakeClock is a controllable clock. The scheduling loop uses it for due-ness
|
||||
// and for every recorded timestamp, so a test decides exactly when a job
|
||||
// becomes due instead of sleeping.
|
||||
type fakeClock struct {
|
||||
mu sync.Mutex
|
||||
now time.Time
|
||||
}
|
||||
|
||||
func newFakeClock() *fakeClock {
|
||||
return &fakeClock{now: time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)}
|
||||
}
|
||||
func (c *fakeClock) Now() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.now
|
||||
}
|
||||
func (c *fakeClock) Advance(d time.Duration) {
|
||||
c.mu.Lock()
|
||||
c.now = c.now.Add(d)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func quietLogger() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(nopWriter{}, &slog.HandlerOptions{Level: slog.LevelError + 1}))
|
||||
}
|
||||
|
||||
type nopWriter struct{}
|
||||
|
||||
func (nopWriter) Write(p []byte) (int, error) { return len(p), nil }
|
||||
|
||||
func testConfig(t *testing.T, clock *fakeClock, store LeaseStore) Config {
|
||||
t.Helper()
|
||||
return Config{
|
||||
Owner: "worker-test", Tick: 10 * time.Millisecond, HeartbeatFile: filepath.Join(t.TempDir(), "healthy"),
|
||||
DrainTimeout: 2 * time.Second, Leases: store, Logger: quietLogger(), Now: clock.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// waitFor polls until condition holds or the deadline passes.
|
||||
func waitFor(t *testing.T, timeout time.Duration, condition func() bool) bool {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if condition() {
|
||||
return true
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
return condition()
|
||||
}
|
||||
|
||||
func TestNewRejectsInvalidConfiguration(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
valid := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
|
||||
Run: func(context.Context) (Outcome, error) { return Outcome{}, nil }}
|
||||
for name, testCase := range map[string]struct {
|
||||
mutate func(*Config)
|
||||
jobs []Job
|
||||
}{
|
||||
"no owner": {mutate: func(c *Config) { c.Owner = "" }, jobs: []Job{valid}},
|
||||
"no lease store": {mutate: func(c *Config) { c.Leases = nil }, jobs: []Job{valid}},
|
||||
"no jobs": {mutate: func(*Config) {}, jobs: nil},
|
||||
"tick above bound": {mutate: func(c *Config) { c.Tick = time.Minute }, jobs: []Job{valid}},
|
||||
"duplicate job": {mutate: func(*Config) {}, jobs: []Job{valid, valid}},
|
||||
"unknown component": {mutate: func(*Config) {}, jobs: []Job{{Name: "x-job", Component: "storage", Interval: time.Minute,
|
||||
Timeout: time.Second, Run: valid.Run}}},
|
||||
"missing run": {mutate: func(*Config) {}, jobs: []Job{{Name: "discovery", Component: systemstatus.ComponentWorker,
|
||||
Interval: time.Minute, Timeout: time.Second}}},
|
||||
"timeout above bound": {mutate: func(*Config) {}, jobs: []Job{{Name: "discovery", Component: systemstatus.ComponentWorker,
|
||||
Interval: time.Minute, Timeout: time.Hour, Run: valid.Run}}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
config := testConfig(t, clock, NewMemoryLeaseStore())
|
||||
testCase.mutate(&config)
|
||||
if _, err := New(config, testCase.jobs...); !errors.Is(err, ErrInvalidConfig) {
|
||||
t.Fatalf("error = %v, want ErrInvalidConfig", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRunsOncePerIntervalWindow(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
var runs atomic.Int64
|
||||
job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 5 * time.Second,
|
||||
Run: func(context.Context) (Outcome, error) {
|
||||
runs.Add(1)
|
||||
return Outcome{Counts: map[string]int64{"items": 1}}, nil
|
||||
}}
|
||||
runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), job)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
|
||||
if !waitFor(t, time.Second, func() bool { return runs.Load() == 1 }) {
|
||||
t.Fatalf("first run count = %d, want 1", runs.Load())
|
||||
}
|
||||
// Many loop iterations pass without the clock moving: the job must not run
|
||||
// again inside its interval.
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
if runs.Load() != 1 {
|
||||
t.Fatalf("run count inside interval = %d, want 1", runs.Load())
|
||||
}
|
||||
clock.Advance(time.Minute)
|
||||
if !waitFor(t, time.Second, func() bool { return runs.Load() == 2 }) {
|
||||
t.Fatalf("run count after interval = %d, want 2", runs.Load())
|
||||
}
|
||||
cancel()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
status := runtime.Status()[0]
|
||||
if status.LastStatus != StatusCompleted || status.Runs != 2 || status.Successes != 2 || status.LastSuccessAt.IsZero() {
|
||||
t.Fatalf("status = %#v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecondRuntimeIsBlockedByTheLeaseAndRepeatedRunsAreIdempotent(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
store := NewMemoryLeaseStore()
|
||||
var first, second atomic.Int64
|
||||
build := func(counter *atomic.Int64, owner string) *Runtime {
|
||||
config := testConfig(t, clock, store)
|
||||
config.Owner = owner
|
||||
runtime, err := New(config, Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 5 * time.Second,
|
||||
Run: func(context.Context) (Outcome, error) { counter.Add(1); return Outcome{}, nil }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return runtime
|
||||
}
|
||||
runtimeA, runtimeB := build(&first, "worker-a"), build(&second, "worker-b")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
doneA, doneB := make(chan error, 1), make(chan error, 1)
|
||||
go func() { doneA <- runtimeA.Run(ctx) }()
|
||||
go func() { doneB <- runtimeB.Run(ctx) }()
|
||||
if !waitFor(t, 2*time.Second, func() bool { return first.Load()+second.Load() >= 1 }) {
|
||||
t.Fatal("no worker executed the job")
|
||||
}
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
if total := first.Load() + second.Load(); total != 1 {
|
||||
t.Fatalf("executions for one window = %d, want 1", total)
|
||||
}
|
||||
// The next window is a different lease key, so exactly one worker runs again.
|
||||
clock.Advance(time.Minute)
|
||||
if !waitFor(t, 2*time.Second, func() bool { return first.Load()+second.Load() == 2 }) {
|
||||
t.Fatalf("executions after second window = %d, want 2", first.Load()+second.Load())
|
||||
}
|
||||
cancel()
|
||||
<-doneA
|
||||
<-doneB
|
||||
if store.Windows() != 2 {
|
||||
t.Fatalf("claimed windows = %d, want 2", store.Windows())
|
||||
}
|
||||
skips := uint64(0)
|
||||
for _, status := range append(runtimeA.Status(), runtimeB.Status()...) {
|
||||
skips += status.Skips
|
||||
}
|
||||
if skips == 0 {
|
||||
t.Fatal("the worker that lost a window did not record a lease contention skip")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneFailingJobDoesNotStopTheOthers(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
var healthy, panicking, failing atomic.Int64
|
||||
jobs := []Job{
|
||||
{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
|
||||
Run: func(context.Context) (Outcome, error) {
|
||||
failing.Add(1)
|
||||
return Outcome{}, errors.New("database is unreachable")
|
||||
}},
|
||||
{Name: "probe-execution", Component: systemstatus.ComponentProbes, Interval: time.Minute, Timeout: time.Second,
|
||||
Run: func(context.Context) (Outcome, error) { panicking.Add(1); panic("probe exploded") }},
|
||||
{Name: "notification-drain", Component: systemstatus.ComponentNotifications, Interval: time.Minute, Timeout: time.Second,
|
||||
Run: func(context.Context) (Outcome, error) { healthy.Add(1); return Outcome{}, nil }},
|
||||
}
|
||||
runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), jobs...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
if !waitFor(t, 2*time.Second, func() bool { return healthy.Load() == 1 && failing.Load() == 1 && panicking.Load() == 1 }) {
|
||||
t.Fatalf("runs healthy=%d failing=%d panicking=%d", healthy.Load(), failing.Load(), panicking.Load())
|
||||
}
|
||||
clock.Advance(time.Minute)
|
||||
if !waitFor(t, 2*time.Second, func() bool { return healthy.Load() == 2 && failing.Load() == 2 }) {
|
||||
t.Fatalf("second window healthy=%d failing=%d", healthy.Load(), failing.Load())
|
||||
}
|
||||
cancel()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
byName := map[string]JobStatus{}
|
||||
for _, status := range runtime.Status() {
|
||||
byName[status.Name] = status
|
||||
}
|
||||
if byName["discovery"].LastStatus != StatusFailed || byName["discovery"].Failures == 0 || byName["discovery"].LastError == "" {
|
||||
t.Fatalf("failing job status = %#v", byName["discovery"])
|
||||
}
|
||||
if !strings.Contains(byName["probe-execution"].LastError, "panicked") {
|
||||
t.Fatalf("panicking job status = %#v", byName["probe-execution"])
|
||||
}
|
||||
if byName["notification-drain"].LastStatus != StatusCompleted || byName["notification-drain"].Successes == 0 {
|
||||
t.Fatalf("healthy job status = %#v", byName["notification-drain"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobTimeoutIsEnforcedAndRecorded(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
released := make(chan struct{})
|
||||
job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
|
||||
Run: func(ctx context.Context) (Outcome, error) {
|
||||
<-ctx.Done()
|
||||
close(released)
|
||||
return Outcome{}, ctx.Err()
|
||||
}}
|
||||
runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), job)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
select {
|
||||
case <-released:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("job was not cancelled by its timeout")
|
||||
}
|
||||
if !waitFor(t, 2*time.Second, func() bool { return runtime.Status()[0].LastStatus == StatusFailed }) {
|
||||
t.Fatalf("status = %#v", runtime.Status()[0])
|
||||
}
|
||||
if reason := runtime.Status()[0].LastReason; reason != "timeout" {
|
||||
t.Fatalf("reason = %q, want timeout", reason)
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
}
|
||||
|
||||
func TestShutdownDrainsInFlightWork(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
started := make(chan struct{})
|
||||
var finished atomic.Bool
|
||||
job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 5 * time.Second,
|
||||
Run: func(ctx context.Context) (Outcome, error) {
|
||||
close(started)
|
||||
select {
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
case <-ctx.Done():
|
||||
return Outcome{}, ctx.Err()
|
||||
}
|
||||
finished.Store(true)
|
||||
return Outcome{}, nil
|
||||
}}
|
||||
runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), job)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
<-started
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("shutdown: %v", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("shutdown did not complete")
|
||||
}
|
||||
if !finished.Load() {
|
||||
t.Fatal("in-flight job was cancelled instead of drained")
|
||||
}
|
||||
if status := runtime.Status()[0]; status.LastStatus != StatusCompleted {
|
||||
t.Fatalf("drained job status = %#v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShutdownCancelsWorkThatOutlastsTheDrainBudget(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
config := testConfig(t, clock, NewMemoryLeaseStore())
|
||||
config.DrainTimeout = time.Second
|
||||
started := make(chan struct{})
|
||||
job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 4 * time.Minute,
|
||||
Run: func(ctx context.Context) (Outcome, error) {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
return Outcome{}, ctx.Err()
|
||||
}}
|
||||
runtime, err := New(config, job)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
<-started
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("shutdown: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("shutdown never terminated")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeatIsWrittenByTheLoopAndSurvivesWriteFailure(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
directory := t.TempDir()
|
||||
path := filepath.Join(directory, "healthy")
|
||||
blocked := make(chan struct{})
|
||||
job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: 4 * time.Minute,
|
||||
Run: func(ctx context.Context) (Outcome, error) {
|
||||
// The loop must keep heartbeating while a job is stuck.
|
||||
<-blocked
|
||||
return Outcome{}, nil
|
||||
}}
|
||||
config := testConfig(t, clock, NewMemoryLeaseStore())
|
||||
config.HeartbeatFile = path
|
||||
runtime, err := New(config, job)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
if !waitFor(t, 2*time.Second, func() bool { return runtime.HeartbeatCount() >= 3 }) {
|
||||
t.Fatalf("heartbeat count = %d while a job is stuck", runtime.HeartbeatCount())
|
||||
}
|
||||
close(blocked)
|
||||
cancel()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("shutdown: %v", err)
|
||||
}
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := time.Parse(time.RFC3339, strings.TrimSpace(string(content))); err != nil {
|
||||
t.Fatalf("heartbeat content %q is not RFC3339: %v", content, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeatFailureIsReportedWithoutStoppingTheLoop(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
config := testConfig(t, clock, NewMemoryLeaseStore())
|
||||
// A directory can never be written as a file.
|
||||
config.HeartbeatFile = t.TempDir()
|
||||
var runs atomic.Int64
|
||||
runtime, err := New(config, Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
|
||||
Run: func(context.Context) (Outcome, error) { runs.Add(1); return Outcome{}, nil }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
if !waitFor(t, 2*time.Second, func() bool { return runs.Load() == 1 }) {
|
||||
t.Fatal("loop stopped after a heartbeat write failure")
|
||||
}
|
||||
if runtime.HeartbeatCount() != 0 {
|
||||
t.Fatalf("heartbeat count = %d, want 0 after write failures", runtime.HeartbeatCount())
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
}
|
||||
|
||||
func TestLeaseFailureIsReportedAsAFailedRun(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
config := testConfig(t, clock, failingLeaseStore{})
|
||||
runtime, err := New(config, Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
|
||||
Run: func(context.Context) (Outcome, error) { t.Error("job ran without a lease"); return Outcome{}, nil }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
if !waitFor(t, 2*time.Second, func() bool { return runtime.Status()[0].LastStatus == StatusFailed }) {
|
||||
t.Fatalf("status = %#v", runtime.Status()[0])
|
||||
}
|
||||
if reason := runtime.Status()[0].LastReason; reason != "lease_unavailable" {
|
||||
t.Fatalf("reason = %q", reason)
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
}
|
||||
|
||||
type failingLeaseStore struct{}
|
||||
|
||||
func (failingLeaseStore) Acquire(context.Context, string, string, time.Time, string, time.Time, time.Duration) (Lease, bool, error) {
|
||||
return Lease{}, false, errors.New("job_runs is unavailable")
|
||||
}
|
||||
func (failingLeaseStore) Complete(context.Context, Lease, string, string, map[string]int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestJobHealthReportsOnlyJobsThatRan(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
release := make(chan struct{})
|
||||
jobs := []Job{
|
||||
{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
|
||||
Run: func(context.Context) (Outcome, error) { return Outcome{}, nil }},
|
||||
{Name: "probe-execution", Component: systemstatus.ComponentProbes, Interval: time.Minute, Timeout: 4 * time.Minute,
|
||||
Run: func(context.Context) (Outcome, error) { <-release; return Outcome{}, nil }},
|
||||
}
|
||||
runtime, err := New(testConfig(t, clock, NewMemoryLeaseStore()), jobs...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
if !waitFor(t, 2*time.Second, func() bool { return len(runtime.JobHealth()) == 1 }) {
|
||||
t.Fatalf("job health = %#v", runtime.JobHealth())
|
||||
}
|
||||
health := runtime.JobHealth()[0]
|
||||
if health.Component != systemstatus.ComponentWorker || health.Status != systemstatus.JobCompleted || health.LastSuccessAt.IsZero() {
|
||||
t.Fatalf("health = %#v", health)
|
||||
}
|
||||
// The still-running probe job has reported nothing, so its component keeps
|
||||
// the Unknown "not recorded" state.
|
||||
snapshot := systemstatus.Build(configFixture(), true, clock.Now(), nil, systemstatus.WithJobs(time.Minute, runtime.JobHealth()...))
|
||||
for _, component := range snapshot.Components {
|
||||
if component.ID == systemstatus.ComponentProbes && component.Reason != "probe_heartbeat_not_recorded" {
|
||||
t.Fatalf("probes component = %#v", component)
|
||||
}
|
||||
if component.ID == systemstatus.ComponentWorker && component.State != systemstatus.StateHealthy {
|
||||
t.Fatalf("worker component = %#v", component)
|
||||
}
|
||||
}
|
||||
close(release)
|
||||
cancel()
|
||||
<-done
|
||||
}
|
||||
|
||||
func TestScheduleUsesBoundedIntervalsAndDisablesMissingWork(t *testing.T) {
|
||||
jobs := Schedule(ScheduleRuns{})
|
||||
if len(jobs) != 4 {
|
||||
t.Fatalf("job count = %d, want 4", len(jobs))
|
||||
}
|
||||
seen := map[string]Job{}
|
||||
for _, job := range jobs {
|
||||
if err := job.validate(); err != nil {
|
||||
t.Fatalf("job %s: %v", job.Name, err)
|
||||
}
|
||||
if job.Timeout > 10*job.Interval {
|
||||
t.Fatalf("job %s timeout %s is unreasonable for interval %s", job.Name, job.Timeout, job.Interval)
|
||||
}
|
||||
seen[job.Name] = job
|
||||
}
|
||||
for _, name := range []string{JobDiscovery, JobAlertEvaluation, JobProbeExecution, JobNotificationDrain} {
|
||||
job, ok := seen[name]
|
||||
if !ok {
|
||||
t.Fatalf("job %s is not scheduled", name)
|
||||
}
|
||||
outcome, err := job.Run(context.Background())
|
||||
if err != nil || !outcome.Disabled || outcome.Reason == "" {
|
||||
t.Fatalf("unconfigured job %s outcome = %#v err = %v", name, outcome, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestASkippedRunNeverBecomesAHealthyComponent(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
store := NewMemoryLeaseStore()
|
||||
job := Job{Name: "discovery", Component: systemstatus.ComponentWorker, Interval: time.Minute, Timeout: time.Second,
|
||||
Run: func(context.Context) (Outcome, error) {
|
||||
return Outcome{Skipped: true, Reason: "source_unavailable"}, nil
|
||||
}}
|
||||
runtime, err := New(testConfig(t, clock, store), job)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
if !waitFor(t, 2*time.Second, func() bool { return len(runtime.JobHealth()) == 1 }) {
|
||||
t.Fatal("the skipped run was not reported")
|
||||
}
|
||||
health := runtime.JobHealth()
|
||||
if !health[0].LastSuccessAt.IsZero() {
|
||||
t.Fatalf("a skipped run recorded a success: %#v", health[0])
|
||||
}
|
||||
snapshot := systemstatus.Build(configFixture(), true, clock.Now(), nil, systemstatus.WithJobs(time.Minute, health...))
|
||||
for _, component := range snapshot.Components {
|
||||
if component.ID == systemstatus.ComponentWorker && component.State == systemstatus.StateHealthy {
|
||||
t.Fatalf("a job that only skips reported healthy: %#v", component)
|
||||
}
|
||||
}
|
||||
if recorded := store.Status(job.JobType(), job.Name, clock.Now().Truncate(job.Interval)); recorded != StatusSkipped {
|
||||
t.Fatalf("recorded lease status = %q, want %q", recorded, StatusSkipped)
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/systemstatus"
|
||||
)
|
||||
|
||||
// The standard job schedule.
|
||||
//
|
||||
// Intervals are chosen from what each job observes and how expensive it is, and
|
||||
// every timeout stays well inside its lease so a slow run cannot be duplicated
|
||||
// by another worker before it finishes:
|
||||
//
|
||||
// - discovery: inventory changes slowly and one pass reads a full container
|
||||
// snapshot for up to 150 containers, so a minute is responsive enough
|
||||
// without re-reading the same snapshot repeatedly;
|
||||
// - alert evaluation: rules declare their own evaluation interval (5s
|
||||
// minimum, 60s typically). Scanning every 15 seconds means a rule is picked
|
||||
// up within 15 seconds of becoming due, while the per-rule lease keyed on
|
||||
// the rule's own interval prevents evaluating it more often than declared;
|
||||
// - probe execution: probe intervals start at 5 seconds, so a 15 second scan
|
||||
// keeps a due probe close to its schedule; the batch itself is bounded to
|
||||
// 300 probes with bounded concurrency;
|
||||
// - notification drain: the outbox retry backoff starts at one second, so a
|
||||
// 10 second drain delivers promptly without polling the database hard.
|
||||
const (
|
||||
DiscoveryInterval = time.Minute
|
||||
DiscoveryTimeout = 45 * time.Second
|
||||
AlertEvaluationInterval = 15 * time.Second
|
||||
AlertEvaluationTimeout = 60 * time.Second
|
||||
ProbeExecutionInterval = 15 * time.Second
|
||||
ProbeExecutionTimeout = 2 * time.Minute
|
||||
NotificationInterval = 10 * time.Second
|
||||
NotificationDrainTimeout = DrainDeadline
|
||||
)
|
||||
|
||||
// Job names, stable because they are persisted in job_runs.
|
||||
const (
|
||||
JobDiscovery = "discovery"
|
||||
JobAlertEvaluation = "alert-evaluation"
|
||||
JobProbeExecution = "probe-execution"
|
||||
JobNotificationDrain = "notification-drain"
|
||||
)
|
||||
|
||||
// ScheduleRuns holds the work each scheduled job performs. A nil entry is still
|
||||
// scheduled, but reports Disabled with a reason, so an unconfigured capability
|
||||
// is visible rather than silently absent.
|
||||
type ScheduleRuns struct {
|
||||
Discovery RunFunc
|
||||
AlertEvaluation RunFunc
|
||||
ProbeExecution RunFunc
|
||||
NotificationDrain RunFunc
|
||||
}
|
||||
|
||||
// Schedule returns the standard worker job set.
|
||||
func Schedule(runs ScheduleRuns) []Job {
|
||||
return []Job{
|
||||
{Name: JobDiscovery, Component: systemstatus.ComponentWorker, Interval: DiscoveryInterval, Timeout: DiscoveryTimeout,
|
||||
Run: orDisabled(runs.Discovery, "discovery_not_configured")},
|
||||
{Name: JobAlertEvaluation, Component: systemstatus.ComponentWorker, Interval: AlertEvaluationInterval, Timeout: AlertEvaluationTimeout,
|
||||
Run: orDisabled(runs.AlertEvaluation, "alert_evaluation_not_configured")},
|
||||
{Name: JobProbeExecution, Component: systemstatus.ComponentProbes, Interval: ProbeExecutionInterval, Timeout: ProbeExecutionTimeout,
|
||||
Run: orDisabled(runs.ProbeExecution, "probe_execution_not_configured")},
|
||||
{Name: JobNotificationDrain, Component: systemstatus.ComponentNotifications, Interval: NotificationInterval, Timeout: NotificationDrainTimeout,
|
||||
Run: orDisabled(runs.NotificationDrain, "notifications_not_configured")},
|
||||
}
|
||||
}
|
||||
|
||||
func orDisabled(run RunFunc, reason string) RunFunc {
|
||||
if run != nil {
|
||||
return run
|
||||
}
|
||||
return func(context.Context) (Outcome, error) { return Outcome{Disabled: true, Reason: reason}, nil }
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/systemstatus"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// ReadJobHealth reads the last recorded outcome of every supplied job from
|
||||
// job_runs and projects it onto the system status model.
|
||||
//
|
||||
// This is the cross-process half of worker observability: the worker records
|
||||
// each run in job_runs, and any reader (the API's system status endpoint, an
|
||||
// operator, a second worker) can reconstruct what the background runtime is
|
||||
// doing without talking to the worker process. A job with no recorded run is
|
||||
// omitted, so its component keeps the Unknown "not recorded" state rather than
|
||||
// being invented as healthy (ADR-0008).
|
||||
func ReadJobHealth(ctx context.Context, pool *pgxpool.Pool, jobs []Job) ([]systemstatus.JobHealth, error) {
|
||||
if pool == nil {
|
||||
return nil, fmt.Errorf("%w: job health reader has no database pool", ErrInvalidConfig)
|
||||
}
|
||||
if len(jobs) == 0 || len(jobs) > maxJobs {
|
||||
return nil, fmt.Errorf("%w: between 1 and %d jobs are required", ErrInvalidConfig, maxJobs)
|
||||
}
|
||||
types := make([]string, 0, len(jobs))
|
||||
components := make(map[string]Job, len(jobs))
|
||||
for _, job := range jobs {
|
||||
types = append(types, job.JobType())
|
||||
components[job.JobType()] = job
|
||||
}
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT DISTINCT ON (job_type) job_type, status, COALESCE(error_code,''), COALESCE(counts->>'reason',''), started_at, completed_at
|
||||
FROM job_runs
|
||||
WHERE job_type = ANY($1::text[])
|
||||
ORDER BY job_type ASC, scheduled_at DESC, started_at DESC NULLS LAST`, types)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read worker job runs: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
health := make([]systemstatus.JobHealth, 0, len(jobs))
|
||||
index := make(map[string]int, len(jobs))
|
||||
for rows.Next() {
|
||||
var jobType, status, errorCode, reason string
|
||||
var startedAt, completedAt *time.Time
|
||||
if err := rows.Scan(&jobType, &status, &errorCode, &reason, &startedAt, &completedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan worker job run: %w", err)
|
||||
}
|
||||
job, known := components[jobType]
|
||||
if !known {
|
||||
continue
|
||||
}
|
||||
item := systemstatus.JobHealth{Component: job.Component, JobKey: job.Name, Status: mapStatus(status), ErrorCode: errorCode, Reason: reason}
|
||||
if completedAt != nil {
|
||||
item.LastRunAt = completedAt.UTC()
|
||||
} else if startedAt != nil {
|
||||
item.LastRunAt = startedAt.UTC()
|
||||
}
|
||||
if startedAt != nil && completedAt != nil {
|
||||
item.Duration = completedAt.Sub(*startedAt)
|
||||
}
|
||||
index[jobType] = len(health)
|
||||
health = append(health, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("read worker job run rows: %w", err)
|
||||
}
|
||||
successRows, err := pool.Query(ctx, `SELECT job_type, max(completed_at) FROM job_runs WHERE job_type = ANY($1::text[]) AND status='completed' AND completed_at IS NOT NULL GROUP BY job_type`, types)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read worker job successes: %w", err)
|
||||
}
|
||||
defer successRows.Close()
|
||||
for successRows.Next() {
|
||||
var jobType string
|
||||
var lastSuccess *time.Time
|
||||
if err := successRows.Scan(&jobType, &lastSuccess); err != nil {
|
||||
return nil, fmt.Errorf("scan worker job success: %w", err)
|
||||
}
|
||||
position, ok := index[jobType]
|
||||
if !ok || lastSuccess == nil {
|
||||
continue
|
||||
}
|
||||
health[position].LastSuccessAt = lastSuccess.UTC()
|
||||
}
|
||||
if err := successRows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("read worker job success rows: %w", err)
|
||||
}
|
||||
return health, nil
|
||||
}
|
||||
Reference in New Issue
Block a user