Publish ITWorx Pulse source

2026-09-03 02:09:19 +02:00
commit bd774932d5
614 changed files with 77116 additions and 0 deletions
@@ -0,0 +1,308 @@
+package probe
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+type Executor interface {
+	Execute(context.Context, Definition) (Result, error)
+}
+
+type RetryableError struct{ Err error }
+
+func (e RetryableError) Error() string {
+	if e.Err == nil {
+		return "retryable probe error"
+	}
+	return e.Err.Error()
+}
+func (e RetryableError) Unwrap() error { return e.Err }
+
+type SchedulerConfig struct {
+	MaxConcurrent  int
+	MaxAttempts    int
+	AttemptTimeout time.Duration
+	RetryBackoff   time.Duration
+	Now            func() time.Time
+}
+
+func (c SchedulerConfig) withDefaults() SchedulerConfig {
+	if c.MaxConcurrent == 0 {
+		c.MaxConcurrent = 16
+	}
+	if c.MaxAttempts == 0 {
+		c.MaxAttempts = 2
+	}
+	if c.AttemptTimeout == 0 {
+		c.AttemptTimeout = 10 * time.Second
+	}
+	if c.RetryBackoff == 0 {
+		c.RetryBackoff = 100 * time.Millisecond
+	}
+	if c.Now == nil {
+		c.Now = func() time.Time { return time.Now().UTC() }
+	}
+	return c
+}
+
+func (c SchedulerConfig) Validate() error {
+	if c.MaxConcurrent < 1 || c.MaxConcurrent > 64 || c.MaxAttempts < 1 || c.MaxAttempts > 3 || c.AttemptTimeout <= 0 || c.AttemptTimeout > 2*time.Minute || c.RetryBackoff < 0 || c.RetryBackoff > time.Minute {
+		return errors.New("probe scheduler configuration is outside safe bounds")
+	}
+	return nil
+}
+
+type Metrics struct {
+	Runs           int64     `json:"runs"`
+	Completed      int64     `json:"completed"`
+	Failed         int64     `json:"failed"`
+	TimedOut       int64     `json:"timedOut"`
+	Retried        int64     `json:"retried"`
+	SkippedOverlap int64     `json:"skippedOverlap"`
+	Active         int64     `json:"active"`
+	LastRunAt      time.Time `json:"lastRunAt"`
+}
+
+type RunReport struct {
+	Results []Result `json:"results"`
+	Metrics Metrics  `json:"metrics"`
+}
+
+type Scheduler struct {
+	executor Executor
+	config   SchedulerConfig
+	sem      chan struct{}
+	mu       sync.Mutex
+	inflight map[string]context.CancelFunc
+	closed   bool
+	wg       sync.WaitGroup
+	metrics  Metrics
+}
+
+func NewScheduler(executor Executor, config SchedulerConfig) (*Scheduler, error) {
+	config = config.withDefaults()
+	if executor == nil {
+		return nil, errors.New("probe executor is required")
+	}
+	if err := config.Validate(); err != nil {
+		return nil, err
+	}
+	return &Scheduler{executor: executor, config: config, sem: make(chan struct{}, config.MaxConcurrent), inflight: make(map[string]context.CancelFunc)}, nil
+}
+
+func (s *Scheduler) Run(ctx context.Context, definitions []Definition) (RunReport, error) {
+	if s == nil {
+		return RunReport{}, errors.New("probe scheduler is nil")
+	}
+	if ctx == nil {
+		return RunReport{}, errors.New("probe scheduler context is nil")
+	}
+	if err := ctx.Err(); err != nil {
+		return RunReport{}, err
+	}
+	items := append([]Definition(nil), definitions...)
+	sort.SliceStable(items, func(i, j int) bool { return items[i].ID < items[j].ID })
+	for _, definition := range items {
+		if !definition.Enabled || definition.ArchivedAt != nil {
+			continue
+		}
+		if err := definition.Validate(); err != nil {
+			return RunReport{}, fmt.Errorf("validate probe %s: %w", definition.ID, err)
+		}
+	}
+	results := make(chan Result, len(items))
+	for _, definition := range items {
+		if definition.Enabled && definition.ArchivedAt == nil {
+			s.begin(definition, ctx, results)
+		}
+	}
+	s.wg.Wait()
+	close(results)
+	report := RunReport{Results: make([]Result, 0, len(results))}
+	for result := range results {
+		report.Results = append(report.Results, result)
+	}
+	sort.SliceStable(report.Results, func(i, j int) bool { return report.Results[i].ProbeID < report.Results[j].ProbeID })
+	s.mu.Lock()
+	report.Metrics = s.metrics
+	s.mu.Unlock()
+	return report, nil
+}
+func (s *Scheduler) begin(definition Definition, parent context.Context, results chan<- Result) bool {
+	probeID := definition.ID
+	s.mu.Lock()
+	if s.closed {
+		s.mu.Unlock()
+		return false
+	}
+	if _, exists := s.inflight[probeID]; exists {
+		s.metrics.SkippedOverlap++
+		s.mu.Unlock()
+		return false
+	}
+	child, cancel := context.WithCancel(parent)
+	s.inflight[probeID] = cancel
+	s.wg.Add(1)
+	s.metrics.Runs++
+	s.metrics.Active++
+	s.mu.Unlock()
+	go func() {
+		defer s.wg.Done()
+		defer func() { s.mu.Lock(); delete(s.inflight, probeID); s.metrics.Active--; s.mu.Unlock() }()
+		select {
+		case s.sem <- struct{}{}:
+		case <-child.Done():
+			result := s.failureResult(probeID, child.Err(), 0)
+			s.recordFailure(result)
+			results <- result
+			return
+		}
+		defer func() { <-s.sem }()
+		result := s.executeDefinition(child, definition)
+		results <- result
+	}()
+	return true
+}
+
+func (s *Scheduler) executeDefinition(ctx context.Context, definition Definition) Result {
+	started := s.config.Now().UTC()
+	var lastErr error
+	var attempts int
+	for attempts = 1; attempts <= s.config.MaxAttempts; attempts++ {
+		attemptCtx, cancel := context.WithTimeout(ctx, s.config.AttemptTimeout)
+		result, err := s.executor.Execute(attemptCtx, definition)
+		deadline := errors.Is(attemptCtx.Err(), context.DeadlineExceeded)
+		cancel()
+		if err == nil {
+			result = normalizeResult(result, definition.ID, attempts, started, s.config.Now)
+			s.recordSuccess(result, false)
+			return result
+		}
+		lastErr = err
+		if errors.Is(ctx.Err(), context.Canceled) || errors.Is(ctx.Err(), context.DeadlineExceeded) {
+			break
+		}
+		if attempts >= s.config.MaxAttempts || (!deadline && !isRetryable(err)) {
+			break
+		}
+		s.mu.Lock()
+		s.metrics.Retried++
+		s.mu.Unlock()
+		if !waitBackoff(ctx, s.config.RetryBackoff) {
+			break
+		}
+	}
+	result := s.failureResult(definition.ID, lastErr, attempts)
+	if errors.Is(ctx.Err(), context.Canceled) || errors.Is(ctx.Err(), context.DeadlineExceeded) {
+		result.ErrorClass = "canceled"
+	}
+	s.recordFailure(result)
+	return result
+}
+
+func (s *Scheduler) failureResult(probeID string, err error, attempts int) Result {
+	result := Result{ProbeID: probeID, State: "unknown", Attempts: attempts, ErrorClass: "execution_error", ErrorMessage: boundedError(err)}
+	if errors.Is(err, context.DeadlineExceeded) {
+		result.ErrorClass = "timeout"
+	}
+	if errors.Is(err, context.Canceled) {
+		result.ErrorClass = "canceled"
+	}
+	return normalizeResult(result, probeID, attempts, s.config.Now(), s.config.Now)
+}
+
+func (s *Scheduler) recordSuccess(result Result, _ bool) {
+	s.mu.Lock()
+	s.metrics.Completed++
+	s.metrics.LastRunAt = result.CompletedAt
+	s.mu.Unlock()
+}
+func (s *Scheduler) recordFailure(result Result) {
+	s.mu.Lock()
+	s.metrics.Failed++
+	if result.ErrorClass == "timeout" {
+		s.metrics.TimedOut++
+	}
+	s.metrics.LastRunAt = result.CompletedAt
+	s.mu.Unlock()
+}
+
+func (s *Scheduler) Shutdown(ctx context.Context) error {
+	if s == nil {
+		return nil
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	s.mu.Lock()
+	if !s.closed {
+		s.closed = true
+		for _, cancel := range s.inflight {
+			cancel()
+		}
+	}
+	s.mu.Unlock()
+	done := make(chan struct{})
+	go func() { s.wg.Wait(); close(done) }()
+	select {
+	case <-done:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (s *Scheduler) Metrics() Metrics { s.mu.Lock(); defer s.mu.Unlock(); return s.metrics }
+
+func normalizeResult(result Result, probeID string, attempts int, started time.Time, now func() time.Time) Result {
+	result.ProbeID = probeID
+	if result.State != "up" && result.State != "degraded" && result.State != "down" && result.State != "unknown" {
+		result.State = "unknown"
+		if result.ErrorClass == "" {
+			result.ErrorClass = "invalid_result"
+		}
+	}
+	result.Attempts = attempts
+	if result.ObservedAt.IsZero() {
+		result.ObservedAt = started.UTC()
+	}
+	if result.CompletedAt.IsZero() {
+		result.CompletedAt = now().UTC()
+	}
+	result.ErrorMessage = boundText(result.ErrorMessage, 256)
+	return result
+}
+
+func isRetryable(err error) bool { var retryable RetryableError; return errors.As(err, &retryable) }
+func waitBackoff(ctx context.Context, delay time.Duration) bool {
+	if delay <= 0 {
+		return true
+	}
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-timer.C:
+		return true
+	case <-ctx.Done():
+		return false
+	}
+}
+func boundedError(err error) string {
+	if err == nil {
+		return ""
+	}
+	return boundText(err.Error(), 256)
+}
+func boundText(value string, max int) string {
+	value = strings.TrimSpace(value)
+	if len(value) > max {
+		return value[:max]
+	}
+	return value
+}