ITWorx-Pulse-Public/internal/probe/scheduler_test.go

package probe

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"testing"
	"time"
)

type executorFunc func(context.Context, Definition) (Result, error)

func (f executorFunc) Execute(ctx context.Context, definition Definition) (Result, error) {
	return f(ctx, definition)
}

func schedulerDefinition(id string) Definition {
	return Definition{ID: id, ServiceID: "service", Name: id, Type: TypeHTTP, Target: Target{Scheme: "https", Host: "example.internal", Port: 443}, Interval: 30 * time.Second, Timeout: 5 * time.Second, Enabled: true, Revision: 1}
}

func TestSchedulerRetriesTimeoutsAndNormalizesResults(t *testing.T) {
	var mu sync.Mutex
	attempts := 0
	executor := executorFunc(func(ctx context.Context, definition Definition) (Result, error) {
		if definition.ID != "probe-1" {
			t.Errorf("definition=%+v", definition)
		}
		mu.Lock()
		attempts++
		current := attempts
		mu.Unlock()
		if current == 1 {
			return Result{}, RetryableError{Err: errors.New("temporary")}
		}
		return Result{State: "up"}, nil
	})
	scheduler, err := NewScheduler(executor, SchedulerConfig{MaxConcurrent: 2, MaxAttempts: 2, RetryBackoff: 0})
	if err != nil {
		t.Fatal(err)
	}
	report, err := scheduler.Run(context.Background(), []Definition{schedulerDefinition("probe-1")})
	if err != nil || len(report.Results) != 1 {
		t.Fatalf("report=%+v err=%v", report, err)
	}
	if report.Results[0].State != "up" || report.Results[0].Attempts != 2 || report.Metrics.Retried != 1 {
		t.Fatalf("report=%+v", report)
	}

	timeoutExecutor := executorFunc(func(ctx context.Context, _ Definition) (Result, error) { <-ctx.Done(); return Result{}, ctx.Err() })
	timeoutScheduler, err := NewScheduler(timeoutExecutor, SchedulerConfig{MaxAttempts: 2, AttemptTimeout: 5 * time.Millisecond, RetryBackoff: 0})
	if err != nil {
		t.Fatal(err)
	}
	timeoutReport, err := timeoutScheduler.Run(context.Background(), []Definition{schedulerDefinition("timeout")})
	if err != nil || timeoutReport.Results[0].ErrorClass != "timeout" || timeoutReport.Results[0].Attempts != 2 || timeoutReport.Metrics.TimedOut != 1 {
		t.Fatalf("timeout=%+v err=%v", timeoutReport, err)
	}
}

func TestSchedulerPreventsOverlappingRunsAndShutsDown(t *testing.T) {
	started := make(chan struct{})
	release := make(chan struct{})
	executor := executorFunc(func(ctx context.Context, _ Definition) (Result, error) {
		closeOnce(started)
		select {
		case <-release:
			return Result{State: "up"}, nil
		case <-ctx.Done():
			return Result{}, ctx.Err()
		}
	})
	scheduler, err := NewScheduler(executor, SchedulerConfig{MaxAttempts: 1, AttemptTimeout: time.Second})
	if err != nil {
		t.Fatal(err)
	}
	first := make(chan RunReport, 1)
	go func() {
		report, _ := scheduler.Run(context.Background(), []Definition{schedulerDefinition("same")})
		first <- report
	}()
	<-started
	second, err := scheduler.Run(context.Background(), []Definition{schedulerDefinition("same")})
	if err != nil || len(second.Results) != 0 || second.Metrics.SkippedOverlap < 1 {
		t.Fatalf("overlap=%+v err=%v", second, err)
	}
	shutdownDone := make(chan error, 1)
	go func() { shutdownDone <- scheduler.Shutdown(context.Background()) }()
	select {
	case err := <-shutdownDone:
		if err != nil {
			t.Fatal(err)
		}
	case <-time.After(time.Second):
		t.Fatal("shutdown did not cancel active probe")
	}
	close(release)
	select {
	case <-first:
	case <-time.After(time.Second):
		t.Fatal("run did not finish after shutdown")
	}
	if err := scheduler.Shutdown(context.Background()); err != nil {
		t.Fatal(err)
	}
}

func TestSchedulerTargetScale300IsBoundedAndDeterministic(t *testing.T) {
	var mu sync.Mutex
	active, maximum := 0, 0
	executor := executorFunc(func(ctx context.Context, definition Definition) (Result, error) {
		mu.Lock()
		active++
		if active > maximum {
			maximum = active
		}
		mu.Unlock()
		select {
		case <-ctx.Done():
			return Result{}, ctx.Err()
		default:
		}
		mu.Lock()
		active--
		mu.Unlock()
		return Result{State: "up"}, nil
	})
	scheduler, err := NewScheduler(executor, SchedulerConfig{MaxConcurrent: 16, MaxAttempts: 1})
	if err != nil {
		t.Fatal(err)
	}
	definitions := make([]Definition, 300)
	for index := range definitions {
		definitions[index] = schedulerDefinition(fmt.Sprintf("probe-%03d", 300-index))
	}
	started := time.Now()
	report, err := scheduler.Run(context.Background(), definitions)
	if err != nil || len(report.Results) != 300 {
		t.Fatalf("count=%d err=%v", len(report.Results), err)
	}
	if maximum > 16 {
		t.Fatalf("maximum concurrency=%d", maximum)
	}
	if time.Since(started) > 2*time.Second {
		t.Fatalf("300-probe run exceeded budget: %s", time.Since(started))
	}
	for index := 1; index < len(report.Results); index++ {
		if report.Results[index-1].ProbeID > report.Results[index].ProbeID {
			t.Fatal("results are not deterministic")
		}
	}
}

func closeOnce(channel chan struct{}) {
	select {
	case <-channel:
	default:
		close(channel)
	}
}