This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
package alertcontrol
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ExpiryJob struct {
|
||||
Store Store
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (job ExpiryJob) Run(ctx context.Context) (ExpiryResult, error) {
|
||||
if job.Now == nil {
|
||||
job.Now = time.Now
|
||||
}
|
||||
return job.Store.Expire(ctx, job.Now().UTC())
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
package alertcontrol
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type Store interface {
|
||||
CreateSilence(context.Context, string, Silence) (Silence, error)
|
||||
ListSilences(context.Context, int, time.Time) ([]Silence, error)
|
||||
RevokeSilence(context.Context, string, string, int64, time.Time) (Silence, error)
|
||||
CreateMaintenance(context.Context, string, MaintenanceWindow) (MaintenanceWindow, error)
|
||||
ListMaintenance(context.Context, int, time.Time) ([]MaintenanceWindow, error)
|
||||
RevokeMaintenance(context.Context, string, string, int64, time.Time) (MaintenanceWindow, error)
|
||||
Expire(context.Context, time.Time) (ExpiryResult, error)
|
||||
}
|
||||
|
||||
type Repository struct{ Pool *pgxpool.Pool }
|
||||
|
||||
type ExpiryResult struct {
|
||||
Silences int `json:"silences"`
|
||||
MaintenanceWindows int `json:"maintenanceWindows"`
|
||||
}
|
||||
|
||||
func (r Repository) CreateSilence(ctx context.Context, actor string, silence Silence) (Silence, error) {
|
||||
if r.Pool == nil {
|
||||
return Silence{}, ErrUnavailable
|
||||
}
|
||||
if actor == "" {
|
||||
return Silence{}, fmt.Errorf("%w: creator is required", ErrInvalid)
|
||||
}
|
||||
if silence.ID == "" {
|
||||
silence.ID = NewID()
|
||||
}
|
||||
if silence.Owner == "" {
|
||||
silence.Owner = actor
|
||||
}
|
||||
if err := silence.Validate(time.Now().UTC()); err != nil {
|
||||
return Silence{}, err
|
||||
}
|
||||
matcher, err := json.Marshal(silence.Matchers)
|
||||
if err != nil {
|
||||
return Silence{}, fmt.Errorf("marshal silence matcher: %w", err)
|
||||
}
|
||||
if silence.Owner == "" {
|
||||
silence.Owner = actor
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Silence{}, fmt.Errorf("begin silence create: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
_, err = tx.Exec(ctx, `INSERT INTO alert_silences (id,name,reason,owner,matchers,starts_at,expires_at,created_by) VALUES ($1::uuid,$2,$3,$4,$5::jsonb,$6,$7,$8)`, silence.ID, silence.Name, silence.Reason, silence.Owner, matcher, silence.StartsAt.UTC(), silence.ExpiresAt.UTC(), actor)
|
||||
if err != nil {
|
||||
return Silence{}, mapError(fmt.Errorf("create silence: %w", err))
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Silence{}, fmt.Errorf("commit silence create: %w", err)
|
||||
}
|
||||
return r.getSilence(ctx, silence.ID, time.Now().UTC())
|
||||
}
|
||||
|
||||
func (r Repository) ListSilences(ctx context.Context, limit int, now time.Time) ([]Silence, error) {
|
||||
if r.Pool == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if limit < 1 || limit > 100 {
|
||||
return nil, fmt.Errorf("%w: invalid list limit", ErrInvalid)
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, `SELECT id,name,reason,owner,matchers,starts_at,expires_at,status,created_by,created_at,revoked_by,revoked_at,expired_at,revision FROM alert_silences ORDER BY starts_at DESC,id DESC LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list silences: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]Silence, 0)
|
||||
for rows.Next() {
|
||||
item, err := scanSilence(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.State = item.StateAt(now.UTC())
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate silences: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (r Repository) RevokeSilence(ctx context.Context, id, actor string, expected int64, now time.Time) (Silence, error) {
|
||||
if r.Pool == nil {
|
||||
return Silence{}, ErrUnavailable
|
||||
}
|
||||
if id == "" || actor == "" {
|
||||
return Silence{}, fmt.Errorf("%w: id and actor are required", ErrInvalid)
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return Silence{}, fmt.Errorf("begin silence revoke: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var query string
|
||||
var args []any
|
||||
if expected > 0 {
|
||||
query = `UPDATE alert_silences SET status='revoked',revoked_by=$2,revoked_at=$3,revision=revision+1 WHERE id=$1 AND status='active' AND revision=$4`
|
||||
args = []any{id, actor, now.UTC(), expected}
|
||||
} else {
|
||||
query = `UPDATE alert_silences SET status='revoked',revoked_by=$2,revoked_at=$3,revision=revision+1 WHERE id=$1 AND status='active'`
|
||||
args = []any{id, actor, now.UTC()}
|
||||
}
|
||||
result, err := tx.Exec(ctx, query, args...)
|
||||
if err != nil {
|
||||
return Silence{}, mapError(fmt.Errorf("revoke silence: %w", err))
|
||||
}
|
||||
if result.RowsAffected() == 0 {
|
||||
return Silence{}, r.revokeFailure(ctx, tx, id, expected, true)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return Silence{}, fmt.Errorf("commit silence revoke: %w", err)
|
||||
}
|
||||
return r.getSilence(ctx, id, now.UTC())
|
||||
}
|
||||
|
||||
func (r Repository) CreateMaintenance(ctx context.Context, actor string, window MaintenanceWindow) (MaintenanceWindow, error) {
|
||||
if r.Pool == nil {
|
||||
return MaintenanceWindow{}, ErrUnavailable
|
||||
}
|
||||
if actor == "" {
|
||||
return MaintenanceWindow{}, fmt.Errorf("%w: creator is required", ErrInvalid)
|
||||
}
|
||||
if window.ID == "" {
|
||||
window.ID = NewID()
|
||||
}
|
||||
if err := window.Validate(time.Now().UTC()); err != nil {
|
||||
return MaintenanceWindow{}, err
|
||||
}
|
||||
selector, err := json.Marshal(window.Selector)
|
||||
if err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("marshal maintenance selector: %w", err)
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("begin maintenance create: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
_, err = tx.Exec(ctx, `INSERT INTO maintenance_windows (id,name,reason,selector,starts_at,ends_at,created_by) VALUES ($1::uuid,$2,$3,$4::jsonb,$5,$6,$7)`, window.ID, window.Name, window.Reason, selector, window.StartsAt.UTC(), window.EndsAt.UTC(), actor)
|
||||
if err != nil {
|
||||
return MaintenanceWindow{}, mapError(fmt.Errorf("create maintenance window: %w", err))
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("commit maintenance create: %w", err)
|
||||
}
|
||||
return r.getMaintenance(ctx, window.ID, time.Now().UTC())
|
||||
}
|
||||
|
||||
func (r Repository) ListMaintenance(ctx context.Context, limit int, now time.Time) ([]MaintenanceWindow, error) {
|
||||
if r.Pool == nil {
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
if limit < 1 || limit > 100 {
|
||||
return nil, fmt.Errorf("%w: invalid list limit", ErrInvalid)
|
||||
}
|
||||
rows, err := r.Pool.Query(ctx, `SELECT id,name,reason,selector,starts_at,ends_at,status,created_by,created_at,revoked_by,revoked_at,expired_at,revision FROM maintenance_windows ORDER BY starts_at DESC,id DESC LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list maintenance windows: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]MaintenanceWindow, 0)
|
||||
for rows.Next() {
|
||||
item, err := scanMaintenance(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.State = item.StateAt(now.UTC())
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate maintenance windows: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (r Repository) RevokeMaintenance(ctx context.Context, id, actor string, expected int64, now time.Time) (MaintenanceWindow, error) {
|
||||
if r.Pool == nil {
|
||||
return MaintenanceWindow{}, ErrUnavailable
|
||||
}
|
||||
if id == "" || actor == "" {
|
||||
return MaintenanceWindow{}, fmt.Errorf("%w: id and actor are required", ErrInvalid)
|
||||
}
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("begin maintenance revoke: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var query string
|
||||
var args []any
|
||||
if expected > 0 {
|
||||
query = `UPDATE maintenance_windows SET status='revoked',revoked_by=$2,revoked_at=$3,revision=revision+1 WHERE id=$1 AND status='active' AND revision=$4`
|
||||
args = []any{id, actor, now.UTC(), expected}
|
||||
} else {
|
||||
query = `UPDATE maintenance_windows SET status='revoked',revoked_by=$2,revoked_at=$3,revision=revision+1 WHERE id=$1 AND status='active'`
|
||||
args = []any{id, actor, now.UTC()}
|
||||
}
|
||||
result, err := tx.Exec(ctx, query, args...)
|
||||
if err != nil {
|
||||
return MaintenanceWindow{}, mapError(fmt.Errorf("revoke maintenance window: %w", err))
|
||||
}
|
||||
if result.RowsAffected() == 0 {
|
||||
return MaintenanceWindow{}, r.revokeFailure(ctx, tx, id, expected, false)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("commit maintenance revoke: %w", err)
|
||||
}
|
||||
return r.getMaintenance(ctx, id, now.UTC())
|
||||
}
|
||||
|
||||
func (r Repository) Expire(ctx context.Context, now time.Time) (ExpiryResult, error) {
|
||||
if r.Pool == nil {
|
||||
return ExpiryResult{}, ErrUnavailable
|
||||
}
|
||||
now = now.UTC()
|
||||
tx, err := r.Pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return ExpiryResult{}, fmt.Errorf("begin control expiry: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var result ExpiryResult
|
||||
if err := tx.QueryRow(ctx, `WITH expired AS (UPDATE alert_silences SET status='expired',expired_at=$1,revision=revision+1 WHERE status='active' AND expires_at <= $1 RETURNING id) SELECT count(*) FROM expired`, now).Scan(&result.Silences); err != nil {
|
||||
return ExpiryResult{}, fmt.Errorf("expire silences: %w", err)
|
||||
}
|
||||
if err := tx.QueryRow(ctx, `WITH expired AS (UPDATE maintenance_windows SET status='expired',expired_at=$1,revision=revision+1 WHERE status='active' AND ends_at <= $1 RETURNING id) SELECT count(*) FROM expired`, now).Scan(&result.MaintenanceWindows); err != nil {
|
||||
return ExpiryResult{}, fmt.Errorf("expire maintenance windows: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ExpiryResult{}, fmt.Errorf("commit control expiry: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r Repository) getSilence(ctx context.Context, id string, now time.Time) (Silence, error) {
|
||||
var item Silence
|
||||
row := r.Pool.QueryRow(ctx, `SELECT id,name,reason,owner,matchers,starts_at,expires_at,status,created_by,created_at,revoked_by,revoked_at,expired_at,revision FROM alert_silences WHERE id=$1`, id)
|
||||
scanned, err := scanSilence(row)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Silence{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Silence{}, fmt.Errorf("get silence: %w", err)
|
||||
}
|
||||
item = scanned
|
||||
item.State = item.StateAt(now.UTC())
|
||||
return item, nil
|
||||
}
|
||||
func (r Repository) getMaintenance(ctx context.Context, id string, now time.Time) (MaintenanceWindow, error) {
|
||||
row := r.Pool.QueryRow(ctx, `SELECT id,name,reason,selector,starts_at,ends_at,status,created_by,created_at,revoked_by,revoked_at,expired_at,revision FROM maintenance_windows WHERE id=$1`, id)
|
||||
item, err := scanMaintenance(row)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return MaintenanceWindow{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("get maintenance window: %w", err)
|
||||
}
|
||||
item.State = item.StateAt(now.UTC())
|
||||
return item, nil
|
||||
}
|
||||
|
||||
type rowScanner interface{ Scan(...any) error }
|
||||
|
||||
func scanSilence(row rowScanner) (Silence, error) {
|
||||
var item Silence
|
||||
var matcher []byte
|
||||
var status string
|
||||
var revokedBy, createdBy *string
|
||||
if err := row.Scan(&item.ID, &item.Name, &item.Reason, &item.Owner, &matcher, &item.StartsAt, &item.ExpiresAt, &status, &createdBy, &item.CreatedAt, &revokedBy, &item.RevokedAt, &item.ExpiredAt, &item.Revision); err != nil {
|
||||
return Silence{}, err
|
||||
}
|
||||
if err := json.Unmarshal(matcher, &item.Matchers); err != nil {
|
||||
return Silence{}, fmt.Errorf("decode silence matcher: %w", err)
|
||||
}
|
||||
item.CreatedBy = valueOrEmpty(createdBy)
|
||||
item.RevokedBy = valueOrEmpty(revokedBy)
|
||||
return item, nil
|
||||
}
|
||||
func scanMaintenance(row rowScanner) (MaintenanceWindow, error) {
|
||||
var item MaintenanceWindow
|
||||
var selector []byte
|
||||
var status string
|
||||
var revokedBy, createdBy *string
|
||||
if err := row.Scan(&item.ID, &item.Name, &item.Reason, &selector, &item.StartsAt, &item.EndsAt, &status, &createdBy, &item.CreatedAt, &revokedBy, &item.RevokedAt, &item.ExpiredAt, &item.Revision); err != nil {
|
||||
return MaintenanceWindow{}, err
|
||||
}
|
||||
if err := json.Unmarshal(selector, &item.Selector); err != nil {
|
||||
return MaintenanceWindow{}, fmt.Errorf("decode maintenance selector: %w", err)
|
||||
}
|
||||
item.CreatedBy = valueOrEmpty(createdBy)
|
||||
item.RevokedBy = valueOrEmpty(revokedBy)
|
||||
return item, nil
|
||||
}
|
||||
func valueOrEmpty(value *string) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
return *value
|
||||
}
|
||||
func (r Repository) revokeFailure(ctx context.Context, tx pgx.Tx, id string, expected int64, silence bool) error {
|
||||
var revision int64
|
||||
var status string
|
||||
table := "maintenance_windows"
|
||||
if silence {
|
||||
table = "alert_silences"
|
||||
}
|
||||
err := tx.QueryRow(ctx, "SELECT revision,status FROM "+table+" WHERE id=$1 FOR UPDATE", id).Scan(&revision, &status)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect control revoke: %w", err)
|
||||
}
|
||||
if expected > 0 && revision != expected {
|
||||
return ErrConflict
|
||||
}
|
||||
return fmt.Errorf("%w: control is %s", ErrConflict, status)
|
||||
}
|
||||
func mapError(err error) error {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
switch pgErr.Code {
|
||||
case "23505":
|
||||
return fmt.Errorf("%w: duplicate control", ErrConflict)
|
||||
case "23514", "22P02":
|
||||
return fmt.Errorf("%w: database constraint", ErrInvalid)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package alertcontrol
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/database"
|
||||
)
|
||||
|
||||
func TestPostgreSQLControlsAreExpiringAuditedAndIdempotent(t *testing.T) {
|
||||
dsn := os.Getenv("PULSE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("PULSE_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 8, MinConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := database.Migrate(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repo := Repository{Pool: pool}
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
silenceID, maintenanceID, expiringID := NewID(), NewID(), NewID()
|
||||
t.Cleanup(func() {
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cleanupCancel()
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM alert_silences WHERE id = ANY($1::uuid[])`, []string{silenceID, expiringID})
|
||||
_, _ = pool.Exec(cleanupCtx, `DELETE FROM maintenance_windows WHERE id = $1::uuid`, maintenanceID)
|
||||
})
|
||||
|
||||
silence, err := repo.CreateSilence(ctx, "operator-1", Silence{ID: silenceID, Name: "planned deploy", Reason: "change window", Owner: "operator-1", Matchers: Matcher{Severities: []string{"critical"}}, StartsAt: now.Add(-time.Minute), ExpiresAt: now.Add(time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if silence.State != StateActive || silence.Revision != 1 {
|
||||
t.Fatalf("unexpected silence: %#v", silence)
|
||||
}
|
||||
window, err := repo.CreateMaintenance(ctx, "operator-1", MaintenanceWindow{ID: maintenanceID, Name: "maintenance", Reason: "firmware", Selector: Matcher{EntityTypes: []string{"host"}}, StartsAt: now.Add(-time.Minute), EndsAt: now.Add(time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if window.State != StateActive {
|
||||
t.Fatalf("maintenance state = %s", window.State)
|
||||
}
|
||||
items, err := repo.ListSilences(ctx, 100, now)
|
||||
if err != nil || len(items) != 1 {
|
||||
t.Fatalf("list silences: %d, %v", len(items), err)
|
||||
}
|
||||
|
||||
expiring, err := repo.CreateSilence(ctx, "operator-1", Silence{ID: expiringID, Name: "short", Reason: "test expiry", Owner: "operator-1", Matchers: Matcher{}, StartsAt: now.Add(-time.Minute), ExpiresAt: now.Add(-time.Second)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if expiring.State != StateExpired {
|
||||
t.Fatalf("expired control before job = %s", expiring.State)
|
||||
}
|
||||
first, err := repo.Expire(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := repo.Expire(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.Silences != 1 || first.MaintenanceWindows != 0 || second != (ExpiryResult{}) {
|
||||
t.Fatalf("expiry not idempotent: first=%#v second=%#v", first, second)
|
||||
}
|
||||
|
||||
concurrentID := NewID()
|
||||
defer func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM alert_silences WHERE id=$1::uuid`, concurrentID)
|
||||
}()
|
||||
if _, err := repo.CreateSilence(ctx, "operator-1", Silence{ID: concurrentID, Name: "concurrent", Reason: "test", Owner: "operator-1", Matchers: Matcher{}, StartsAt: now.Add(-time.Minute), ExpiresAt: now.Add(-time.Second)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
results := make(chan ExpiryResult, 2)
|
||||
errorsCh := make(chan error, 2)
|
||||
for i := 0; i < 2; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
result, err := repo.Expire(ctx, now.Add(time.Second))
|
||||
if err != nil {
|
||||
errorsCh <- err
|
||||
return
|
||||
}
|
||||
results <- result
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
close(errorsCh)
|
||||
total := 0
|
||||
for result := range results {
|
||||
total += result.Silences + result.MaintenanceWindows
|
||||
}
|
||||
for err := range errorsCh {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if total != 1 {
|
||||
t.Fatalf("concurrent expiry count = %d, want 1", total)
|
||||
}
|
||||
|
||||
if _, err := repo.CreateSilence(ctx, "operator-1", Silence{ID: silenceID, Name: "duplicate", Reason: "duplicate", Owner: "operator-1", Matchers: Matcher{}, StartsAt: now, ExpiresAt: now.Add(time.Hour)}); err == nil {
|
||||
t.Fatal("duplicate silence should fail")
|
||||
}
|
||||
if _, err := repo.RevokeSilence(ctx, silenceID, "operator-1", silence.Revision, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := repo.RevokeSilence(ctx, silenceID, "operator-1", silence.Revision+1, now); err == nil {
|
||||
t.Fatal("repeated revoke should conflict")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package alertcontrol
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
func RunExpiryLoop(ctx context.Context, store Store, interval time.Duration, logger *slog.Logger) {
|
||||
if store == nil || interval <= 0 {
|
||||
return
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
job := ExpiryJob{Store: store}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if _, err := job.Run(ctx); err != nil && logger != nil {
|
||||
logger.Warn("alert control expiry failed", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package alertcontrol
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxName = 160
|
||||
MaxReason = 500
|
||||
MaxOwner = 255
|
||||
MaxMatcherKeys = 20
|
||||
MaxMatcherValues = 50
|
||||
MaxDuration = 365 * 24 * time.Hour
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalid = errors.New("invalid alert control")
|
||||
ErrNotFound = errors.New("alert control not found")
|
||||
ErrUnavailable = errors.New("alert control repository is unavailable")
|
||||
ErrConflict = errors.New("alert control has already changed")
|
||||
keyPattern = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_.:/-]{0,63}$`)
|
||||
)
|
||||
|
||||
type Matcher struct {
|
||||
RuleIDs []string `json:"ruleIds,omitempty"`
|
||||
EntityIDs []string `json:"entityIds,omitempty"`
|
||||
EntityTypes []string `json:"entityTypes,omitempty"`
|
||||
Severities []string `json:"severities,omitempty"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
}
|
||||
|
||||
type Signal struct {
|
||||
InstanceID string `json:"instanceId"`
|
||||
RuleID string `json:"ruleId"`
|
||||
EntityID string `json:"entityId,omitempty"`
|
||||
Severity string `json:"severity"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
}
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateScheduled State = "scheduled"
|
||||
StateActive State = "active"
|
||||
StateExpired State = "expired"
|
||||
StateRevoked State = "revoked"
|
||||
)
|
||||
|
||||
type Silence struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Reason string `json:"reason"`
|
||||
Owner string `json:"owner"`
|
||||
Matchers Matcher `json:"matchers"`
|
||||
StartsAt time.Time `json:"startsAt"`
|
||||
ExpiresAt time.Time `json:"expiresAt"`
|
||||
State State `json:"state"`
|
||||
CreatedBy string `json:"createdBy"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
RevokedBy string `json:"revokedBy,omitempty"`
|
||||
RevokedAt *time.Time `json:"revokedAt,omitempty"`
|
||||
ExpiredAt *time.Time `json:"expiredAt,omitempty"`
|
||||
Revision int64 `json:"revision"`
|
||||
}
|
||||
|
||||
type MaintenanceWindow struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Reason string `json:"reason"`
|
||||
Selector Matcher `json:"selector"`
|
||||
StartsAt time.Time `json:"startsAt"`
|
||||
EndsAt time.Time `json:"endsAt"`
|
||||
State State `json:"state"`
|
||||
CreatedBy string `json:"createdBy"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
RevokedBy string `json:"revokedBy,omitempty"`
|
||||
RevokedAt *time.Time `json:"revokedAt,omitempty"`
|
||||
ExpiredAt *time.Time `json:"expiredAt,omitempty"`
|
||||
Revision int64 `json:"revision"`
|
||||
}
|
||||
|
||||
type Preview struct {
|
||||
Matched bool `json:"matched"`
|
||||
MatchedCount int `json:"matchedCount"`
|
||||
InstanceIDs []string `json:"instanceIds"`
|
||||
}
|
||||
|
||||
func NewID() string {
|
||||
b := make([]byte, 16)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "00000000-0000-4000-8000-000000000000"
|
||||
}
|
||||
b[6] = (b[6] & 0x0f) | 0x40
|
||||
b[8] = (b[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%s-%s-%s-%s-%s", hex.EncodeToString(b[0:4]), hex.EncodeToString(b[4:6]), hex.EncodeToString(b[6:8]), hex.EncodeToString(b[8:10]), hex.EncodeToString(b[10:16]))
|
||||
}
|
||||
|
||||
func (m Matcher) Validate() error {
|
||||
count := len(m.RuleIDs) + len(m.EntityIDs) + len(m.EntityTypes) + len(m.Severities) + len(m.Labels)
|
||||
if count > MaxMatcherKeys {
|
||||
return fmt.Errorf("%w: too many matcher keys", ErrInvalid)
|
||||
}
|
||||
for _, values := range [][]string{m.RuleIDs, m.EntityIDs, m.EntityTypes, m.Severities} {
|
||||
if len(values) > MaxMatcherValues {
|
||||
return fmt.Errorf("%w: too many matcher values", ErrInvalid)
|
||||
}
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
if err := validateValue(value); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, ok := seen[value]; ok {
|
||||
return fmt.Errorf("%w: duplicate matcher value", ErrInvalid)
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(m.Labels) > MaxMatcherValues {
|
||||
return fmt.Errorf("%w: too many label matchers", ErrInvalid)
|
||||
}
|
||||
for key, value := range m.Labels {
|
||||
if !keyPattern.MatchString(key) {
|
||||
return fmt.Errorf("%w: invalid label key", ErrInvalid)
|
||||
}
|
||||
if err := validateValue(value); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateValue(value string) error {
|
||||
if value == "" || len(value) > 160 || strings.ContainsAny(value, "\r\n\x00") {
|
||||
return fmt.Errorf("%w: invalid matcher value", ErrInvalid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m Matcher) Matches(signal Signal) bool {
|
||||
return containsOrWildcard(m.RuleIDs, signal.RuleID) && containsOrWildcard(m.EntityIDs, signal.EntityID) && containsOrWildcard(m.Severities, signal.Severity) && containsOrWildcard(m.EntityTypes, signal.Labels["entity.type"]) && labelsMatch(m.Labels, signal.Labels)
|
||||
}
|
||||
|
||||
func containsOrWildcard(values []string, value string) bool {
|
||||
if len(values) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, item := range values {
|
||||
if item == value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func labelsMatch(expected, actual map[string]string) bool {
|
||||
for key, value := range expected {
|
||||
if actual[key] != value {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s Silence) Validate(now time.Time) error {
|
||||
if err := validateCommon(s.Name, s.Reason, s.StartsAt, s.ExpiresAt, s.Matchers); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.Owner == "" || len(s.Owner) > MaxOwner {
|
||||
return fmt.Errorf("%w: invalid owner", ErrInvalid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w MaintenanceWindow) Validate(now time.Time) error {
|
||||
return validateCommon(w.Name, w.Reason, w.StartsAt, w.EndsAt, w.Selector)
|
||||
}
|
||||
|
||||
func validateCommon(name, reason string, starts, ends time.Time, matcher Matcher) error {
|
||||
if strings.TrimSpace(name) == "" || len(name) > MaxName || strings.ContainsAny(name, "\r\n\x00") {
|
||||
return fmt.Errorf("%w: invalid name", ErrInvalid)
|
||||
}
|
||||
if strings.TrimSpace(reason) == "" || len(reason) > MaxReason || strings.ContainsAny(reason, "\r\n\x00") {
|
||||
return fmt.Errorf("%w: invalid reason", ErrInvalid)
|
||||
}
|
||||
if starts.IsZero() || ends.IsZero() || !ends.After(starts) || ends.Sub(starts) > MaxDuration {
|
||||
return fmt.Errorf("%w: invalid expiry window", ErrInvalid)
|
||||
}
|
||||
if err := matcher.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s Silence) StateAt(now time.Time) State {
|
||||
return temporalState(s.StartsAt, s.ExpiresAt, s.RevokedAt, s.ExpiredAt, now)
|
||||
}
|
||||
func (w MaintenanceWindow) StateAt(now time.Time) State {
|
||||
return temporalState(w.StartsAt, w.EndsAt, w.RevokedAt, w.ExpiredAt, now)
|
||||
}
|
||||
func temporalState(starts, ends time.Time, revoked, expired *time.Time, now time.Time) State {
|
||||
if revoked != nil {
|
||||
return StateRevoked
|
||||
}
|
||||
if expired != nil || !now.Before(ends) {
|
||||
return StateExpired
|
||||
}
|
||||
if now.Before(starts) {
|
||||
return StateScheduled
|
||||
}
|
||||
return StateActive
|
||||
}
|
||||
|
||||
func (s Silence) Matches(signal Signal, now time.Time) bool {
|
||||
return s.StateAt(now) == StateActive && s.Matchers.Matches(signal)
|
||||
}
|
||||
func (w MaintenanceWindow) Matches(signal Signal, now time.Time) bool {
|
||||
return w.StateAt(now) == StateActive && w.Selector.Matches(signal)
|
||||
}
|
||||
|
||||
func PreviewSignals(m Matcher, signals []Signal) (Preview, error) {
|
||||
if err := m.Validate(); err != nil {
|
||||
return Preview{}, err
|
||||
}
|
||||
ids := make([]string, 0, len(signals))
|
||||
seen := make(map[string]struct{}, len(signals))
|
||||
for _, signal := range signals {
|
||||
if signal.InstanceID == "" {
|
||||
return Preview{}, fmt.Errorf("%w: signal instance id is required", ErrInvalid)
|
||||
}
|
||||
if m.Matches(signal) {
|
||||
if _, ok := seen[signal.InstanceID]; !ok {
|
||||
ids = append(ids, signal.InstanceID)
|
||||
seen[signal.InstanceID] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return Preview{Matched: len(ids) > 0, MatchedCount: len(ids), InstanceIDs: ids}, nil
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package alertcontrol
|
||||
|
||||
import "testing"
|
||||
|
||||
func BenchmarkPreviewSignals1000(b *testing.B) {
|
||||
matcher := Matcher{Severities: []string{"critical"}, Labels: map[string]string{"source.type": "prometheus"}}
|
||||
signals := make([]Signal, 1000)
|
||||
for index := range signals {
|
||||
signals[index] = Signal{InstanceID: NewID(), Severity: "critical", Labels: map[string]string{"source.type": "prometheus"}}
|
||||
}
|
||||
b.ResetTimer()
|
||||
for index := 0; index < b.N; index++ {
|
||||
if _, err := PreviewSignals(matcher, signals); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package alertcontrol
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMatcherIsBoundedAndDeterministic(t *testing.T) {
|
||||
m := Matcher{RuleIDs: []string{"rule-a"}, EntityTypes: []string{"host"}, Labels: map[string]string{"source.type": "prometheus"}}
|
||||
if err := m.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
signal := Signal{InstanceID: "instance-1", RuleID: "rule-a", Severity: "critical", Labels: map[string]string{"entity.type": "host", "source.type": "prometheus"}}
|
||||
if !m.Matches(signal) {
|
||||
t.Fatal("matcher should match signal")
|
||||
}
|
||||
signal.Labels["source.type"] = "agent"
|
||||
if m.Matches(signal) {
|
||||
t.Fatal("matcher should reject different labels")
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlStateAndPreview(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
|
||||
control := Silence{ID: "silence-1", Name: "Deploy", Reason: "planned change", Owner: "operator", Matchers: Matcher{Severities: []string{"critical"}}, StartsAt: now, ExpiresAt: now.Add(time.Hour)}
|
||||
if got := control.StateAt(now); got != StateActive {
|
||||
t.Fatalf("state at start = %s", got)
|
||||
}
|
||||
if got := control.StateAt(now.Add(time.Hour)); got != StateExpired {
|
||||
t.Fatalf("state at expiry = %s", got)
|
||||
}
|
||||
if err := control.Validate(now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
preview, err := PreviewSignals(control.Matchers, []Signal{{InstanceID: "b", Severity: "critical"}, {InstanceID: "a", Severity: "critical"}, {InstanceID: "x", Severity: "attention"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if preview.MatchedCount != 2 || preview.InstanceIDs[0] != "a" || preview.InstanceIDs[1] != "b" {
|
||||
t.Fatalf("unexpected preview: %#v", preview)
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlValidationRejectsUnboundedExpiryAndMatchers(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
window := MaintenanceWindow{Name: "window", Reason: "reason", Selector: Matcher{Labels: map[string]string{"bad key": "value"}}, StartsAt: now, EndsAt: now.Add(2 * MaxDuration)}
|
||||
if err := window.Validate(now); err == nil {
|
||||
t.Fatal("expected validation error")
|
||||
}
|
||||
matcher := Matcher{RuleIDs: make([]string, MaxMatcherValues+1)}
|
||||
if err := matcher.Validate(); err == nil {
|
||||
t.Fatal("expected matcher bound error")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user