This commit is contained in:
@@ -0,0 +1,219 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxEvents = 1000
|
||||
MaxAttributes = 20
|
||||
)
|
||||
|
||||
type Severity string
|
||||
|
||||
const (
|
||||
SeverityInfo Severity = "info"
|
||||
SeverityAttention Severity = "attention"
|
||||
SeverityWarning Severity = "warning"
|
||||
SeverityCritical Severity = "critical"
|
||||
)
|
||||
|
||||
type Event struct {
|
||||
ID string
|
||||
EventType string
|
||||
Severity Severity
|
||||
EntityID string
|
||||
SourceID string
|
||||
OccurredAt time.Time
|
||||
ReceivedAt time.Time
|
||||
DedupKey string
|
||||
Summary string
|
||||
Attributes map[string]string
|
||||
}
|
||||
|
||||
type RestartResult struct {
|
||||
EntityID string
|
||||
CurrentState string
|
||||
Count int
|
||||
Window time.Duration
|
||||
Threshold int
|
||||
Firing bool
|
||||
Reason string
|
||||
}
|
||||
|
||||
type ResourceSample struct {
|
||||
ID string
|
||||
Label string
|
||||
CPUPercent float64
|
||||
MemoryBytes uint64
|
||||
}
|
||||
|
||||
type StatusItem struct {
|
||||
ID string
|
||||
Label string
|
||||
State string
|
||||
Reason string
|
||||
}
|
||||
|
||||
func Normalize(events []Event, now time.Time) ([]Event, error) {
|
||||
if len(events) > MaxEvents {
|
||||
return nil, errors.New("lifecycle event count exceeds bounds")
|
||||
}
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
now = now.UTC()
|
||||
seen := make(map[string]struct{}, len(events))
|
||||
result := make([]Event, 0, len(events))
|
||||
for _, event := range events {
|
||||
if strings.TrimSpace(event.ID) == "" || strings.TrimSpace(event.EventType) == "" || strings.TrimSpace(event.SourceID) == "" || strings.TrimSpace(event.DedupKey) == "" || strings.TrimSpace(event.Summary) == "" || event.OccurredAt.IsZero() {
|
||||
return nil, errors.New("invalid lifecycle event")
|
||||
}
|
||||
if len(event.Attributes) > MaxAttributes {
|
||||
return nil, errors.New("lifecycle event attributes exceed bounds")
|
||||
}
|
||||
if event.OccurredAt.After(now.Add(time.Minute)) {
|
||||
return nil, errors.New("lifecycle event is materially in the future")
|
||||
}
|
||||
event.OccurredAt = event.OccurredAt.UTC()
|
||||
if event.ReceivedAt.IsZero() {
|
||||
event.ReceivedAt = now
|
||||
}
|
||||
event.ReceivedAt = event.ReceivedAt.UTC()
|
||||
key := event.SourceID + "\x00" + event.DedupKey + "\x00" + event.OccurredAt.Format(time.RFC3339Nano)
|
||||
if _, exists := seen[key]; exists {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
event.Attributes = copyAttributes(event.Attributes)
|
||||
result = append(result, event)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if !result[i].OccurredAt.Equal(result[j].OccurredAt) {
|
||||
return result[i].OccurredAt.After(result[j].OccurredAt)
|
||||
}
|
||||
if result[i].SourceID != result[j].SourceID {
|
||||
return result[i].SourceID < result[j].SourceID
|
||||
}
|
||||
return result[i].ID < result[j].ID
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func ContainerTransitionEvents(sourceID, entityID string, before, after container.Container, now, received time.Time) []Event {
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
if received.IsZero() {
|
||||
received = now
|
||||
}
|
||||
events := make([]Event, 0, 4)
|
||||
add := func(kind, severity Severity, suffix, summary string, attrs map[string]string) {
|
||||
dedup := entityID + ":" + suffix + ":" + now.UTC().Format(time.RFC3339Nano)
|
||||
events = append(events, Event{ID: deterministicID(sourceID, dedup), EventType: "container." + suffix, Severity: severity, EntityID: entityID, SourceID: sourceID, OccurredAt: now.UTC(), ReceivedAt: received.UTC(), DedupKey: dedup, Summary: summary, Attributes: attrs})
|
||||
_ = kind
|
||||
}
|
||||
if before.State != after.State {
|
||||
add("state", SeverityAttention, "state_changed", "Container state changed.", map[string]string{"before": before.State, "after": after.State})
|
||||
}
|
||||
if before.Health != after.Health {
|
||||
add("health", SeverityAttention, "health_changed", "Container health changed.", map[string]string{"before": before.Health, "after": after.Health})
|
||||
}
|
||||
if after.RestartCount > before.RestartCount {
|
||||
add("restart", SeverityWarning, "restart", "Container restarted.", map[string]string{"count": fmt.Sprint(after.RestartCount)})
|
||||
}
|
||||
if before.IntentionalStop != after.IntentionalStop {
|
||||
add("stop", SeverityInfo, "intentional_stop_changed", "Intentional stop state changed.", map[string]string{"before": fmt.Sprint(before.IntentionalStop), "after": fmt.Sprint(after.IntentionalStop)})
|
||||
}
|
||||
return events
|
||||
}
|
||||
|
||||
func EvaluateRestartLoop(events []Event, entityID, currentState string, now time.Time, window time.Duration, threshold int) (RestartResult, error) {
|
||||
if strings.TrimSpace(entityID) == "" {
|
||||
return RestartResult{}, errors.New("restart loop entity id is required")
|
||||
}
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
if window == 0 {
|
||||
window = 10 * time.Minute
|
||||
}
|
||||
if threshold == 0 {
|
||||
threshold = 3
|
||||
}
|
||||
if window <= 0 || window > 24*time.Hour || threshold < 2 || threshold > 100 {
|
||||
return RestartResult{}, errors.New("restart loop policy is outside bounds")
|
||||
}
|
||||
cutoff := now.UTC().Add(-window)
|
||||
count := 0
|
||||
for _, event := range events {
|
||||
if event.EntityID == entityID && event.EventType == "container.restart" && !event.OccurredAt.Before(cutoff) && !event.OccurredAt.After(now.UTC().Add(time.Minute)) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
result := RestartResult{EntityID: entityID, CurrentState: currentState, Count: count, Window: window, Threshold: threshold, Firing: strings.EqualFold(currentState, "running") && count >= threshold}
|
||||
if result.Firing {
|
||||
result.Reason = "restart_loop"
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func TopN(samples []ResourceSample, n int, by string) ([]ResourceSample, error) {
|
||||
if n < 1 || n > 100 || (by != "cpu" && by != "memory") {
|
||||
return nil, errors.New("top-n policy is invalid")
|
||||
}
|
||||
if len(samples) > 1000 {
|
||||
return nil, errors.New("top-n sample count exceeds bounds")
|
||||
}
|
||||
result := append([]ResourceSample(nil), samples...)
|
||||
sort.SliceStable(result, func(i, j int) bool {
|
||||
if by == "cpu" && result[i].CPUPercent != result[j].CPUPercent {
|
||||
return result[i].CPUPercent > result[j].CPUPercent
|
||||
}
|
||||
if by == "memory" && result[i].MemoryBytes != result[j].MemoryBytes {
|
||||
return result[i].MemoryBytes > result[j].MemoryBytes
|
||||
}
|
||||
if result[i].ID != result[j].ID {
|
||||
return result[i].ID < result[j].ID
|
||||
}
|
||||
return result[i].Label < result[j].Label
|
||||
})
|
||||
if len(result) > n {
|
||||
result = result[:n]
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func StatusGrid(items []StatusItem) []StatusItem {
|
||||
result := append([]StatusItem(nil), items...)
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].State != result[j].State {
|
||||
return result[i].State < result[j].State
|
||||
}
|
||||
return result[i].ID < result[j].ID
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func deterministicID(sourceID, key string) string {
|
||||
sum := sha256.Sum256([]byte("itworx-pulse/event/v1/" + sourceID + "\x00" + key))
|
||||
b := sum[:16]
|
||||
b[6] = (b[6] & 0x0f) | 0x50
|
||||
b[8] = (b[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%s-%s-%s-%s-%s", hex.EncodeToString(b[0:4]), hex.EncodeToString(b[4:6]), hex.EncodeToString(b[6:8]), hex.EncodeToString(b[8:10]), hex.EncodeToString(b[10:16]))
|
||||
}
|
||||
func copyAttributes(input map[string]string) map[string]string {
|
||||
result := make(map[string]string, len(input))
|
||||
for key, value := range input {
|
||||
result[key] = value
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/container"
|
||||
)
|
||||
|
||||
func TestRestartLoopFiresWhileRunningAndEventsDeduplicate(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 22, 0, 0, 0, time.UTC)
|
||||
before := container.Container{ID: "runtime", State: "running", Health: "healthy", RestartCount: 1}
|
||||
after := before
|
||||
after.RestartCount = 2
|
||||
generated := ContainerTransitionEvents("agent", "entity", before, after, now, now)
|
||||
generated = append(generated, generated...)
|
||||
normalized, err := Normalize(generated, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(normalized) != 1 || normalized[0].EventType != "container.restart" {
|
||||
t.Fatalf("events=%+v", normalized)
|
||||
}
|
||||
all := []Event{
|
||||
{ID: "1", EventType: "container.restart", EntityID: "entity", SourceID: "agent", OccurredAt: now.Add(-3 * time.Minute), ReceivedAt: now, DedupKey: "r1", Summary: "restart"},
|
||||
{ID: "2", EventType: "container.restart", EntityID: "entity", SourceID: "agent", OccurredAt: now.Add(-2 * time.Minute), ReceivedAt: now, DedupKey: "r2", Summary: "restart"},
|
||||
{ID: "3", EventType: "container.restart", EntityID: "entity", SourceID: "agent", OccurredAt: now.Add(-time.Minute), ReceivedAt: now, DedupKey: "r3", Summary: "restart"},
|
||||
}
|
||||
result, err := EvaluateRestartLoop(all, "entity", "running", now, 10*time.Minute, 3)
|
||||
if err != nil || !result.Firing || result.Reason != "restart_loop" {
|
||||
t.Fatalf("result=%+v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
func TestRestartLoopDoesNotFireForStoppedContainer(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 22, 0, 0, 0, time.UTC)
|
||||
result, err := EvaluateRestartLoop(nil, "entity", "exited", now, 10*time.Minute, 3)
|
||||
if err != nil || result.Firing {
|
||||
t.Fatalf("result=%+v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
func TestTopNStableUnderTiesAndStatusGridSorted(t *testing.T) {
|
||||
ranked, err := TopN([]ResourceSample{{ID: "b", Label: "B", CPUPercent: 10}, {ID: "a", Label: "A", CPUPercent: 10}, {ID: "c", Label: "C", CPUPercent: 20}}, 2, "cpu")
|
||||
if err != nil || len(ranked) != 2 || ranked[0].ID != "c" || ranked[1].ID != "a" {
|
||||
t.Fatalf("ranked=%+v err=%v", ranked, err)
|
||||
}
|
||||
grid := StatusGrid([]StatusItem{{ID: "b", State: "healthy"}, {ID: "a", State: "degraded"}, {ID: "c", State: "degraded"}})
|
||||
if len(grid) != 3 || grid[0].ID != "a" || grid[1].ID != "c" {
|
||||
t.Fatalf("grid=%+v", grid)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user