Publish ITWorx Pulse source

2026-09-03 02:09:19 +02:00
commit bd774932d5
614 changed files with 77116 additions and 0 deletions
@@ -0,0 +1,209 @@
+package reconciliation
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+const (
+	containerServiceExternalType  = "container-service"
+	containerInstanceExternalType = "container-instance"
+)
+
+type ContainerObservation struct {
+	SourceID    string
+	RuntimeID   string
+	Name        string
+	Project     string
+	Service     string
+	ImageDigest string
+	ObservedAt  time.Time
+}
+
+type ContainerAlias struct {
+	EntityID    string
+	SourceID    string
+	RuntimeID   string
+	Name        string
+	Project     string
+	Service     string
+	ImageDigest string
+	FirstSeenAt time.Time
+	LastSeenAt  time.Time
+	Active      bool
+	// TombstonedAt records when the runtime alias stopped being observed. It is
+	// the zero time while the alias is active and is stamped once, so repeated
+	// snapshots keep the original tombstone moment.
+	TombstonedAt time.Time
+}
+
+type ContainerRecreation struct {
+	EntityID          string
+	PreviousRuntimeID string
+	CurrentRuntimeID  string
+	EventType         string
+	EventEntityID     string
+	OccurredAt        time.Time
+}
+
+type ContainerIdentityResult struct {
+	Aliases   []ContainerAlias
+	Current   []ContainerAlias
+	Added     []string
+	Updated   []string
+	Recreated []ContainerRecreation
+	Warnings  []string
+}
+
+func ReconcileContainers(observations []ContainerObservation, previous []ContainerAlias, now time.Time) (ContainerIdentityResult, error) {
+	if now.IsZero() {
+		now = time.Now().UTC()
+	}
+	now = now.UTC()
+	records := append([]ContainerAlias(nil), previous...)
+	for i := range records {
+		if records[i].SourceID == "" || records[i].RuntimeID == "" || records[i].EntityID == "" {
+			return ContainerIdentityResult{}, errors.New("invalid previous container alias")
+		}
+		records[i].FirstSeenAt = records[i].FirstSeenAt.UTC()
+		records[i].LastSeenAt = records[i].LastSeenAt.UTC()
+		records[i].TombstonedAt = records[i].TombstonedAt.UTC()
+		records[i].Active = false
+	}
+	exact := make(map[string][]int)
+	stable := make(map[string][]int)
+	for i, record := range records {
+		exact[containerAliasKey(record.SourceID, record.RuntimeID)] = append(exact[containerAliasKey(record.SourceID, record.RuntimeID)], i)
+		if key := containerStableKey(record.SourceID, record.Project, record.Service); key != "" {
+			stable[key] = append(stable[key], i)
+		}
+	}
+	seen := make(map[string]struct{}, len(observations))
+	result := ContainerIdentityResult{}
+	for _, observation := range observations {
+		if err := validateContainerObservation(observation); err != nil {
+			return ContainerIdentityResult{}, err
+		}
+		runtimeKey := containerAliasKey(observation.SourceID, observation.RuntimeID)
+		if _, ok := seen[runtimeKey]; ok {
+			return ContainerIdentityResult{}, fmt.Errorf("duplicate container observation %q", runtimeKey)
+		}
+		seen[runtimeKey] = struct{}{}
+		var record ContainerAlias
+
+		if candidates := exact[runtimeKey]; len(candidates) == 1 {
+			record = records[candidates[0]]
+
+			record.Active = true
+			record.Name = observation.Name
+			record.Project = observation.Project
+			record.Service = observation.Service
+			record.ImageDigest = observation.ImageDigest
+			record.LastSeenAt = observation.ObservedAt.UTC()
+			result.Updated = append(result.Updated, record.EntityID)
+		} else if key := containerStableKey(observation.SourceID, observation.Project, observation.Service); key != "" && len(stable[key]) == 1 {
+			old := records[stable[key][0]]
+			old.Active = false
+			record = ContainerAlias{
+				EntityID: old.EntityID, SourceID: observation.SourceID, RuntimeID: observation.RuntimeID,
+				Name: observation.Name, Project: observation.Project, Service: observation.Service,
+				ImageDigest: observation.ImageDigest, FirstSeenAt: old.FirstSeenAt, LastSeenAt: observation.ObservedAt.UTC(), Active: true,
+			}
+			records = append(records, record)
+			recreation := ContainerRecreation{
+				EntityID: record.EntityID, PreviousRuntimeID: old.RuntimeID, CurrentRuntimeID: record.RuntimeID,
+				EventType: "container.recreated", EventEntityID: record.EntityID, OccurredAt: observation.ObservedAt.UTC(),
+			}
+			result.Recreated = append(result.Recreated, recreation)
+			result.Updated = append(result.Updated, record.EntityID)
+			continue
+		} else {
+			externalType := containerInstanceExternalType
+			externalID := observation.RuntimeID
+			if key := containerStableKey(observation.SourceID, observation.Project, observation.Service); key != "" {
+				if len(stable[key]) > 1 {
+					result.Warnings = append(result.Warnings, "AMBIGUOUS_COMPOSE_IDENTITY_NO_MERGE")
+				} else {
+					externalType = containerServiceExternalType
+					externalID = observation.Project + "\x00" + observation.Service
+				}
+			}
+			entityID := StableEntityID(observation.SourceID, externalType, externalID)
+			record = ContainerAlias{
+				EntityID: entityID, SourceID: observation.SourceID, RuntimeID: observation.RuntimeID,
+				Name: observation.Name, Project: observation.Project, Service: observation.Service,
+				ImageDigest: observation.ImageDigest, FirstSeenAt: observation.ObservedAt.UTC(),
+				LastSeenAt: observation.ObservedAt.UTC(), Active: true,
+			}
+			records = append(records, record)
+			result.Added = append(result.Added, entityID)
+			continue
+		}
+		records = replaceAlias(records, record)
+	}
+	// A runtime alias that is no longer observed is tombstoned rather than
+	// deleted, and keeps the moment it was first found missing.
+	for i := range records {
+		if records[i].Active {
+			records[i].TombstonedAt = time.Time{}
+		} else if records[i].TombstonedAt.IsZero() {
+			records[i].TombstonedAt = now
+		}
+	}
+	sort.Strings(result.Added)
+	sort.Strings(result.Updated)
+	sort.Slice(result.Recreated, func(i, j int) bool {
+		if result.Recreated[i].EntityID != result.Recreated[j].EntityID {
+			return result.Recreated[i].EntityID < result.Recreated[j].EntityID
+		}
+		return result.Recreated[i].CurrentRuntimeID < result.Recreated[j].CurrentRuntimeID
+	})
+	sort.Strings(result.Warnings)
+	sort.Slice(records, func(i, j int) bool {
+		if records[i].EntityID != records[j].EntityID {
+			return records[i].EntityID < records[j].EntityID
+		}
+		return records[i].RuntimeID < records[j].RuntimeID
+	})
+	result.Aliases = records
+	for _, record := range records {
+		if record.Active {
+			result.Current = append(result.Current, record)
+		}
+	}
+	return result, nil
+}
+
+func validateContainerObservation(observation ContainerObservation) error {
+	if strings.TrimSpace(observation.SourceID) == "" || strings.TrimSpace(observation.RuntimeID) == "" || strings.TrimSpace(observation.Name) == "" || observation.ObservedAt.IsZero() {
+		return errors.New("invalid container observation")
+	}
+	if (strings.TrimSpace(observation.Project) == "") != (strings.TrimSpace(observation.Service) == "") {
+		return errors.New("compose project and service must be supplied together")
+	}
+	return nil
+}
+
+func containerAliasKey(sourceID, runtimeID string) string {
+	return sourceID + "\x00" + runtimeID
+}
+
+func containerStableKey(sourceID, project, service string) string {
+	if strings.TrimSpace(project) == "" || strings.TrimSpace(service) == "" {
+		return ""
+	}
+	return sourceID + "\x00" + project + "\x00" + service
+}
+
+func replaceAlias(records []ContainerAlias, updated ContainerAlias) []ContainerAlias {
+	for i := range records {
+		if records[i].EntityID == updated.EntityID && records[i].RuntimeID == updated.RuntimeID {
+			records[i] = updated
+			return records
+		}
+	}
+	return append(records, updated)
+}
@@ -0,0 +1,136 @@
+package reconciliation
+
+import (
+	"testing"
+	"time"
+)
+
+func TestComposeRecreationKeepsLogicalEntityAndLinksEvent(t *testing.T) {
+	now := time.Date(2026, time.August, 1, 22, 0, 0, 0, time.UTC)
+	first, err := ReconcileContainers([]ContainerObservation{{
+		SourceID: "agent", RuntimeID: "runtime-a", Name: "pulse-api-1", Project: "pulse", Service: "api",
+		ImageDigest: "sha256:old", ObservedAt: now,
+	}}, nil, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := ReconcileContainers([]ContainerObservation{{
+		SourceID: "agent", RuntimeID: "runtime-b", Name: "pulse-api-1", Project: "pulse", Service: "api",
+		ImageDigest: "sha256:new", ObservedAt: now.Add(time.Minute),
+	}}, first.Aliases, now.Add(time.Minute))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(second.Recreated) != 1 || second.Recreated[0].EntityID != first.Current[0].EntityID || second.Recreated[0].EventEntityID != first.Current[0].EntityID {
+		t.Fatalf("recreation did not preserve logical event identity: %+v", second)
+	}
+	if len(second.Aliases) != 2 || len(second.Current) != 1 || second.Current[0].RuntimeID != "runtime-b" {
+		t.Fatalf("runtime history not retained: %+v", second)
+	}
+	if second.Current[0].ImageDigest != "sha256:new" {
+		t.Fatalf("new runtime facts not applied: %+v", second.Current[0])
+	}
+}
+
+func TestReusedNameWithoutEvidenceDoesNotMerge(t *testing.T) {
+	now := time.Date(2026, time.August, 1, 22, 0, 0, 0, time.UTC)
+	first, err := ReconcileContainers([]ContainerObservation{{
+		SourceID: "agent", RuntimeID: "runtime-a", Name: "worker", ObservedAt: now,
+	}}, nil, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := ReconcileContainers([]ContainerObservation{{
+		SourceID: "agent", RuntimeID: "runtime-b", Name: "worker", ObservedAt: now.Add(time.Minute),
+	}}, first.Aliases, now.Add(time.Minute))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(second.Current) != 1 || second.Current[0].EntityID == first.Current[0].EntityID {
+		t.Fatalf("reused name was merged without evidence: %+v", second)
+	}
+	if len(second.Recreated) != 0 {
+		t.Fatalf("name-only reuse created recreation: %+v", second.Recreated)
+	}
+}
+
+func TestIdempotentComposeSnapshotDoesNotDuplicateHistory(t *testing.T) {
+	now := time.Date(2026, time.August, 1, 22, 0, 0, 0, time.UTC)
+	obs := []ContainerObservation{{SourceID: "agent", RuntimeID: "runtime-a", Name: "api", Project: "pulse", Service: "api", ObservedAt: now}}
+	first, err := ReconcileContainers(obs, nil, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := ReconcileContainers(obs, first.Aliases, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(second.Aliases) != 1 || len(second.Recreated) != 0 || len(second.Added) != 0 || len(second.Updated) != 1 {
+		t.Fatalf("repeat was not idempotent: %+v", second)
+	}
+	if first.Current[0].EntityID != second.Current[0].EntityID {
+		t.Fatal("logical ID changed on repeated observation")
+	}
+}
+
+func TestAmbiguousComposeIdentityRefusesMerge(t *testing.T) {
+	now := time.Date(2026, time.August, 1, 22, 0, 0, 0, time.UTC)
+	previous := []ContainerAlias{
+		{EntityID: "entity-a", SourceID: "agent", RuntimeID: "runtime-a", Project: "pulse", Service: "api", Active: true, FirstSeenAt: now, LastSeenAt: now},
+		{EntityID: "entity-b", SourceID: "agent", RuntimeID: "runtime-b", Project: "pulse", Service: "api", Active: true, FirstSeenAt: now, LastSeenAt: now},
+	}
+	result, err := ReconcileContainers([]ContainerObservation{{SourceID: "agent", RuntimeID: "runtime-c", Name: "api", Project: "pulse", Service: "api", ObservedAt: now.Add(time.Minute)}}, previous, now.Add(time.Minute))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(result.Recreated) != 0 || len(result.Warnings) != 1 || result.Current[0].EntityID == "entity-a" || result.Current[0].EntityID == "entity-b" {
+		t.Fatalf("ambiguous identity merged: %+v", result)
+	}
+}
+
+func TestSupersededRuntimeAliasIsTombstonedOnce(t *testing.T) {
+	now := time.Date(2026, time.August, 1, 22, 0, 0, 0, time.UTC)
+	first, err := ReconcileContainers([]ContainerObservation{{
+		SourceID: "agent", RuntimeID: "runtime-a", Name: "pulse-api-1", Project: "pulse", Service: "api", ObservedAt: now,
+	}}, nil, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !first.Current[0].TombstonedAt.IsZero() {
+		t.Fatalf("active alias carries a tombstone time: %+v", first.Current[0])
+	}
+	recreated := now.Add(time.Minute).In(time.FixedZone("CEST", 2*60*60))
+	second, err := ReconcileContainers([]ContainerObservation{{
+		SourceID: "agent", RuntimeID: "runtime-b", Name: "pulse-api-1", Project: "pulse", Service: "api", ObservedAt: recreated,
+	}}, first.Aliases, recreated)
+	if err != nil {
+		t.Fatal(err)
+	}
+	old := aliasByRuntime(t, second.Aliases, "runtime-a")
+	if old.Active || !old.TombstonedAt.Equal(recreated) || old.TombstonedAt.Location() != time.UTC {
+		t.Fatalf("superseded alias was not tombstoned in UTC: %+v", old)
+	}
+	if current := aliasByRuntime(t, second.Aliases, "runtime-b"); !current.Active || !current.TombstonedAt.IsZero() {
+		t.Fatalf("current alias carries a tombstone time: %+v", current)
+	}
+	third, err := ReconcileContainers([]ContainerObservation{{
+		SourceID: "agent", RuntimeID: "runtime-b", Name: "pulse-api-1", Project: "pulse", Service: "api", ObservedAt: recreated.Add(time.Hour),
+	}}, second.Aliases, recreated.Add(time.Hour))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if again := aliasByRuntime(t, third.Aliases, "runtime-a"); !again.TombstonedAt.Equal(recreated) {
+		t.Fatalf("tombstone time moved on a repeated snapshot: %+v", again)
+	}
+}
+
+func aliasByRuntime(t *testing.T, aliases []ContainerAlias, runtimeID string) ContainerAlias {
+	t.Helper()
+	for _, alias := range aliases {
+		if alias.RuntimeID == runtimeID {
+			return alias
+		}
+	}
+	t.Fatalf("alias %q not found in %+v", runtimeID, aliases)
+	return ContainerAlias{}
+}
@@ -0,0 +1,103 @@
+package reconciliation
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/itworx/pulse/internal/datasource"
+)
+
+type Observation struct {
+	SourceID, ExternalType, ExternalID, EntityType, CanonicalName, DisplayName string
+	ObservedAt                                                                 time.Time
+}
+
+type Existing struct {
+	EntityID                           string
+	SourceID, ExternalType, ExternalID string
+	CanonicalName, DisplayName         string
+	Tombstoned                         bool
+	// TombstonedAt records when the entity was tombstoned. It is the zero time
+	// while the entity is live and maps onto entities.tombstoned_at.
+	TombstonedAt time.Time
+}
+type Override struct{ EntityID, FieldName, Value string }
+type Result struct {
+	Entities                   []Existing
+	Added, Updated, Tombstoned []string
+	Aliases                    map[string]string
+	Warnings                   []string
+}
+
+func StableEntityID(sourceID, externalType, externalID string) string {
+	hash := sha256.Sum256([]byte("itworx-pulse/entity/v1/" + sourceID + "\x00" + externalType + "\x00" + externalID))
+	b := hash[:16]
+	b[6] = (b[6] & 0x0f) | 0x50
+	b[8] = (b[8] & 0x3f) | 0x80
+	return fmt.Sprintf("%s-%s-%s-%s-%s", hex.EncodeToString(b[0:4]), hex.EncodeToString(b[4:6]), hex.EncodeToString(b[6:8]), hex.EncodeToString(b[8:10]), hex.EncodeToString(b[10:16]))
+}
+
+func Reconcile(sourceID string, health datasource.HealthState, observations []Observation, previous []Existing, overrides []Override, now time.Time) (Result, error) {
+	if strings.TrimSpace(sourceID) == "" {
+		return Result{}, errors.New("source id is required")
+	}
+	if health == datasource.HealthUnknown || health == datasource.HealthDisabled {
+		return Result{Entities: previous, Warnings: []string{"SOURCE_NOT_HEALTHY_NO_TOMBSTONES"}}, nil
+	}
+	seen := make(map[string]struct{}, len(observations))
+	result := Result{Aliases: make(map[string]string)}
+	overrideMap := make(map[string]string)
+	for _, o := range overrides {
+		overrideMap[o.EntityID+"\x00"+o.FieldName] = o.Value
+	}
+	previousByAlias := make(map[string]Existing)
+	for _, e := range previous {
+		previousByAlias[e.SourceID+"\x00"+e.ExternalType+"\x00"+e.ExternalID] = e
+	}
+	for _, o := range observations {
+		if o.SourceID != sourceID || o.ExternalType == "" || o.ExternalID == "" || o.EntityType == "" || o.CanonicalName == "" || o.ObservedAt.IsZero() {
+			return Result{}, errors.New("invalid observation")
+		}
+		key := o.SourceID + "\x00" + o.ExternalType + "\x00" + o.ExternalID
+		if _, ok := seen[key]; ok {
+			return Result{}, fmt.Errorf("duplicate observation %q", key)
+		}
+		seen[key] = struct{}{}
+		id := StableEntityID(o.SourceID, o.ExternalType, o.ExternalID)
+		if old, ok := previousByAlias[key]; ok {
+			id = old.EntityID
+		}
+		display := o.DisplayName
+		if value, ok := overrideMap[id+"\x00displayName"]; ok {
+			display = value
+		}
+		entity := Existing{EntityID: id, SourceID: o.SourceID, ExternalType: o.ExternalType, ExternalID: o.ExternalID, CanonicalName: o.CanonicalName, DisplayName: display}
+		result.Entities = append(result.Entities, entity)
+		result.Aliases[key] = id
+		if _, ok := previousByAlias[key]; ok {
+			result.Updated = append(result.Updated, id)
+		} else {
+			result.Added = append(result.Added, id)
+		}
+	}
+	if now.IsZero() {
+		now = time.Now().UTC()
+	}
+	now = now.UTC()
+	for _, old := range previous {
+		key := old.SourceID + "\x00" + old.ExternalType + "\x00" + old.ExternalID
+		if old.SourceID == sourceID {
+			if _, ok := seen[key]; !ok && !old.Tombstoned {
+				old.Tombstoned = true
+				old.TombstonedAt = now
+				result.Entities = append(result.Entities, old)
+				result.Tombstoned = append(result.Tombstoned, old.EntityID)
+			}
+		}
+	}
+	return result, nil
+}
@@ -0,0 +1,84 @@
+package reconciliation
+
+import (
+	"testing"
+	"time"
+
+	"github.com/itworx/pulse/internal/datasource"
+)
+
+func TestRepeatedSnapshotIsIdempotent(t *testing.T) {
+	now := time.Now().UTC()
+	obs := []Observation{{SourceID: "source", ExternalType: "container", ExternalID: "abc", EntityType: "container", CanonicalName: "app", DisplayName: "App", ObservedAt: now}}
+	first, err := Reconcile("source", datasource.HealthHealthy, obs, nil, nil, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := Reconcile("source", datasource.HealthHealthy, obs, first.Entities, nil, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(second.Added) != 0 || len(second.Updated) != 1 || first.Aliases["source\x00container\x00abc"] != second.Aliases["source\x00container\x00abc"] {
+		t.Fatalf("not idempotent: first=%+v second=%+v", first, second)
+	}
+}
+func TestSourceOutageDoesNotTombstone(t *testing.T) {
+	now := time.Now().UTC()
+	previous := []Existing{{EntityID: StableEntityID("source", "container", "abc"), SourceID: "source", ExternalType: "container", ExternalID: "abc"}}
+	result, err := Reconcile("source", datasource.HealthUnknown, nil, previous, nil, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(result.Tombstoned) != 0 || len(result.Entities) != 1 {
+		t.Fatalf("outage changed inventory: %+v", result)
+	}
+}
+func TestOverrideSurvivesDiscovery(t *testing.T) {
+	now := time.Now().UTC()
+	id := StableEntityID("source", "container", "abc")
+	result, err := Reconcile("source", datasource.HealthHealthy, []Observation{{SourceID: "source", ExternalType: "container", ExternalID: "abc", EntityType: "container", CanonicalName: "app", DisplayName: "Discovered", ObservedAt: now}}, nil, []Override{{EntityID: id, FieldName: "displayName", Value: "Manual"}}, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Aliases["source\x00container\x00abc"] != id || result.Entities[0].DisplayName != "Manual" {
+		t.Fatalf("override was not preserved: %+v", result)
+	}
+}
+
+func TestMissingObservationStampsTombstoneTime(t *testing.T) {
+	now := time.Date(2026, time.August, 1, 22, 0, 0, 0, time.UTC)
+	obs := []Observation{{SourceID: "source", ExternalType: "container", ExternalID: "abc", EntityType: "container", CanonicalName: "app", DisplayName: "App", ObservedAt: now}}
+	first, err := Reconcile("source", datasource.HealthHealthy, obs, nil, nil, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !first.Entities[0].TombstonedAt.IsZero() {
+		t.Fatalf("live entity carries a tombstone time: %+v", first.Entities[0])
+	}
+	later := now.Add(time.Hour).In(time.FixedZone("CEST", 2*60*60))
+	second, err := Reconcile("source", datasource.HealthHealthy, nil, first.Entities, nil, later)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(second.Tombstoned) != 1 || len(second.Entities) != 1 {
+		t.Fatalf("expected a single tombstone: %+v", second)
+	}
+	entity := second.Entities[0]
+	if !entity.Tombstoned || !entity.TombstonedAt.Equal(later) || entity.TombstonedAt.Location() != time.UTC {
+		t.Fatalf("tombstone time was not stamped in UTC: %+v", entity)
+	}
+	third, err := Reconcile("source", datasource.HealthHealthy, nil, second.Entities, nil, later.Add(time.Hour))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(third.Tombstoned) != 0 {
+		t.Fatalf("already tombstoned entity was tombstoned again: %+v", third)
+	}
+	revived, err := Reconcile("source", datasource.HealthHealthy, obs, second.Entities, nil, later.Add(2*time.Hour))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if revived.Entities[0].Tombstoned || !revived.Entities[0].TombstonedAt.IsZero() {
+		t.Fatalf("revived entity kept its tombstone: %+v", revived.Entities[0])
+	}
+}