Publish ITWorx Pulse source
Public source validation / validate (push) Failing after 3m8s

This commit is contained in:
ITWorx Pulse release export
2026-09-03 02:09:19 +02:00
commit bd774932d5
614 changed files with 77116 additions and 0 deletions
+282
View File
@@ -0,0 +1,282 @@
package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
)
type EntityFilter struct {
Limit int
AfterName, AfterID string
Search string
EntityType string
Status string
Direction string
}
type EntitySummary struct {
ID string `json:"id"`
EntityType string `json:"entityType"`
CanonicalName string `json:"canonicalName"`
DisplayName string `json:"displayName"`
Status string `json:"status"`
FirstSeenAt *time.Time `json:"firstSeenAt"`
LastSeenAt *time.Time `json:"lastSeenAt,omitempty"`
TombstonedAt *time.Time `json:"tombstonedAt,omitempty"`
FactCount int `json:"factCount"`
OverrideCount int `json:"overrideCount"`
RelationCount int `json:"relationCount"`
SourceCount int `json:"sourceCount"`
StaleFactCount int `json:"staleFactCount"`
}
type AliasView struct {
SourceID string `json:"sourceId"`
SourceName string `json:"sourceName"`
ExternalType string `json:"externalType"`
ExternalID string `json:"externalId"`
}
type FactView struct {
FieldName string `json:"fieldName"`
SourceID string `json:"sourceId"`
SourceName string `json:"sourceName"`
Value json.RawMessage `json:"value"`
ObservedAt time.Time `json:"observedAt"`
Confidence float64 `json:"confidence"`
ValidUntil *time.Time `json:"validUntil,omitempty"`
Stale bool `json:"stale"`
}
type OverrideView struct {
FieldName string `json:"fieldName"`
Value json.RawMessage `json:"value"`
UserID string `json:"userId,omitempty"`
UpdatedAt time.Time `json:"updatedAt"`
}
type EffectiveValue struct {
FieldName string `json:"fieldName"`
Value json.RawMessage `json:"value"`
Origin string `json:"origin"`
SourceID string `json:"sourceId,omitempty"`
SourceName string `json:"sourceName,omitempty"`
ObservedAt *time.Time `json:"observedAt,omitempty"`
Confidence *float64 `json:"confidence,omitempty"`
Stale bool `json:"stale"`
OverriddenAt *time.Time `json:"overriddenAt,omitempty"`
}
type RelationView struct {
ID string `json:"id"`
Direction string `json:"direction"`
RelationType string `json:"relationType"`
PeerID string `json:"peerId"`
PeerType string `json:"peerType"`
PeerName string `json:"peerName"`
PeerStatus string `json:"peerStatus"`
PeerTombstonedAt *time.Time `json:"peerTombstonedAt,omitempty"`
SourceID string `json:"sourceId"`
SourceName string `json:"sourceName"`
Confidence float64 `json:"confidence"`
Confirmed bool `json:"confirmed"`
FirstSeenAt *time.Time `json:"firstSeenAt"`
LastSeenAt *time.Time `json:"lastSeenAt,omitempty"`
TombstonedAt *time.Time `json:"tombstonedAt,omitempty"`
}
type EntityDetail struct {
Entity EntitySummary `json:"entity"`
Aliases []AliasView `json:"aliases"`
Facts []FactView `json:"facts"`
Overrides []OverrideView `json:"overrides"`
Effective []EffectiveValue `json:"effectiveValues"`
Relations []RelationView `json:"relations"`
}
func (f EntityFilter) Validate() error {
if f.Limit < 1 || f.Limit > 100 {
return errors.New("entity page limit must be between 1 and 100")
}
if len(f.Search) > 120 || len(f.EntityType) > 80 || len(f.Status) > 40 {
return errors.New("entity filters exceed bounds")
}
if f.Direction != "asc" && f.Direction != "desc" {
return errors.New("entity sort direction must be asc or desc")
}
return nil
}
func (r *Repository) SearchEntities(ctx context.Context, filter EntityFilter) ([]EntitySummary, error) {
if err := filter.Validate(); err != nil {
return nil, err
}
rows, err := r.pool.Query(ctx, `WITH projected AS (
SELECT e.id::text,e.entity_type,e.canonical_name,
COALESCE(NULLIF(od.value #>> '{}',''),e.display_name) AS effective_name,
COALESCE(NULLIF(os.value #>> '{}',''),e.status) AS effective_status,
e.first_seen_at,e.last_seen_at,e.tombstoned_at,
(SELECT count(*) FROM entity_facts f WHERE f.entity_id=e.id)::int AS fact_count,
(SELECT count(*) FROM entity_overrides o WHERE o.entity_id=e.id)::int AS override_count,
(SELECT count(*) FROM entity_relations rel WHERE rel.source_entity_id=e.id OR rel.target_entity_id=e.id)::int AS relation_count,
(SELECT count(DISTINCT source_id) FROM entity_facts f WHERE f.entity_id=e.id)::int AS source_count,
(SELECT count(*) FROM entity_facts f WHERE f.entity_id=e.id AND f.valid_until IS NOT NULL AND f.valid_until < now())::int AS stale_fact_count
FROM entities e
LEFT JOIN entity_overrides od ON od.entity_id=e.id AND od.field_name='displayName'
LEFT JOIN entity_overrides os ON os.entity_id=e.id AND os.field_name='status'
WHERE e.tombstoned_at IS NULL)
SELECT id,entity_type,canonical_name,effective_name,effective_status,first_seen_at,last_seen_at,tombstoned_at,fact_count,override_count,relation_count,source_count,stale_fact_count
FROM projected
WHERE ($1='' OR lower(effective_name) LIKE '%'||lower($1)||'%' OR lower(canonical_name) LIKE '%'||lower($1)||'%')
AND ($2='' OR entity_type=$2) AND ($3='' OR effective_status=$3)
AND ($4='' OR ($6='asc' AND (lower(effective_name),id) > (lower($4),$5)) OR ($6='desc' AND (lower(effective_name),id) < (lower($4),$5)))
ORDER BY CASE WHEN $6='asc' THEN lower(effective_name) END ASC, CASE WHEN $6='desc' THEN lower(effective_name) END DESC,
CASE WHEN $6='asc' THEN id END ASC, CASE WHEN $6='desc' THEN id END DESC LIMIT $7`, strings.TrimSpace(filter.Search), filter.EntityType, filter.Status, filter.AfterName, filter.AfterID, filter.Direction, filter.Limit+1)
if err != nil {
return nil, fmt.Errorf("search entities: %w", err)
}
defer rows.Close()
result := make([]EntitySummary, 0, filter.Limit+1)
for rows.Next() {
var item EntitySummary
if err := rows.Scan(&item.ID, &item.EntityType, &item.CanonicalName, &item.DisplayName, &item.Status, &item.FirstSeenAt, &item.LastSeenAt, &item.TombstonedAt, &item.FactCount, &item.OverrideCount, &item.RelationCount, &item.SourceCount, &item.StaleFactCount); err != nil {
return nil, fmt.Errorf("scan entity search: %w", err)
}
result = append(result, item)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("search entity rows: %w", err)
}
return result, nil
}
func (r *Repository) GetEntityDetail(ctx context.Context, id string) (EntityDetail, error) {
rows, err := r.SearchEntitiesByID(ctx, id)
if err != nil {
return EntityDetail{}, err
}
if len(rows) == 0 {
return EntityDetail{}, fmt.Errorf("get entity detail: %w", ErrNotFound)
}
detail := EntityDetail{Entity: rows[0], Aliases: []AliasView{}, Facts: []FactView{}, Overrides: []OverrideView{}, Effective: []EffectiveValue{}, Relations: []RelationView{}}
aliasRows, err := r.pool.Query(ctx, `SELECT a.source_id::text,s.name,a.external_type,a.external_id FROM entity_aliases a JOIN data_sources s ON s.id=a.source_id WHERE a.entity_id=$1 ORDER BY s.name,a.external_type,a.external_id`, id)
if err != nil {
return EntityDetail{}, fmt.Errorf("list entity aliases: %w", err)
}
for aliasRows.Next() {
var item AliasView
if err := aliasRows.Scan(&item.SourceID, &item.SourceName, &item.ExternalType, &item.ExternalID); err != nil {
aliasRows.Close()
return EntityDetail{}, err
}
detail.Aliases = append(detail.Aliases, item)
}
aliasRows.Close()
if err := aliasRows.Err(); err != nil {
return EntityDetail{}, err
}
factRows, err := r.pool.Query(ctx, `SELECT f.field_name,f.source_id::text,s.name,f.value,f.observed_at,f.confidence,f.valid_until,(f.valid_until IS NOT NULL AND f.valid_until<now()) FROM entity_facts f JOIN data_sources s ON s.id=f.source_id WHERE f.entity_id=$1 ORDER BY f.field_name,(f.valid_until IS NOT NULL AND f.valid_until<now()),f.confidence DESC,f.observed_at DESC,f.source_id`, id)
if err != nil {
return EntityDetail{}, fmt.Errorf("list entity fact views: %w", err)
}
for factRows.Next() {
var item FactView
if err := factRows.Scan(&item.FieldName, &item.SourceID, &item.SourceName, &item.Value, &item.ObservedAt, &item.Confidence, &item.ValidUntil, &item.Stale); err != nil {
factRows.Close()
return EntityDetail{}, err
}
detail.Facts = append(detail.Facts, item)
}
factRows.Close()
if err := factRows.Err(); err != nil {
return EntityDetail{}, err
}
overrideRows, err := r.pool.Query(ctx, `SELECT field_name,value,COALESCE(user_id::text,''),updated_at FROM entity_overrides WHERE entity_id=$1 ORDER BY field_name`, id)
if err != nil {
return EntityDetail{}, fmt.Errorf("list entity overrides: %w", err)
}
for overrideRows.Next() {
var item OverrideView
if err := overrideRows.Scan(&item.FieldName, &item.Value, &item.UserID, &item.UpdatedAt); err != nil {
overrideRows.Close()
return EntityDetail{}, err
}
detail.Overrides = append(detail.Overrides, item)
}
overrideRows.Close()
if err := overrideRows.Err(); err != nil {
return EntityDetail{}, err
}
detail.Effective = effectiveValues(detail.Facts, detail.Overrides)
relationRows, err := r.pool.Query(ctx, `SELECT rel.id::text,CASE WHEN rel.source_entity_id=$1 THEN 'outgoing' ELSE 'incoming' END,rel.relation_type,peer.id::text,peer.entity_type,peer.display_name,CASE WHEN peer.tombstoned_at IS NULL THEN peer.status ELSE 'Ontbrekende entiteit' END,peer.tombstoned_at,rel.source_id::text,s.name,rel.confidence,rel.confirmed,rel.first_seen_at,rel.last_seen_at,rel.tombstoned_at FROM entity_relations rel JOIN entities peer ON peer.id=CASE WHEN rel.source_entity_id=$1 THEN rel.target_entity_id ELSE rel.source_entity_id END JOIN data_sources s ON s.id=rel.source_id WHERE rel.source_entity_id=$1 OR rel.target_entity_id=$1 ORDER BY rel.relation_type,peer.display_name,rel.id`, id)
if err != nil {
return EntityDetail{}, fmt.Errorf("list entity relation views: %w", err)
}
for relationRows.Next() {
var item RelationView
if err := relationRows.Scan(&item.ID, &item.Direction, &item.RelationType, &item.PeerID, &item.PeerType, &item.PeerName, &item.PeerStatus, &item.PeerTombstonedAt, &item.SourceID, &item.SourceName, &item.Confidence, &item.Confirmed, &item.FirstSeenAt, &item.LastSeenAt, &item.TombstonedAt); err != nil {
relationRows.Close()
return EntityDetail{}, err
}
detail.Relations = append(detail.Relations, item)
}
relationRows.Close()
if err := relationRows.Err(); err != nil {
return EntityDetail{}, err
}
return detail, nil
}
var ErrNotFound = errors.New("inventory entity not found")
func (r *Repository) SearchEntitiesByID(ctx context.Context, id string) ([]EntitySummary, error) {
rows, err := r.pool.Query(ctx, `SELECT e.id::text,e.entity_type,e.canonical_name,COALESCE(NULLIF(od.value #>> '{}',''),e.display_name),COALESCE(NULLIF(os.value #>> '{}',''),e.status),e.first_seen_at,e.last_seen_at,e.tombstoned_at,(SELECT count(*) FROM entity_facts f WHERE f.entity_id=e.id)::int,(SELECT count(*) FROM entity_overrides o WHERE o.entity_id=e.id)::int,(SELECT count(*) FROM entity_relations rel WHERE rel.source_entity_id=e.id OR rel.target_entity_id=e.id)::int,(SELECT count(DISTINCT source_id) FROM entity_facts f WHERE f.entity_id=e.id)::int,(SELECT count(*) FROM entity_facts f WHERE f.entity_id=e.id AND f.valid_until IS NOT NULL AND f.valid_until<now())::int FROM entities e LEFT JOIN entity_overrides od ON od.entity_id=e.id AND od.field_name='displayName' LEFT JOIN entity_overrides os ON os.entity_id=e.id AND os.field_name='status' WHERE e.id=$1`, id)
if err != nil {
return nil, err
}
defer rows.Close()
var result []EntitySummary
for rows.Next() {
var item EntitySummary
if err := rows.Scan(&item.ID, &item.EntityType, &item.CanonicalName, &item.DisplayName, &item.Status, &item.FirstSeenAt, &item.LastSeenAt, &item.TombstonedAt, &item.FactCount, &item.OverrideCount, &item.RelationCount, &item.SourceCount, &item.StaleFactCount); err != nil {
return nil, err
}
result = append(result, item)
}
return result, rows.Err()
}
func effectiveValues(facts []FactView, overrides []OverrideView) []EffectiveValue {
overrideByField := make(map[string]OverrideView, len(overrides))
for _, item := range overrides {
overrideByField[item.FieldName] = item
}
fields := make(map[string]struct{}, len(facts)+len(overrides))
for _, item := range facts {
fields[item.FieldName] = struct{}{}
}
for _, item := range overrides {
fields[item.FieldName] = struct{}{}
}
names := make([]string, 0, len(fields))
for name := range fields {
names = append(names, name)
}
sort.Strings(names)
result := make([]EffectiveValue, 0, len(names))
for _, name := range names {
if item, ok := overrideByField[name]; ok {
at := item.UpdatedAt
result = append(result, EffectiveValue{FieldName: name, Value: item.Value, Origin: "override", OverriddenAt: &at})
continue
}
for _, fact := range facts {
if fact.FieldName == name {
observed, confidence := fact.ObservedAt, fact.Confidence
result = append(result, EffectiveValue{FieldName: name, Value: fact.Value, Origin: "discovered", SourceID: fact.SourceID, SourceName: fact.SourceName, ObservedAt: &observed, Confidence: &confidence, Stale: fact.Stale})
break
}
}
}
return result
}
+28
View File
@@ -0,0 +1,28 @@
package inventory
import (
"encoding/json"
"testing"
"time"
)
func TestEffectiveValuesPreserveFactsAndPreferOverrides(t *testing.T) {
now := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
facts := []FactView{
{FieldName: "image", SourceID: "primary", SourceName: "Unraid", Value: json.RawMessage(`"stable"`), ObservedAt: now, Confidence: 1},
{FieldName: "image", SourceID: "secondary", SourceName: "Agent", Value: json.RawMessage(`"older"`), ObservedAt: now.Add(-time.Hour), Confidence: .8},
{FieldName: "status", SourceID: "primary", SourceName: "Unraid", Value: json.RawMessage(`"running"`), ObservedAt: now, Confidence: 1},
}
overrides := []OverrideView{{FieldName: "image", Value: json.RawMessage(`"manual"`), UpdatedAt: now.Add(time.Minute)}}
effective := effectiveValues(facts, overrides)
if len(effective) != 2 || len(facts) != 3 {
t.Fatalf("effective=%d facts=%d", len(effective), len(facts))
}
if effective[0].FieldName != "image" || effective[0].Origin != "override" || string(effective[0].Value) != `"manual"` {
t.Fatalf("override did not win: %+v", effective[0])
}
if effective[1].FieldName != "status" || effective[1].Origin != "discovered" || effective[1].SourceName != "Unraid" {
t.Fatalf("discovered provenance lost: %+v", effective[1])
}
}
+301
View File
@@ -0,0 +1,301 @@
package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
)
type Repository struct{ pool *pgxpool.Pool }
// PersistDiscovery atomically applies one source snapshot. It updates discovered
// entities/facts/aliases/relations only; user overrides are deliberately untouched.
func (r *Repository) PersistDiscovery(ctx context.Context, entity Entity, alias Alias, facts []Fact, relations []Relation) error {
return r.InTx(ctx, func(txctx context.Context, tx pgx.Tx) error {
if err := validateEntity(entity); err != nil {
return err
}
if _, err := tx.Exec(txctx, `INSERT INTO entities (id,entity_type,canonical_name,display_name,status,first_seen_at,last_seen_at,tombstoned_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT (id) DO UPDATE SET entity_type=EXCLUDED.entity_type,canonical_name=EXCLUDED.canonical_name,display_name=EXCLUDED.display_name,status=EXCLUDED.status,last_seen_at=EXCLUDED.last_seen_at,tombstoned_at=EXCLUDED.tombstoned_at`, entity.ID, entity.EntityType, entity.CanonicalName, entity.DisplayName, entity.Status, entity.FirstSeenAt, entity.LastSeenAt, entity.TombstonedAt); err != nil {
return fmt.Errorf("persist discovery entity: %w", err)
}
if err := upsertAlias(txctx, tx, alias); err != nil {
return err
}
for _, fact := range facts {
if err := upsertFact(txctx, tx, fact); err != nil {
return err
}
}
for _, relation := range relations {
if err := upsertRelation(txctx, tx, relation); err != nil {
return err
}
}
return nil
})
}
func NewRepository(pool *pgxpool.Pool) (*Repository, error) {
if pool == nil {
return nil, errors.New("inventory repository requires a database pool")
}
return &Repository{pool: pool}, nil
}
func (r *Repository) InTx(ctx context.Context, fn func(context.Context, pgx.Tx) error) error {
tx, err := r.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return fmt.Errorf("begin inventory transaction: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
if err := fn(ctx, tx); err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
return fmt.Errorf("commit inventory transaction: %w", err)
}
return nil
}
func (r *Repository) CreateEntity(ctx context.Context, entity Entity) error {
if err := validateEntity(entity); err != nil {
return err
}
_, err := r.pool.Exec(ctx, `INSERT INTO entities (id, entity_type, canonical_name, display_name, status, first_seen_at, last_seen_at, tombstoned_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`, entity.ID, entity.EntityType, entity.CanonicalName, entity.DisplayName, entity.Status, entity.FirstSeenAt, entity.LastSeenAt, entity.TombstonedAt)
if err != nil {
return fmt.Errorf("create entity: %w", err)
}
return nil
}
func (r *Repository) UpsertEntity(ctx context.Context, entity Entity) error {
if err := validateEntity(entity); err != nil {
return err
}
_, err := r.pool.Exec(ctx, `INSERT INTO entities (id, entity_type, canonical_name, display_name, status, first_seen_at, last_seen_at, tombstoned_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT (id) DO UPDATE SET entity_type=EXCLUDED.entity_type, canonical_name=EXCLUDED.canonical_name, display_name=EXCLUDED.display_name, status=EXCLUDED.status, last_seen_at=EXCLUDED.last_seen_at, tombstoned_at=EXCLUDED.tombstoned_at`, entity.ID, entity.EntityType, entity.CanonicalName, entity.DisplayName, entity.Status, entity.FirstSeenAt, entity.LastSeenAt, entity.TombstonedAt)
if err != nil {
return fmt.Errorf("upsert entity: %w", err)
}
return nil
}
func (r *Repository) GetEntity(ctx context.Context, id string) (Entity, error) {
var e Entity
err := r.pool.QueryRow(ctx, `SELECT id, entity_type, canonical_name, display_name, status, first_seen_at, last_seen_at, tombstoned_at FROM entities WHERE id=$1`, id).Scan(&e.ID, &e.EntityType, &e.CanonicalName, &e.DisplayName, &e.Status, &e.FirstSeenAt, &e.LastSeenAt, &e.TombstonedAt)
if err != nil {
return Entity{}, fmt.Errorf("get entity: %w", err)
}
return e, nil
}
func (r *Repository) ListEntities(ctx context.Context, limit int) ([]Entity, error) {
if limit < 1 || limit > 100000 {
return nil, errors.New("entity list limit must be between 1 and 100000")
}
rows, err := r.pool.Query(ctx, `SELECT id, entity_type, canonical_name, display_name, status, first_seen_at, last_seen_at, tombstoned_at FROM entities ORDER BY canonical_name ASC, id ASC LIMIT $1`, limit)
if err != nil {
return nil, fmt.Errorf("list entities: %w", err)
}
defer rows.Close()
result := make([]Entity, 0)
for rows.Next() {
var e Entity
if err := rows.Scan(&e.ID, &e.EntityType, &e.CanonicalName, &e.DisplayName, &e.Status, &e.FirstSeenAt, &e.LastSeenAt, &e.TombstonedAt); err != nil {
return nil, fmt.Errorf("scan entity: %w", err)
}
result = append(result, e)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list entities rows: %w", err)
}
return result, nil
}
func (r *Repository) ListEntitiesAfter(ctx context.Context, limit int, afterName, afterID string) ([]Entity, error) {
if limit < 1 || limit > 100 {
return nil, errors.New("entity page limit must be between 1 and 100")
}
rows, err := r.pool.Query(ctx, `SELECT id, entity_type, canonical_name, display_name, status, first_seen_at, last_seen_at, tombstoned_at FROM entities WHERE ($1 = '' OR canonical_name > $1 OR (canonical_name = $1 AND id::text > $2)) ORDER BY canonical_name ASC, id ASC LIMIT $3`, afterName, afterID, limit)
if err != nil {
return nil, fmt.Errorf("list entity page: %w", err)
}
defer rows.Close()
result := make([]Entity, 0, limit)
for rows.Next() {
var e Entity
if err := rows.Scan(&e.ID, &e.EntityType, &e.CanonicalName, &e.DisplayName, &e.Status, &e.FirstSeenAt, &e.LastSeenAt, &e.TombstonedAt); err != nil {
return nil, fmt.Errorf("scan entity page: %w", err)
}
result = append(result, e)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list entity page rows: %w", err)
}
return result, nil
}
func (r *Repository) UpsertAlias(ctx context.Context, alias Alias) error {
return upsertAlias(ctx, r.pool, alias)
}
func (r *Repository) ListFacts(ctx context.Context, entityID string) ([]Fact, error) {
rows, err := r.pool.Query(ctx, `SELECT entity_id,field_name,source_id,value,observed_at,confidence,valid_until FROM entity_facts WHERE entity_id=$1 ORDER BY field_name ASC, source_id ASC`, entityID)
if err != nil {
return nil, fmt.Errorf("list entity facts: %w", err)
}
defer rows.Close()
result := make([]Fact, 0)
for rows.Next() {
var f Fact
if err := rows.Scan(&f.EntityID, &f.FieldName, &f.SourceID, &f.Value, &f.ObservedAt, &f.Confidence, &f.ValidUntil); err != nil {
return nil, fmt.Errorf("scan entity fact: %w", err)
}
result = append(result, f)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list entity facts rows: %w", err)
}
return result, nil
}
func (r *Repository) GetOverride(ctx context.Context, entityID, field string) (Override, error) {
var o Override
err := r.pool.QueryRow(ctx, `SELECT entity_id,field_name,value,COALESCE(user_id::text,''),updated_at FROM entity_overrides WHERE entity_id=$1 AND field_name=$2`, entityID, field).Scan(&o.EntityID, &o.FieldName, &o.Value, &o.UserID, &o.UpdatedAt)
if err != nil {
return Override{}, fmt.Errorf("get entity override: %w", err)
}
return o, nil
}
func (r *Repository) ListRelations(ctx context.Context, entityID string) ([]Relation, error) {
rows, err := r.pool.Query(ctx, `SELECT id,source_entity_id,relation_type,target_entity_id,source_id,confidence,confirmed,first_seen_at,last_seen_at,tombstoned_at FROM entity_relations WHERE source_entity_id=$1 OR target_entity_id=$1 ORDER BY relation_type ASC,id ASC`, entityID)
if err != nil {
return nil, fmt.Errorf("list entity relations: %w", err)
}
defer rows.Close()
result := make([]Relation, 0)
for rows.Next() {
var rel Relation
if err := rows.Scan(&rel.ID, &rel.SourceEntityID, &rel.RelationType, &rel.TargetEntityID, &rel.SourceID, &rel.Confidence, &rel.Confirmed, &rel.FirstSeenAt, &rel.LastSeenAt, &rel.TombstonedAt); err != nil {
return nil, fmt.Errorf("scan entity relation: %w", err)
}
result = append(result, rel)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list entity relations rows: %w", err)
}
return result, nil
}
func (r *Repository) DeleteFact(ctx context.Context, entityID, fieldName, sourceID string) error {
_, err := r.pool.Exec(ctx, `DELETE FROM entity_facts WHERE entity_id=$1 AND field_name=$2 AND source_id=$3`, entityID, fieldName, sourceID)
if err != nil {
return fmt.Errorf("delete entity fact: %w", err)
}
return nil
}
func (r *Repository) DeleteRelation(ctx context.Context, sourceEntityID, relationType, targetEntityID, sourceID string) error {
_, err := r.pool.Exec(ctx, `DELETE FROM entity_relations WHERE source_entity_id=$1 AND relation_type=$2 AND target_entity_id=$3 AND source_id=$4`, sourceEntityID, relationType, targetEntityID, sourceID)
if err != nil {
return fmt.Errorf("delete entity relation: %w", err)
}
return nil
}
func (r *Repository) UpsertFact(ctx context.Context, fact Fact) error {
return upsertFact(ctx, r.pool, fact)
}
func (r *Repository) UpsertOverride(ctx context.Context, override Override) error {
return upsertOverride(ctx, r.pool, override)
}
func (r *Repository) DeleteOverride(ctx context.Context, entityID, field string) error {
_, err := r.pool.Exec(ctx, `DELETE FROM entity_overrides WHERE entity_id=$1 AND field_name=$2`, entityID, field)
if err != nil {
return fmt.Errorf("delete override: %w", err)
}
return nil
}
func (r *Repository) UpsertRelation(ctx context.Context, relation Relation) error {
return upsertRelation(ctx, r.pool, relation)
}
func upsertAlias(ctx context.Context, q interface {
Exec(context.Context, string, ...any) (pgconn.CommandTag, error)
}, a Alias) error {
if err := a.Validate(); err != nil {
return err
}
_, err := q.Exec(ctx, `INSERT INTO entity_aliases (entity_id,source_id,external_type,external_id) VALUES ($1,$2,$3,$4) ON CONFLICT (source_id,external_type,external_id) DO UPDATE SET entity_id=EXCLUDED.entity_id`, a.EntityID, a.SourceID, a.ExternalType, a.ExternalID)
if err != nil {
return fmt.Errorf("upsert entity alias: %w", err)
}
return nil
}
func upsertFact(ctx context.Context, q interface {
Exec(context.Context, string, ...any) (pgconn.CommandTag, error)
}, f Fact) error {
if err := f.Validate(); err != nil {
return err
}
_, err := q.Exec(ctx, `INSERT INTO entity_facts (entity_id,field_name,source_id,value,observed_at,confidence,valid_until) VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT (entity_id,field_name,source_id) DO UPDATE SET value=EXCLUDED.value,observed_at=EXCLUDED.observed_at,confidence=EXCLUDED.confidence,valid_until=EXCLUDED.valid_until`, f.EntityID, f.FieldName, f.SourceID, f.Value, f.ObservedAt, f.Confidence, f.ValidUntil)
if err != nil {
return fmt.Errorf("upsert entity fact: %w", err)
}
return nil
}
func upsertOverride(ctx context.Context, q interface {
Exec(context.Context, string, ...any) (pgconn.CommandTag, error)
}, o Override) error {
if err := o.Validate(); err != nil {
return err
}
_, err := q.Exec(ctx, `INSERT INTO entity_overrides (entity_id,field_name,value,user_id,updated_at) VALUES ($1,$2,$3,$4,$5) ON CONFLICT (entity_id,field_name) DO UPDATE SET value=EXCLUDED.value,user_id=EXCLUDED.user_id,updated_at=EXCLUDED.updated_at`, o.EntityID, o.FieldName, o.Value, nullableID(o.UserID), o.UpdatedAt)
if err != nil {
return fmt.Errorf("upsert entity override: %w", err)
}
return nil
}
func nullableID(value string) any {
if value == "" {
return nil
}
return value
}
func upsertRelation(ctx context.Context, q interface {
Exec(context.Context, string, ...any) (pgconn.CommandTag, error)
}, rel Relation) error {
if rel.ID == "" || rel.SourceEntityID == "" || rel.TargetEntityID == "" || rel.SourceID == "" || rel.RelationType == "" || rel.FirstSeenAt == nil || rel.Confidence < 0 || rel.Confidence > 1 {
return errors.New("invalid entity relation")
}
_, err := q.Exec(ctx, `INSERT INTO entity_relations (id,source_entity_id,relation_type,target_entity_id,source_id,confidence,confirmed,first_seen_at,last_seen_at,tombstoned_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT (source_entity_id,relation_type,target_entity_id,source_id) DO UPDATE SET confidence=EXCLUDED.confidence,confirmed=EXCLUDED.confirmed,last_seen_at=EXCLUDED.last_seen_at,tombstoned_at=EXCLUDED.tombstoned_at`, rel.ID, rel.SourceEntityID, rel.RelationType, rel.TargetEntityID, rel.SourceID, rel.Confidence, rel.Confirmed, rel.FirstSeenAt, rel.LastSeenAt, rel.TombstonedAt)
if err != nil {
return fmt.Errorf("upsert entity relation: %w", err)
}
return nil
}
func validateEntity(e Entity) error {
if e.ID == "" || e.EntityType == "" || e.CanonicalName == "" || e.DisplayName == "" || e.Status == "" || e.FirstSeenAt == nil {
return errors.New("invalid entity")
}
if e.LastSeenAt != nil && e.LastSeenAt.Before(*e.FirstSeenAt) {
return errors.New("entity last seen precedes first seen")
}
return nil
}
func MarshalValue(v any) ([]byte, error) {
b, err := json.Marshal(v)
if err != nil {
return nil, fmt.Errorf("marshal inventory value: %w", err)
}
return b, nil
}
@@ -0,0 +1,303 @@
package inventory
import (
"context"
"errors"
"fmt"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/itworx/pulse/internal/database"
"github.com/jackc/pgx/v5"
)
func TestInventoryRepositoryPostgreSQL(t *testing.T) {
dsn := inventoryIntegrationDSN()
if dsn == "" {
t.Skip("PULSE_TEST_DATABASE_URL is not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 8, MinConns: 1})
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := database.Ping(ctx, pool); err != nil {
t.Fatal(err)
}
if err := database.Migrate(ctx, pool); err != nil {
t.Fatal(err)
}
r, err := NewRepository(pool)
if err != nil {
t.Fatal(err)
}
const sourceA = "00000000-0000-0000-0000-0000000000a1"
const sourceB = "00000000-0000-0000-0000-0000000000b1"
const entityID = "00000000-0000-0000-0000-0000000000e1"
_, _ = pool.Exec(ctx, `DELETE FROM entity_relations WHERE source_id IN ($1,$2); DELETE FROM entity_facts WHERE source_id IN ($1,$2); DELETE FROM entity_aliases WHERE source_id IN ($1,$2); DELETE FROM entities WHERE id=$3; DELETE FROM data_sources WHERE id IN ($1,$2)`, sourceA, sourceB, entityID)
for _, source := range []string{sourceA, sourceB} {
if _, err := pool.Exec(ctx, `INSERT INTO data_sources (id,type,name,configuration_ref) VALUES ($1,'exporter',$2,'test') ON CONFLICT (id) DO NOTHING`, source, source); err != nil {
t.Fatal(err)
}
}
now := time.Now().UTC().Truncate(time.Microsecond)
later := now.Add(time.Second)
entity := Entity{ID: entityID, EntityType: "container", CanonicalName: "container.test", DisplayName: "Test container", Status: "unknown", FirstSeenAt: &now, LastSeenAt: &now}
if err := r.PersistDiscovery(ctx, entity, Alias{EntityID: entityID, SourceID: sourceA, ExternalType: "container", ExternalID: "abc"}, []Fact{{EntityID: entityID, FieldName: "image", SourceID: sourceA, Value: []byte(`"one"`), ObservedAt: now, Confidence: 1}}, nil); err != nil {
t.Fatal(err)
}
entity.DisplayName = "Updated"
entity.LastSeenAt = &later
if err := r.UpsertEntity(ctx, entity); err != nil {
t.Fatal(err)
}
got, err := r.GetEntity(ctx, entityID)
if err != nil || got.DisplayName != "Updated" {
t.Fatalf("get entity = %+v, err=%v", got, err)
}
if err := r.UpsertFact(ctx, Fact{EntityID: entityID, FieldName: "image", SourceID: sourceB, Value: []byte(`"two"`), ObservedAt: now, Confidence: .9}); err != nil {
t.Fatal(err)
}
if err := r.UpsertOverride(ctx, Override{EntityID: entityID, FieldName: "displayName", Value: []byte(`"Manual"`), UpdatedAt: later}); err != nil {
t.Fatal(err)
}
if err := r.UpsertRelation(ctx, Relation{ID: "00000000-0000-0000-0000-0000000000f1", SourceEntityID: entityID, TargetEntityID: "00000000-0000-0000-0000-0000000000e2", RelationType: "contains", SourceID: sourceA, Confidence: 1, FirstSeenAt: &now}); err == nil {
t.Fatal("expected FK failure for missing target entity")
}
if err := r.InTx(ctx, func(txctx context.Context, tx pgx.Tx) error {
if err := upsertFact(txctx, tx, Fact{EntityID: entityID, FieldName: "rollback", SourceID: sourceA, Value: []byte(`true`), ObservedAt: now, Confidence: 1}); err != nil {
return err
}
return errors.New("force rollback")
}); err == nil {
t.Fatal("expected transaction callback error")
}
count := 0
if err := pool.QueryRow(ctx, `SELECT count(*) FROM entity_facts WHERE field_name='rollback'`).Scan(&count); err != nil {
t.Fatal(err)
}
if count != 0 {
t.Fatal("rolled back fact persisted")
}
if err := r.UpsertAlias(ctx, Alias{EntityID: entityID, SourceID: sourceA, ExternalType: "container", ExternalID: "abc"}); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `DELETE FROM entities WHERE id=$1`, entityID); err != nil {
t.Fatal(err)
}
var aliases, facts, overrides int
if err := pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM entity_aliases WHERE entity_id=$1),(SELECT count(*) FROM entity_facts WHERE entity_id=$1),(SELECT count(*) FROM entity_overrides WHERE entity_id=$1)`, entityID).Scan(&aliases, &facts, &overrides); err != nil {
t.Fatal(err)
}
if aliases != 0 || facts != 0 || overrides != 0 {
t.Fatalf("cascade counts aliases=%d facts=%d overrides=%d", aliases, facts, overrides)
}
}
func TestInventoryRepositoryTargetScaleAndConcurrentUpserts(t *testing.T) {
dsn := inventoryIntegrationDSN()
if dsn == "" {
t.Skip("PULSE_TEST_DATABASE_URL is not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 10, MinConns: 1})
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := database.Migrate(ctx, pool); err != nil {
t.Fatal(err)
}
r, err := NewRepository(pool)
if err != nil {
t.Fatal(err)
}
_, _ = pool.Exec(ctx, `DELETE FROM entities WHERE canonical_name LIKE 'm2-04-scale-%'`)
now := time.Now().UTC().Truncate(time.Microsecond)
for i := 1; i <= 490; i++ {
id := fmt.Sprintf("00000000-0000-0000-0000-%012d", i)
typeName := "container"
if i > 150 && i <= 190 {
typeName = "disk"
}
if i > 190 {
typeName = "probe"
}
if err := r.UpsertEntity(ctx, Entity{ID: id, EntityType: typeName, CanonicalName: fmt.Sprintf("m2-04-scale-%04d", i), DisplayName: fmt.Sprintf("Scale %04d", i), Status: "unknown", FirstSeenAt: &now, LastSeenAt: &now}); err != nil {
t.Fatal(err)
}
}
var wg sync.WaitGroup
errs := make(chan error, 10)
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
defer wg.Done()
id := "00000000-0000-0000-0000-000000000001"
for j := 0; j < 10; j++ {
if err := r.UpsertEntity(ctx, Entity{ID: id, EntityType: "container", CanonicalName: "m2-04-scale-0001", DisplayName: "Concurrent", Status: "unknown", FirstSeenAt: &now, LastSeenAt: &now}); err != nil {
errs <- err
return
}
}
}()
}
wg.Wait()
close(errs)
for err := range errs {
t.Fatal(err)
}
started := time.Now()
entities, err := r.ListEntities(ctx, 600)
if err != nil {
t.Fatal(err)
}
elapsed := time.Since(started)
if len(entities) < 490 {
t.Fatalf("target-scale entity count=%d", len(entities))
}
if elapsed >= 500*time.Millisecond {
t.Fatalf("target-scale inventory list took %s", elapsed)
}
_, _ = pool.Exec(ctx, `DELETE FROM entities WHERE canonical_name LIKE 'm2-04-scale-%'`)
}
func TestInventoryReadModelPostgreSQL(t *testing.T) {
dsn := inventoryIntegrationDSN()
if dsn == "" {
t.Skip("PULSE_TEST_DATABASE_URL is not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
pool, err := database.NewPool(ctx, database.Config{URL: dsn, MaxConns: 8, MinConns: 1})
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := database.Migrate(ctx, pool); err != nil {
t.Fatal(err)
}
r, err := NewRepository(pool)
if err != nil {
t.Fatal(err)
}
const sourceA = "11000000-0000-0000-0000-0000000000a1"
const sourceB = "11000000-0000-0000-0000-0000000000b1"
_, _ = pool.Exec(ctx, `DELETE FROM entities WHERE canonical_name LIKE 'm11-05-read-%'`)
defer func() {
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cleanupCancel()
if _, cleanupErr := pool.Exec(cleanupCtx, `DELETE FROM entities WHERE canonical_name LIKE 'm11-05-read-%'`); cleanupErr != nil {
t.Errorf("clean read-model integration entities: %v", cleanupErr)
}
}()
for _, source := range []string{sourceA, sourceB} {
if _, err := pool.Exec(ctx, `INSERT INTO data_sources (id,type,name,configuration_ref) VALUES ($1,'exporter',$2,'integration') ON CONFLICT (id) DO UPDATE SET name=EXCLUDED.name`, source, source); err != nil {
t.Fatal(err)
}
}
now := time.Now().UTC().Truncate(time.Microsecond)
for index := 1; index <= 490; index++ {
id := fmt.Sprintf("11000000-0000-0000-0000-%012d", index)
entityType := "container"
if index > 150 && index <= 190 {
entityType = "disk"
} else if index > 190 {
entityType = "probe"
}
if err := r.UpsertEntity(ctx, Entity{ID: id, EntityType: entityType, CanonicalName: fmt.Sprintf("m11-05-read-%03d", index), DisplayName: fmt.Sprintf("Read entity %03d", index), Status: "operational", FirstSeenAt: &now, LastSeenAt: &now}); err != nil {
t.Fatal(err)
}
}
const entityID = "11000000-0000-0000-0000-000000000001"
const peerID = "11000000-0000-0000-0000-000000000002"
staleAt := now.Add(-time.Minute)
if err := r.UpsertFact(ctx, Fact{EntityID: entityID, FieldName: "image", SourceID: sourceA, Value: []byte(`"discovered-a"`), ObservedAt: now, Confidence: 1}); err != nil {
t.Fatal(err)
}
if err := r.UpsertFact(ctx, Fact{EntityID: entityID, FieldName: "image", SourceID: sourceB, Value: []byte(`"discovered-b"`), ObservedAt: now.Add(-time.Hour), Confidence: .8, ValidUntil: &staleAt}); err != nil {
t.Fatal(err)
}
if err := r.UpsertOverride(ctx, Override{EntityID: entityID, FieldName: "image", Value: []byte(`"manual"`), UpdatedAt: now}); err != nil {
t.Fatal(err)
}
if err := r.UpsertOverride(ctx, Override{EntityID: entityID, FieldName: "status", Value: []byte(`"attention"`), UpdatedAt: now}); err != nil {
t.Fatal(err)
}
if err := r.UpsertRelation(ctx, Relation{ID: "11000000-0000-0000-0000-0000000000f1", SourceEntityID: entityID, TargetEntityID: peerID, RelationType: "depends_on", SourceID: sourceA, Confidence: .9, FirstSeenAt: &now}); err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
filter := EntityFilter{Limit: 17, Search: "m11-05-read-", Direction: "asc"}
var slowestPage time.Duration
for {
started := time.Now()
page, err := r.SearchEntities(ctx, filter)
if elapsed := time.Since(started); elapsed > slowestPage {
slowestPage = elapsed
}
if err != nil {
t.Fatal(err)
}
hasMore := len(page) > filter.Limit
if hasMore {
page = page[:filter.Limit]
}
for _, item := range page {
if seen[item.ID] {
t.Fatalf("duplicate paginated entity %s", item.ID)
}
seen[item.ID] = true
}
if !hasMore {
break
}
last := page[len(page)-1]
filter.AfterName, filter.AfterID = strings.ToLower(last.DisplayName), last.ID
}
if len(seen) != 490 {
t.Fatalf("paginated entities=%d want=490", len(seen))
}
if slowestPage >= 500*time.Millisecond {
t.Fatalf("slowest read-model target page took %s", slowestPage)
}
disks, err := r.SearchEntities(ctx, EntityFilter{Limit: 100, Search: "m11-05-read-", EntityType: "disk", Direction: "desc"})
if err != nil || len(disks) != 40 {
t.Fatalf("disk filter count=%d err=%v", len(disks), err)
}
attention, err := r.SearchEntities(ctx, EntityFilter{Limit: 10, Search: "m11-05-read-001", Status: "attention", Direction: "asc"})
if err != nil || len(attention) != 1 || attention[0].Status != "attention" {
t.Fatalf("effective status filter=%+v err=%v", attention, err)
}
detail, err := r.GetEntityDetail(ctx, entityID)
if err != nil {
t.Fatal(err)
}
if len(detail.Facts) != 2 || detail.Entity.SourceCount != 2 || detail.Entity.StaleFactCount != 1 || len(detail.Relations) != 1 {
t.Fatalf("detail projection=%+v", detail)
}
if len(detail.Effective) != 2 || detail.Effective[0].Origin != "override" || string(detail.Effective[0].Value) != `"manual"` {
t.Fatalf("effective values=%+v", detail.Effective)
}
if err := r.PersistDiscovery(ctx, Entity{ID: entityID, EntityType: "container", CanonicalName: "m11-05-read-001", DisplayName: "Discovery rerun", Status: "operational", FirstSeenAt: &now, LastSeenAt: &now}, Alias{EntityID: entityID, SourceID: sourceA, ExternalType: "container", ExternalID: "read-001"}, []Fact{{EntityID: entityID, FieldName: "image", SourceID: sourceA, Value: []byte(`"new-discovery"`), ObservedAt: now.Add(time.Minute), Confidence: 1}}, nil); err != nil {
t.Fatal(err)
}
detail, err = r.GetEntityDetail(ctx, entityID)
if err != nil || detail.Effective[0].Origin != "override" || string(detail.Effective[0].Value) != `"manual"` {
t.Fatalf("discovery overwrote override: detail=%+v err=%v", detail, err)
}
}
func inventoryIntegrationDSN() string {
if dsn := os.Getenv("PULSE_TEST_DATABASE_URL"); dsn != "" {
return dsn
}
return os.Getenv("PULSE_DATABASE_URL")
}
+53
View File
@@ -0,0 +1,53 @@
package inventory
import (
"errors"
"strings"
"time"
)
type Entity struct {
ID, EntityType, CanonicalName, DisplayName, Status string
FirstSeenAt, LastSeenAt, TombstonedAt *time.Time
}
type Alias struct{ EntityID, SourceID, ExternalType, ExternalID string }
type Fact struct {
EntityID, FieldName, SourceID string
Value []byte
ObservedAt time.Time
Confidence float64
ValidUntil *time.Time
}
type Override struct {
EntityID, FieldName, UserID string
Value []byte
UpdatedAt time.Time
}
type Relation struct {
ID, SourceEntityID, RelationType, TargetEntityID, SourceID string
Confidence float64
Confirmed bool
FirstSeenAt, LastSeenAt, TombstonedAt *time.Time
}
func (a Alias) Validate() error {
if strings.TrimSpace(a.EntityID) == "" || strings.TrimSpace(a.SourceID) == "" || strings.TrimSpace(a.ExternalType) == "" || strings.TrimSpace(a.ExternalID) == "" {
return errors.New("alias identity fields are required")
}
if len(a.ExternalType) > 80 || len(a.ExternalID) > 512 {
return errors.New("alias fields exceed bounds")
}
return nil
}
func (f Fact) Validate() error {
if f.EntityID == "" || f.FieldName == "" || f.SourceID == "" || len(f.Value) == 0 || f.Confidence < 0 || f.Confidence > 1 || f.ObservedAt.IsZero() {
return errors.New("invalid entity fact")
}
return nil
}
func (o Override) Validate() error {
if o.EntityID == "" || o.FieldName == "" || len(o.Value) == 0 {
return errors.New("invalid entity override")
}
return nil
}