ITWorx Pulse release export
bd774932d5
Public source validation / validate (push) Failing after 3m8s
302 lines
13 KiB
Go
302 lines
13 KiB
Go
package inventory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgconn"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
type Repository struct{ pool *pgxpool.Pool }
|
|
|
|
// PersistDiscovery atomically applies one source snapshot. It updates discovered
|
|
// entities/facts/aliases/relations only; user overrides are deliberately untouched.
|
|
func (r *Repository) PersistDiscovery(ctx context.Context, entity Entity, alias Alias, facts []Fact, relations []Relation) error {
|
|
return r.InTx(ctx, func(txctx context.Context, tx pgx.Tx) error {
|
|
if err := validateEntity(entity); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.Exec(txctx, `INSERT INTO entities (id,entity_type,canonical_name,display_name,status,first_seen_at,last_seen_at,tombstoned_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT (id) DO UPDATE SET entity_type=EXCLUDED.entity_type,canonical_name=EXCLUDED.canonical_name,display_name=EXCLUDED.display_name,status=EXCLUDED.status,last_seen_at=EXCLUDED.last_seen_at,tombstoned_at=EXCLUDED.tombstoned_at`, entity.ID, entity.EntityType, entity.CanonicalName, entity.DisplayName, entity.Status, entity.FirstSeenAt, entity.LastSeenAt, entity.TombstonedAt); err != nil {
|
|
return fmt.Errorf("persist discovery entity: %w", err)
|
|
}
|
|
if err := upsertAlias(txctx, tx, alias); err != nil {
|
|
return err
|
|
}
|
|
for _, fact := range facts {
|
|
if err := upsertFact(txctx, tx, fact); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for _, relation := range relations {
|
|
if err := upsertRelation(txctx, tx, relation); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func NewRepository(pool *pgxpool.Pool) (*Repository, error) {
|
|
if pool == nil {
|
|
return nil, errors.New("inventory repository requires a database pool")
|
|
}
|
|
return &Repository{pool: pool}, nil
|
|
}
|
|
|
|
func (r *Repository) InTx(ctx context.Context, fn func(context.Context, pgx.Tx) error) error {
|
|
tx, err := r.pool.BeginTx(ctx, pgx.TxOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("begin inventory transaction: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback(ctx) }()
|
|
if err := fn(ctx, tx); err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return fmt.Errorf("commit inventory transaction: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Repository) CreateEntity(ctx context.Context, entity Entity) error {
|
|
if err := validateEntity(entity); err != nil {
|
|
return err
|
|
}
|
|
_, err := r.pool.Exec(ctx, `INSERT INTO entities (id, entity_type, canonical_name, display_name, status, first_seen_at, last_seen_at, tombstoned_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`, entity.ID, entity.EntityType, entity.CanonicalName, entity.DisplayName, entity.Status, entity.FirstSeenAt, entity.LastSeenAt, entity.TombstonedAt)
|
|
if err != nil {
|
|
return fmt.Errorf("create entity: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Repository) UpsertEntity(ctx context.Context, entity Entity) error {
|
|
if err := validateEntity(entity); err != nil {
|
|
return err
|
|
}
|
|
_, err := r.pool.Exec(ctx, `INSERT INTO entities (id, entity_type, canonical_name, display_name, status, first_seen_at, last_seen_at, tombstoned_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT (id) DO UPDATE SET entity_type=EXCLUDED.entity_type, canonical_name=EXCLUDED.canonical_name, display_name=EXCLUDED.display_name, status=EXCLUDED.status, last_seen_at=EXCLUDED.last_seen_at, tombstoned_at=EXCLUDED.tombstoned_at`, entity.ID, entity.EntityType, entity.CanonicalName, entity.DisplayName, entity.Status, entity.FirstSeenAt, entity.LastSeenAt, entity.TombstonedAt)
|
|
if err != nil {
|
|
return fmt.Errorf("upsert entity: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Repository) GetEntity(ctx context.Context, id string) (Entity, error) {
|
|
var e Entity
|
|
err := r.pool.QueryRow(ctx, `SELECT id, entity_type, canonical_name, display_name, status, first_seen_at, last_seen_at, tombstoned_at FROM entities WHERE id=$1`, id).Scan(&e.ID, &e.EntityType, &e.CanonicalName, &e.DisplayName, &e.Status, &e.FirstSeenAt, &e.LastSeenAt, &e.TombstonedAt)
|
|
if err != nil {
|
|
return Entity{}, fmt.Errorf("get entity: %w", err)
|
|
}
|
|
return e, nil
|
|
}
|
|
|
|
func (r *Repository) ListEntities(ctx context.Context, limit int) ([]Entity, error) {
|
|
if limit < 1 || limit > 100000 {
|
|
return nil, errors.New("entity list limit must be between 1 and 100000")
|
|
}
|
|
rows, err := r.pool.Query(ctx, `SELECT id, entity_type, canonical_name, display_name, status, first_seen_at, last_seen_at, tombstoned_at FROM entities ORDER BY canonical_name ASC, id ASC LIMIT $1`, limit)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list entities: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
result := make([]Entity, 0)
|
|
for rows.Next() {
|
|
var e Entity
|
|
if err := rows.Scan(&e.ID, &e.EntityType, &e.CanonicalName, &e.DisplayName, &e.Status, &e.FirstSeenAt, &e.LastSeenAt, &e.TombstonedAt); err != nil {
|
|
return nil, fmt.Errorf("scan entity: %w", err)
|
|
}
|
|
result = append(result, e)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("list entities rows: %w", err)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (r *Repository) ListEntitiesAfter(ctx context.Context, limit int, afterName, afterID string) ([]Entity, error) {
|
|
if limit < 1 || limit > 100 {
|
|
return nil, errors.New("entity page limit must be between 1 and 100")
|
|
}
|
|
rows, err := r.pool.Query(ctx, `SELECT id, entity_type, canonical_name, display_name, status, first_seen_at, last_seen_at, tombstoned_at FROM entities WHERE ($1 = '' OR canonical_name > $1 OR (canonical_name = $1 AND id::text > $2)) ORDER BY canonical_name ASC, id ASC LIMIT $3`, afterName, afterID, limit)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list entity page: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
result := make([]Entity, 0, limit)
|
|
for rows.Next() {
|
|
var e Entity
|
|
if err := rows.Scan(&e.ID, &e.EntityType, &e.CanonicalName, &e.DisplayName, &e.Status, &e.FirstSeenAt, &e.LastSeenAt, &e.TombstonedAt); err != nil {
|
|
return nil, fmt.Errorf("scan entity page: %w", err)
|
|
}
|
|
result = append(result, e)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("list entity page rows: %w", err)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (r *Repository) UpsertAlias(ctx context.Context, alias Alias) error {
|
|
return upsertAlias(ctx, r.pool, alias)
|
|
}
|
|
|
|
func (r *Repository) ListFacts(ctx context.Context, entityID string) ([]Fact, error) {
|
|
rows, err := r.pool.Query(ctx, `SELECT entity_id,field_name,source_id,value,observed_at,confidence,valid_until FROM entity_facts WHERE entity_id=$1 ORDER BY field_name ASC, source_id ASC`, entityID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list entity facts: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
result := make([]Fact, 0)
|
|
for rows.Next() {
|
|
var f Fact
|
|
if err := rows.Scan(&f.EntityID, &f.FieldName, &f.SourceID, &f.Value, &f.ObservedAt, &f.Confidence, &f.ValidUntil); err != nil {
|
|
return nil, fmt.Errorf("scan entity fact: %w", err)
|
|
}
|
|
result = append(result, f)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("list entity facts rows: %w", err)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (r *Repository) GetOverride(ctx context.Context, entityID, field string) (Override, error) {
|
|
var o Override
|
|
err := r.pool.QueryRow(ctx, `SELECT entity_id,field_name,value,COALESCE(user_id::text,''),updated_at FROM entity_overrides WHERE entity_id=$1 AND field_name=$2`, entityID, field).Scan(&o.EntityID, &o.FieldName, &o.Value, &o.UserID, &o.UpdatedAt)
|
|
if err != nil {
|
|
return Override{}, fmt.Errorf("get entity override: %w", err)
|
|
}
|
|
return o, nil
|
|
}
|
|
|
|
func (r *Repository) ListRelations(ctx context.Context, entityID string) ([]Relation, error) {
|
|
rows, err := r.pool.Query(ctx, `SELECT id,source_entity_id,relation_type,target_entity_id,source_id,confidence,confirmed,first_seen_at,last_seen_at,tombstoned_at FROM entity_relations WHERE source_entity_id=$1 OR target_entity_id=$1 ORDER BY relation_type ASC,id ASC`, entityID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list entity relations: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
result := make([]Relation, 0)
|
|
for rows.Next() {
|
|
var rel Relation
|
|
if err := rows.Scan(&rel.ID, &rel.SourceEntityID, &rel.RelationType, &rel.TargetEntityID, &rel.SourceID, &rel.Confidence, &rel.Confirmed, &rel.FirstSeenAt, &rel.LastSeenAt, &rel.TombstonedAt); err != nil {
|
|
return nil, fmt.Errorf("scan entity relation: %w", err)
|
|
}
|
|
result = append(result, rel)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("list entity relations rows: %w", err)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (r *Repository) DeleteFact(ctx context.Context, entityID, fieldName, sourceID string) error {
|
|
_, err := r.pool.Exec(ctx, `DELETE FROM entity_facts WHERE entity_id=$1 AND field_name=$2 AND source_id=$3`, entityID, fieldName, sourceID)
|
|
if err != nil {
|
|
return fmt.Errorf("delete entity fact: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
func (r *Repository) DeleteRelation(ctx context.Context, sourceEntityID, relationType, targetEntityID, sourceID string) error {
|
|
_, err := r.pool.Exec(ctx, `DELETE FROM entity_relations WHERE source_entity_id=$1 AND relation_type=$2 AND target_entity_id=$3 AND source_id=$4`, sourceEntityID, relationType, targetEntityID, sourceID)
|
|
if err != nil {
|
|
return fmt.Errorf("delete entity relation: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
func (r *Repository) UpsertFact(ctx context.Context, fact Fact) error {
|
|
return upsertFact(ctx, r.pool, fact)
|
|
}
|
|
func (r *Repository) UpsertOverride(ctx context.Context, override Override) error {
|
|
return upsertOverride(ctx, r.pool, override)
|
|
}
|
|
func (r *Repository) DeleteOverride(ctx context.Context, entityID, field string) error {
|
|
_, err := r.pool.Exec(ctx, `DELETE FROM entity_overrides WHERE entity_id=$1 AND field_name=$2`, entityID, field)
|
|
if err != nil {
|
|
return fmt.Errorf("delete override: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
func (r *Repository) UpsertRelation(ctx context.Context, relation Relation) error {
|
|
return upsertRelation(ctx, r.pool, relation)
|
|
}
|
|
|
|
func upsertAlias(ctx context.Context, q interface {
|
|
Exec(context.Context, string, ...any) (pgconn.CommandTag, error)
|
|
}, a Alias) error {
|
|
if err := a.Validate(); err != nil {
|
|
return err
|
|
}
|
|
_, err := q.Exec(ctx, `INSERT INTO entity_aliases (entity_id,source_id,external_type,external_id) VALUES ($1,$2,$3,$4) ON CONFLICT (source_id,external_type,external_id) DO UPDATE SET entity_id=EXCLUDED.entity_id`, a.EntityID, a.SourceID, a.ExternalType, a.ExternalID)
|
|
if err != nil {
|
|
return fmt.Errorf("upsert entity alias: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func upsertFact(ctx context.Context, q interface {
|
|
Exec(context.Context, string, ...any) (pgconn.CommandTag, error)
|
|
}, f Fact) error {
|
|
if err := f.Validate(); err != nil {
|
|
return err
|
|
}
|
|
_, err := q.Exec(ctx, `INSERT INTO entity_facts (entity_id,field_name,source_id,value,observed_at,confidence,valid_until) VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT (entity_id,field_name,source_id) DO UPDATE SET value=EXCLUDED.value,observed_at=EXCLUDED.observed_at,confidence=EXCLUDED.confidence,valid_until=EXCLUDED.valid_until`, f.EntityID, f.FieldName, f.SourceID, f.Value, f.ObservedAt, f.Confidence, f.ValidUntil)
|
|
if err != nil {
|
|
return fmt.Errorf("upsert entity fact: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func upsertOverride(ctx context.Context, q interface {
|
|
Exec(context.Context, string, ...any) (pgconn.CommandTag, error)
|
|
}, o Override) error {
|
|
if err := o.Validate(); err != nil {
|
|
return err
|
|
}
|
|
_, err := q.Exec(ctx, `INSERT INTO entity_overrides (entity_id,field_name,value,user_id,updated_at) VALUES ($1,$2,$3,$4,$5) ON CONFLICT (entity_id,field_name) DO UPDATE SET value=EXCLUDED.value,user_id=EXCLUDED.user_id,updated_at=EXCLUDED.updated_at`, o.EntityID, o.FieldName, o.Value, nullableID(o.UserID), o.UpdatedAt)
|
|
if err != nil {
|
|
return fmt.Errorf("upsert entity override: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func nullableID(value string) any {
|
|
if value == "" {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
|
|
func upsertRelation(ctx context.Context, q interface {
|
|
Exec(context.Context, string, ...any) (pgconn.CommandTag, error)
|
|
}, rel Relation) error {
|
|
if rel.ID == "" || rel.SourceEntityID == "" || rel.TargetEntityID == "" || rel.SourceID == "" || rel.RelationType == "" || rel.FirstSeenAt == nil || rel.Confidence < 0 || rel.Confidence > 1 {
|
|
return errors.New("invalid entity relation")
|
|
}
|
|
_, err := q.Exec(ctx, `INSERT INTO entity_relations (id,source_entity_id,relation_type,target_entity_id,source_id,confidence,confirmed,first_seen_at,last_seen_at,tombstoned_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT (source_entity_id,relation_type,target_entity_id,source_id) DO UPDATE SET confidence=EXCLUDED.confidence,confirmed=EXCLUDED.confirmed,last_seen_at=EXCLUDED.last_seen_at,tombstoned_at=EXCLUDED.tombstoned_at`, rel.ID, rel.SourceEntityID, rel.RelationType, rel.TargetEntityID, rel.SourceID, rel.Confidence, rel.Confirmed, rel.FirstSeenAt, rel.LastSeenAt, rel.TombstonedAt)
|
|
if err != nil {
|
|
return fmt.Errorf("upsert entity relation: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateEntity(e Entity) error {
|
|
if e.ID == "" || e.EntityType == "" || e.CanonicalName == "" || e.DisplayName == "" || e.Status == "" || e.FirstSeenAt == nil {
|
|
return errors.New("invalid entity")
|
|
}
|
|
if e.LastSeenAt != nil && e.LastSeenAt.Before(*e.FirstSeenAt) {
|
|
return errors.New("entity last seen precedes first seen")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func MarshalValue(v any) ([]byte, error) {
|
|
b, err := json.Marshal(v)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("marshal inventory value: %w", err)
|
|
}
|
|
return b, nil
|
|
}
|