Publish ITWorx Pulse source

2026-09-03 02:09:19 +02:00
commit bd774932d5
614 changed files with 77116 additions and 0 deletions
@@ -0,0 +1,382 @@
+package share
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"strings"
+	"time"
+)
+
+const ContractVersion = "v1"
+
+const (
+	StateHealthy  = "healthy"
+	StateUnknown  = "unknown"
+	Fresh         = "fresh"
+	Stale         = "stale"
+	Unavailable   = "unavailable"
+	SizeAvailable = "available"
+	SizeCached    = "cached"
+	SizeStale     = "stale"
+	SizeUnknown   = "unknown"
+)
+
+type Limits struct {
+	MaxShares     int
+	MaxHistory    int
+	MaxPlacements int
+}
+
+func (l Limits) withDefaults() Limits {
+	if l.MaxShares == 0 {
+		l.MaxShares = 256
+	}
+	if l.MaxHistory == 0 {
+		l.MaxHistory = 128
+	}
+	if l.MaxPlacements == 0 {
+		l.MaxPlacements = 32
+	}
+	return l
+}
+func (l Limits) Validate() error {
+	if l.MaxShares < 1 || l.MaxShares > 1024 || l.MaxHistory < 1 || l.MaxHistory > 512 || l.MaxPlacements < 1 || l.MaxPlacements > 128 {
+		return errors.New("share limits are outside safe bounds")
+	}
+	return nil
+}
+
+type ScanPolicy struct {
+	MaxSharesPerRun int
+	CacheTTL        time.Duration
+	FreshnessMaxAge time.Duration
+}
+
+func (p ScanPolicy) withDefaults() ScanPolicy {
+	if p.MaxSharesPerRun == 0 {
+		p.MaxSharesPerRun = 32
+	}
+	if p.CacheTTL == 0 {
+		p.CacheTTL = 5 * time.Minute
+	}
+	if p.FreshnessMaxAge == 0 {
+		p.FreshnessMaxAge = 15 * time.Minute
+	}
+	return p
+}
+func (p ScanPolicy) Validate() error {
+	if p.MaxSharesPerRun < 1 || p.MaxSharesPerRun > 256 || p.CacheTTL <= 0 || p.CacheTTL > 24*time.Hour || p.FreshnessMaxAge <= 0 || p.FreshnessMaxAge > 7*24*time.Hour {
+		return errors.New("share scan policy is outside safe bounds")
+	}
+	return nil
+}
+
+type Policy struct {
+	FreshnessMaxAge time.Duration
+	Scan            ScanPolicy
+}
+
+func (p Policy) withDefaults() Policy {
+	if p.FreshnessMaxAge == 0 {
+		p.FreshnessMaxAge = 2 * time.Minute
+	}
+	p.Scan = p.Scan.withDefaults()
+	return p
+}
+func (p Policy) Validate() error {
+	if p.FreshnessMaxAge <= 0 || p.FreshnessMaxAge > 24*time.Hour || p.Scan.Validate() != nil {
+		return errors.New("share policy is outside safe bounds")
+	}
+	return nil
+}
+
+type Source struct {
+	ID                string    `json:"id"`
+	Type              string    `json:"type"`
+	CapabilityVersion string    `json:"capabilityVersion"`
+	ObservedAt        time.Time `json:"observedAt"`
+	ReceivedAt        time.Time `json:"receivedAt"`
+	Freshness         string    `json:"freshness"`
+	State             string    `json:"state"`
+	Reason            string    `json:"reason,omitempty"`
+}
+type StoragePolicy struct {
+	Allocation  string `json:"allocation,omitempty"`
+	CachePolicy string `json:"cachePolicy,omitempty"`
+	PrimaryPool string `json:"primaryPool,omitempty"`
+	CachePool   string `json:"cachePool,omitempty"`
+}
+type RawPlacement struct {
+	PoolID string `json:"poolId"`
+	Bytes  uint64 `json:"bytes"`
+}
+type Placement struct {
+	PoolID string `json:"poolId"`
+	Bytes  uint64 `json:"bytes"`
+}
+type RawGrowthPoint struct {
+	ObservedAt time.Time `json:"observedAt"`
+	UsedBytes  uint64    `json:"usedBytes"`
+}
+type GrowthPoint struct {
+	ObservedAt      time.Time `json:"observedAt"`
+	UsedBytes       uint64    `json:"usedBytes"`
+	DeltaBytes      int64     `json:"deltaBytes"`
+	RateBytesPerDay float64   `json:"rateBytesPerDay"`
+}
+type RawShare struct {
+	ID             string           `json:"id"`
+	Name           string           `json:"name"`
+	StoragePolicy  StoragePolicy    `json:"storagePolicy"`
+	UsedBytes      uint64           `json:"usedBytes"`
+	SizeObservedAt time.Time        `json:"sizeObservedAt"`
+	SizeState      string           `json:"sizeState"`
+	Placements     []RawPlacement   `json:"placements,omitempty"`
+	GrowthHistory  []RawGrowthPoint `json:"growthHistory,omitempty"`
+}
+type Share struct {
+	ID             string        `json:"id"`
+	Name           string        `json:"name"`
+	StoragePolicy  StoragePolicy `json:"storagePolicy"`
+	UsedBytes      uint64        `json:"usedBytes"`
+	SizeObservedAt time.Time     `json:"sizeObservedAt"`
+	SizeState      string        `json:"sizeState"`
+	Placements     []Placement   `json:"placements,omitempty"`
+	GrowthHistory  []GrowthPoint `json:"growthHistory,omitempty"`
+}
+type ScanPlan struct {
+	GeneratedAt     time.Time `json:"generatedAt"`
+	MaxSharesPerRun int       `json:"maxSharesPerRun"`
+	DueShareIDs     []string  `json:"dueShareIds,omitempty"`
+	DeferredCount   int       `json:"deferredCount"`
+	CacheTTLSeconds int       `json:"cacheTtlSeconds"`
+}
+type RawSnapshot struct {
+	Source     Source     `json:"source"`
+	Shares     []RawShare `json:"shares"`
+	ObservedAt time.Time  `json:"observedAt"`
+	ReceivedAt time.Time  `json:"receivedAt"`
+}
+type Snapshot struct {
+	ContractVersion string    `json:"contractVersion"`
+	Source          Source    `json:"source"`
+	Shares          []Share   `json:"shares"`
+	Total           int       `json:"total"`
+	Scan            ScanPlan  `json:"scan"`
+	ObservedAt      time.Time `json:"observedAt"`
+	ReceivedAt      time.Time `json:"receivedAt"`
+}
+type Provider interface {
+	Snapshot(context.Context) (Snapshot, error)
+}
+type RawProvider interface {
+	Snapshot(context.Context) (RawSnapshot, error)
+}
+type Adapter struct {
+	Source RawProvider
+	Limits Limits
+	Policy Policy
+	Now    func() time.Time
+}
+
+func (a Adapter) Snapshot(ctx context.Context) (Snapshot, error) {
+	if err := ctx.Err(); err != nil {
+		return Snapshot{}, err
+	}
+	if a.Source == nil {
+		return UnknownSnapshot(time.Now().UTC(), "shares", "unraid", "source_unavailable"), nil
+	}
+	raw, err := a.Source.Snapshot(ctx)
+	if err != nil {
+		return Snapshot{}, err
+	}
+	now := time.Now().UTC()
+	if a.Now != nil {
+		now = a.Now()
+	}
+	return Normalize(raw, now, a.Limits, a.Policy)
+}
+func UnknownSnapshot(now time.Time, id, typ, reason string) Snapshot {
+	if now.IsZero() {
+		now = time.Now().UTC()
+	}
+	now = now.UTC()
+	return Snapshot{ContractVersion: ContractVersion, Source: Source{ID: id, Type: typ, CapabilityVersion: ContractVersion, ReceivedAt: now, Freshness: Unavailable, State: StateUnknown, Reason: reason}, Shares: []Share{}, Scan: ScanPlan{GeneratedAt: now}, ObservedAt: now, ReceivedAt: now}
+}
+
+func Normalize(raw RawSnapshot, now time.Time, limits Limits, policy Policy) (Snapshot, error) {
+	limits = limits.withDefaults()
+	policy = policy.withDefaults()
+	if err := limits.Validate(); err != nil {
+		return Snapshot{}, err
+	}
+	if err := policy.Validate(); err != nil {
+		return Snapshot{}, err
+	}
+	if now.IsZero() {
+		now = time.Now().UTC()
+	}
+	if raw.ReceivedAt.IsZero() {
+		raw.ReceivedAt = now
+	}
+	if raw.ObservedAt.IsZero() {
+		raw.ObservedAt = raw.ReceivedAt
+	}
+	if raw.ObservedAt.After(now.Add(time.Minute)) {
+		return Snapshot{}, errors.New("share observation is materially in the future")
+	}
+	if len(raw.Shares) > limits.MaxShares {
+		return Snapshot{}, errors.New("share payload exceeds bounds")
+	}
+	shares := make([]Share, 0, len(raw.Shares))
+	for _, item := range raw.Shares {
+		normalized, err := normalizeShare(item, now, limits, policy)
+		if err != nil {
+			return Snapshot{}, err
+		}
+		shares = append(shares, normalized)
+	}
+	sort.Slice(shares, func(i, j int) bool {
+		if shares[i].Name != shares[j].Name {
+			return shares[i].Name < shares[j].Name
+		}
+		return shares[i].ID < shares[j].ID
+	})
+	source := raw.Source
+	if source.ID == "" {
+		source.ID = "shares"
+	}
+	if source.Type == "" {
+		source.Type = "unraid"
+	}
+	if source.CapabilityVersion == "" {
+		source.CapabilityVersion = ContractVersion
+	}
+	source.ObservedAt = raw.ObservedAt.UTC()
+	source.ReceivedAt = raw.ReceivedAt.UTC()
+	source.Freshness = Fresh
+	source.State = StateHealthy
+	if now.Sub(raw.ObservedAt) > policy.FreshnessMaxAge {
+		source.Freshness = Stale
+		source.State = StateUnknown
+		source.Reason = "stale_source"
+		for i := range shares {
+			shares[i].SizeState = SizeUnknown
+		}
+	}
+	scan := PlanScan(now, shares, policy.Scan)
+	return Snapshot{ContractVersion: ContractVersion, Source: source, Shares: shares, Total: len(shares), Scan: scan, ObservedAt: raw.ObservedAt.UTC(), ReceivedAt: raw.ReceivedAt.UTC()}, nil
+}
+
+func normalizeShare(item RawShare, now time.Time, limits Limits, policy Policy) (Share, error) {
+	if strings.TrimSpace(item.ID) == "" || strings.TrimSpace(item.Name) == "" || len(item.ID) > 128 || len(item.Name) > 255 {
+		return Share{}, errors.New("share identity is invalid")
+	}
+	if len(item.Placements) > limits.MaxPlacements || len(item.GrowthHistory) > limits.MaxHistory {
+		return Share{}, errors.New("share collection exceeds bounds")
+	}
+	sizeState := normalizeSizeState(item.SizeState)
+	if item.SizeObservedAt.IsZero() {
+		sizeState = SizeUnknown
+	} else if item.SizeObservedAt.After(now.Add(time.Minute)) {
+		return Share{}, errors.New("share size timestamp is invalid")
+	} else if now.Sub(item.SizeObservedAt) > policy.Scan.FreshnessMaxAge && sizeState != SizeUnknown {
+		sizeState = SizeStale
+	}
+	result := Share{ID: item.ID, Name: item.Name, StoragePolicy: normalizeStoragePolicy(item.StoragePolicy), UsedBytes: item.UsedBytes, SizeObservedAt: item.SizeObservedAt.UTC(), SizeState: sizeState}
+	result.Placements = make([]Placement, 0, len(item.Placements))
+	for _, placement := range item.Placements {
+		if strings.TrimSpace(placement.PoolID) == "" || len(placement.PoolID) > 128 {
+			return Share{}, errors.New("share placement identity is invalid")
+		}
+		result.Placements = append(result.Placements, Placement(placement))
+	}
+	sort.Slice(result.Placements, func(i, j int) bool { return result.Placements[i].PoolID < result.Placements[j].PoolID })
+	points := make([]RawGrowthPoint, len(item.GrowthHistory))
+	copy(points, item.GrowthHistory)
+	sort.SliceStable(points, func(i, j int) bool { return points[i].ObservedAt.Before(points[j].ObservedAt) })
+	result.GrowthHistory = make([]GrowthPoint, 0, len(points))
+	for index, point := range points {
+		if point.ObservedAt.IsZero() || point.ObservedAt.After(now.Add(time.Minute)) {
+			return Share{}, errors.New("share growth timestamp is invalid")
+		}
+		growth := GrowthPoint{ObservedAt: point.ObservedAt.UTC(), UsedBytes: point.UsedBytes}
+		if index > 0 {
+			previous := points[index-1]
+			duration := point.ObservedAt.Sub(previous.ObservedAt)
+			if duration > 0 {
+				delta := signedDelta(point.UsedBytes, previous.UsedBytes)
+				growth.DeltaBytes = delta
+				growth.RateBytesPerDay = float64(delta) / (duration.Hours() / 24)
+			}
+		}
+		result.GrowthHistory = append(result.GrowthHistory, growth)
+	}
+	return result, nil
+}
+func signedDelta(current, previous uint64) int64 {
+	if current >= previous {
+		delta := current - previous
+		if delta > uint64(1<<63-1) {
+			return int64(1<<63 - 1)
+		}
+		return int64(delta)
+	}
+	delta := previous - current
+	if delta > uint64(1<<63) {
+		return -int64(1<<63-1) - 1
+	}
+	return -int64(delta)
+}
+func normalizeStoragePolicy(value StoragePolicy) StoragePolicy {
+	return StoragePolicy{Allocation: bounded(value.Allocation, "unknown"), CachePolicy: bounded(value.CachePolicy, "unknown"), PrimaryPool: bounded(value.PrimaryPool, ""), CachePool: bounded(value.CachePool, "")}
+}
+func normalizeSizeState(value string) string {
+	switch strings.ToLower(strings.TrimSpace(value)) {
+	case SizeAvailable:
+		return SizeAvailable
+	case SizeCached:
+		return SizeCached
+	case SizeStale:
+		return SizeStale
+	default:
+		return SizeUnknown
+	}
+}
+func PlanScan(now time.Time, shares []Share, policy ScanPolicy) ScanPlan {
+	policy = policy.withDefaults()
+	if now.IsZero() {
+		now = time.Now().UTC()
+	}
+	candidates := make([]string, 0, len(shares))
+	for _, item := range shares {
+		if item.SizeState == SizeUnknown || item.SizeObservedAt.IsZero() || now.Sub(item.SizeObservedAt) >= policy.CacheTTL {
+			candidates = append(candidates, item.ID)
+		}
+	}
+	sort.Strings(candidates)
+	due := candidates
+	if len(due) > policy.MaxSharesPerRun {
+		due = due[:policy.MaxSharesPerRun]
+	}
+	return ScanPlan{GeneratedAt: now.UTC(), MaxSharesPerRun: policy.MaxSharesPerRun, DueShareIDs: due, DeferredCount: len(candidates) - len(due), CacheTTLSeconds: int(policy.CacheTTL / time.Second)}
+}
+func bounded(value, fallback string) string {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return fallback
+	}
+	if len(value) > 128 {
+		return value[:128]
+	}
+	return value
+}
+func ShareByID(snapshot Snapshot, id string) (Share, bool) {
+	for _, item := range snapshot.Shares {
+		if item.ID == id {
+			return item, true
+		}
+	}
+	return Share{}, false
+}
@@ -0,0 +1,71 @@
+package share
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+)
+
+type rawProvider struct {
+	snapshot RawSnapshot
+	err      error
+}
+
+func (p rawProvider) Snapshot(context.Context) (RawSnapshot, error) { return p.snapshot, p.err }
+
+func baseRaw(now time.Time) RawSnapshot {
+	return RawSnapshot{Source: Source{ID: "fixture-shares", Type: "fixture"}, Shares: []RawShare{{ID: "share-media", Name: "Media", StoragePolicy: StoragePolicy{Allocation: "high-water", CachePolicy: "prefer", PrimaryPool: "array", CachePool: "cache"}, UsedBytes: 700, SizeObservedAt: now, SizeState: SizeCached, Placements: []RawPlacement{{PoolID: "cache", Bytes: 200}, {PoolID: "array", Bytes: 500}}, GrowthHistory: []RawGrowthPoint{{ObservedAt: now.Add(-24 * time.Hour), UsedBytes: 500}, {ObservedAt: now, UsedBytes: 700}}}}, ObservedAt: now, ReceivedAt: now}
+}
+
+func TestNormalizePreservesPolicyPlacementsAndAccurateGrowth(t *testing.T) {
+	now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
+	got, err := Normalize(baseRaw(now), now, Limits{}, Policy{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	item := got.Shares[0]
+	if item.StoragePolicy.CachePool != "cache" || item.Placements[0].PoolID != "array" || item.GrowthHistory[1].DeltaBytes != 200 || item.GrowthHistory[1].RateBytesPerDay != 200 {
+		t.Fatalf("share=%+v", item)
+	}
+}
+func TestPlanScanBoundsExpensiveRefreshes(t *testing.T) {
+	now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
+	shares := make([]Share, 0, 64)
+	for i := 0; i < 64; i++ {
+		shares = append(shares, Share{ID: fmt.Sprintf("share-%02d", i), Name: fmt.Sprintf("Share %02d", i), SizeState: SizeUnknown})
+	}
+	plan := PlanScan(now, shares, ScanPolicy{MaxSharesPerRun: 16, CacheTTL: time.Minute})
+	if len(plan.DueShareIDs) != 16 || plan.DeferredCount != 48 || plan.CacheTTLSeconds != 60 {
+		t.Fatalf("plan=%+v", plan)
+	}
+	if plan.DueShareIDs[0] != "share-00" || plan.DueShareIDs[15] != "share-15" {
+		t.Fatalf("ids=%v", plan.DueShareIDs)
+	}
+}
+func TestNormalizeStaleAndInvalidContentBoundaries(t *testing.T) {
+	now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
+	raw := baseRaw(now.Add(-3 * time.Minute))
+	got, err := Normalize(raw, now, Limits{}, Policy{FreshnessMaxAge: time.Minute})
+	if err != nil || got.Source.State != StateUnknown || got.Shares[0].SizeState != SizeUnknown {
+		t.Fatalf("stale=%+v err=%v", got, err)
+	}
+	raw = baseRaw(now)
+	raw.Shares[0].GrowthHistory[1].ObservedAt = now.Add(2 * time.Hour)
+	if _, err = Normalize(raw, now, Limits{}, Policy{}); err == nil {
+		t.Fatal("expected future growth validation")
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	_, err = (Adapter{Source: rawProvider{snapshot: raw}}).Snapshot(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("err=%v", err)
+	}
+}
+func TestAdapterUnknownFallback(t *testing.T) {
+	got, err := (Adapter{Now: func() time.Time { return time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC) }}).Snapshot(context.Background())
+	if err != nil || got.Source.Reason != "source_unavailable" || got.Source.State != StateUnknown {
+		t.Fatalf("snapshot=%+v err=%v", got, err)
+	}
+}