Publish ITWorx Pulse source

2026-09-03 02:09:19 +02:00
commit bd774932d5
614 changed files with 77116 additions and 0 deletions
@@ -0,0 +1,418 @@
+package systemstatus
+
+import (
+	"sort"
+	"time"
+
+	"github.com/itworx/pulse/internal/buildinfo"
+	"github.com/itworx/pulse/internal/config"
+	"github.com/itworx/pulse/internal/datasource"
+)
+
+type State string
+
+const (
+	StateHealthy  State = "healthy"
+	StateDegraded State = "degraded"
+	StateUnknown  State = "unknown"
+	StateDisabled State = "disabled"
+)
+
+type Component struct {
+	ID            string     `json:"id"`
+	State         State      `json:"state"`
+	Reason        string     `json:"reason"`
+	ObservedAt    *time.Time `json:"observedAt,omitempty"`
+	LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
+}
+
+type Backup struct {
+	State         State      `json:"state"`
+	Reason        string     `json:"reason"`
+	LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
+	AgeSeconds    *int64     `json:"ageSeconds,omitempty"`
+	VerifiedAt    *time.Time `json:"verifiedAt,omitempty"`
+}
+
+type Release struct {
+	Version          string     `json:"version"`
+	Commit           string     `json:"commit"`
+	BuiltAt          *time.Time `json:"builtAt,omitempty"`
+	MigrationVersion string     `json:"migrationVersion"`
+}
+
+type SourceLag struct {
+	SourceID       string     `json:"sourceId"`
+	State          State      `json:"state"`
+	Reason         string     `json:"reason"`
+	LastObservedAt *time.Time `json:"lastObservedAt,omitempty"`
+	AgeSeconds     *int64     `json:"ageSeconds,omitempty"`
+}
+
+type Snapshot struct {
+	Version      string      `json:"version"`
+	Release      Release     `json:"release"`
+	GeneratedAt  time.Time   `json:"generatedAt"`
+	OverallState State       `json:"overallState"`
+	Components   []Component `json:"components"`
+	Backup       Backup      `json:"backup"`
+	SourceLag    []SourceLag `json:"sourceLag"`
+	AuditEvents  *int64      `json:"auditEvents,omitempty"`
+}
+
+// Background job component identifiers. A background job reports its outcome
+// against one of these, and the component keeps its "never reported" Unknown
+// state until a real outcome arrives (ADR-0008).
+const (
+	ComponentWorker        = "worker"
+	ComponentProbes        = "probes"
+	ComponentNotifications = "notifications"
+)
+
+// Job outcome statuses a background job may report.
+const (
+	JobCompleted = "completed"
+	JobFailed    = "failed"
+	JobRunning   = "running"
+	// JobDisabled marks a job that cannot run because the feature it drives is
+	// not configured. It is reported as Disabled, never as Healthy.
+	JobDisabled = "disabled"
+)
+
+// defaultJobMaxAge bounds how old a successful background job run may be before
+// the component it feeds falls back to Unknown. It is deliberately several
+// multiples of the slowest worker job interval so a single slow cycle does not
+// flap the status.
+const defaultJobMaxAge = 5 * time.Minute
+
+// JobHealth is the last reported outcome of one background job. It is produced
+// by the worker runtime (persisted in job_runs) and read back here so the
+// worker, probe and notification components report observed behaviour instead
+// of a hardcoded placeholder.
+type JobHealth struct {
+	Component     string
+	JobKey        string
+	Status        string
+	Reason        string
+	ErrorCode     string
+	LastRunAt     time.Time
+	LastSuccessAt time.Time
+	Duration      time.Duration
+}
+
+type options struct {
+	jobs                 []JobHealth
+	jobMaxAge            time.Duration
+	sources              map[string]SourceHealth
+	backup               *Backup
+	migration            string
+	authenticatedSession bool
+}
+
+const backupMaxAge = 26 * time.Hour
+
+func WithMigrationVersion(version string) Option { return func(o *options) { o.migration = version } }
+
+// WithAuthenticatedSession records request-local proof that the protected
+// status endpoint was reached through a valid Pulse session.
+func WithAuthenticatedSession(authenticated bool) Option {
+	return func(o *options) { o.authenticatedSession = authenticated }
+}
+
+func WithBackupObservation(lastSuccess, verifiedAt time.Time, verificationErr error) Option {
+	return func(o *options) {
+		backup := Backup{State: StateUnknown, Reason: "no_verified_backup"}
+		if verificationErr != nil {
+			backup.State, backup.Reason = StateDegraded, "backup_verification_failed"
+		}
+		if !lastSuccess.IsZero() {
+			value := lastSuccess.UTC()
+			backup.LastSuccessAt = &value
+		}
+		if !verifiedAt.IsZero() {
+			value := verifiedAt.UTC()
+			backup.VerifiedAt = &value
+		}
+		o.backup = &backup
+	}
+}
+
+// SourceHealth is a runtime observation for one configured source or source-backed
+// capability group. The API supplies these observations after querying the real
+// Prometheus adapter and persisted agent snapshots.
+type SourceHealth struct {
+	ID            string
+	State         State
+	Reason        string
+	ObservedAt    time.Time
+	LastSuccessAt time.Time
+}
+
+// Option adjusts how a snapshot is built. Options are variadic so existing
+// callers that have no background job information keep compiling and keep
+// getting the conservative "not recorded" Unknown result.
+type Option func(*options)
+
+// WithJobs supplies the most recent background job outcomes. maxAge bounds how
+// long a successful run stays usable; zero selects the default.
+func WithJobs(maxAge time.Duration, jobs ...JobHealth) Option {
+	return func(o *options) {
+		o.jobs = append(o.jobs, jobs...)
+		if maxAge > 0 {
+			o.jobMaxAge = maxAge
+		}
+	}
+}
+
+// WithSources overrides configuration-only placeholders with observed runtime health.
+// Unknown identifiers are ignored by Build, keeping the public component set closed.
+func WithSources(sources ...SourceHealth) Option {
+	return func(o *options) {
+		if o.sources == nil {
+			o.sources = make(map[string]SourceHealth)
+		}
+		for _, source := range sources {
+			if source.ID != "" {
+				o.sources[source.ID] = source
+			}
+		}
+	}
+}
+
+// FromDatasource maps the shared datasource contract onto the self-observability
+// contract without allowing stale Healthy values to survive EffectiveState.
+func FromDatasource(id string, health datasource.SourceHealth, now time.Time) SourceHealth {
+	state := StateUnknown
+	switch health.EffectiveState(now.UTC()) {
+	case datasource.HealthHealthy:
+		state = StateHealthy
+	case datasource.HealthDegraded:
+		state = StateDegraded
+	case datasource.HealthDisabled:
+		state = StateDisabled
+	}
+	fallback := "source_health_unknown"
+	if state == StateHealthy {
+		fallback = "source_sampled"
+	} else if state == StateDegraded {
+		fallback = "source_degraded"
+	} else if state == StateDisabled {
+		fallback = "not_configured"
+	}
+	return SourceHealth{ID: id, State: state, Reason: reasonOr(health.ReasonCode, fallback), ObservedAt: health.ObservedAt.UTC(), LastSuccessAt: health.LastSuccess.UTC()}
+}
+
+func Build(cfg config.Config, databaseReady bool, now time.Time, auditEvents *int64, opts ...Option) Snapshot {
+	if now.IsZero() {
+		now = time.Now().UTC()
+	}
+	now = now.UTC()
+	settings := options{jobMaxAge: defaultJobMaxAge, sources: make(map[string]SourceHealth)}
+	for _, apply := range opts {
+		if apply != nil {
+			apply(&settings)
+		}
+	}
+	components := []Component{
+		state(databaseReady, "database_ready", "database_unavailable"),
+		jobComponent(ComponentWorker, "heartbeat_not_recorded", settings, now),
+		jobComponent(ComponentProbes, "probe_heartbeat_not_recorded", settings, now),
+		jobComponent(ComponentNotifications, "delivery_health_not_sampled", settings, now),
+	}
+	prometheusComponent := configuredSourceComponent("prometheus", cfg.PrometheusURL != "", "not_configured", settings)
+	components = append(components, prometheusComponent, mirrorComponent("query", prometheusComponent, "prometheus_not_configured"))
+	// The API consumes Unraid through pulse-agent snapshots. A fresh observation is
+	// stronger evidence than absent API-local Unraid credentials and must win.
+	unraidConfigured := cfg.UnraidURL != "" && cfg.UnraidAPIToken != ""
+	unraidComponent := configuredSourceComponent("unraid", unraidConfigured, "not_configured", settings)
+	storageComponent := configuredSourceComponent("storage", unraidConfigured, "source_not_configured", settings)
+	components = append(components, unraidComponent, storageComponent)
+	if cfg.AuthMode == "oidc" && cfg.OIDCIssuer != "" && cfg.OIDCClientID != "" {
+		oidc := Component{ID: "oidc", State: StateUnknown, Reason: "authenticated_session_not_observed"}
+		if settings.authenticatedSession {
+			oidc.State, oidc.Reason = StateHealthy, "authenticated_session"
+			observed := now
+			oidc.ObservedAt, oidc.LastSuccessAt = &observed, &observed
+		}
+		components = append(components, oidc)
+	} else {
+		components = append(components, Component{ID: "oidc", State: StateDisabled, Reason: "not_configured"})
+	}
+	sort.Slice(components, func(i, j int) bool { return components[i].ID < components[j].ID })
+	backup := Backup{State: StateUnknown, Reason: "no_verified_backup"}
+	if settings.backup != nil {
+		backup = *settings.backup
+	}
+	if backup.LastSuccessAt != nil {
+		age := int64(now.Sub(backup.LastSuccessAt.UTC()).Seconds())
+		if age < 0 {
+			age = 0
+		}
+		backup.AgeSeconds = &age
+		if backup.State != StateDegraded {
+			if time.Duration(age)*time.Second <= backupMaxAge {
+				backup.State, backup.Reason = StateHealthy, "backup_verified"
+			} else {
+				backup.State, backup.Reason = StateUnknown, "backup_stale"
+			}
+		}
+	}
+	sourceLag := make([]SourceLag, 0, 2)
+	for _, component := range []Component{prometheusComponent, unraidComponent} {
+		if component.State == StateDisabled && settings.sources[component.ID].ID == "" {
+			continue
+		}
+		lag := SourceLag{SourceID: component.ID, State: component.State, Reason: component.Reason, LastObservedAt: component.ObservedAt}
+		if component.ObservedAt != nil {
+			age := int64(now.Sub(*component.ObservedAt).Seconds())
+			if age < 0 {
+				age = 0
+			}
+			lag.AgeSeconds = &age
+		}
+		sourceLag = append(sourceLag, lag)
+	}
+	state := StateHealthy
+	for _, component := range components {
+		if component.State == StateDegraded {
+			state = StateDegraded
+			break
+		}
+		if component.State == StateUnknown && state == StateHealthy {
+			state = StateUnknown
+		}
+		if component.State == StateDisabled && requiredComponent(component.ID) && state == StateHealthy {
+			state = StateUnknown
+		}
+	}
+	if backup.State == StateDegraded {
+		state = StateDegraded
+	}
+	return Snapshot{Version: buildinfo.Version, Release: Release{Version: buildinfo.Version, Commit: buildinfo.Commit, BuiltAt: buildinfo.BuiltAt(), MigrationVersion: settings.migration}, GeneratedAt: now, OverallState: state, Components: components, Backup: backup, SourceLag: sourceLag, AuditEvents: auditEvents}
+}
+
+func configuredSourceComponent(id string, configured bool, disabledReason string, settings options) Component {
+	if observed, ok := settings.sources[id]; ok {
+		component := Component{ID: id, State: observed.State, Reason: reasonOr(observed.Reason, "source_health_unknown")}
+		if !observed.ObservedAt.IsZero() {
+			value := observed.ObservedAt.UTC()
+			component.ObservedAt = &value
+		}
+		if !observed.LastSuccessAt.IsZero() {
+			value := observed.LastSuccessAt.UTC()
+			component.LastSuccessAt = &value
+		}
+		return component
+	}
+	if !configured {
+		return Component{ID: id, State: StateDisabled, Reason: disabledReason}
+	}
+	return Component{ID: id, State: StateUnknown, Reason: "source_health_not_sampled"}
+}
+
+func mirrorComponent(id string, source Component, disabledReason string) Component {
+	result := source
+	result.ID = id
+	if source.State == StateDisabled {
+		result.Reason = disabledReason
+	}
+	return result
+}
+
+func requiredComponent(id string) bool {
+	switch id {
+	case "database", ComponentWorker, "prometheus", "query", "unraid", "storage":
+		return true
+	default:
+		return false
+	}
+}
+
+// jobComponent folds every reported job for one component into a single state.
+// The worst reported state wins, a job that has never reported keeps the
+// component Unknown with its original reason, and no combination of inputs can
+// produce Healthy without a recent successful run (ADR-0008).
+func jobComponent(id, neverReportedReason string, settings options, now time.Time) Component {
+	component := Component{ID: id, State: StateUnknown, Reason: neverReportedReason}
+	reported := false
+	for _, job := range settings.jobs {
+		if job.Component != id {
+			continue
+		}
+		candidate := jobState(job, settings.jobMaxAge, now)
+		if !reported || stateRank(candidate.State) > stateRank(component.State) {
+			component = candidate
+			reported = true
+		}
+	}
+	return component
+}
+
+func jobState(job JobHealth, maxAge time.Duration, now time.Time) Component {
+	component := Component{ID: job.Component, State: StateUnknown, Reason: "job_outcome_not_recorded"}
+	if !job.LastRunAt.IsZero() {
+		observed := job.LastRunAt.UTC()
+		component.ObservedAt = &observed
+	}
+	if !job.LastSuccessAt.IsZero() {
+		success := job.LastSuccessAt.UTC()
+		component.LastSuccessAt = &success
+	}
+	switch job.Status {
+	case JobDisabled:
+		component.State = StateDisabled
+		component.Reason = reasonOr(job.Reason, "not_configured")
+		return component
+	case JobFailed:
+		component.State = StateDegraded
+		component.Reason = reasonOr(job.ErrorCode, reasonOr(job.Reason, "last_run_failed"))
+		return component
+	case JobCompleted:
+		if job.LastSuccessAt.IsZero() {
+			component.Reason = "last_success_not_recorded"
+			return component
+		}
+		if maxAge > 0 && now.Sub(job.LastSuccessAt.UTC()) > maxAge {
+			component.Reason = "last_success_stale"
+			return component
+		}
+		component.State = StateHealthy
+		component.Reason = reasonOr(job.Reason, "last_run_succeeded")
+		return component
+	case JobRunning:
+		component.Reason = "run_in_progress"
+		return component
+	default:
+		return component
+	}
+}
+
+func reasonOr(value, fallback string) string {
+	if value == "" {
+		return fallback
+	}
+	if len(value) > 160 {
+		return value[:160]
+	}
+	return value
+}
+
+func stateRank(value State) int {
+	switch value {
+	case StateDegraded:
+		return 3
+	case StateUnknown:
+		return 2
+	case StateHealthy:
+		return 1
+	default:
+		return 0
+	}
+}
+
+func state(ready bool, healthyReason, unknownReason string) Component {
+	if ready {
+		return Component{ID: "database", State: StateHealthy, Reason: healthyReason}
+	}
+	return Component{ID: "database", State: StateUnknown, Reason: unknownReason}
+}
@@ -0,0 +1,195 @@
+package systemstatus
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/itworx/pulse/internal/config"
+	"github.com/itworx/pulse/internal/datasource"
+)
+
+func TestBuildNeverReportsUnsampledSourcesAsHealthy(t *testing.T) {
+	snapshot := Build(config.Config{AuthMode: "mock", PrometheusURL: "http://prometheus", UnraidURL: "http://unraid", UnraidAPIToken: "configured"}, true, time.Unix(100, 0), nil)
+	if snapshot.OverallState != StateUnknown {
+		t.Fatalf("overall state = %q, want unknown", snapshot.OverallState)
+	}
+	for _, component := range snapshot.Components {
+		if component.ID == "prometheus" || component.ID == "unraid" {
+			if component.State == StateHealthy {
+				t.Fatalf("unsampled source %s reported healthy", component.ID)
+			}
+		}
+	}
+	if snapshot.Backup.State != StateUnknown || snapshot.Backup.Reason != "no_verified_backup" {
+		t.Fatalf("backup status = %#v", snapshot.Backup)
+	}
+}
+
+func TestObservedAgentSourcesOverrideMissingAPILocalCredentials(t *testing.T) {
+	now := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
+	observed := datasource.SourceHealth{
+		State: datasource.HealthHealthy, ObservedAt: now.Add(-time.Second), ReceivedAt: now, LastSuccess: now.Add(-time.Second),
+		Policy: datasource.FreshnessPolicy{MaxAge: time.Minute}, ReasonCode: "source_sampled",
+	}
+	snapshot := Build(config.Config{AuthMode: "mock"}, true, now, nil, WithSources(
+		FromDatasource("unraid", observed, now),
+		FromDatasource("storage", observed, now),
+	))
+	for _, id := range []string{"unraid", "storage"} {
+		component := componentByID(snapshot, id)
+		if component.State != StateHealthy || component.Reason != "source_sampled" || component.ObservedAt == nil {
+			t.Fatalf("component %s = %#v", id, component)
+		}
+	}
+	if len(snapshot.SourceLag) != 1 || snapshot.SourceLag[0].SourceID != "unraid" || snapshot.SourceLag[0].State != StateHealthy {
+		t.Fatalf("source lag = %#v", snapshot.SourceLag)
+	}
+}
+
+func TestConfiguredOIDCReflectsAuthenticatedProtectedRequest(t *testing.T) {
+	now := time.Date(2026, 8, 12, 9, 0, 0, 0, time.UTC)
+	cfg := config.Config{AuthMode: "oidc", OIDCIssuer: "https://auth.example", OIDCClientID: "pulse"}
+
+	withoutSession := Build(cfg, true, now, nil)
+	if got := componentByID(withoutSession, "oidc"); got.State != StateUnknown || got.Reason != "authenticated_session_not_observed" {
+		t.Fatalf("OIDC without request-local session proof = %#v", got)
+	}
+
+	withSession := Build(cfg, true, now, nil, WithAuthenticatedSession(true))
+	got := componentByID(withSession, "oidc")
+	if got.State != StateHealthy || got.Reason != "authenticated_session" || got.ObservedAt == nil || got.LastSuccessAt == nil {
+		t.Fatalf("OIDC with authenticated session = %#v", got)
+	}
+}
+
+func TestRequiredDisabledSourcePreventsHealthyOverallState(t *testing.T) {
+	now := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
+	jobs := []JobHealth{
+		{Component: ComponentWorker, Status: JobCompleted, LastRunAt: now, LastSuccessAt: now},
+		{Component: ComponentProbes, Status: JobCompleted, LastRunAt: now, LastSuccessAt: now},
+		{Component: ComponentNotifications, Status: JobCompleted, LastRunAt: now, LastSuccessAt: now},
+	}
+	snapshot := Build(config.Config{AuthMode: "mock"}, true, now, nil, WithJobs(time.Minute, jobs...))
+	if snapshot.OverallState != StateUnknown {
+		t.Fatalf("overall state = %q, want unknown", snapshot.OverallState)
+	}
+}
+
+func TestBackupObservationNeverTreatsStaleOrFailedAsHealthy(t *testing.T) {
+	now := time.Date(2026, 8, 12, 4, 0, 0, 0, time.UTC)
+	fresh := Build(config.Config{AuthMode: "mock"}, true, now, nil, WithBackupObservation(now.Add(-time.Hour), now.Add(-time.Minute), nil), WithMigrationVersion("0019_capacity_history"))
+	if fresh.Backup.State != StateHealthy || fresh.Backup.Reason != "backup_verified" || fresh.Backup.AgeSeconds == nil || fresh.Backup.VerifiedAt == nil || fresh.Backup.VerifiedAt.Equal(*fresh.Backup.LastSuccessAt) || fresh.Release.MigrationVersion != "0019_capacity_history" {
+		t.Fatalf("fresh snapshot = %#v", fresh)
+	}
+	stale := Build(config.Config{AuthMode: "mock"}, true, now, nil, WithBackupObservation(now.Add(-48*time.Hour), now, nil))
+	if stale.Backup.State == StateHealthy || stale.Backup.Reason != "backup_stale" {
+		t.Fatalf("stale backup = %#v", stale.Backup)
+	}
+	failed := Build(config.Config{AuthMode: "mock"}, true, now, nil, WithBackupObservation(time.Time{}, time.Time{}, errors.New("checksum mismatch")))
+	if failed.Backup.State != StateDegraded || failed.Backup.Reason != "backup_verification_failed" || failed.OverallState != StateDegraded {
+		t.Fatalf("failed backup = %#v", failed.Backup)
+	}
+}
+
+func TestBuildSortsComponentsAndDisablesUnconfiguredSources(t *testing.T) {
+	snapshot := Build(config.Config{AuthMode: "mock"}, false, time.Unix(100, 0), nil)
+	if snapshot.Components[0].ID != "database" {
+		t.Fatalf("components are not deterministic: %#v", snapshot.Components)
+	}
+	for _, component := range snapshot.Components {
+		if component.ID == "prometheus" || component.ID == "unraid" {
+			if component.State != StateDisabled {
+				t.Fatalf("component %s = %q, want disabled", component.ID, component.State)
+			}
+		}
+	}
+}
+
+func componentByID(snapshot Snapshot, id string) Component {
+	for _, component := range snapshot.Components {
+		if component.ID == id {
+			return component
+		}
+	}
+	return Component{}
+}
+
+func TestBackgroundJobComponentsReportRealOutcomes(t *testing.T) {
+	now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
+	cfg := config.Config{AuthMode: "mock"}
+	for name, testCase := range map[string]struct {
+		jobs       []JobHealth
+		wantState  State
+		wantReason string
+	}{
+		"never reported": {
+			jobs:      nil,
+			wantState: StateUnknown, wantReason: "heartbeat_not_recorded",
+		},
+		"recent success": {
+			jobs:      []JobHealth{{Component: ComponentWorker, JobKey: "discovery", Status: JobCompleted, LastRunAt: now.Add(-time.Minute), LastSuccessAt: now.Add(-time.Minute)}},
+			wantState: StateHealthy, wantReason: "last_run_succeeded",
+		},
+		"failure is degraded": {
+			jobs:      []JobHealth{{Component: ComponentWorker, JobKey: "discovery", Status: JobFailed, ErrorCode: "lease_unavailable", LastRunAt: now}},
+			wantState: StateDegraded, wantReason: "lease_unavailable",
+		},
+		"stale success is unknown": {
+			jobs:      []JobHealth{{Component: ComponentWorker, JobKey: "discovery", Status: JobCompleted, LastRunAt: now.Add(-time.Hour), LastSuccessAt: now.Add(-time.Hour)}},
+			wantState: StateUnknown, wantReason: "last_success_stale",
+		},
+		"completed without a success is unknown": {
+			jobs:      []JobHealth{{Component: ComponentWorker, JobKey: "discovery", Status: JobCompleted, LastRunAt: now}},
+			wantState: StateUnknown, wantReason: "last_success_not_recorded",
+		},
+		"not configured is disabled": {
+			jobs:      []JobHealth{{Component: ComponentWorker, JobKey: "discovery", Status: JobDisabled, Reason: "container_source_not_registered", LastRunAt: now}},
+			wantState: StateDisabled, wantReason: "container_source_not_registered",
+		},
+		"worst job wins": {
+			jobs: []JobHealth{
+				{Component: ComponentWorker, JobKey: "discovery", Status: JobCompleted, LastRunAt: now, LastSuccessAt: now},
+				{Component: ComponentWorker, JobKey: "alert-evaluation", Status: JobFailed, ErrorCode: "run_failed", LastRunAt: now},
+			},
+			wantState: StateDegraded, wantReason: "run_failed",
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			snapshot := Build(cfg, true, now, nil, WithJobs(5*time.Minute, testCase.jobs...))
+			worker := componentByID(snapshot, ComponentWorker)
+			if worker.State != testCase.wantState || worker.Reason != testCase.wantReason {
+				t.Fatalf("worker component = %#v", worker)
+			}
+			if worker.State == StateHealthy && worker.LastSuccessAt == nil {
+				t.Fatal("a healthy component must carry the observation that justifies it")
+			}
+			// Components without a reported job keep their conservative default.
+			probes := componentByID(snapshot, ComponentProbes)
+			if probes.State != StateUnknown || probes.Reason != "probe_heartbeat_not_recorded" {
+				t.Fatalf("probes component = %#v", probes)
+			}
+		})
+	}
+}
+
+func TestOverallStateStaysUnknownUntilEveryComponentReports(t *testing.T) {
+	now := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
+	cfg := config.Config{AuthMode: "mock"}
+	jobs := []JobHealth{
+		{Component: ComponentWorker, Status: JobCompleted, LastRunAt: now, LastSuccessAt: now},
+		{Component: ComponentProbes, Status: JobCompleted, LastRunAt: now, LastSuccessAt: now},
+		{Component: ComponentNotifications, Status: JobCompleted, LastRunAt: now, LastSuccessAt: now},
+	}
+	if state := Build(cfg, true, now, nil, WithJobs(time.Minute, jobs[:1]...)).OverallState; state != StateUnknown {
+		t.Fatalf("overall state with one reporting job = %q", state)
+	}
+	snapshot := Build(cfg, true, now, nil, WithJobs(time.Minute, jobs...))
+	for _, component := range snapshot.Components {
+		if component.ID == ComponentWorker || component.ID == ComponentProbes || component.ID == ComponentNotifications {
+			if component.State != StateHealthy {
+				t.Fatalf("component %s = %#v", component.ID, component)
+			}
+		}
+	}
+}