package systemstatus import ( "sort" "time" "github.com/itworx/pulse/internal/buildinfo" "github.com/itworx/pulse/internal/config" "github.com/itworx/pulse/internal/datasource" ) type State string const ( StateHealthy State = "healthy" StateDegraded State = "degraded" StateUnknown State = "unknown" StateDisabled State = "disabled" ) type Component struct { ID string `json:"id"` State State `json:"state"` Reason string `json:"reason"` ObservedAt *time.Time `json:"observedAt,omitempty"` LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"` } type Backup struct { State State `json:"state"` Reason string `json:"reason"` LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"` AgeSeconds *int64 `json:"ageSeconds,omitempty"` VerifiedAt *time.Time `json:"verifiedAt,omitempty"` } type Release struct { Version string `json:"version"` Commit string `json:"commit"` BuiltAt *time.Time `json:"builtAt,omitempty"` MigrationVersion string `json:"migrationVersion"` } type SourceLag struct { SourceID string `json:"sourceId"` State State `json:"state"` Reason string `json:"reason"` LastObservedAt *time.Time `json:"lastObservedAt,omitempty"` AgeSeconds *int64 `json:"ageSeconds,omitempty"` } type Snapshot struct { Version string `json:"version"` Release Release `json:"release"` GeneratedAt time.Time `json:"generatedAt"` OverallState State `json:"overallState"` Components []Component `json:"components"` Backup Backup `json:"backup"` SourceLag []SourceLag `json:"sourceLag"` AuditEvents *int64 `json:"auditEvents,omitempty"` } // Background job component identifiers. A background job reports its outcome // against one of these, and the component keeps its "never reported" Unknown // state until a real outcome arrives (ADR-0008). const ( ComponentWorker = "worker" ComponentProbes = "probes" ComponentNotifications = "notifications" ) // Job outcome statuses a background job may report. const ( JobCompleted = "completed" JobFailed = "failed" JobRunning = "running" // JobDisabled marks a job that cannot run because the feature it drives is // not configured. It is reported as Disabled, never as Healthy. JobDisabled = "disabled" ) // defaultJobMaxAge bounds how old a successful background job run may be before // the component it feeds falls back to Unknown. It is deliberately several // multiples of the slowest worker job interval so a single slow cycle does not // flap the status. const defaultJobMaxAge = 5 * time.Minute // JobHealth is the last reported outcome of one background job. It is produced // by the worker runtime (persisted in job_runs) and read back here so the // worker, probe and notification components report observed behaviour instead // of a hardcoded placeholder. type JobHealth struct { Component string JobKey string Status string Reason string ErrorCode string LastRunAt time.Time LastSuccessAt time.Time Duration time.Duration } type options struct { jobs []JobHealth jobMaxAge time.Duration sources map[string]SourceHealth backup *Backup migration string authenticatedSession bool } const backupMaxAge = 26 * time.Hour func WithMigrationVersion(version string) Option { return func(o *options) { o.migration = version } } // WithAuthenticatedSession records request-local proof that the protected // status endpoint was reached through a valid Pulse session. func WithAuthenticatedSession(authenticated bool) Option { return func(o *options) { o.authenticatedSession = authenticated } } func WithBackupObservation(lastSuccess, verifiedAt time.Time, verificationErr error) Option { return func(o *options) { backup := Backup{State: StateUnknown, Reason: "no_verified_backup"} if verificationErr != nil { backup.State, backup.Reason = StateDegraded, "backup_verification_failed" } if !lastSuccess.IsZero() { value := lastSuccess.UTC() backup.LastSuccessAt = &value } if !verifiedAt.IsZero() { value := verifiedAt.UTC() backup.VerifiedAt = &value } o.backup = &backup } } // SourceHealth is a runtime observation for one configured source or source-backed // capability group. The API supplies these observations after querying the real // Prometheus adapter and persisted agent snapshots. type SourceHealth struct { ID string State State Reason string ObservedAt time.Time LastSuccessAt time.Time } // Option adjusts how a snapshot is built. Options are variadic so existing // callers that have no background job information keep compiling and keep // getting the conservative "not recorded" Unknown result. type Option func(*options) // WithJobs supplies the most recent background job outcomes. maxAge bounds how // long a successful run stays usable; zero selects the default. func WithJobs(maxAge time.Duration, jobs ...JobHealth) Option { return func(o *options) { o.jobs = append(o.jobs, jobs...) if maxAge > 0 { o.jobMaxAge = maxAge } } } // WithSources overrides configuration-only placeholders with observed runtime health. // Unknown identifiers are ignored by Build, keeping the public component set closed. func WithSources(sources ...SourceHealth) Option { return func(o *options) { if o.sources == nil { o.sources = make(map[string]SourceHealth) } for _, source := range sources { if source.ID != "" { o.sources[source.ID] = source } } } } // FromDatasource maps the shared datasource contract onto the self-observability // contract without allowing stale Healthy values to survive EffectiveState. func FromDatasource(id string, health datasource.SourceHealth, now time.Time) SourceHealth { state := StateUnknown switch health.EffectiveState(now.UTC()) { case datasource.HealthHealthy: state = StateHealthy case datasource.HealthDegraded: state = StateDegraded case datasource.HealthDisabled: state = StateDisabled } fallback := "source_health_unknown" if state == StateHealthy { fallback = "source_sampled" } else if state == StateDegraded { fallback = "source_degraded" } else if state == StateDisabled { fallback = "not_configured" } return SourceHealth{ID: id, State: state, Reason: reasonOr(health.ReasonCode, fallback), ObservedAt: health.ObservedAt.UTC(), LastSuccessAt: health.LastSuccess.UTC()} } func Build(cfg config.Config, databaseReady bool, now time.Time, auditEvents *int64, opts ...Option) Snapshot { if now.IsZero() { now = time.Now().UTC() } now = now.UTC() settings := options{jobMaxAge: defaultJobMaxAge, sources: make(map[string]SourceHealth)} for _, apply := range opts { if apply != nil { apply(&settings) } } components := []Component{ state(databaseReady, "database_ready", "database_unavailable"), jobComponent(ComponentWorker, "heartbeat_not_recorded", settings, now), jobComponent(ComponentProbes, "probe_heartbeat_not_recorded", settings, now), jobComponent(ComponentNotifications, "delivery_health_not_sampled", settings, now), } prometheusComponent := configuredSourceComponent("prometheus", cfg.PrometheusURL != "", "not_configured", settings) components = append(components, prometheusComponent, mirrorComponent("query", prometheusComponent, "prometheus_not_configured")) // The API consumes Unraid through pulse-agent snapshots. A fresh observation is // stronger evidence than absent API-local Unraid credentials and must win. unraidConfigured := cfg.UnraidURL != "" && cfg.UnraidAPIToken != "" unraidComponent := configuredSourceComponent("unraid", unraidConfigured, "not_configured", settings) storageComponent := configuredSourceComponent("storage", unraidConfigured, "source_not_configured", settings) components = append(components, unraidComponent, storageComponent) if cfg.AuthMode == "oidc" && cfg.OIDCIssuer != "" && cfg.OIDCClientID != "" { oidc := Component{ID: "oidc", State: StateUnknown, Reason: "authenticated_session_not_observed"} if settings.authenticatedSession { oidc.State, oidc.Reason = StateHealthy, "authenticated_session" observed := now oidc.ObservedAt, oidc.LastSuccessAt = &observed, &observed } components = append(components, oidc) } else { components = append(components, Component{ID: "oidc", State: StateDisabled, Reason: "not_configured"}) } sort.Slice(components, func(i, j int) bool { return components[i].ID < components[j].ID }) backup := Backup{State: StateUnknown, Reason: "no_verified_backup"} if settings.backup != nil { backup = *settings.backup } if backup.LastSuccessAt != nil { age := int64(now.Sub(backup.LastSuccessAt.UTC()).Seconds()) if age < 0 { age = 0 } backup.AgeSeconds = &age if backup.State != StateDegraded { if time.Duration(age)*time.Second <= backupMaxAge { backup.State, backup.Reason = StateHealthy, "backup_verified" } else { backup.State, backup.Reason = StateUnknown, "backup_stale" } } } sourceLag := make([]SourceLag, 0, 2) for _, component := range []Component{prometheusComponent, unraidComponent} { if component.State == StateDisabled && settings.sources[component.ID].ID == "" { continue } lag := SourceLag{SourceID: component.ID, State: component.State, Reason: component.Reason, LastObservedAt: component.ObservedAt} if component.ObservedAt != nil { age := int64(now.Sub(*component.ObservedAt).Seconds()) if age < 0 { age = 0 } lag.AgeSeconds = &age } sourceLag = append(sourceLag, lag) } state := StateHealthy for _, component := range components { if component.State == StateDegraded { state = StateDegraded break } if component.State == StateUnknown && state == StateHealthy { state = StateUnknown } if component.State == StateDisabled && requiredComponent(component.ID) && state == StateHealthy { state = StateUnknown } } if backup.State == StateDegraded { state = StateDegraded } return Snapshot{Version: buildinfo.Version, Release: Release{Version: buildinfo.Version, Commit: buildinfo.Commit, BuiltAt: buildinfo.BuiltAt(), MigrationVersion: settings.migration}, GeneratedAt: now, OverallState: state, Components: components, Backup: backup, SourceLag: sourceLag, AuditEvents: auditEvents} } func configuredSourceComponent(id string, configured bool, disabledReason string, settings options) Component { if observed, ok := settings.sources[id]; ok { component := Component{ID: id, State: observed.State, Reason: reasonOr(observed.Reason, "source_health_unknown")} if !observed.ObservedAt.IsZero() { value := observed.ObservedAt.UTC() component.ObservedAt = &value } if !observed.LastSuccessAt.IsZero() { value := observed.LastSuccessAt.UTC() component.LastSuccessAt = &value } return component } if !configured { return Component{ID: id, State: StateDisabled, Reason: disabledReason} } return Component{ID: id, State: StateUnknown, Reason: "source_health_not_sampled"} } func mirrorComponent(id string, source Component, disabledReason string) Component { result := source result.ID = id if source.State == StateDisabled { result.Reason = disabledReason } return result } func requiredComponent(id string) bool { switch id { case "database", ComponentWorker, "prometheus", "query", "unraid", "storage": return true default: return false } } // jobComponent folds every reported job for one component into a single state. // The worst reported state wins, a job that has never reported keeps the // component Unknown with its original reason, and no combination of inputs can // produce Healthy without a recent successful run (ADR-0008). func jobComponent(id, neverReportedReason string, settings options, now time.Time) Component { component := Component{ID: id, State: StateUnknown, Reason: neverReportedReason} reported := false for _, job := range settings.jobs { if job.Component != id { continue } candidate := jobState(job, settings.jobMaxAge, now) if !reported || stateRank(candidate.State) > stateRank(component.State) { component = candidate reported = true } } return component } func jobState(job JobHealth, maxAge time.Duration, now time.Time) Component { component := Component{ID: job.Component, State: StateUnknown, Reason: "job_outcome_not_recorded"} if !job.LastRunAt.IsZero() { observed := job.LastRunAt.UTC() component.ObservedAt = &observed } if !job.LastSuccessAt.IsZero() { success := job.LastSuccessAt.UTC() component.LastSuccessAt = &success } switch job.Status { case JobDisabled: component.State = StateDisabled component.Reason = reasonOr(job.Reason, "not_configured") return component case JobFailed: component.State = StateDegraded component.Reason = reasonOr(job.ErrorCode, reasonOr(job.Reason, "last_run_failed")) return component case JobCompleted: if job.LastSuccessAt.IsZero() { component.Reason = "last_success_not_recorded" return component } if maxAge > 0 && now.Sub(job.LastSuccessAt.UTC()) > maxAge { component.Reason = "last_success_stale" return component } component.State = StateHealthy component.Reason = reasonOr(job.Reason, "last_run_succeeded") return component case JobRunning: component.Reason = "run_in_progress" return component default: return component } } func reasonOr(value, fallback string) string { if value == "" { return fallback } if len(value) > 160 { return value[:160] } return value } func stateRank(value State) int { switch value { case StateDegraded: return 3 case StateUnknown: return 2 case StateHealthy: return 1 default: return 0 } } func state(ready bool, healthyReason, unknownReason string) Component { if ready { return Component{ID: "database", State: StateHealthy, Reason: healthyReason} } return Component{ID: "database", State: StateUnknown, Reason: unknownReason} }