This commit is contained in:
ITWorx Pulse release export
@@ -0,0 +1,418 @@
|
||||
package systemstatus
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/buildinfo"
|
||||
"github.com/itworx/pulse/internal/config"
|
||||
"github.com/itworx/pulse/internal/datasource"
|
||||
)
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateHealthy State = "healthy"
|
||||
StateDegraded State = "degraded"
|
||||
StateUnknown State = "unknown"
|
||||
StateDisabled State = "disabled"
|
||||
)
|
||||
|
||||
type Component struct {
|
||||
ID string `json:"id"`
|
||||
State State `json:"state"`
|
||||
Reason string `json:"reason"`
|
||||
ObservedAt *time.Time `json:"observedAt,omitempty"`
|
||||
LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
|
||||
}
|
||||
|
||||
type Backup struct {
|
||||
State State `json:"state"`
|
||||
Reason string `json:"reason"`
|
||||
LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
|
||||
AgeSeconds *int64 `json:"ageSeconds,omitempty"`
|
||||
VerifiedAt *time.Time `json:"verifiedAt,omitempty"`
|
||||
}
|
||||
|
||||
type Release struct {
|
||||
Version string `json:"version"`
|
||||
Commit string `json:"commit"`
|
||||
BuiltAt *time.Time `json:"builtAt,omitempty"`
|
||||
MigrationVersion string `json:"migrationVersion"`
|
||||
}
|
||||
|
||||
type SourceLag struct {
|
||||
SourceID string `json:"sourceId"`
|
||||
State State `json:"state"`
|
||||
Reason string `json:"reason"`
|
||||
LastObservedAt *time.Time `json:"lastObservedAt,omitempty"`
|
||||
AgeSeconds *int64 `json:"ageSeconds,omitempty"`
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
Version string `json:"version"`
|
||||
Release Release `json:"release"`
|
||||
GeneratedAt time.Time `json:"generatedAt"`
|
||||
OverallState State `json:"overallState"`
|
||||
Components []Component `json:"components"`
|
||||
Backup Backup `json:"backup"`
|
||||
SourceLag []SourceLag `json:"sourceLag"`
|
||||
AuditEvents *int64 `json:"auditEvents,omitempty"`
|
||||
}
|
||||
|
||||
// Background job component identifiers. A background job reports its outcome
|
||||
// against one of these, and the component keeps its "never reported" Unknown
|
||||
// state until a real outcome arrives (ADR-0008).
|
||||
const (
|
||||
ComponentWorker = "worker"
|
||||
ComponentProbes = "probes"
|
||||
ComponentNotifications = "notifications"
|
||||
)
|
||||
|
||||
// Job outcome statuses a background job may report.
|
||||
const (
|
||||
JobCompleted = "completed"
|
||||
JobFailed = "failed"
|
||||
JobRunning = "running"
|
||||
// JobDisabled marks a job that cannot run because the feature it drives is
|
||||
// not configured. It is reported as Disabled, never as Healthy.
|
||||
JobDisabled = "disabled"
|
||||
)
|
||||
|
||||
// defaultJobMaxAge bounds how old a successful background job run may be before
|
||||
// the component it feeds falls back to Unknown. It is deliberately several
|
||||
// multiples of the slowest worker job interval so a single slow cycle does not
|
||||
// flap the status.
|
||||
const defaultJobMaxAge = 5 * time.Minute
|
||||
|
||||
// JobHealth is the last reported outcome of one background job. It is produced
|
||||
// by the worker runtime (persisted in job_runs) and read back here so the
|
||||
// worker, probe and notification components report observed behaviour instead
|
||||
// of a hardcoded placeholder.
|
||||
type JobHealth struct {
|
||||
Component string
|
||||
JobKey string
|
||||
Status string
|
||||
Reason string
|
||||
ErrorCode string
|
||||
LastRunAt time.Time
|
||||
LastSuccessAt time.Time
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
type options struct {
|
||||
jobs []JobHealth
|
||||
jobMaxAge time.Duration
|
||||
sources map[string]SourceHealth
|
||||
backup *Backup
|
||||
migration string
|
||||
authenticatedSession bool
|
||||
}
|
||||
|
||||
const backupMaxAge = 26 * time.Hour
|
||||
|
||||
func WithMigrationVersion(version string) Option { return func(o *options) { o.migration = version } }
|
||||
|
||||
// WithAuthenticatedSession records request-local proof that the protected
|
||||
// status endpoint was reached through a valid Pulse session.
|
||||
func WithAuthenticatedSession(authenticated bool) Option {
|
||||
return func(o *options) { o.authenticatedSession = authenticated }
|
||||
}
|
||||
|
||||
func WithBackupObservation(lastSuccess, verifiedAt time.Time, verificationErr error) Option {
|
||||
return func(o *options) {
|
||||
backup := Backup{State: StateUnknown, Reason: "no_verified_backup"}
|
||||
if verificationErr != nil {
|
||||
backup.State, backup.Reason = StateDegraded, "backup_verification_failed"
|
||||
}
|
||||
if !lastSuccess.IsZero() {
|
||||
value := lastSuccess.UTC()
|
||||
backup.LastSuccessAt = &value
|
||||
}
|
||||
if !verifiedAt.IsZero() {
|
||||
value := verifiedAt.UTC()
|
||||
backup.VerifiedAt = &value
|
||||
}
|
||||
o.backup = &backup
|
||||
}
|
||||
}
|
||||
|
||||
// SourceHealth is a runtime observation for one configured source or source-backed
|
||||
// capability group. The API supplies these observations after querying the real
|
||||
// Prometheus adapter and persisted agent snapshots.
|
||||
type SourceHealth struct {
|
||||
ID string
|
||||
State State
|
||||
Reason string
|
||||
ObservedAt time.Time
|
||||
LastSuccessAt time.Time
|
||||
}
|
||||
|
||||
// Option adjusts how a snapshot is built. Options are variadic so existing
|
||||
// callers that have no background job information keep compiling and keep
|
||||
// getting the conservative "not recorded" Unknown result.
|
||||
type Option func(*options)
|
||||
|
||||
// WithJobs supplies the most recent background job outcomes. maxAge bounds how
|
||||
// long a successful run stays usable; zero selects the default.
|
||||
func WithJobs(maxAge time.Duration, jobs ...JobHealth) Option {
|
||||
return func(o *options) {
|
||||
o.jobs = append(o.jobs, jobs...)
|
||||
if maxAge > 0 {
|
||||
o.jobMaxAge = maxAge
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithSources overrides configuration-only placeholders with observed runtime health.
|
||||
// Unknown identifiers are ignored by Build, keeping the public component set closed.
|
||||
func WithSources(sources ...SourceHealth) Option {
|
||||
return func(o *options) {
|
||||
if o.sources == nil {
|
||||
o.sources = make(map[string]SourceHealth)
|
||||
}
|
||||
for _, source := range sources {
|
||||
if source.ID != "" {
|
||||
o.sources[source.ID] = source
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FromDatasource maps the shared datasource contract onto the self-observability
|
||||
// contract without allowing stale Healthy values to survive EffectiveState.
|
||||
func FromDatasource(id string, health datasource.SourceHealth, now time.Time) SourceHealth {
|
||||
state := StateUnknown
|
||||
switch health.EffectiveState(now.UTC()) {
|
||||
case datasource.HealthHealthy:
|
||||
state = StateHealthy
|
||||
case datasource.HealthDegraded:
|
||||
state = StateDegraded
|
||||
case datasource.HealthDisabled:
|
||||
state = StateDisabled
|
||||
}
|
||||
fallback := "source_health_unknown"
|
||||
if state == StateHealthy {
|
||||
fallback = "source_sampled"
|
||||
} else if state == StateDegraded {
|
||||
fallback = "source_degraded"
|
||||
} else if state == StateDisabled {
|
||||
fallback = "not_configured"
|
||||
}
|
||||
return SourceHealth{ID: id, State: state, Reason: reasonOr(health.ReasonCode, fallback), ObservedAt: health.ObservedAt.UTC(), LastSuccessAt: health.LastSuccess.UTC()}
|
||||
}
|
||||
|
||||
func Build(cfg config.Config, databaseReady bool, now time.Time, auditEvents *int64, opts ...Option) Snapshot {
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
now = now.UTC()
|
||||
settings := options{jobMaxAge: defaultJobMaxAge, sources: make(map[string]SourceHealth)}
|
||||
for _, apply := range opts {
|
||||
if apply != nil {
|
||||
apply(&settings)
|
||||
}
|
||||
}
|
||||
components := []Component{
|
||||
state(databaseReady, "database_ready", "database_unavailable"),
|
||||
jobComponent(ComponentWorker, "heartbeat_not_recorded", settings, now),
|
||||
jobComponent(ComponentProbes, "probe_heartbeat_not_recorded", settings, now),
|
||||
jobComponent(ComponentNotifications, "delivery_health_not_sampled", settings, now),
|
||||
}
|
||||
prometheusComponent := configuredSourceComponent("prometheus", cfg.PrometheusURL != "", "not_configured", settings)
|
||||
components = append(components, prometheusComponent, mirrorComponent("query", prometheusComponent, "prometheus_not_configured"))
|
||||
// The API consumes Unraid through pulse-agent snapshots. A fresh observation is
|
||||
// stronger evidence than absent API-local Unraid credentials and must win.
|
||||
unraidConfigured := cfg.UnraidURL != "" && cfg.UnraidAPIToken != ""
|
||||
unraidComponent := configuredSourceComponent("unraid", unraidConfigured, "not_configured", settings)
|
||||
storageComponent := configuredSourceComponent("storage", unraidConfigured, "source_not_configured", settings)
|
||||
components = append(components, unraidComponent, storageComponent)
|
||||
if cfg.AuthMode == "oidc" && cfg.OIDCIssuer != "" && cfg.OIDCClientID != "" {
|
||||
oidc := Component{ID: "oidc", State: StateUnknown, Reason: "authenticated_session_not_observed"}
|
||||
if settings.authenticatedSession {
|
||||
oidc.State, oidc.Reason = StateHealthy, "authenticated_session"
|
||||
observed := now
|
||||
oidc.ObservedAt, oidc.LastSuccessAt = &observed, &observed
|
||||
}
|
||||
components = append(components, oidc)
|
||||
} else {
|
||||
components = append(components, Component{ID: "oidc", State: StateDisabled, Reason: "not_configured"})
|
||||
}
|
||||
sort.Slice(components, func(i, j int) bool { return components[i].ID < components[j].ID })
|
||||
backup := Backup{State: StateUnknown, Reason: "no_verified_backup"}
|
||||
if settings.backup != nil {
|
||||
backup = *settings.backup
|
||||
}
|
||||
if backup.LastSuccessAt != nil {
|
||||
age := int64(now.Sub(backup.LastSuccessAt.UTC()).Seconds())
|
||||
if age < 0 {
|
||||
age = 0
|
||||
}
|
||||
backup.AgeSeconds = &age
|
||||
if backup.State != StateDegraded {
|
||||
if time.Duration(age)*time.Second <= backupMaxAge {
|
||||
backup.State, backup.Reason = StateHealthy, "backup_verified"
|
||||
} else {
|
||||
backup.State, backup.Reason = StateUnknown, "backup_stale"
|
||||
}
|
||||
}
|
||||
}
|
||||
sourceLag := make([]SourceLag, 0, 2)
|
||||
for _, component := range []Component{prometheusComponent, unraidComponent} {
|
||||
if component.State == StateDisabled && settings.sources[component.ID].ID == "" {
|
||||
continue
|
||||
}
|
||||
lag := SourceLag{SourceID: component.ID, State: component.State, Reason: component.Reason, LastObservedAt: component.ObservedAt}
|
||||
if component.ObservedAt != nil {
|
||||
age := int64(now.Sub(*component.ObservedAt).Seconds())
|
||||
if age < 0 {
|
||||
age = 0
|
||||
}
|
||||
lag.AgeSeconds = &age
|
||||
}
|
||||
sourceLag = append(sourceLag, lag)
|
||||
}
|
||||
state := StateHealthy
|
||||
for _, component := range components {
|
||||
if component.State == StateDegraded {
|
||||
state = StateDegraded
|
||||
break
|
||||
}
|
||||
if component.State == StateUnknown && state == StateHealthy {
|
||||
state = StateUnknown
|
||||
}
|
||||
if component.State == StateDisabled && requiredComponent(component.ID) && state == StateHealthy {
|
||||
state = StateUnknown
|
||||
}
|
||||
}
|
||||
if backup.State == StateDegraded {
|
||||
state = StateDegraded
|
||||
}
|
||||
return Snapshot{Version: buildinfo.Version, Release: Release{Version: buildinfo.Version, Commit: buildinfo.Commit, BuiltAt: buildinfo.BuiltAt(), MigrationVersion: settings.migration}, GeneratedAt: now, OverallState: state, Components: components, Backup: backup, SourceLag: sourceLag, AuditEvents: auditEvents}
|
||||
}
|
||||
|
||||
func configuredSourceComponent(id string, configured bool, disabledReason string, settings options) Component {
|
||||
if observed, ok := settings.sources[id]; ok {
|
||||
component := Component{ID: id, State: observed.State, Reason: reasonOr(observed.Reason, "source_health_unknown")}
|
||||
if !observed.ObservedAt.IsZero() {
|
||||
value := observed.ObservedAt.UTC()
|
||||
component.ObservedAt = &value
|
||||
}
|
||||
if !observed.LastSuccessAt.IsZero() {
|
||||
value := observed.LastSuccessAt.UTC()
|
||||
component.LastSuccessAt = &value
|
||||
}
|
||||
return component
|
||||
}
|
||||
if !configured {
|
||||
return Component{ID: id, State: StateDisabled, Reason: disabledReason}
|
||||
}
|
||||
return Component{ID: id, State: StateUnknown, Reason: "source_health_not_sampled"}
|
||||
}
|
||||
|
||||
func mirrorComponent(id string, source Component, disabledReason string) Component {
|
||||
result := source
|
||||
result.ID = id
|
||||
if source.State == StateDisabled {
|
||||
result.Reason = disabledReason
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func requiredComponent(id string) bool {
|
||||
switch id {
|
||||
case "database", ComponentWorker, "prometheus", "query", "unraid", "storage":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// jobComponent folds every reported job for one component into a single state.
|
||||
// The worst reported state wins, a job that has never reported keeps the
|
||||
// component Unknown with its original reason, and no combination of inputs can
|
||||
// produce Healthy without a recent successful run (ADR-0008).
|
||||
func jobComponent(id, neverReportedReason string, settings options, now time.Time) Component {
|
||||
component := Component{ID: id, State: StateUnknown, Reason: neverReportedReason}
|
||||
reported := false
|
||||
for _, job := range settings.jobs {
|
||||
if job.Component != id {
|
||||
continue
|
||||
}
|
||||
candidate := jobState(job, settings.jobMaxAge, now)
|
||||
if !reported || stateRank(candidate.State) > stateRank(component.State) {
|
||||
component = candidate
|
||||
reported = true
|
||||
}
|
||||
}
|
||||
return component
|
||||
}
|
||||
|
||||
func jobState(job JobHealth, maxAge time.Duration, now time.Time) Component {
|
||||
component := Component{ID: job.Component, State: StateUnknown, Reason: "job_outcome_not_recorded"}
|
||||
if !job.LastRunAt.IsZero() {
|
||||
observed := job.LastRunAt.UTC()
|
||||
component.ObservedAt = &observed
|
||||
}
|
||||
if !job.LastSuccessAt.IsZero() {
|
||||
success := job.LastSuccessAt.UTC()
|
||||
component.LastSuccessAt = &success
|
||||
}
|
||||
switch job.Status {
|
||||
case JobDisabled:
|
||||
component.State = StateDisabled
|
||||
component.Reason = reasonOr(job.Reason, "not_configured")
|
||||
return component
|
||||
case JobFailed:
|
||||
component.State = StateDegraded
|
||||
component.Reason = reasonOr(job.ErrorCode, reasonOr(job.Reason, "last_run_failed"))
|
||||
return component
|
||||
case JobCompleted:
|
||||
if job.LastSuccessAt.IsZero() {
|
||||
component.Reason = "last_success_not_recorded"
|
||||
return component
|
||||
}
|
||||
if maxAge > 0 && now.Sub(job.LastSuccessAt.UTC()) > maxAge {
|
||||
component.Reason = "last_success_stale"
|
||||
return component
|
||||
}
|
||||
component.State = StateHealthy
|
||||
component.Reason = reasonOr(job.Reason, "last_run_succeeded")
|
||||
return component
|
||||
case JobRunning:
|
||||
component.Reason = "run_in_progress"
|
||||
return component
|
||||
default:
|
||||
return component
|
||||
}
|
||||
}
|
||||
|
||||
func reasonOr(value, fallback string) string {
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
if len(value) > 160 {
|
||||
return value[:160]
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func stateRank(value State) int {
|
||||
switch value {
|
||||
case StateDegraded:
|
||||
return 3
|
||||
case StateUnknown:
|
||||
return 2
|
||||
case StateHealthy:
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func state(ready bool, healthyReason, unknownReason string) Component {
|
||||
if ready {
|
||||
return Component{ID: "database", State: StateHealthy, Reason: healthyReason}
|
||||
}
|
||||
return Component{ID: "database", State: StateUnknown, Reason: unknownReason}
|
||||
}
|
||||
Reference in New Issue
Block a user