ITWorx-Pulse-Public/internal/systemstatus/status.go

package systemstatus

import (
	"sort"
	"time"

	"github.com/itworx/pulse/internal/buildinfo"
	"github.com/itworx/pulse/internal/config"
	"github.com/itworx/pulse/internal/datasource"
)

type State string

const (
	StateHealthy  State = "healthy"
	StateDegraded State = "degraded"
	StateUnknown  State = "unknown"
	StateDisabled State = "disabled"
)

type Component struct {
	ID            string     `json:"id"`
	State         State      `json:"state"`
	Reason        string     `json:"reason"`
	ObservedAt    *time.Time `json:"observedAt,omitempty"`
	LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
}

type Backup struct {
	State         State      `json:"state"`
	Reason        string     `json:"reason"`
	LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
	AgeSeconds    *int64     `json:"ageSeconds,omitempty"`
	VerifiedAt    *time.Time `json:"verifiedAt,omitempty"`
}

type Release struct {
	Version          string     `json:"version"`
	Commit           string     `json:"commit"`
	BuiltAt          *time.Time `json:"builtAt,omitempty"`
	MigrationVersion string     `json:"migrationVersion"`
}

type SourceLag struct {
	SourceID       string     `json:"sourceId"`
	State          State      `json:"state"`
	Reason         string     `json:"reason"`
	LastObservedAt *time.Time `json:"lastObservedAt,omitempty"`
	AgeSeconds     *int64     `json:"ageSeconds,omitempty"`
}

type Snapshot struct {
	Version      string      `json:"version"`
	Release      Release     `json:"release"`
	GeneratedAt  time.Time   `json:"generatedAt"`
	OverallState State       `json:"overallState"`
	Components   []Component `json:"components"`
	Backup       Backup      `json:"backup"`
	SourceLag    []SourceLag `json:"sourceLag"`
	AuditEvents  *int64      `json:"auditEvents,omitempty"`
}

// Background job component identifiers. A background job reports its outcome
// against one of these, and the component keeps its "never reported" Unknown
// state until a real outcome arrives (ADR-0008).
const (
	ComponentWorker        = "worker"
	ComponentProbes        = "probes"
	ComponentNotifications = "notifications"
)

// Job outcome statuses a background job may report.
const (
	JobCompleted = "completed"
	JobFailed    = "failed"
	JobRunning   = "running"
	// JobDisabled marks a job that cannot run because the feature it drives is
	// not configured. It is reported as Disabled, never as Healthy.
	JobDisabled = "disabled"
)

// defaultJobMaxAge bounds how old a successful background job run may be before
// the component it feeds falls back to Unknown. It is deliberately several
// multiples of the slowest worker job interval so a single slow cycle does not
// flap the status.
const defaultJobMaxAge = 5 * time.Minute

// JobHealth is the last reported outcome of one background job. It is produced
// by the worker runtime (persisted in job_runs) and read back here so the
// worker, probe and notification components report observed behaviour instead
// of a hardcoded placeholder.
type JobHealth struct {
	Component     string
	JobKey        string
	Status        string
	Reason        string
	ErrorCode     string
	LastRunAt     time.Time
	LastSuccessAt time.Time
	Duration      time.Duration
}

type options struct {
	jobs                 []JobHealth
	jobMaxAge            time.Duration
	sources              map[string]SourceHealth
	backup               *Backup
	migration            string
	authenticatedSession bool
}

const backupMaxAge = 26 * time.Hour

func WithMigrationVersion(version string) Option { return func(o *options) { o.migration = version } }

// WithAuthenticatedSession records request-local proof that the protected
// status endpoint was reached through a valid Pulse session.
func WithAuthenticatedSession(authenticated bool) Option {
	return func(o *options) { o.authenticatedSession = authenticated }
}

func WithBackupObservation(lastSuccess, verifiedAt time.Time, verificationErr error) Option {
	return func(o *options) {
		backup := Backup{State: StateUnknown, Reason: "no_verified_backup"}
		if verificationErr != nil {
			backup.State, backup.Reason = StateDegraded, "backup_verification_failed"
		}
		if !lastSuccess.IsZero() {
			value := lastSuccess.UTC()
			backup.LastSuccessAt = &value
		}
		if !verifiedAt.IsZero() {
			value := verifiedAt.UTC()
			backup.VerifiedAt = &value
		}
		o.backup = &backup
	}
}

// SourceHealth is a runtime observation for one configured source or source-backed
// capability group. The API supplies these observations after querying the real
// Prometheus adapter and persisted agent snapshots.
type SourceHealth struct {
	ID            string
	State         State
	Reason        string
	ObservedAt    time.Time
	LastSuccessAt time.Time
}

// Option adjusts how a snapshot is built. Options are variadic so existing
// callers that have no background job information keep compiling and keep
// getting the conservative "not recorded" Unknown result.
type Option func(*options)

// WithJobs supplies the most recent background job outcomes. maxAge bounds how
// long a successful run stays usable; zero selects the default.
func WithJobs(maxAge time.Duration, jobs ...JobHealth) Option {
	return func(o *options) {
		o.jobs = append(o.jobs, jobs...)
		if maxAge > 0 {
			o.jobMaxAge = maxAge
		}
	}
}

// WithSources overrides configuration-only placeholders with observed runtime health.
// Unknown identifiers are ignored by Build, keeping the public component set closed.
func WithSources(sources ...SourceHealth) Option {
	return func(o *options) {
		if o.sources == nil {
			o.sources = make(map[string]SourceHealth)
		}
		for _, source := range sources {
			if source.ID != "" {
				o.sources[source.ID] = source
			}
		}
	}
}

// FromDatasource maps the shared datasource contract onto the self-observability
// contract without allowing stale Healthy values to survive EffectiveState.
func FromDatasource(id string, health datasource.SourceHealth, now time.Time) SourceHealth {
	state := StateUnknown
	switch health.EffectiveState(now.UTC()) {
	case datasource.HealthHealthy:
		state = StateHealthy
	case datasource.HealthDegraded:
		state = StateDegraded
	case datasource.HealthDisabled:
		state = StateDisabled
	}
	fallback := "source_health_unknown"
	if state == StateHealthy {
		fallback = "source_sampled"
	} else if state == StateDegraded {
		fallback = "source_degraded"
	} else if state == StateDisabled {
		fallback = "not_configured"
	}
	return SourceHealth{ID: id, State: state, Reason: reasonOr(health.ReasonCode, fallback), ObservedAt: health.ObservedAt.UTC(), LastSuccessAt: health.LastSuccess.UTC()}
}

func Build(cfg config.Config, databaseReady bool, now time.Time, auditEvents *int64, opts ...Option) Snapshot {
	if now.IsZero() {
		now = time.Now().UTC()
	}
	now = now.UTC()
	settings := options{jobMaxAge: defaultJobMaxAge, sources: make(map[string]SourceHealth)}
	for _, apply := range opts {
		if apply != nil {
			apply(&settings)
		}
	}
	components := []Component{
		state(databaseReady, "database_ready", "database_unavailable"),
		jobComponent(ComponentWorker, "heartbeat_not_recorded", settings, now),
		jobComponent(ComponentProbes, "probe_heartbeat_not_recorded", settings, now),
		jobComponent(ComponentNotifications, "delivery_health_not_sampled", settings, now),
	}
	prometheusComponent := configuredSourceComponent("prometheus", cfg.PrometheusURL != "", "not_configured", settings)
	components = append(components, prometheusComponent, mirrorComponent("query", prometheusComponent, "prometheus_not_configured"))
	// The API consumes Unraid through pulse-agent snapshots. A fresh observation is
	// stronger evidence than absent API-local Unraid credentials and must win.
	unraidConfigured := cfg.UnraidURL != "" && cfg.UnraidAPIToken != ""
	unraidComponent := configuredSourceComponent("unraid", unraidConfigured, "not_configured", settings)
	storageComponent := configuredSourceComponent("storage", unraidConfigured, "source_not_configured", settings)
	components = append(components, unraidComponent, storageComponent)
	if cfg.AuthMode == "oidc" && cfg.OIDCIssuer != "" && cfg.OIDCClientID != "" {
		oidc := Component{ID: "oidc", State: StateUnknown, Reason: "authenticated_session_not_observed"}
		if settings.authenticatedSession {
			oidc.State, oidc.Reason = StateHealthy, "authenticated_session"
			observed := now
			oidc.ObservedAt, oidc.LastSuccessAt = &observed, &observed
		}
		components = append(components, oidc)
	} else {
		components = append(components, Component{ID: "oidc", State: StateDisabled, Reason: "not_configured"})
	}
	sort.Slice(components, func(i, j int) bool { return components[i].ID < components[j].ID })
	backup := Backup{State: StateUnknown, Reason: "no_verified_backup"}
	if settings.backup != nil {
		backup = *settings.backup
	}
	if backup.LastSuccessAt != nil {
		age := int64(now.Sub(backup.LastSuccessAt.UTC()).Seconds())
		if age < 0 {
			age = 0
		}
		backup.AgeSeconds = &age
		if backup.State != StateDegraded {
			if time.Duration(age)*time.Second <= backupMaxAge {
				backup.State, backup.Reason = StateHealthy, "backup_verified"
			} else {
				backup.State, backup.Reason = StateUnknown, "backup_stale"
			}
		}
	}
	sourceLag := make([]SourceLag, 0, 2)
	for _, component := range []Component{prometheusComponent, unraidComponent} {
		if component.State == StateDisabled && settings.sources[component.ID].ID == "" {
			continue
		}
		lag := SourceLag{SourceID: component.ID, State: component.State, Reason: component.Reason, LastObservedAt: component.ObservedAt}
		if component.ObservedAt != nil {
			age := int64(now.Sub(*component.ObservedAt).Seconds())
			if age < 0 {
				age = 0
			}
			lag.AgeSeconds = &age
		}
		sourceLag = append(sourceLag, lag)
	}
	state := StateHealthy
	for _, component := range components {
		if component.State == StateDegraded {
			state = StateDegraded
			break
		}
		if component.State == StateUnknown && state == StateHealthy {
			state = StateUnknown
		}
		if component.State == StateDisabled && requiredComponent(component.ID) && state == StateHealthy {
			state = StateUnknown
		}
	}
	if backup.State == StateDegraded {
		state = StateDegraded
	}
	return Snapshot{Version: buildinfo.Version, Release: Release{Version: buildinfo.Version, Commit: buildinfo.Commit, BuiltAt: buildinfo.BuiltAt(), MigrationVersion: settings.migration}, GeneratedAt: now, OverallState: state, Components: components, Backup: backup, SourceLag: sourceLag, AuditEvents: auditEvents}
}

func configuredSourceComponent(id string, configured bool, disabledReason string, settings options) Component {
	if observed, ok := settings.sources[id]; ok {
		component := Component{ID: id, State: observed.State, Reason: reasonOr(observed.Reason, "source_health_unknown")}
		if !observed.ObservedAt.IsZero() {
			value := observed.ObservedAt.UTC()
			component.ObservedAt = &value
		}
		if !observed.LastSuccessAt.IsZero() {
			value := observed.LastSuccessAt.UTC()
			component.LastSuccessAt = &value
		}
		return component
	}
	if !configured {
		return Component{ID: id, State: StateDisabled, Reason: disabledReason}
	}
	return Component{ID: id, State: StateUnknown, Reason: "source_health_not_sampled"}
}

func mirrorComponent(id string, source Component, disabledReason string) Component {
	result := source
	result.ID = id
	if source.State == StateDisabled {
		result.Reason = disabledReason
	}
	return result
}

func requiredComponent(id string) bool {
	switch id {
	case "database", ComponentWorker, "prometheus", "query", "unraid", "storage":
		return true
	default:
		return false
	}
}

// jobComponent folds every reported job for one component into a single state.
// The worst reported state wins, a job that has never reported keeps the
// component Unknown with its original reason, and no combination of inputs can
// produce Healthy without a recent successful run (ADR-0008).
func jobComponent(id, neverReportedReason string, settings options, now time.Time) Component {
	component := Component{ID: id, State: StateUnknown, Reason: neverReportedReason}
	reported := false
	for _, job := range settings.jobs {
		if job.Component != id {
			continue
		}
		candidate := jobState(job, settings.jobMaxAge, now)
		if !reported || stateRank(candidate.State) > stateRank(component.State) {
			component = candidate
			reported = true
		}
	}
	return component
}

func jobState(job JobHealth, maxAge time.Duration, now time.Time) Component {
	component := Component{ID: job.Component, State: StateUnknown, Reason: "job_outcome_not_recorded"}
	if !job.LastRunAt.IsZero() {
		observed := job.LastRunAt.UTC()
		component.ObservedAt = &observed
	}
	if !job.LastSuccessAt.IsZero() {
		success := job.LastSuccessAt.UTC()
		component.LastSuccessAt = &success
	}
	switch job.Status {
	case JobDisabled:
		component.State = StateDisabled
		component.Reason = reasonOr(job.Reason, "not_configured")
		return component
	case JobFailed:
		component.State = StateDegraded
		component.Reason = reasonOr(job.ErrorCode, reasonOr(job.Reason, "last_run_failed"))
		return component
	case JobCompleted:
		if job.LastSuccessAt.IsZero() {
			component.Reason = "last_success_not_recorded"
			return component
		}
		if maxAge > 0 && now.Sub(job.LastSuccessAt.UTC()) > maxAge {
			component.Reason = "last_success_stale"
			return component
		}
		component.State = StateHealthy
		component.Reason = reasonOr(job.Reason, "last_run_succeeded")
		return component
	case JobRunning:
		component.Reason = "run_in_progress"
		return component
	default:
		return component
	}
}

func reasonOr(value, fallback string) string {
	if value == "" {
		return fallback
	}
	if len(value) > 160 {
		return value[:160]
	}
	return value
}

func stateRank(value State) int {
	switch value {
	case StateDegraded:
		return 3
	case StateUnknown:
		return 2
	case StateHealthy:
		return 1
	default:
		return 0
	}
}

func state(ready bool, healthyReason, unknownReason string) Component {
	if ready {
		return Component{ID: "database", State: StateHealthy, Reason: healthyReason}
	}
	return Component{ID: "database", State: StateUnknown, Reason: unknownReason}
}