This commit is contained in:
ITWorx Pulse release export
@@ -0,0 +1,411 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/probe"
|
||||
)
|
||||
|
||||
const ContractVersion = "v1"
|
||||
|
||||
const (
|
||||
CapabilityAvailable = "available"
|
||||
CapabilityUnavailable = "unavailable"
|
||||
CapabilityUnsupported = "unsupported"
|
||||
ConfigurationConfigured = "configured"
|
||||
ConfigurationNotConfigured = "not_configured"
|
||||
ConfigurationUnknown = "unknown"
|
||||
)
|
||||
|
||||
type StatusPolicy struct {
|
||||
FreshnessMaxAge time.Duration
|
||||
MaxServices int
|
||||
MaxHistory int
|
||||
}
|
||||
|
||||
func (p StatusPolicy) withDefaults() StatusPolicy {
|
||||
if p.FreshnessMaxAge == 0 {
|
||||
p.FreshnessMaxAge = 2 * time.Minute
|
||||
}
|
||||
if p.MaxServices == 0 {
|
||||
p.MaxServices = 150
|
||||
}
|
||||
if p.MaxHistory == 0 {
|
||||
p.MaxHistory = 100
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (p StatusPolicy) Validate() error {
|
||||
p = p.withDefaults()
|
||||
if p.FreshnessMaxAge <= 0 || p.FreshnessMaxAge > 24*time.Hour || p.MaxServices < 1 || p.MaxServices > 1000 || p.MaxHistory < 1 || p.MaxHistory > 500 {
|
||||
return errors.New("service status policy is outside safe bounds")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ProbeHistory struct {
|
||||
ProbeID string
|
||||
Results []probe.Result
|
||||
}
|
||||
|
||||
type ProbeConfig struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
IntervalSeconds int `json:"intervalSeconds"`
|
||||
TimeoutSeconds int `json:"timeoutSeconds"`
|
||||
Enabled bool `json:"enabled"`
|
||||
FollowRedirects bool `json:"followRedirects"`
|
||||
VerifyTLS bool `json:"verifyTls"`
|
||||
Revision int64 `json:"revision"`
|
||||
}
|
||||
|
||||
type ServiceInput struct {
|
||||
Service Service
|
||||
Probes []ProbeHistory
|
||||
ProbeConfigs []ProbeConfig
|
||||
LatestCertificate *probe.Certificate
|
||||
}
|
||||
|
||||
type ServiceStatus struct {
|
||||
ID string `json:"id"`
|
||||
EntityID string `json:"entityId,omitempty"`
|
||||
SourceID string `json:"sourceId,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
State string `json:"state"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
LastResultAt *time.Time `json:"lastResultAt,omitempty"`
|
||||
LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
|
||||
LastFailureAt *time.Time `json:"lastFailureAt,omitempty"`
|
||||
ResponseTimeMS *int `json:"responseTimeMs,omitempty"`
|
||||
AvailabilityPercent *float64 `json:"availabilityPercent,omitempty"`
|
||||
SampleCount int `json:"sampleCount"`
|
||||
SuccessfulSampleCount int `json:"successfulSampleCount"`
|
||||
History []probe.Result `json:"history,omitempty"`
|
||||
Probes []ProbeConfig `json:"probes,omitempty"`
|
||||
Certificate *probe.Certificate `json:"certificate,omitempty"`
|
||||
Revision int64 `json:"revision"`
|
||||
ArchivedAt *time.Time `json:"archivedAt,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
ContractVersion string `json:"contractVersion"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
CapabilityState string `json:"capabilityState"`
|
||||
ConfigurationState string `json:"configurationState"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Services []ServiceStatus `json:"services"`
|
||||
Total int `json:"total"`
|
||||
Events []Event `json:"events,omitempty"`
|
||||
}
|
||||
|
||||
type Provider interface {
|
||||
Snapshot(context.Context) (Snapshot, error)
|
||||
}
|
||||
|
||||
type UnknownProvider struct {
|
||||
Reason string
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (p UnknownProvider) Snapshot(ctx context.Context) (Snapshot, error) {
|
||||
if ctx == nil {
|
||||
return Snapshot{}, errors.New("service status context is nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if p.Now != nil {
|
||||
now = p.Now().UTC()
|
||||
}
|
||||
reason := p.Reason
|
||||
if strings.TrimSpace(reason) == "" {
|
||||
reason = "source_unavailable"
|
||||
}
|
||||
return UnknownSnapshot(now, reason), nil
|
||||
}
|
||||
|
||||
func UnknownSnapshot(now time.Time, reason string) Snapshot {
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
capability := CapabilityUnavailable
|
||||
if reason == "unsupported" {
|
||||
capability = CapabilityUnsupported
|
||||
}
|
||||
return Snapshot{ContractVersion: ContractVersion, ObservedAt: now.UTC(), CapabilityState: capability, ConfigurationState: ConfigurationUnknown, Reason: boundText(reason, 128), Services: []ServiceStatus{}, Total: 0, Events: []Event{}}
|
||||
}
|
||||
|
||||
func BuildSnapshot(now time.Time, inputs []ServiceInput, policy StatusPolicy) (Snapshot, error) {
|
||||
policy = policy.withDefaults()
|
||||
if err := policy.Validate(); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
now = now.UTC()
|
||||
if len(inputs) > policy.MaxServices {
|
||||
return Snapshot{}, errors.New("service count exceeds bounds")
|
||||
}
|
||||
statuses := make([]ServiceStatus, 0, len(inputs))
|
||||
seen := make(map[string]struct{}, len(inputs))
|
||||
for _, input := range inputs {
|
||||
if err := input.Service.Validate(); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
if _, exists := seen[input.Service.ID]; exists {
|
||||
return Snapshot{}, errors.New("duplicate service identity")
|
||||
}
|
||||
seen[input.Service.ID] = struct{}{}
|
||||
status := projectStatus(now, input, policy)
|
||||
statuses = append(statuses, status)
|
||||
}
|
||||
sort.Slice(statuses, func(i, j int) bool { return statuses[i].ID < statuses[j].ID })
|
||||
configuration := ConfigurationConfigured
|
||||
reason := ""
|
||||
if len(statuses) == 0 {
|
||||
configuration = ConfigurationNotConfigured
|
||||
reason = "no_services_configured"
|
||||
}
|
||||
return Snapshot{ContractVersion: ContractVersion, ObservedAt: now, CapabilityState: CapabilityAvailable, ConfigurationState: configuration, Reason: reason, Services: statuses, Total: len(statuses), Events: []Event{}}, nil
|
||||
}
|
||||
|
||||
func projectStatus(now time.Time, input ServiceInput, policy StatusPolicy) ServiceStatus {
|
||||
status := ServiceStatus{ID: input.Service.ID, EntityID: input.Service.EntityID, SourceID: input.Service.SourceID, Name: input.Service.Name, Description: input.Service.Description, State: StateUnknown, Reason: "no_probe_result", Revision: input.Service.Revision, ArchivedAt: input.Service.ArchivedAt, CreatedAt: input.Service.CreatedAt.UTC(), UpdatedAt: input.Service.UpdatedAt.UTC(), History: make([]probe.Result, 0), Probes: boundProbeConfigs(input.ProbeConfigs), Certificate: boundCertificate(input.LatestCertificate)}
|
||||
if len(status.Probes) == 0 {
|
||||
status.Reason = "no_probe_configured"
|
||||
} else {
|
||||
enabled := false
|
||||
for _, config := range status.Probes {
|
||||
enabled = enabled || config.Enabled
|
||||
}
|
||||
if !enabled {
|
||||
status.Reason = "probes_disabled"
|
||||
}
|
||||
}
|
||||
all := make([]probe.Result, 0)
|
||||
for _, history := range input.Probes {
|
||||
for _, result := range history.Results {
|
||||
if result.ProbeID == "" {
|
||||
result.ProbeID = history.ProbeID
|
||||
}
|
||||
if result.ProbeID == "" || result.ObservedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
all = append(all, boundResult(result))
|
||||
}
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool {
|
||||
if !all[i].ObservedAt.Equal(all[j].ObservedAt) {
|
||||
return all[i].ObservedAt.After(all[j].ObservedAt)
|
||||
}
|
||||
if all[i].ProbeID != all[j].ProbeID {
|
||||
return all[i].ProbeID < all[j].ProbeID
|
||||
}
|
||||
return all[i].ID < all[j].ID
|
||||
})
|
||||
if len(all) > policy.MaxHistory {
|
||||
all = all[:policy.MaxHistory]
|
||||
}
|
||||
status.History = append(status.History, all...)
|
||||
if len(all) == 0 {
|
||||
return status
|
||||
}
|
||||
latest := all[0]
|
||||
lastResultAt := latest.ObservedAt.UTC()
|
||||
status.LastResultAt = &lastResultAt
|
||||
status.State = normalizedState(latest.State)
|
||||
status.Reason = latest.ErrorClass
|
||||
if status.State == StateUp {
|
||||
status.Reason = ""
|
||||
} else if status.State == StateDegraded && status.Reason == "" {
|
||||
status.Reason = "probe_degraded"
|
||||
}
|
||||
if now.Sub(latest.ObservedAt) > policy.FreshnessMaxAge || latest.ObservedAt.After(now.Add(time.Minute)) {
|
||||
status.State = StateUnknown
|
||||
status.Reason = "stale_probe"
|
||||
}
|
||||
if latest.ResponseTimeMS != nil {
|
||||
value := *latest.ResponseTimeMS
|
||||
status.ResponseTimeMS = &value
|
||||
}
|
||||
known := 0
|
||||
successful := 0
|
||||
for _, result := range all {
|
||||
state := normalizedState(result.State)
|
||||
switch state {
|
||||
case StateUp, StateDegraded:
|
||||
known++
|
||||
successful++
|
||||
case StateDown, StateUnknown:
|
||||
known++
|
||||
}
|
||||
if state == StateUp {
|
||||
observed := result.ObservedAt.UTC()
|
||||
if status.LastSuccessAt == nil || observed.After(*status.LastSuccessAt) {
|
||||
status.LastSuccessAt = &observed
|
||||
}
|
||||
}
|
||||
if state == StateDown || state == StateUnknown {
|
||||
observed := result.ObservedAt.UTC()
|
||||
if status.LastFailureAt == nil || observed.After(*status.LastFailureAt) {
|
||||
status.LastFailureAt = &observed
|
||||
}
|
||||
}
|
||||
}
|
||||
status.SampleCount = known
|
||||
status.SuccessfulSampleCount = successful
|
||||
if known > 0 {
|
||||
value := float64(successful) * 100 / float64(known)
|
||||
status.AvailabilityPercent = &value
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func boundProbeConfigs(configs []ProbeConfig) []ProbeConfig {
|
||||
bounded := append([]ProbeConfig(nil), configs...)
|
||||
sort.SliceStable(bounded, func(i, j int) bool {
|
||||
if bounded[i].ID != bounded[j].ID {
|
||||
return bounded[i].ID < bounded[j].ID
|
||||
}
|
||||
return bounded[i].Name < bounded[j].Name
|
||||
})
|
||||
if len(bounded) > 100 {
|
||||
bounded = bounded[:100]
|
||||
}
|
||||
for index := range bounded {
|
||||
bounded[index].ID = boundText(bounded[index].ID, 64)
|
||||
bounded[index].Name = boundText(bounded[index].Name, 160)
|
||||
bounded[index].Type = boundText(bounded[index].Type, 16)
|
||||
if bounded[index].IntervalSeconds < 0 {
|
||||
bounded[index].IntervalSeconds = 0
|
||||
}
|
||||
if bounded[index].TimeoutSeconds < 0 {
|
||||
bounded[index].TimeoutSeconds = 0
|
||||
}
|
||||
}
|
||||
return bounded
|
||||
}
|
||||
func boundResult(result probe.Result) probe.Result {
|
||||
result.ObservedAt = result.ObservedAt.UTC()
|
||||
result.CompletedAt = result.CompletedAt.UTC()
|
||||
result.ErrorClass = boundText(result.ErrorClass, 64)
|
||||
result.ErrorMessage = boundText(result.ErrorMessage, 256)
|
||||
if result.ResponseTimeMS != nil && *result.ResponseTimeMS < 0 {
|
||||
result.ResponseTimeMS = nil
|
||||
}
|
||||
if result.Attributes != nil {
|
||||
keys := make([]string, 0, len(result.Attributes))
|
||||
for key := range result.Attributes {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
if len(keys) > 16 {
|
||||
keys = keys[:16]
|
||||
}
|
||||
bounded := make(map[string]any, len(keys))
|
||||
for _, key := range keys {
|
||||
bounded[boundText(key, 64)] = boundAttribute(result.Attributes[key])
|
||||
}
|
||||
result.Attributes = bounded
|
||||
}
|
||||
if result.Certificate != nil {
|
||||
result.Certificate = boundCertificate(result.Certificate)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func boundCertificate(value *probe.Certificate) *probe.Certificate {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
certificate := *value
|
||||
certificate.ID = boundText(certificate.ID, 128)
|
||||
certificate.ServiceID = boundText(certificate.ServiceID, 128)
|
||||
certificate.EndpointID = boundText(certificate.EndpointID, 128)
|
||||
certificate.ObservedAt = certificate.ObservedAt.UTC()
|
||||
if certificate.ExpiresAt != nil {
|
||||
expiresAt := certificate.ExpiresAt.UTC()
|
||||
certificate.ExpiresAt = &expiresAt
|
||||
}
|
||||
certificate.Issuer = boundText(certificate.Issuer, 256)
|
||||
certificate.Subject = boundText(certificate.Subject, 256)
|
||||
certificate.VerificationState = boundText(certificate.VerificationState, 32)
|
||||
return &certificate
|
||||
}
|
||||
|
||||
func boundAttribute(value any) any {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return boundText(typed, 256)
|
||||
case bool, int, int32, int64, float32, float64, nil:
|
||||
return typed
|
||||
default:
|
||||
return "[redacted]"
|
||||
}
|
||||
}
|
||||
func normalizedState(state string) string {
|
||||
switch state {
|
||||
case StateUp, StateDegraded, StateDown, StateUnknown:
|
||||
return state
|
||||
default:
|
||||
return StateUnknown
|
||||
}
|
||||
}
|
||||
|
||||
func boundText(value string, max int) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if len(value) > max {
|
||||
return value[:max]
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
ServiceID string `json:"serviceId"`
|
||||
FromState string `json:"fromState"`
|
||||
ToState string `json:"toState"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
OccurredAt time.Time `json:"occurredAt"`
|
||||
}
|
||||
|
||||
func TransitionEvents(previous, current Snapshot) []Event {
|
||||
previousByID := make(map[string]ServiceStatus, len(previous.Services))
|
||||
for _, item := range previous.Services {
|
||||
previousByID[item.ID] = item
|
||||
}
|
||||
events := make([]Event, 0)
|
||||
for _, item := range current.Services {
|
||||
before, exists := previousByID[item.ID]
|
||||
if !exists || before.State == item.State {
|
||||
continue
|
||||
}
|
||||
eventType := "service.state_changed"
|
||||
if item.State == StateDown {
|
||||
eventType = "service.down"
|
||||
} else if item.State == StateUp && (before.State == StateDown || before.State == StateUnknown) {
|
||||
eventType = "service.recovered"
|
||||
}
|
||||
eventTime := current.ObservedAt.UTC()
|
||||
event := Event{Type: eventType, ServiceID: item.ID, FromState: before.State, ToState: item.State, Reason: boundText(item.Reason, 128), OccurredAt: eventTime}
|
||||
digest := sha256.Sum256([]byte(item.ID + "\x00" + before.State + "\x00" + item.State + "\x00" + eventTime.Format(time.RFC3339Nano)))
|
||||
event.ID = hex.EncodeToString(digest[:])
|
||||
events = append(events, event)
|
||||
}
|
||||
sort.Slice(events, func(i, j int) bool { return events[i].ServiceID < events[j].ServiceID })
|
||||
return events
|
||||
}
|
||||
Reference in New Issue
Block a user