Public source validation / validate (push) Failing after 3m8s
412 lines
13 KiB
Go
412 lines
13 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/itworx/pulse/internal/probe"
|
|
)
|
|
|
|
const ContractVersion = "v1"
|
|
|
|
const (
|
|
CapabilityAvailable = "available"
|
|
CapabilityUnavailable = "unavailable"
|
|
CapabilityUnsupported = "unsupported"
|
|
ConfigurationConfigured = "configured"
|
|
ConfigurationNotConfigured = "not_configured"
|
|
ConfigurationUnknown = "unknown"
|
|
)
|
|
|
|
type StatusPolicy struct {
|
|
FreshnessMaxAge time.Duration
|
|
MaxServices int
|
|
MaxHistory int
|
|
}
|
|
|
|
func (p StatusPolicy) withDefaults() StatusPolicy {
|
|
if p.FreshnessMaxAge == 0 {
|
|
p.FreshnessMaxAge = 2 * time.Minute
|
|
}
|
|
if p.MaxServices == 0 {
|
|
p.MaxServices = 150
|
|
}
|
|
if p.MaxHistory == 0 {
|
|
p.MaxHistory = 100
|
|
}
|
|
return p
|
|
}
|
|
|
|
func (p StatusPolicy) Validate() error {
|
|
p = p.withDefaults()
|
|
if p.FreshnessMaxAge <= 0 || p.FreshnessMaxAge > 24*time.Hour || p.MaxServices < 1 || p.MaxServices > 1000 || p.MaxHistory < 1 || p.MaxHistory > 500 {
|
|
return errors.New("service status policy is outside safe bounds")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type ProbeHistory struct {
|
|
ProbeID string
|
|
Results []probe.Result
|
|
}
|
|
|
|
type ProbeConfig struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Type string `json:"type"`
|
|
IntervalSeconds int `json:"intervalSeconds"`
|
|
TimeoutSeconds int `json:"timeoutSeconds"`
|
|
Enabled bool `json:"enabled"`
|
|
FollowRedirects bool `json:"followRedirects"`
|
|
VerifyTLS bool `json:"verifyTls"`
|
|
Revision int64 `json:"revision"`
|
|
}
|
|
|
|
type ServiceInput struct {
|
|
Service Service
|
|
Probes []ProbeHistory
|
|
ProbeConfigs []ProbeConfig
|
|
LatestCertificate *probe.Certificate
|
|
}
|
|
|
|
type ServiceStatus struct {
|
|
ID string `json:"id"`
|
|
EntityID string `json:"entityId,omitempty"`
|
|
SourceID string `json:"sourceId,omitempty"`
|
|
Name string `json:"name"`
|
|
Description string `json:"description"`
|
|
State string `json:"state"`
|
|
Reason string `json:"reason,omitempty"`
|
|
LastResultAt *time.Time `json:"lastResultAt,omitempty"`
|
|
LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
|
|
LastFailureAt *time.Time `json:"lastFailureAt,omitempty"`
|
|
ResponseTimeMS *int `json:"responseTimeMs,omitempty"`
|
|
AvailabilityPercent *float64 `json:"availabilityPercent,omitempty"`
|
|
SampleCount int `json:"sampleCount"`
|
|
SuccessfulSampleCount int `json:"successfulSampleCount"`
|
|
History []probe.Result `json:"history,omitempty"`
|
|
Probes []ProbeConfig `json:"probes,omitempty"`
|
|
Certificate *probe.Certificate `json:"certificate,omitempty"`
|
|
Revision int64 `json:"revision"`
|
|
ArchivedAt *time.Time `json:"archivedAt,omitempty"`
|
|
CreatedAt time.Time `json:"createdAt"`
|
|
UpdatedAt time.Time `json:"updatedAt"`
|
|
}
|
|
|
|
type Snapshot struct {
|
|
ContractVersion string `json:"contractVersion"`
|
|
ObservedAt time.Time `json:"observedAt"`
|
|
CapabilityState string `json:"capabilityState"`
|
|
ConfigurationState string `json:"configurationState"`
|
|
Reason string `json:"reason,omitempty"`
|
|
Services []ServiceStatus `json:"services"`
|
|
Total int `json:"total"`
|
|
Events []Event `json:"events,omitempty"`
|
|
}
|
|
|
|
type Provider interface {
|
|
Snapshot(context.Context) (Snapshot, error)
|
|
}
|
|
|
|
type UnknownProvider struct {
|
|
Reason string
|
|
Now func() time.Time
|
|
}
|
|
|
|
func (p UnknownProvider) Snapshot(ctx context.Context) (Snapshot, error) {
|
|
if ctx == nil {
|
|
return Snapshot{}, errors.New("service status context is nil")
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
now := time.Now().UTC()
|
|
if p.Now != nil {
|
|
now = p.Now().UTC()
|
|
}
|
|
reason := p.Reason
|
|
if strings.TrimSpace(reason) == "" {
|
|
reason = "source_unavailable"
|
|
}
|
|
return UnknownSnapshot(now, reason), nil
|
|
}
|
|
|
|
func UnknownSnapshot(now time.Time, reason string) Snapshot {
|
|
if now.IsZero() {
|
|
now = time.Now().UTC()
|
|
}
|
|
capability := CapabilityUnavailable
|
|
if reason == "unsupported" {
|
|
capability = CapabilityUnsupported
|
|
}
|
|
return Snapshot{ContractVersion: ContractVersion, ObservedAt: now.UTC(), CapabilityState: capability, ConfigurationState: ConfigurationUnknown, Reason: boundText(reason, 128), Services: []ServiceStatus{}, Total: 0, Events: []Event{}}
|
|
}
|
|
|
|
func BuildSnapshot(now time.Time, inputs []ServiceInput, policy StatusPolicy) (Snapshot, error) {
|
|
policy = policy.withDefaults()
|
|
if err := policy.Validate(); err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
if now.IsZero() {
|
|
now = time.Now().UTC()
|
|
}
|
|
now = now.UTC()
|
|
if len(inputs) > policy.MaxServices {
|
|
return Snapshot{}, errors.New("service count exceeds bounds")
|
|
}
|
|
statuses := make([]ServiceStatus, 0, len(inputs))
|
|
seen := make(map[string]struct{}, len(inputs))
|
|
for _, input := range inputs {
|
|
if err := input.Service.Validate(); err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
if _, exists := seen[input.Service.ID]; exists {
|
|
return Snapshot{}, errors.New("duplicate service identity")
|
|
}
|
|
seen[input.Service.ID] = struct{}{}
|
|
status := projectStatus(now, input, policy)
|
|
statuses = append(statuses, status)
|
|
}
|
|
sort.Slice(statuses, func(i, j int) bool { return statuses[i].ID < statuses[j].ID })
|
|
configuration := ConfigurationConfigured
|
|
reason := ""
|
|
if len(statuses) == 0 {
|
|
configuration = ConfigurationNotConfigured
|
|
reason = "no_services_configured"
|
|
}
|
|
return Snapshot{ContractVersion: ContractVersion, ObservedAt: now, CapabilityState: CapabilityAvailable, ConfigurationState: configuration, Reason: reason, Services: statuses, Total: len(statuses), Events: []Event{}}, nil
|
|
}
|
|
|
|
func projectStatus(now time.Time, input ServiceInput, policy StatusPolicy) ServiceStatus {
|
|
status := ServiceStatus{ID: input.Service.ID, EntityID: input.Service.EntityID, SourceID: input.Service.SourceID, Name: input.Service.Name, Description: input.Service.Description, State: StateUnknown, Reason: "no_probe_result", Revision: input.Service.Revision, ArchivedAt: input.Service.ArchivedAt, CreatedAt: input.Service.CreatedAt.UTC(), UpdatedAt: input.Service.UpdatedAt.UTC(), History: make([]probe.Result, 0), Probes: boundProbeConfigs(input.ProbeConfigs), Certificate: boundCertificate(input.LatestCertificate)}
|
|
if len(status.Probes) == 0 {
|
|
status.Reason = "no_probe_configured"
|
|
} else {
|
|
enabled := false
|
|
for _, config := range status.Probes {
|
|
enabled = enabled || config.Enabled
|
|
}
|
|
if !enabled {
|
|
status.Reason = "probes_disabled"
|
|
}
|
|
}
|
|
all := make([]probe.Result, 0)
|
|
for _, history := range input.Probes {
|
|
for _, result := range history.Results {
|
|
if result.ProbeID == "" {
|
|
result.ProbeID = history.ProbeID
|
|
}
|
|
if result.ProbeID == "" || result.ObservedAt.IsZero() {
|
|
continue
|
|
}
|
|
all = append(all, boundResult(result))
|
|
}
|
|
}
|
|
sort.SliceStable(all, func(i, j int) bool {
|
|
if !all[i].ObservedAt.Equal(all[j].ObservedAt) {
|
|
return all[i].ObservedAt.After(all[j].ObservedAt)
|
|
}
|
|
if all[i].ProbeID != all[j].ProbeID {
|
|
return all[i].ProbeID < all[j].ProbeID
|
|
}
|
|
return all[i].ID < all[j].ID
|
|
})
|
|
if len(all) > policy.MaxHistory {
|
|
all = all[:policy.MaxHistory]
|
|
}
|
|
status.History = append(status.History, all...)
|
|
if len(all) == 0 {
|
|
return status
|
|
}
|
|
latest := all[0]
|
|
lastResultAt := latest.ObservedAt.UTC()
|
|
status.LastResultAt = &lastResultAt
|
|
status.State = normalizedState(latest.State)
|
|
status.Reason = latest.ErrorClass
|
|
if status.State == StateUp {
|
|
status.Reason = ""
|
|
} else if status.State == StateDegraded && status.Reason == "" {
|
|
status.Reason = "probe_degraded"
|
|
}
|
|
if now.Sub(latest.ObservedAt) > policy.FreshnessMaxAge || latest.ObservedAt.After(now.Add(time.Minute)) {
|
|
status.State = StateUnknown
|
|
status.Reason = "stale_probe"
|
|
}
|
|
if latest.ResponseTimeMS != nil {
|
|
value := *latest.ResponseTimeMS
|
|
status.ResponseTimeMS = &value
|
|
}
|
|
known := 0
|
|
successful := 0
|
|
for _, result := range all {
|
|
state := normalizedState(result.State)
|
|
switch state {
|
|
case StateUp, StateDegraded:
|
|
known++
|
|
successful++
|
|
case StateDown, StateUnknown:
|
|
known++
|
|
}
|
|
if state == StateUp {
|
|
observed := result.ObservedAt.UTC()
|
|
if status.LastSuccessAt == nil || observed.After(*status.LastSuccessAt) {
|
|
status.LastSuccessAt = &observed
|
|
}
|
|
}
|
|
if state == StateDown || state == StateUnknown {
|
|
observed := result.ObservedAt.UTC()
|
|
if status.LastFailureAt == nil || observed.After(*status.LastFailureAt) {
|
|
status.LastFailureAt = &observed
|
|
}
|
|
}
|
|
}
|
|
status.SampleCount = known
|
|
status.SuccessfulSampleCount = successful
|
|
if known > 0 {
|
|
value := float64(successful) * 100 / float64(known)
|
|
status.AvailabilityPercent = &value
|
|
}
|
|
return status
|
|
}
|
|
|
|
func boundProbeConfigs(configs []ProbeConfig) []ProbeConfig {
|
|
bounded := append([]ProbeConfig(nil), configs...)
|
|
sort.SliceStable(bounded, func(i, j int) bool {
|
|
if bounded[i].ID != bounded[j].ID {
|
|
return bounded[i].ID < bounded[j].ID
|
|
}
|
|
return bounded[i].Name < bounded[j].Name
|
|
})
|
|
if len(bounded) > 100 {
|
|
bounded = bounded[:100]
|
|
}
|
|
for index := range bounded {
|
|
bounded[index].ID = boundText(bounded[index].ID, 64)
|
|
bounded[index].Name = boundText(bounded[index].Name, 160)
|
|
bounded[index].Type = boundText(bounded[index].Type, 16)
|
|
if bounded[index].IntervalSeconds < 0 {
|
|
bounded[index].IntervalSeconds = 0
|
|
}
|
|
if bounded[index].TimeoutSeconds < 0 {
|
|
bounded[index].TimeoutSeconds = 0
|
|
}
|
|
}
|
|
return bounded
|
|
}
|
|
func boundResult(result probe.Result) probe.Result {
|
|
result.ObservedAt = result.ObservedAt.UTC()
|
|
result.CompletedAt = result.CompletedAt.UTC()
|
|
result.ErrorClass = boundText(result.ErrorClass, 64)
|
|
result.ErrorMessage = boundText(result.ErrorMessage, 256)
|
|
if result.ResponseTimeMS != nil && *result.ResponseTimeMS < 0 {
|
|
result.ResponseTimeMS = nil
|
|
}
|
|
if result.Attributes != nil {
|
|
keys := make([]string, 0, len(result.Attributes))
|
|
for key := range result.Attributes {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
if len(keys) > 16 {
|
|
keys = keys[:16]
|
|
}
|
|
bounded := make(map[string]any, len(keys))
|
|
for _, key := range keys {
|
|
bounded[boundText(key, 64)] = boundAttribute(result.Attributes[key])
|
|
}
|
|
result.Attributes = bounded
|
|
}
|
|
if result.Certificate != nil {
|
|
result.Certificate = boundCertificate(result.Certificate)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func boundCertificate(value *probe.Certificate) *probe.Certificate {
|
|
if value == nil {
|
|
return nil
|
|
}
|
|
certificate := *value
|
|
certificate.ID = boundText(certificate.ID, 128)
|
|
certificate.ServiceID = boundText(certificate.ServiceID, 128)
|
|
certificate.EndpointID = boundText(certificate.EndpointID, 128)
|
|
certificate.ObservedAt = certificate.ObservedAt.UTC()
|
|
if certificate.ExpiresAt != nil {
|
|
expiresAt := certificate.ExpiresAt.UTC()
|
|
certificate.ExpiresAt = &expiresAt
|
|
}
|
|
certificate.Issuer = boundText(certificate.Issuer, 256)
|
|
certificate.Subject = boundText(certificate.Subject, 256)
|
|
certificate.VerificationState = boundText(certificate.VerificationState, 32)
|
|
return &certificate
|
|
}
|
|
|
|
func boundAttribute(value any) any {
|
|
switch typed := value.(type) {
|
|
case string:
|
|
return boundText(typed, 256)
|
|
case bool, int, int32, int64, float32, float64, nil:
|
|
return typed
|
|
default:
|
|
return "[redacted]"
|
|
}
|
|
}
|
|
func normalizedState(state string) string {
|
|
switch state {
|
|
case StateUp, StateDegraded, StateDown, StateUnknown:
|
|
return state
|
|
default:
|
|
return StateUnknown
|
|
}
|
|
}
|
|
|
|
func boundText(value string, max int) string {
|
|
value = strings.TrimSpace(value)
|
|
if len(value) > max {
|
|
return value[:max]
|
|
}
|
|
return value
|
|
}
|
|
|
|
type Event struct {
|
|
ID string `json:"id"`
|
|
Type string `json:"type"`
|
|
ServiceID string `json:"serviceId"`
|
|
FromState string `json:"fromState"`
|
|
ToState string `json:"toState"`
|
|
Reason string `json:"reason,omitempty"`
|
|
OccurredAt time.Time `json:"occurredAt"`
|
|
}
|
|
|
|
func TransitionEvents(previous, current Snapshot) []Event {
|
|
previousByID := make(map[string]ServiceStatus, len(previous.Services))
|
|
for _, item := range previous.Services {
|
|
previousByID[item.ID] = item
|
|
}
|
|
events := make([]Event, 0)
|
|
for _, item := range current.Services {
|
|
before, exists := previousByID[item.ID]
|
|
if !exists || before.State == item.State {
|
|
continue
|
|
}
|
|
eventType := "service.state_changed"
|
|
if item.State == StateDown {
|
|
eventType = "service.down"
|
|
} else if item.State == StateUp && (before.State == StateDown || before.State == StateUnknown) {
|
|
eventType = "service.recovered"
|
|
}
|
|
eventTime := current.ObservedAt.UTC()
|
|
event := Event{Type: eventType, ServiceID: item.ID, FromState: before.State, ToState: item.State, Reason: boundText(item.Reason, 128), OccurredAt: eventTime}
|
|
digest := sha256.Sum256([]byte(item.ID + "\x00" + before.State + "\x00" + item.State + "\x00" + eventTime.Format(time.RFC3339Nano)))
|
|
event.ID = hex.EncodeToString(digest[:])
|
|
events = append(events, event)
|
|
}
|
|
sort.Slice(events, func(i, j int) bool { return events[i].ServiceID < events[j].ServiceID })
|
|
return events
|
|
}
|