This commit is contained in:
@@ -0,0 +1,46 @@
|
||||
//go:build linux
|
||||
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"math"
|
||||
"syscall"
|
||||
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
)
|
||||
|
||||
// Bits of the adjtimex status word, defined here because the standard syscall package
|
||||
// does not export them on every architecture.
|
||||
const (
|
||||
staUnsync = 0x0040 // clock is not synchronised to a reference
|
||||
staNano = 0x2000 // offset and precision are in nanoseconds, not microseconds
|
||||
timeError = 5 // adjtimex state: the clock is not synchronised
|
||||
)
|
||||
|
||||
// readClockSync asks the kernel about clock discipline with adjtimex(2) in read mode.
|
||||
//
|
||||
// A zeroed Timex has Modes == 0, which makes the call a pure read: it adjusts nothing.
|
||||
// Docker's default seccomp profile still gates adjtimex behind CAP_SYS_TIME, which the
|
||||
// agent drops, so the expected in-container result is EPERM. The caller turns that into
|
||||
// a warning; see Collector.clock.
|
||||
func readClockSync() (host.RawTime, error) {
|
||||
timex := syscall.Timex{}
|
||||
state, err := syscall.Adjtimex(&timex)
|
||||
if err != nil {
|
||||
return host.RawTime{}, err
|
||||
}
|
||||
divisor := 1e6
|
||||
if timex.Status&staNano != 0 {
|
||||
divisor = 1e9
|
||||
}
|
||||
offset := math.Abs(float64(timex.Offset)) / divisor
|
||||
if math.IsNaN(offset) || math.IsInf(offset, 0) {
|
||||
offset = 0
|
||||
}
|
||||
return host.RawTime{
|
||||
Synchronized: timex.Status&staUnsync == 0 && state != timeError,
|
||||
// The kernel exposes no stratum; that belongs to the NTP daemon, which the
|
||||
// agent deliberately does not talk to.
|
||||
OffsetSeconds: offset,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
//go:build !linux
|
||||
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
)
|
||||
|
||||
// readClockSync has no portable equivalent off Linux; the caller degrades to a warning.
|
||||
func readClockSync() (host.RawTime, error) {
|
||||
return host.RawTime{}, errors.New("hostcollect: clock synchronisation probe is only available on linux")
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
// Package hostcollect turns the kernel's read-only procfs and sysfs views into the
|
||||
// bounded domain snapshots defined by internal/host and internal/process.
|
||||
//
|
||||
// Everything here is read-only by construction: the collector opens files under a
|
||||
// configurable procfs/sysfs root and never writes, executes, or opens a socket. That
|
||||
// is what lets pulse-agent run with cap_drop: [ALL] and satisfy ADR-0005 — no Docker
|
||||
// socket, no privileged host access, no mutation path.
|
||||
//
|
||||
// The roots are injectable so the parsers can be exercised against committed fixture
|
||||
// directories: the test environment is never the target host, and a collector that can
|
||||
// only be tested on the real machine is a collector that is not tested at all.
|
||||
//
|
||||
// Two properties of /proc drive most of the design:
|
||||
//
|
||||
// - Utilisation is a delta, not a reading. /proc/stat and /proc/<pid>/stat expose
|
||||
// monotonic counters in USER_HZ ticks, so a single sample cannot yield a percentage.
|
||||
// The collector keeps the previous sample and reports percentages only from the
|
||||
// second collection onwards.
|
||||
// - /proc is a live view of a moving system. Processes disappear between readdir and
|
||||
// open, counters reset when a kernel counter wraps or a task is replaced by a new
|
||||
// one with the same PID. Every such case degrades one field, never the collection.
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
"github.com/itworx/pulse/internal/process"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultProcRoot is the standard procfs mount point.
|
||||
DefaultProcRoot = "/proc"
|
||||
// DefaultSysRoot is the standard sysfs mount point.
|
||||
DefaultSysRoot = "/sys"
|
||||
// defaultClockTicks is the USER_HZ value the kernel exposes CPU times in. It is
|
||||
// 100 on every supported Linux/architecture combination Pulse targets; sysconf is
|
||||
// unavailable without cgo, so it stays configurable instead of guessed silently.
|
||||
defaultClockTicks = 100
|
||||
// defaultPageSizeFallback is used only when the runtime reports a nonsensical page
|
||||
// size; RSS in /proc/<pid>/stat is counted in pages, not bytes.
|
||||
defaultPageSizeFallback = 4096
|
||||
|
||||
maxFileBytes = 1 << 20
|
||||
maxProcessFileBytes = 64 << 10
|
||||
// maxCmdlineBytes bounds how much of a command line is read at all. Only the
|
||||
// program name is ever kept, so arguments — which routinely carry tokens and
|
||||
// passwords — are never copied into a snapshot.
|
||||
maxCmdlineBytes = 4 << 10
|
||||
maxNameBytes = 255
|
||||
)
|
||||
|
||||
// Warning codes attached to a host snapshot when one optional field degrades. They are
|
||||
// stable identifiers, safe to log and to render, and never contain host data.
|
||||
const (
|
||||
WarningCPUUnavailable = "cpu_source_unavailable"
|
||||
WarningCPUFirstSample = "cpu_awaiting_second_sample"
|
||||
WarningCPUCounterReset = "cpu_counter_reset"
|
||||
WarningCPUTopologyChanged = "cpu_topology_changed"
|
||||
WarningCPUCoresTruncated = "cpu_cores_truncated"
|
||||
WarningLoadUnavailable = "load_source_unavailable"
|
||||
WarningNetworkUnavailable = "network_source_unavailable"
|
||||
WarningNetworkTruncated = "network_interfaces_truncated"
|
||||
WarningMountsUnavailable = "filesystem_source_unavailable"
|
||||
WarningFilesystemPartial = "filesystem_partially_unavailable"
|
||||
WarningFilesystemTruncated = "filesystems_truncated"
|
||||
WarningFilesystemDisabled = "filesystem_root_not_configured"
|
||||
WarningKernelUnavailable = "kernel_version_unavailable"
|
||||
WarningClockProbeAssumed = "clock_sync_unverified_assumed_synchronized"
|
||||
)
|
||||
|
||||
// FilesystemUsage is the bounded result of one statfs call.
|
||||
type FilesystemUsage struct {
|
||||
CapacityBytes uint64
|
||||
UsedBytes uint64
|
||||
InodesTotal uint64
|
||||
InodesUsed uint64
|
||||
}
|
||||
|
||||
// Options configures a Collector. The zero value is usable and reads the real host.
|
||||
type Options struct {
|
||||
// ProcRoot is the procfs mount point, "/proc" by default. In a container it is the
|
||||
// read-only bind mount of the host's /proc (see deploy/compose.yaml).
|
||||
ProcRoot string
|
||||
// SysRoot is the sysfs mount point, "/sys" by default. Only used for interface
|
||||
// operational state today.
|
||||
SysRoot string
|
||||
// FilesystemRoot prefixes every mount point before statfs. It is empty by default,
|
||||
// which disables filesystem collection entirely: inside a container the host mount
|
||||
// points listed in /proc/mounts do not resolve, and statfs of an identically named
|
||||
// path would silently report the container's own overlay instead of the host's
|
||||
// array. Set it to "/" on a host, or to the prefix a host root is mounted at.
|
||||
FilesystemRoot string
|
||||
// HostName overrides the collected host name. /proc/sys/kernel/hostname is read
|
||||
// through the reader's UTS namespace, so inside a container it returns the
|
||||
// container's name even when the host's /proc is bind mounted.
|
||||
HostName string
|
||||
// ClockTicks is USER_HZ; 100 when unset.
|
||||
ClockTicks float64
|
||||
// PageSize is the memory page size in bytes; the runtime value when unset.
|
||||
PageSize int
|
||||
// SourceID identifies this collector in the snapshot's Source block.
|
||||
SourceID string
|
||||
// HostLimits bounds the host snapshot (cores, filesystems, interfaces, warnings).
|
||||
HostLimits host.Limits
|
||||
// ProcessLimits bounds the process inventory.
|
||||
ProcessLimits process.Limits
|
||||
// Now returns the current time; time.Now when unset.
|
||||
Now func() time.Time
|
||||
// StatFS reads usage for one mount point; a real statfs syscall when unset.
|
||||
StatFS func(path string) (FilesystemUsage, error)
|
||||
// ClockSync reports host clock synchronisation; adjtimex(2) in read mode when unset.
|
||||
ClockSync func() (host.RawTime, error)
|
||||
}
|
||||
|
||||
func (o Options) withDefaults() Options {
|
||||
if o.ProcRoot == "" {
|
||||
o.ProcRoot = DefaultProcRoot
|
||||
}
|
||||
if o.SysRoot == "" {
|
||||
o.SysRoot = DefaultSysRoot
|
||||
}
|
||||
if o.ClockTicks <= 0 {
|
||||
o.ClockTicks = defaultClockTicks
|
||||
}
|
||||
if o.PageSize <= 0 {
|
||||
o.PageSize = os.Getpagesize()
|
||||
}
|
||||
if o.PageSize <= 0 {
|
||||
o.PageSize = defaultPageSizeFallback
|
||||
}
|
||||
if o.SourceID == "" {
|
||||
o.SourceID = "host"
|
||||
}
|
||||
o.HostLimits = hostLimitsWithDefaults(o.HostLimits)
|
||||
o.ProcessLimits = processLimitsWithDefaults(o.ProcessLimits)
|
||||
if o.Now == nil {
|
||||
o.Now = time.Now
|
||||
}
|
||||
if o.StatFS == nil {
|
||||
o.StatFS = statFS
|
||||
}
|
||||
if o.ClockSync == nil {
|
||||
o.ClockSync = readClockSync
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// hostLimitsWithDefaults mirrors the defaults internal/host applies during
|
||||
// normalization; the collector must bound its output before the domain sees it, and
|
||||
// the domain's own defaulting is unexported.
|
||||
func hostLimitsWithDefaults(limits host.Limits) host.Limits {
|
||||
if limits.MaxCores == 0 {
|
||||
limits.MaxCores = 256
|
||||
}
|
||||
if limits.MaxFilesystems == 0 {
|
||||
limits.MaxFilesystems = 256
|
||||
}
|
||||
if limits.MaxInterfaces == 0 {
|
||||
limits.MaxInterfaces = 128
|
||||
}
|
||||
if limits.MaxWarnings == 0 {
|
||||
limits.MaxWarnings = 20
|
||||
}
|
||||
return limits
|
||||
}
|
||||
|
||||
func processLimitsWithDefaults(limits process.Limits) process.Limits {
|
||||
if limits.MaxRows == 0 {
|
||||
limits.MaxRows = 1000
|
||||
}
|
||||
if limits.MaxPageSize == 0 {
|
||||
limits.MaxPageSize = 100
|
||||
}
|
||||
return limits
|
||||
}
|
||||
|
||||
// Collector reads host and process telemetry. It is safe for concurrent use; the
|
||||
// previous samples it needs for delta calculation are guarded by a mutex.
|
||||
type Collector struct {
|
||||
options Options
|
||||
|
||||
mu sync.Mutex
|
||||
cpu *cpuSample
|
||||
processes map[processKey]processCPUSample
|
||||
}
|
||||
|
||||
// New validates the options and returns a ready collector.
|
||||
func New(options Options) (*Collector, error) {
|
||||
options = options.withDefaults()
|
||||
if !filepath.IsAbs(options.ProcRoot) && !isTestPath(options.ProcRoot) {
|
||||
return nil, fmt.Errorf("hostcollect: proc root %q must be an absolute path", options.ProcRoot)
|
||||
}
|
||||
if err := options.HostLimits.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("hostcollect: %w", err)
|
||||
}
|
||||
if err := options.ProcessLimits.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("hostcollect: %w", err)
|
||||
}
|
||||
return &Collector{options: options, processes: map[processKey]processCPUSample{}}, nil
|
||||
}
|
||||
|
||||
// isTestPath allows relative fixture roots so tests can use testdata directories
|
||||
// without constructing absolute paths.
|
||||
func isTestPath(path string) bool { return path != "" && !filepath.IsAbs(path) }
|
||||
|
||||
// Host reads one bounded host snapshot. It fails only when a field the domain requires
|
||||
// is unreadable (identity, uptime, memory); every optional field degrades into a
|
||||
// warning so a single unreadable file cannot blank the whole capability.
|
||||
func (c *Collector) Host(ctx context.Context) (host.RawSnapshot, error) {
|
||||
if c == nil {
|
||||
return host.RawSnapshot{}, errors.New("hostcollect: collector is nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return host.RawSnapshot{}, err
|
||||
}
|
||||
now := c.options.Now().UTC()
|
||||
warnings := newWarningSet(c.options.HostLimits.MaxWarnings)
|
||||
|
||||
identity, err := c.identity()
|
||||
if err != nil {
|
||||
return host.RawSnapshot{}, err
|
||||
}
|
||||
uptimeSeconds, err := c.uptimeSeconds()
|
||||
if err != nil {
|
||||
return host.RawSnapshot{}, err
|
||||
}
|
||||
memory, err := c.memory()
|
||||
if err != nil {
|
||||
return host.RawSnapshot{}, err
|
||||
}
|
||||
|
||||
bootTime := now.Add(-time.Duration(uptimeSeconds * float64(time.Second))).UTC()
|
||||
|
||||
load, err := c.loadAverage()
|
||||
if err != nil {
|
||||
warnings.add(WarningLoadUnavailable)
|
||||
}
|
||||
cpu := c.cpuUsage(now, warnings)
|
||||
network := c.network(warnings)
|
||||
filesystems := c.filesystems(warnings)
|
||||
clock := c.clock(warnings)
|
||||
|
||||
return host.RawSnapshot{
|
||||
Source: host.Source{
|
||||
ID: c.options.SourceID,
|
||||
Type: "agent",
|
||||
CapabilityVersion: host.ContractVersion,
|
||||
ObservedAt: now,
|
||||
},
|
||||
Identity: identity,
|
||||
UptimeSeconds: uptimeSeconds,
|
||||
BootTime: &bootTime,
|
||||
CPU: cpu,
|
||||
Load: load,
|
||||
Memory: memory,
|
||||
Filesystems: filesystems,
|
||||
Network: network,
|
||||
Time: clock,
|
||||
ObservedAt: now,
|
||||
Warnings: warnings.list(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Collector) identity() (host.HostIdentity, error) {
|
||||
identity := host.HostIdentity{Name: c.options.HostName, Arch: runtime.GOARCH}
|
||||
if identity.Name == "" {
|
||||
name, err := readTrimmed(c.procPath("sys", "kernel", "hostname"), maxProcessFileBytes)
|
||||
if err != nil {
|
||||
return host.HostIdentity{}, fmt.Errorf("hostcollect: read host name: %w", err)
|
||||
}
|
||||
identity.Name = name
|
||||
}
|
||||
if identity.Name == "" {
|
||||
return host.HostIdentity{}, errors.New("hostcollect: host name is empty")
|
||||
}
|
||||
if len(identity.Name) > maxNameBytes {
|
||||
identity.Name = identity.Name[:maxNameBytes]
|
||||
}
|
||||
if kernel, err := readTrimmed(c.procPath("sys", "kernel", "osrelease"), maxProcessFileBytes); err == nil {
|
||||
identity.Kernel = truncate(kernel, maxNameBytes)
|
||||
}
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
// clock reports host clock synchronisation. adjtimex(2) in read mode (Modes == 0)
|
||||
// mutates nothing, but Docker's default seccomp profile gates it behind CAP_SYS_TIME,
|
||||
// which the agent drops. When the probe is unavailable the collector reports the clock
|
||||
// as synchronised and records a warning rather than reporting an unsynchronised clock:
|
||||
// the domain maps "not synchronised" to Degraded with an explicit operator-facing
|
||||
// reason, and asserting a clock fault we never observed would be a false alarm on every
|
||||
// healthy host. The warning keeps the uncertainty visible in the snapshot.
|
||||
func (c *Collector) clock(warnings *warningSet) host.RawTime {
|
||||
clock, err := c.options.ClockSync()
|
||||
if err != nil {
|
||||
warnings.add(WarningClockProbeAssumed)
|
||||
return host.RawTime{Synchronized: true}
|
||||
}
|
||||
if clock.OffsetSeconds < 0 {
|
||||
clock.OffsetSeconds = -clock.OffsetSeconds
|
||||
}
|
||||
return clock
|
||||
}
|
||||
|
||||
func (c *Collector) procPath(elements ...string) string {
|
||||
return filepath.Join(append([]string{c.options.ProcRoot}, elements...)...)
|
||||
}
|
||||
|
||||
func (c *Collector) sysPath(elements ...string) string {
|
||||
return filepath.Join(append([]string{c.options.SysRoot}, elements...)...)
|
||||
}
|
||||
|
||||
// warningSet collects deduplicated, bounded warning codes.
|
||||
type warningSet struct {
|
||||
max int
|
||||
seen map[string]struct{}
|
||||
items []string
|
||||
}
|
||||
|
||||
func newWarningSet(max int) *warningSet {
|
||||
if max < 1 {
|
||||
max = 1
|
||||
}
|
||||
return &warningSet{max: max, seen: map[string]struct{}{}}
|
||||
}
|
||||
|
||||
func (w *warningSet) add(code string) {
|
||||
if code == "" || len(w.items) >= w.max {
|
||||
return
|
||||
}
|
||||
if _, exists := w.seen[code]; exists {
|
||||
return
|
||||
}
|
||||
w.seen[code] = struct{}{}
|
||||
w.items = append(w.items, code)
|
||||
}
|
||||
|
||||
func (w *warningSet) list() []string {
|
||||
if len(w.items) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]string(nil), w.items...)
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
)
|
||||
|
||||
// copyTree copies a fixture directory into a writable temporary root so a test can
|
||||
// mutate individual /proc files between collections, which is the only way to exercise
|
||||
// delta based metrics.
|
||||
func copyTree(t *testing.T, source string) string {
|
||||
t.Helper()
|
||||
destination := t.TempDir()
|
||||
err := filepath.WalkDir(source, func(path string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
relative, relErr := filepath.Rel(source, path)
|
||||
if relErr != nil {
|
||||
return relErr
|
||||
}
|
||||
target := filepath.Join(destination, relative)
|
||||
if entry.IsDir() {
|
||||
return os.MkdirAll(target, 0o755)
|
||||
}
|
||||
data, readErr := os.ReadFile(path) //nolint:gosec // fixture path is test-controlled
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
return os.WriteFile(target, data, 0o600)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("copy fixture tree: %v", err)
|
||||
}
|
||||
return destination
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("create directory: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeClock struct{ current time.Time }
|
||||
|
||||
func (c *fakeClock) now() time.Time { return c.current }
|
||||
|
||||
func (c *fakeClock) advance(d time.Duration) { c.current = c.current.Add(d) }
|
||||
|
||||
func newTestCollector(t *testing.T, options Options) (*Collector, *fakeClock) {
|
||||
t.Helper()
|
||||
clock := &fakeClock{current: time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)}
|
||||
if options.ProcRoot == "" {
|
||||
options.ProcRoot = filepath.Join("testdata", "proc-healthy")
|
||||
}
|
||||
if options.SysRoot == "" {
|
||||
options.SysRoot = filepath.Join("testdata", "sys-healthy")
|
||||
}
|
||||
if options.Now == nil {
|
||||
options.Now = clock.now
|
||||
}
|
||||
if options.ClockSync == nil {
|
||||
options.ClockSync = func() (host.RawTime, error) {
|
||||
return host.RawTime{Synchronized: true, OffsetSeconds: 0.0012}, nil
|
||||
}
|
||||
}
|
||||
collector, err := New(options)
|
||||
if err != nil {
|
||||
t.Fatalf("New returned error: %v", err)
|
||||
}
|
||||
return collector, clock
|
||||
}
|
||||
|
||||
func TestCPUUsageNeedsTwoSamples(t *testing.T) {
|
||||
root := copyTree(t, filepath.Join("testdata", "proc-healthy"))
|
||||
collector, clock := newTestCollector(t, Options{ProcRoot: root})
|
||||
|
||||
first := newWarningSet(20)
|
||||
if cpu := collector.cpuUsage(clock.now(), first); cpu.TotalPercent != nil || cpu.PerCore != nil {
|
||||
t.Fatalf("first sample must not report utilisation: %+v", cpu)
|
||||
}
|
||||
if !containsString(first.list(), WarningCPUFirstSample) {
|
||||
t.Fatalf("expected the first sample warning, got %v", first.list())
|
||||
}
|
||||
|
||||
// Second sample: 200 additional ticks of which 100 idle and 20 iowait, so 40% busy.
|
||||
writeFile(t, filepath.Join(root, "stat"), strings.Join([]string{
|
||||
"cpu 1234647 8901 234567 45679001 12365 0 6789 1234 5000 100",
|
||||
"cpu0 617323 4450 117283 22839500 6182 0 3394 617 2500 50",
|
||||
"cpu1 617324 4451 117284 22839501 6183 0 3395 617 2500 50",
|
||||
"",
|
||||
}, "\n"))
|
||||
clock.advance(10 * time.Second)
|
||||
|
||||
second := newWarningSet(20)
|
||||
cpu := collector.cpuUsage(clock.now(), second)
|
||||
if cpu.TotalPercent == nil {
|
||||
t.Fatal("second sample must report utilisation")
|
||||
}
|
||||
if got := *cpu.TotalPercent; got < 39.9 || got > 40.1 {
|
||||
t.Fatalf("total = %v, want 40", got)
|
||||
}
|
||||
if cpu.IOWaitPercent == nil || *cpu.IOWaitPercent < 9.9 || *cpu.IOWaitPercent > 10.1 {
|
||||
t.Fatalf("iowait = %v, want 10", cpu.IOWaitPercent)
|
||||
}
|
||||
if len(cpu.PerCore) != 2 {
|
||||
t.Fatalf("per core = %v", cpu.PerCore)
|
||||
}
|
||||
if warnings := second.list(); len(warnings) != 0 {
|
||||
t.Fatalf("unexpected warnings: %v", warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPUUsageReportsNothingWhenCountersWrap(t *testing.T) {
|
||||
root := copyTree(t, filepath.Join("testdata", "proc-healthy"))
|
||||
collector, clock := newTestCollector(t, Options{ProcRoot: root})
|
||||
collector.cpuUsage(clock.now(), newWarningSet(20))
|
||||
|
||||
writeFile(t, filepath.Join(root, "stat"), "cpu 10 1 2 3 4 0 5 0 0 0\ncpu0 5 0 1 1 2 0 2 0 0 0\ncpu1 5 1 1 2 2 0 3 0 0 0\n")
|
||||
clock.advance(10 * time.Second)
|
||||
|
||||
warnings := newWarningSet(20)
|
||||
cpu := collector.cpuUsage(clock.now(), warnings)
|
||||
if cpu.TotalPercent != nil || cpu.PerCore != nil {
|
||||
t.Fatalf("a wrapped counter must not produce a percentage: %+v", cpu)
|
||||
}
|
||||
if !containsString(warnings.list(), WarningCPUCounterReset) {
|
||||
t.Fatalf("expected a counter reset warning, got %v", warnings.list())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPUUsageKeepsTotalWhenCoreCountChanges(t *testing.T) {
|
||||
root := copyTree(t, filepath.Join("testdata", "proc-healthy"))
|
||||
collector, clock := newTestCollector(t, Options{ProcRoot: root})
|
||||
collector.cpuUsage(clock.now(), newWarningSet(20))
|
||||
|
||||
writeFile(t, filepath.Join(root, "stat"), strings.Join([]string{
|
||||
"cpu 1234647 8901 234567 45679001 12365 0 6789 1234 5000 100",
|
||||
"cpu0 617323 4450 117283 22839500 6182 0 3394 617 2500 50",
|
||||
"cpu1 617324 4451 117284 22839501 6183 0 3395 617 2500 50",
|
||||
"cpu2 0 0 0 0 0 0 0 0 0 0",
|
||||
"",
|
||||
}, "\n"))
|
||||
clock.advance(10 * time.Second)
|
||||
|
||||
warnings := newWarningSet(20)
|
||||
cpu := collector.cpuUsage(clock.now(), warnings)
|
||||
if cpu.TotalPercent == nil {
|
||||
t.Fatal("expected the aggregate to survive a topology change")
|
||||
}
|
||||
if cpu.PerCore != nil {
|
||||
t.Fatalf("per core values are not comparable across a topology change: %v", cpu.PerCore)
|
||||
}
|
||||
if !containsString(warnings.list(), WarningCPUTopologyChanged) {
|
||||
t.Fatalf("expected a topology warning, got %v", warnings.list())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPUUsageTruncatesToCoreLimit(t *testing.T) {
|
||||
root := copyTree(t, filepath.Join("testdata", "proc-healthy"))
|
||||
collector, clock := newTestCollector(t, Options{ProcRoot: root, HostLimits: host.Limits{MaxCores: 1, MaxFilesystems: 4, MaxInterfaces: 4, MaxWarnings: 5}})
|
||||
collector.cpuUsage(clock.now(), newWarningSet(20))
|
||||
|
||||
writeFile(t, filepath.Join(root, "stat"), strings.Join([]string{
|
||||
"cpu 1234647 8901 234567 45679001 12365 0 6789 1234 5000 100",
|
||||
"cpu0 617323 4450 117283 22839500 6182 0 3394 617 2500 50",
|
||||
"cpu1 617324 4451 117284 22839501 6183 0 3395 617 2500 50",
|
||||
"",
|
||||
}, "\n"))
|
||||
clock.advance(10 * time.Second)
|
||||
|
||||
warnings := newWarningSet(20)
|
||||
cpu := collector.cpuUsage(clock.now(), warnings)
|
||||
if len(cpu.PerCore) != 1 {
|
||||
t.Fatalf("per core = %v, want one entry", cpu.PerCore)
|
||||
}
|
||||
if !containsString(warnings.list(), WarningCPUCoresTruncated) {
|
||||
t.Fatalf("expected a truncation warning, got %v", warnings.list())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPUUsageDegradesWhenProcStatIsUnreadable(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
collector, clock := newTestCollector(t, Options{ProcRoot: root})
|
||||
warnings := newWarningSet(20)
|
||||
if cpu := collector.cpuUsage(clock.now(), warnings); cpu.TotalPercent != nil {
|
||||
t.Fatalf("unexpected utilisation: %+v", cpu)
|
||||
}
|
||||
if !containsString(warnings.list(), WarningCPUUnavailable) {
|
||||
t.Fatalf("expected an unavailable warning, got %v", warnings.list())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetworkSkipsLoopbackAndReadsOperationalState(t *testing.T) {
|
||||
collector, _ := newTestCollector(t, Options{})
|
||||
warnings := newWarningSet(20)
|
||||
interfaces := collector.network(warnings)
|
||||
if len(interfaces) != 2 {
|
||||
t.Fatalf("interfaces = %+v, want br0 and eth0", interfaces)
|
||||
}
|
||||
if interfaces[0].Name != "br0" || interfaces[1].Name != "eth0" {
|
||||
t.Fatalf("interfaces are not sorted: %+v", interfaces)
|
||||
}
|
||||
if interfaces[1].State != "up" {
|
||||
t.Fatalf("eth0 state = %q, want up", interfaces[1].State)
|
||||
}
|
||||
if interfaces[0].State != "unknown" {
|
||||
t.Fatalf("br0 state = %q", interfaces[0].State)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetworkTruncatesToInterfaceLimit(t *testing.T) {
|
||||
collector, _ := newTestCollector(t, Options{HostLimits: host.Limits{MaxCores: 8, MaxFilesystems: 8, MaxInterfaces: 1, MaxWarnings: 5}})
|
||||
warnings := newWarningSet(20)
|
||||
if interfaces := collector.network(warnings); len(interfaces) != 1 {
|
||||
t.Fatalf("interfaces = %+v, want one", interfaces)
|
||||
}
|
||||
if !containsString(warnings.list(), WarningNetworkTruncated) {
|
||||
t.Fatalf("expected a truncation warning, got %v", warnings.list())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemsAreDisabledWithoutAnExplicitRoot(t *testing.T) {
|
||||
collector, _ := newTestCollector(t, Options{StatFS: func(string) (FilesystemUsage, error) {
|
||||
t.Fatal("statfs must not be called when no filesystem root is configured")
|
||||
return FilesystemUsage{}, nil
|
||||
}})
|
||||
warnings := newWarningSet(20)
|
||||
if filesystems := collector.filesystems(warnings); filesystems != nil {
|
||||
t.Fatalf("expected no filesystems, got %+v", filesystems)
|
||||
}
|
||||
if !containsString(warnings.list(), WarningFilesystemDisabled) {
|
||||
t.Fatalf("expected the disabled warning, got %v", warnings.list())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemsSkipPseudoMountsAndSurviveStatFailures(t *testing.T) {
|
||||
requested := []string{}
|
||||
collector, _ := newTestCollector(t, Options{
|
||||
FilesystemRoot: "/host/root",
|
||||
StatFS: func(path string) (FilesystemUsage, error) {
|
||||
path = filepath.ToSlash(path)
|
||||
requested = append(requested, path)
|
||||
if strings.HasSuffix(path, "/mnt/disk2") {
|
||||
return FilesystemUsage{}, errors.New("stale nfs handle")
|
||||
}
|
||||
if strings.HasSuffix(path, "/boot") {
|
||||
return FilesystemUsage{CapacityBytes: 0}, nil
|
||||
}
|
||||
return FilesystemUsage{CapacityBytes: 1000, UsedBytes: 400, InodesTotal: 100, InodesUsed: 40}, nil
|
||||
},
|
||||
})
|
||||
warnings := newWarningSet(20)
|
||||
filesystems := collector.filesystems(warnings)
|
||||
|
||||
mounts := make([]string, 0, len(filesystems))
|
||||
for _, item := range filesystems {
|
||||
mounts = append(mounts, item.Mount)
|
||||
}
|
||||
want := []string{"/", "/mnt/cache", "/mnt/disk1", "/mnt/disks/Media Backup", "/mnt/user"}
|
||||
if strings.Join(mounts, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("mounts = %v, want %v", mounts, want)
|
||||
}
|
||||
for _, path := range requested {
|
||||
if !strings.HasPrefix(path, "/host/root") {
|
||||
t.Fatalf("statfs called outside the configured root: %q", path)
|
||||
}
|
||||
}
|
||||
if !containsString(warnings.list(), WarningFilesystemPartial) {
|
||||
t.Fatalf("expected a partial warning, got %v", warnings.list())
|
||||
}
|
||||
if filesystems[0].Inodes == nil || filesystems[0].Inodes.Used != 40 {
|
||||
t.Fatalf("inodes missing: %+v", filesystems[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemsTruncateToLimit(t *testing.T) {
|
||||
collector, _ := newTestCollector(t, Options{
|
||||
FilesystemRoot: "/",
|
||||
HostLimits: host.Limits{MaxCores: 8, MaxFilesystems: 2, MaxInterfaces: 8, MaxWarnings: 5},
|
||||
StatFS: func(string) (FilesystemUsage, error) {
|
||||
return FilesystemUsage{CapacityBytes: 1000, UsedBytes: 100}, nil
|
||||
},
|
||||
})
|
||||
warnings := newWarningSet(20)
|
||||
if filesystems := collector.filesystems(warnings); len(filesystems) != 2 {
|
||||
t.Fatalf("filesystems = %d, want 2", len(filesystems))
|
||||
}
|
||||
if !containsString(warnings.list(), WarningFilesystemTruncated) {
|
||||
t.Fatalf("expected a truncation warning, got %v", warnings.list())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostSnapshotIsAcceptedByTheDomain(t *testing.T) {
|
||||
collector, clock := newTestCollector(t, Options{
|
||||
FilesystemRoot: "/",
|
||||
StatFS: func(string) (FilesystemUsage, error) {
|
||||
return FilesystemUsage{CapacityBytes: 4000, UsedBytes: 2500, InodesTotal: 200, InodesUsed: 30}, nil
|
||||
},
|
||||
})
|
||||
raw, err := collector.Host(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Host returned error: %v", err)
|
||||
}
|
||||
if raw.Identity.Name != "tower" || raw.Identity.Kernel != "6.1.79-Unraid" {
|
||||
t.Fatalf("unexpected identity: %+v", raw.Identity)
|
||||
}
|
||||
if raw.UptimeSeconds != 351282.31 {
|
||||
t.Fatalf("uptime = %v", raw.UptimeSeconds)
|
||||
}
|
||||
if raw.BootTime == nil || !raw.BootTime.Equal(clock.now().Add(-time.Duration(351282.31*float64(time.Second)))) {
|
||||
t.Fatalf("boot time = %v", raw.BootTime)
|
||||
}
|
||||
if raw.Memory.TotalBytes == 0 || raw.Memory.AvailableBytes == 0 {
|
||||
t.Fatalf("memory not collected: %+v", raw.Memory)
|
||||
}
|
||||
if raw.Load.One != 1.52 {
|
||||
t.Fatalf("load = %+v", raw.Load)
|
||||
}
|
||||
if !raw.Time.Synchronized {
|
||||
t.Fatalf("time = %+v", raw.Time)
|
||||
}
|
||||
if !containsString(raw.Warnings, WarningCPUFirstSample) {
|
||||
t.Fatalf("expected the first sample warning, got %v", raw.Warnings)
|
||||
}
|
||||
// The collector's output must pass domain normalization unchanged; that is the
|
||||
// contract the API reads through.
|
||||
if _, err := host.Normalize(raw, clock.now(), host.Limits{}, host.Policy{}); err != nil {
|
||||
t.Fatalf("host.Normalize rejected the collected snapshot: %v", err)
|
||||
}
|
||||
payload, err := json.Marshal(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
if len(payload) == 0 {
|
||||
t.Fatal("empty payload")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostFailsWhenRequiredSourcesAreMalformed(t *testing.T) {
|
||||
collector, _ := newTestCollector(t, Options{
|
||||
ProcRoot: filepath.Join("testdata", "proc-messy"),
|
||||
SysRoot: filepath.Join("testdata", "sys-healthy"),
|
||||
})
|
||||
if _, err := collector.Host(context.Background()); err == nil {
|
||||
t.Fatal("expected a malformed uptime to fail the whole host collection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostDegradesLoadAndClockIntoWarnings(t *testing.T) {
|
||||
root := copyTree(t, filepath.Join("testdata", "proc-healthy"))
|
||||
writeFile(t, filepath.Join(root, "loadavg"), "broken\n")
|
||||
collector, clock := newTestCollector(t, Options{
|
||||
ProcRoot: root,
|
||||
ClockSync: func() (host.RawTime, error) { return host.RawTime{}, errors.New("operation not permitted") },
|
||||
})
|
||||
raw, err := collector.Host(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Host returned error: %v", err)
|
||||
}
|
||||
if !containsString(raw.Warnings, WarningLoadUnavailable) || !containsString(raw.Warnings, WarningClockProbeAssumed) {
|
||||
t.Fatalf("warnings = %v", raw.Warnings)
|
||||
}
|
||||
if !raw.Time.Synchronized {
|
||||
t.Fatal("an unverifiable clock must not be reported as a confirmed clock fault")
|
||||
}
|
||||
if _, err := host.Normalize(raw, clock.now(), host.Limits{}, host.Policy{}); err != nil {
|
||||
t.Fatalf("host.Normalize rejected the degraded snapshot: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostHonoursContextCancellation(t *testing.T) {
|
||||
collector, _ := newTestCollector(t, Options{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := collector.Host(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("error = %v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostNameOverrideWinsOverTheNamespaceHostname(t *testing.T) {
|
||||
collector, _ := newTestCollector(t, Options{
|
||||
ProcRoot: filepath.Join("testdata", "proc-messy"),
|
||||
HostName: "tower",
|
||||
})
|
||||
identity, err := collector.identity()
|
||||
if err != nil {
|
||||
t.Fatalf("identity returned error: %v", err)
|
||||
}
|
||||
if identity.Name != "tower" {
|
||||
t.Fatalf("name = %q, want the override", identity.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRejectsUnsafeLimits(t *testing.T) {
|
||||
if _, err := New(Options{HostLimits: host.Limits{MaxCores: 9999}}); err == nil {
|
||||
t.Fatal("expected out of bounds host limits to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWarningSetIsDeduplicatedAndBounded(t *testing.T) {
|
||||
warnings := newWarningSet(2)
|
||||
warnings.add(WarningCPUUnavailable)
|
||||
warnings.add(WarningCPUUnavailable)
|
||||
warnings.add(WarningLoadUnavailable)
|
||||
warnings.add(WarningNetworkUnavailable)
|
||||
if list := warnings.list(); len(list) != 2 {
|
||||
t.Fatalf("warnings = %v, want two", list)
|
||||
}
|
||||
}
|
||||
|
||||
func containsString(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if value == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
)
|
||||
|
||||
// cpuTimes holds the three aggregates a utilisation delta needs, all in USER_HZ ticks.
|
||||
type cpuTimes struct {
|
||||
total uint64
|
||||
idle uint64
|
||||
iowait uint64
|
||||
}
|
||||
|
||||
type cpuSample struct {
|
||||
at time.Time
|
||||
all cpuTimes
|
||||
cores []cpuTimes
|
||||
valid bool
|
||||
}
|
||||
|
||||
// parseProcStat reads the cpu aggregate and per-core lines of /proc/stat.
|
||||
//
|
||||
// Column layout (proc(5)): user nice system idle iowait irq softirq steal guest
|
||||
// guest_nice. guest and guest_nice are deliberately excluded from the total: the
|
||||
// kernel already counts guest time inside user and nice, so including them again
|
||||
// inflates the denominator and understates utilisation. Old kernels publish fewer
|
||||
// columns, so anything past idle is optional.
|
||||
func parseProcStat(data []byte) (cpuSample, error) {
|
||||
sample := cpuSample{}
|
||||
type indexedCore struct {
|
||||
index int
|
||||
times cpuTimes
|
||||
}
|
||||
cores := make([]indexedCore, 0, 8)
|
||||
for _, line := range lines(data) {
|
||||
if !strings.HasPrefix(line, "cpu") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 {
|
||||
continue
|
||||
}
|
||||
times, ok := parseCPUTimes(fields[1:])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if fields[0] == "cpu" {
|
||||
sample.all = times
|
||||
sample.valid = true
|
||||
continue
|
||||
}
|
||||
index, err := strconv.Atoi(strings.TrimPrefix(fields[0], "cpu"))
|
||||
if err != nil || index < 0 {
|
||||
continue
|
||||
}
|
||||
cores = append(cores, indexedCore{index: index, times: times})
|
||||
}
|
||||
if !sample.valid {
|
||||
return cpuSample{}, errNoCPUAggregate
|
||||
}
|
||||
sort.Slice(cores, func(i, j int) bool { return cores[i].index < cores[j].index })
|
||||
sample.cores = make([]cpuTimes, 0, len(cores))
|
||||
for _, core := range cores {
|
||||
sample.cores = append(sample.cores, core.times)
|
||||
}
|
||||
return sample, nil
|
||||
}
|
||||
|
||||
// parseCPUTimes sums the first eight jiffy columns; a non-numeric column makes the
|
||||
// whole line untrustworthy.
|
||||
func parseCPUTimes(fields []string) (cpuTimes, bool) {
|
||||
times := cpuTimes{}
|
||||
limit := len(fields)
|
||||
if limit > 8 {
|
||||
limit = 8
|
||||
}
|
||||
for index := 0; index < limit; index++ {
|
||||
value, ok := parseUint(fields[index])
|
||||
if !ok {
|
||||
return cpuTimes{}, false
|
||||
}
|
||||
times.total += value
|
||||
switch index {
|
||||
case 3:
|
||||
times.idle = value
|
||||
case 4:
|
||||
times.iowait = value
|
||||
}
|
||||
}
|
||||
return times, true
|
||||
}
|
||||
|
||||
// cpuUsage samples /proc/stat and reports utilisation relative to the previous sample.
|
||||
// The first call after start (or after a counter reset) reports no percentages at all
|
||||
// rather than a fabricated value: an absolute counter carries no utilisation.
|
||||
func (c *Collector) cpuUsage(now time.Time, warnings *warningSet) host.RawCPU {
|
||||
data, err := readLimited(c.procPath("stat"), maxFileBytes)
|
||||
if err != nil {
|
||||
warnings.add(WarningCPUUnavailable)
|
||||
return host.RawCPU{}
|
||||
}
|
||||
current, err := parseProcStat(data)
|
||||
if err != nil {
|
||||
warnings.add(WarningCPUUnavailable)
|
||||
return host.RawCPU{}
|
||||
}
|
||||
current.at = now
|
||||
|
||||
c.mu.Lock()
|
||||
previous := c.cpu
|
||||
c.cpu = ¤t
|
||||
c.mu.Unlock()
|
||||
|
||||
if previous == nil || !previous.valid {
|
||||
warnings.add(WarningCPUFirstSample)
|
||||
return host.RawCPU{}
|
||||
}
|
||||
busy, iowait, ok := utilisation(previous.all, current.all)
|
||||
if !ok {
|
||||
warnings.add(WarningCPUCounterReset)
|
||||
return host.RawCPU{}
|
||||
}
|
||||
result := host.RawCPU{TotalPercent: &busy, IOWaitPercent: &iowait}
|
||||
if len(previous.cores) != len(current.cores) {
|
||||
warnings.add(WarningCPUTopologyChanged)
|
||||
return result
|
||||
}
|
||||
cores := make([]float64, 0, len(current.cores))
|
||||
for index := range current.cores {
|
||||
corePercent, _, coreOK := utilisation(previous.cores[index], current.cores[index])
|
||||
if !coreOK {
|
||||
warnings.add(WarningCPUCounterReset)
|
||||
return result
|
||||
}
|
||||
cores = append(cores, corePercent)
|
||||
}
|
||||
if len(cores) > c.options.HostLimits.MaxCores {
|
||||
cores = cores[:c.options.HostLimits.MaxCores]
|
||||
warnings.add(WarningCPUCoresTruncated)
|
||||
}
|
||||
result.PerCore = cores
|
||||
return result
|
||||
}
|
||||
|
||||
// utilisation converts two counter readings into busy and iowait percentages. Any
|
||||
// counter going backwards means the counter wrapped or the source was replaced (a
|
||||
// container restart, a CPU hot-unplug, a fixture root swap); the delta is then
|
||||
// meaningless and is reported as unusable instead of as a huge or negative spike.
|
||||
func utilisation(previous, current cpuTimes) (busyPercent, iowaitPercent float64, ok bool) {
|
||||
if current.total < previous.total || current.idle < previous.idle || current.iowait < previous.iowait {
|
||||
return 0, 0, false
|
||||
}
|
||||
totalDelta := current.total - previous.total
|
||||
if totalDelta == 0 {
|
||||
return 0, 0, false
|
||||
}
|
||||
idleDelta := current.idle - previous.idle
|
||||
iowaitDelta := current.iowait - previous.iowait
|
||||
if idleDelta+iowaitDelta > totalDelta {
|
||||
return 0, 0, false
|
||||
}
|
||||
busy := float64(totalDelta-idleDelta-iowaitDelta) / float64(totalDelta) * 100
|
||||
iowait := float64(iowaitDelta) / float64(totalDelta) * 100
|
||||
return clampPercent(busy), clampPercent(iowait), true
|
||||
}
|
||||
|
||||
func clampPercent(value float64) float64 {
|
||||
if value < 0 {
|
||||
return 0
|
||||
}
|
||||
if value > 100 {
|
||||
return 100
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package hostcollect
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
errNoCPUAggregate = errors.New("hostcollect: /proc/stat has no cpu aggregate line")
|
||||
errNoMemTotal = errors.New("hostcollect: /proc/meminfo has no usable MemTotal")
|
||||
errBadLoadAverage = errors.New("hostcollect: /proc/loadavg is malformed")
|
||||
errBadUptime = errors.New("hostcollect: /proc/uptime is malformed")
|
||||
)
|
||||
@@ -0,0 +1,139 @@
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
)
|
||||
|
||||
// pseudoFilesystems never describe usable capacity; reporting them would fill the
|
||||
// filesystem list with kernel bookkeeping and push real volumes past the limit.
|
||||
var pseudoFilesystems = map[string]struct{}{
|
||||
"autofs": {}, "bpf": {}, "binfmt_misc": {}, "cgroup": {}, "cgroup2": {},
|
||||
"configfs": {}, "debugfs": {}, "devpts": {}, "devtmpfs": {}, "efivarfs": {},
|
||||
"fuse.gvfsd-fuse": {}, "fusectl": {}, "hugetlbfs": {}, "mqueue": {}, "nsfs": {},
|
||||
"overlay": {}, "proc": {}, "pstore": {}, "ramfs": {}, "rpc_pipefs": {},
|
||||
"securityfs": {}, "selinuxfs": {}, "squashfs": {}, "sysfs": {}, "tmpfs": {},
|
||||
"tracefs": {},
|
||||
}
|
||||
|
||||
type mountEntry struct {
|
||||
device string
|
||||
mount string
|
||||
fsType string
|
||||
}
|
||||
|
||||
// parseMounts reads /proc/mounts. Fields are space separated and the kernel escapes
|
||||
// space, tab, newline and backslash inside the device and mount point as octal
|
||||
// sequences, so an Unraid share called "/mnt/user/Media Backup" arrives as
|
||||
// "/mnt/user/Media\040Backup" and must be unescaped before it can be used as a path.
|
||||
func parseMounts(data []byte) []mountEntry {
|
||||
all := lines(data)
|
||||
entries := make([]mountEntry, 0, len(all))
|
||||
for _, line := range all {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 3 {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, mountEntry{
|
||||
device: unescapeMountField(fields[0]),
|
||||
mount: unescapeMountField(fields[1]),
|
||||
fsType: fields[2],
|
||||
})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func unescapeMountField(value string) string {
|
||||
if !strings.Contains(value, `\`) {
|
||||
return value
|
||||
}
|
||||
var builder strings.Builder
|
||||
builder.Grow(len(value))
|
||||
for index := 0; index < len(value); index++ {
|
||||
if value[index] == '\\' && index+3 < len(value) {
|
||||
if decoded, err := strconv.ParseUint(value[index+1:index+4], 8, 8); err == nil {
|
||||
builder.WriteByte(byte(decoded))
|
||||
index += 3
|
||||
continue
|
||||
}
|
||||
}
|
||||
builder.WriteByte(value[index])
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// filesystems reports capacity and inode usage per real mount point.
|
||||
//
|
||||
// It is disabled unless FilesystemRoot is configured. Inside a container the host's
|
||||
// /proc/mounts lists host paths that do not exist in the container's mount namespace;
|
||||
// calling statfs on the same string would either fail or — worse, for "/" — silently
|
||||
// measure the container's own overlay and report it as the host's root filesystem.
|
||||
// Mounting a host root read-only and pointing FilesystemRoot at it is an explicit,
|
||||
// auditable decision rather than an accident of naming.
|
||||
func (c *Collector) filesystems(warnings *warningSet) []host.RawFilesystem {
|
||||
if c.options.FilesystemRoot == "" {
|
||||
warnings.add(WarningFilesystemDisabled)
|
||||
return nil
|
||||
}
|
||||
data, err := readLimited(c.procPath("mounts"), maxFileBytes)
|
||||
if err != nil {
|
||||
warnings.add(WarningMountsUnavailable)
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{})
|
||||
results := make([]host.RawFilesystem, 0, 16)
|
||||
failures := 0
|
||||
for _, entry := range parseMounts(data) {
|
||||
if _, pseudo := pseudoFilesystems[entry.fsType]; pseudo {
|
||||
continue
|
||||
}
|
||||
if entry.mount == "" || len(entry.mount) > 512 {
|
||||
continue
|
||||
}
|
||||
if _, duplicate := seen[entry.mount]; duplicate {
|
||||
continue
|
||||
}
|
||||
seen[entry.mount] = struct{}{}
|
||||
usage, statErr := c.options.StatFS(filepath.Join(c.options.FilesystemRoot, entry.mount))
|
||||
if statErr != nil {
|
||||
failures++
|
||||
continue
|
||||
}
|
||||
if usage.CapacityBytes == 0 {
|
||||
continue
|
||||
}
|
||||
if usage.UsedBytes > usage.CapacityBytes {
|
||||
usage.UsedBytes = usage.CapacityBytes
|
||||
}
|
||||
item := host.RawFilesystem{
|
||||
Mount: entry.mount,
|
||||
Filesystem: truncate(entry.fsType, 64),
|
||||
CapacityBytes: usage.CapacityBytes,
|
||||
UsedBytes: usage.UsedBytes,
|
||||
}
|
||||
if usage.InodesTotal > 0 {
|
||||
used := usage.InodesUsed
|
||||
if used > usage.InodesTotal {
|
||||
used = usage.InodesTotal
|
||||
}
|
||||
item.Inodes = &host.RawInodes{Total: usage.InodesTotal, Used: used}
|
||||
}
|
||||
results = append(results, item)
|
||||
}
|
||||
if failures > 0 {
|
||||
warnings.add(WarningFilesystemPartial)
|
||||
}
|
||||
sort.Slice(results, func(i, j int) bool { return results[i].Mount < results[j].Mount })
|
||||
if len(results) > c.options.HostLimits.MaxFilesystems {
|
||||
results = results[:c.options.HostLimits.MaxFilesystems]
|
||||
warnings.add(WarningFilesystemTruncated)
|
||||
}
|
||||
if len(results) == 0 {
|
||||
return nil
|
||||
}
|
||||
return results
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
)
|
||||
|
||||
// parseLoadAverage reads the three load figures from /proc/loadavg. The remaining
|
||||
// fields (runnable/total tasks and last PID) are deliberately ignored: the domain has
|
||||
// no place for them and a process count from a PID namespace would be misleading.
|
||||
func parseLoadAverage(data []byte) (host.RawLoad, error) {
|
||||
fields := strings.Fields(string(data))
|
||||
if len(fields) < 3 {
|
||||
return host.RawLoad{}, errBadLoadAverage
|
||||
}
|
||||
values := [3]float64{}
|
||||
for index := 0; index < 3; index++ {
|
||||
value, ok := parseFloat(fields[index])
|
||||
if !ok || value < 0 || math.IsNaN(value) || math.IsInf(value, 0) {
|
||||
return host.RawLoad{}, errBadLoadAverage
|
||||
}
|
||||
values[index] = value
|
||||
}
|
||||
return host.RawLoad{One: values[0], Five: values[1], Fifteen: values[2]}, nil
|
||||
}
|
||||
|
||||
func (c *Collector) loadAverage() (host.RawLoad, error) {
|
||||
data, err := readLimited(c.procPath("loadavg"), maxProcessFileBytes)
|
||||
if err != nil {
|
||||
return host.RawLoad{}, fmt.Errorf("hostcollect: read loadavg: %w", err)
|
||||
}
|
||||
return parseLoadAverage(data)
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
)
|
||||
|
||||
// parseMeminfo converts /proc/meminfo into bytes.
|
||||
//
|
||||
// meminfo is kB based ("MemTotal: 16316420 kB"), where kB means KiB, and a handful of
|
||||
// counters carry no unit at all. Treating either as bytes is the classic off-by-1024
|
||||
// bug, so the unit is read per line instead of assumed.
|
||||
func parseMeminfo(data []byte) (host.RawMemory, error) {
|
||||
values := make(map[string]uint64, 8)
|
||||
for _, line := range lines(data) {
|
||||
key, rest, found := strings.Cut(line, ":")
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(rest)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
amount, ok := parseUint(fields[0])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if len(fields) > 1 && strings.EqualFold(fields[1], "kB") {
|
||||
const kib = 1024
|
||||
if amount > (1<<64-1)/kib {
|
||||
continue
|
||||
}
|
||||
amount *= kib
|
||||
}
|
||||
values[strings.TrimSpace(key)] = amount
|
||||
}
|
||||
total := values["MemTotal"]
|
||||
if total == 0 {
|
||||
return host.RawMemory{}, errNoMemTotal
|
||||
}
|
||||
available, ok := values["MemAvailable"]
|
||||
if !ok {
|
||||
// Kernels before 3.14 have no MemAvailable. The traditional approximation is
|
||||
// free plus the reclaimable page cache; it is an estimate, never above total.
|
||||
available = values["MemFree"] + values["Buffers"] + values["Cached"] + values["SReclaimable"]
|
||||
}
|
||||
if available > total {
|
||||
available = total
|
||||
}
|
||||
memory := host.RawMemory{TotalBytes: total, AvailableBytes: available}
|
||||
swapTotal := values["SwapTotal"]
|
||||
if swapTotal > 0 {
|
||||
swapFree := values["SwapFree"]
|
||||
if swapFree > swapTotal {
|
||||
swapFree = swapTotal
|
||||
}
|
||||
memory.SwapTotalBytes = swapTotal
|
||||
memory.SwapUsedBytes = swapTotal - swapFree
|
||||
}
|
||||
return memory, nil
|
||||
}
|
||||
|
||||
func (c *Collector) memory() (host.RawMemory, error) {
|
||||
data, err := readLimited(c.procPath("meminfo"), maxFileBytes)
|
||||
if err != nil {
|
||||
return host.RawMemory{}, fmt.Errorf("hostcollect: read meminfo: %w", err)
|
||||
}
|
||||
return parseMeminfo(data)
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/itworx/pulse/internal/host"
|
||||
)
|
||||
|
||||
// loopbackInterface is excluded: its counters describe the host talking to itself and
|
||||
// only add noise to a network overview.
|
||||
const loopbackInterface = "lo"
|
||||
|
||||
// parseNetDev reads /proc/net/dev.
|
||||
//
|
||||
// The format is two header lines followed by " name: rx... tx...". The name is
|
||||
// separated by a colon that is NOT always followed by a space — a sufficiently large
|
||||
// rx byte counter runs straight into it ("eth0:18446744073709551615 ...") — so the
|
||||
// line is split on the first colon rather than on whitespace.
|
||||
func parseNetDev(data []byte) []host.RawNetworkInterface {
|
||||
all := lines(data)
|
||||
interfaces := make([]host.RawNetworkInterface, 0, len(all))
|
||||
for _, line := range all {
|
||||
name, rest, found := strings.Cut(line, ":")
|
||||
name = strings.TrimSpace(name)
|
||||
if !found || name == "" || strings.Contains(name, " ") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(rest)
|
||||
if len(fields) < 16 {
|
||||
continue
|
||||
}
|
||||
values := make([]uint64, 16)
|
||||
malformed := false
|
||||
for index := 0; index < 16; index++ {
|
||||
value, ok := parseUint(fields[index])
|
||||
if !ok {
|
||||
malformed = true
|
||||
break
|
||||
}
|
||||
values[index] = value
|
||||
}
|
||||
if malformed {
|
||||
continue
|
||||
}
|
||||
interfaces = append(interfaces, host.RawNetworkInterface{
|
||||
Name: truncate(name, 128),
|
||||
RxBytes: values[0],
|
||||
RxErrors: values[2],
|
||||
RxDrops: values[3],
|
||||
TxBytes: values[8],
|
||||
TxErrors: values[10],
|
||||
TxDrops: values[11],
|
||||
})
|
||||
}
|
||||
return interfaces
|
||||
}
|
||||
|
||||
func (c *Collector) network(warnings *warningSet) []host.RawNetworkInterface {
|
||||
data, err := readLimited(c.procPath("net", "dev"), maxFileBytes)
|
||||
if err != nil {
|
||||
warnings.add(WarningNetworkUnavailable)
|
||||
return nil
|
||||
}
|
||||
parsed := parseNetDev(data)
|
||||
interfaces := make([]host.RawNetworkInterface, 0, len(parsed))
|
||||
for _, item := range parsed {
|
||||
if item.Name == loopbackInterface {
|
||||
continue
|
||||
}
|
||||
item.State = c.interfaceState(item.Name)
|
||||
interfaces = append(interfaces, item)
|
||||
}
|
||||
sort.Slice(interfaces, func(i, j int) bool { return interfaces[i].Name < interfaces[j].Name })
|
||||
if len(interfaces) > c.options.HostLimits.MaxInterfaces {
|
||||
interfaces = interfaces[:c.options.HostLimits.MaxInterfaces]
|
||||
warnings.add(WarningNetworkTruncated)
|
||||
}
|
||||
if len(interfaces) == 0 {
|
||||
return nil
|
||||
}
|
||||
return interfaces
|
||||
}
|
||||
|
||||
// interfaceState reads the operational state sysfs publishes ("up", "down",
|
||||
// "unknown"). A missing file is normal for virtual interfaces and simply yields no
|
||||
// state rather than a warning.
|
||||
func (c *Collector) interfaceState(name string) string {
|
||||
if strings.ContainsAny(name, "/\\") {
|
||||
return ""
|
||||
}
|
||||
state, err := readTrimmed(c.sysPath("class", "net", name, "operstate"), 64)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return truncate(state, 32)
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func readFixture(t *testing.T, elements ...string) []byte {
|
||||
t.Helper()
|
||||
path := filepath.Join(append([]string{"testdata"}, elements...)...)
|
||||
data, err := os.ReadFile(path) //nolint:gosec // fixture path is test-controlled
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture %s: %v", path, err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func TestParseProcStatReadsAggregateAndCores(t *testing.T) {
|
||||
sample, err := parseProcStat(readFixture(t, "proc-healthy", "stat"))
|
||||
if err != nil {
|
||||
t.Fatalf("parseProcStat returned error: %v", err)
|
||||
}
|
||||
// user+nice+system+idle+iowait+irq+softirq+steal, with guest and guest_nice left
|
||||
// out because the kernel already counts guest time inside user and nice.
|
||||
const wantTotal = 1234567 + 8901 + 234567 + 45678901 + 12345 + 0 + 6789 + 1234
|
||||
if sample.all.total != wantTotal {
|
||||
t.Fatalf("aggregate total = %d, want %d", sample.all.total, wantTotal)
|
||||
}
|
||||
if sample.all.idle != 45678901 || sample.all.iowait != 12345 {
|
||||
t.Fatalf("unexpected idle/iowait: %+v", sample.all)
|
||||
}
|
||||
if len(sample.cores) != 2 {
|
||||
t.Fatalf("cores = %d, want 2", len(sample.cores))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseProcStatToleratesOldKernelsAndGarbage(t *testing.T) {
|
||||
sample, err := parseProcStat(readFixture(t, "proc-messy", "stat"))
|
||||
if err != nil {
|
||||
t.Fatalf("parseProcStat returned error: %v", err)
|
||||
}
|
||||
if sample.all.total != 1000+200+300+4000 {
|
||||
t.Fatalf("unexpected total for a four column kernel: %d", sample.all.total)
|
||||
}
|
||||
// cpu0 and cpu2 parse; "cpu-bogus" and the non-numeric "cpu9" line do not.
|
||||
if len(sample.cores) != 2 {
|
||||
t.Fatalf("cores = %d, want 2", len(sample.cores))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseProcStatRequiresAggregate(t *testing.T) {
|
||||
if _, err := parseProcStat([]byte("intr 1 2 3\nctxt 4\n")); err == nil {
|
||||
t.Fatal("expected an error when /proc/stat has no cpu line")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUtilisationComputesDeltaNotAbsoluteValue(t *testing.T) {
|
||||
previous := cpuTimes{total: 1000, idle: 800, iowait: 100}
|
||||
current := cpuTimes{total: 1200, idle: 900, iowait: 150}
|
||||
busy, iowait, ok := utilisation(previous, current)
|
||||
if !ok {
|
||||
t.Fatal("expected a usable delta")
|
||||
}
|
||||
if busy != 25 {
|
||||
t.Fatalf("busy = %v, want 25", busy)
|
||||
}
|
||||
if iowait != 25 {
|
||||
t.Fatalf("iowait = %v, want 25", iowait)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUtilisationRejectsCounterWrapAndStandstill(t *testing.T) {
|
||||
cases := map[string]struct{ previous, current cpuTimes }{
|
||||
"total wrapped": {cpuTimes{total: 18446744073709551000, idle: 10, iowait: 1}, cpuTimes{total: 400, idle: 20, iowait: 2}},
|
||||
"idle wrapped": {cpuTimes{total: 1000, idle: 900, iowait: 10}, cpuTimes{total: 1100, idle: 5, iowait: 11}},
|
||||
"iowait wrapped": {cpuTimes{total: 1000, idle: 800, iowait: 100}, cpuTimes{total: 1100, idle: 850, iowait: 4}},
|
||||
"no elapsed": {cpuTimes{total: 1000, idle: 800, iowait: 100}, cpuTimes{total: 1000, idle: 800, iowait: 100}},
|
||||
"idle exceeds": {cpuTimes{total: 1000, idle: 800, iowait: 100}, cpuTimes{total: 1010, idle: 900, iowait: 120}},
|
||||
}
|
||||
for name, testCase := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, _, ok := utilisation(testCase.previous, testCase.current); ok {
|
||||
t.Fatal("expected the delta to be rejected as unusable")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMeminfoConvertsKibibytes(t *testing.T) {
|
||||
memory, err := parseMeminfo(readFixture(t, "proc-healthy", "meminfo"))
|
||||
if err != nil {
|
||||
t.Fatalf("parseMeminfo returned error: %v", err)
|
||||
}
|
||||
if memory.TotalBytes != 32819484*1024 {
|
||||
t.Fatalf("total = %d, want %d", memory.TotalBytes, 32819484*1024)
|
||||
}
|
||||
if memory.AvailableBytes != 24680240*1024 {
|
||||
t.Fatalf("available = %d", memory.AvailableBytes)
|
||||
}
|
||||
if memory.SwapTotalBytes != 8388604*1024 || memory.SwapUsedBytes != (8388604-8000000)*1024 {
|
||||
t.Fatalf("unexpected swap: %+v", memory)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMeminfoFallsBackWhenMemAvailableIsAbsent(t *testing.T) {
|
||||
memory, err := parseMeminfo(readFixture(t, "proc-messy", "meminfo"))
|
||||
if err != nil {
|
||||
t.Fatalf("parseMeminfo returned error: %v", err)
|
||||
}
|
||||
want := uint64(123456+23456+2000000+100000) * 1024
|
||||
if memory.AvailableBytes != want {
|
||||
t.Fatalf("available = %d, want %d", memory.AvailableBytes, want)
|
||||
}
|
||||
if memory.SwapTotalBytes != 0 || memory.SwapUsedBytes != 0 {
|
||||
t.Fatalf("swapless host reported swap: %+v", memory)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMeminfoRequiresMemTotal(t *testing.T) {
|
||||
if _, err := parseMeminfo([]byte("MemFree: 100 kB\n")); err == nil {
|
||||
t.Fatal("expected an error without MemTotal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLoadAverage(t *testing.T) {
|
||||
load, err := parseLoadAverage(readFixture(t, "proc-healthy", "loadavg"))
|
||||
if err != nil {
|
||||
t.Fatalf("parseLoadAverage returned error: %v", err)
|
||||
}
|
||||
if load.One != 1.52 || load.Five != 2.08 || load.Fifteen != 2.35 {
|
||||
t.Fatalf("unexpected load: %+v", load)
|
||||
}
|
||||
for _, malformed := range []string{"not-a-load\n", "1.0 2.0\n", "-1 2 3\n", ""} {
|
||||
if _, err := parseLoadAverage([]byte(malformed)); err == nil {
|
||||
t.Fatalf("expected an error for %q", malformed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseUptime(t *testing.T) {
|
||||
seconds, err := parseUptime(readFixture(t, "proc-healthy", "uptime"))
|
||||
if err != nil {
|
||||
t.Fatalf("parseUptime returned error: %v", err)
|
||||
}
|
||||
if seconds != 351282.31 {
|
||||
t.Fatalf("uptime = %v", seconds)
|
||||
}
|
||||
for _, malformed := range []string{"", "nonsense\n", "-5 10\n", "999999999999 1\n"} {
|
||||
if _, err := parseUptime([]byte(malformed)); err == nil {
|
||||
t.Fatalf("expected an error for %q", malformed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNetDevHandlesMissingSpaceAfterColon(t *testing.T) {
|
||||
interfaces := parseNetDev(readFixture(t, "proc-healthy", "net", "dev"))
|
||||
byName := map[string]uint64{}
|
||||
for _, item := range interfaces {
|
||||
byName[item.Name] = item.RxBytes
|
||||
}
|
||||
// eth0's receive counter runs straight into the colon in the fixture.
|
||||
if byName["eth0"] != 18446744073709551615 {
|
||||
t.Fatalf("eth0 rx = %d", byName["eth0"])
|
||||
}
|
||||
if _, present := byName["wlan0"]; present {
|
||||
t.Fatal("a row with a non-numeric counter must be dropped, not zeroed")
|
||||
}
|
||||
if _, present := byName["tap0"]; present {
|
||||
t.Fatal("a truncated row must be dropped")
|
||||
}
|
||||
if len(interfaces) != 3 {
|
||||
t.Fatalf("interfaces = %d, want lo, eth0 and br0", len(interfaces))
|
||||
}
|
||||
for _, item := range interfaces {
|
||||
if item.Name == "eth0" && (item.RxErrors != 12 || item.RxDrops != 3 || item.TxErrors != 1 || item.TxDrops != 7) {
|
||||
t.Fatalf("unexpected eth0 error counters: %+v", item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMountsUnescapesOctalSequences(t *testing.T) {
|
||||
entries := parseMounts(readFixture(t, "proc-healthy", "mounts"))
|
||||
found := false
|
||||
for _, entry := range entries {
|
||||
if entry.mount == "/mnt/disks/Media Backup" {
|
||||
found = true
|
||||
if entry.fsType != "btrfs" {
|
||||
t.Fatalf("unexpected fs type: %q", entry.fsType)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("expected the escaped mount point to be decoded")
|
||||
}
|
||||
if len(entries) != 12 {
|
||||
t.Fatalf("entries = %d, want 12", len(entries))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/process"
|
||||
)
|
||||
|
||||
// maxProcessCPUPercent matches the bound internal/process enforces. A busy multi-core
|
||||
// task legitimately exceeds 100%.
|
||||
const maxProcessCPUPercent = 10000
|
||||
|
||||
// processKey identifies a task across scans. The PID alone is not enough: PIDs are
|
||||
// reused, and charging a new process with the CPU time of the dead one it replaced
|
||||
// would show a fresh task at an absurd utilisation. The kernel's start time makes the
|
||||
// identity stable.
|
||||
type processKey struct {
|
||||
pid int
|
||||
startTime uint64
|
||||
}
|
||||
|
||||
type processCPUSample struct {
|
||||
at time.Time
|
||||
ticks uint64
|
||||
}
|
||||
|
||||
// processStat is the subset of /proc/<pid>/stat the inventory needs.
|
||||
type processStat struct {
|
||||
pid int
|
||||
comm string
|
||||
state string
|
||||
ticks uint64
|
||||
startTime uint64
|
||||
rssPages uint64
|
||||
}
|
||||
|
||||
func (s processStat) key() processKey { return processKey{pid: s.pid, startTime: s.startTime} }
|
||||
|
||||
// parseProcessStat reads /proc/<pid>/stat.
|
||||
//
|
||||
// The second field is the executable name in parentheses and it is neither escaped nor
|
||||
// quoted: a process can legitimately be called ") (" or "my app (2)". Splitting the
|
||||
// line on whitespace therefore corrupts every field after it. The parser instead cuts
|
||||
// at the LAST closing parenthesis, which is the only reliable delimiter.
|
||||
func parseProcessStat(data []byte, pid int) (processStat, error) {
|
||||
text := string(data)
|
||||
open := strings.IndexByte(text, '(')
|
||||
closing := strings.LastIndexByte(text, ')')
|
||||
if open < 0 || closing < open {
|
||||
return processStat{}, errors.New("hostcollect: process stat has no comm field")
|
||||
}
|
||||
comm := text[open+1 : closing]
|
||||
fields := strings.Fields(text[closing+1:])
|
||||
// Field 3 (state) becomes index 0 here; the inventory needs up to field 24 (rss).
|
||||
const (
|
||||
stateIndex = 0
|
||||
utimeIndex = 11
|
||||
stimeIndex = 12
|
||||
startTimeIndex = 19
|
||||
rssIndex = 21
|
||||
)
|
||||
if len(fields) <= rssIndex {
|
||||
return processStat{}, errors.New("hostcollect: process stat is truncated")
|
||||
}
|
||||
utime, utimeOK := parseUint(fields[utimeIndex])
|
||||
stime, stimeOK := parseUint(fields[stimeIndex])
|
||||
startTime, startOK := parseUint(fields[startTimeIndex])
|
||||
if !utimeOK || !stimeOK || !startOK {
|
||||
return processStat{}, errors.New("hostcollect: process stat has malformed counters")
|
||||
}
|
||||
// rss is signed in the kernel's own printf but never negative in practice; a
|
||||
// negative or malformed value degrades to zero rather than failing the process.
|
||||
rss, _ := parseUint(fields[rssIndex])
|
||||
return processStat{
|
||||
pid: pid,
|
||||
comm: sanitizeName(comm),
|
||||
state: processState(fields[stateIndex]),
|
||||
ticks: utime + stime,
|
||||
startTime: startTime,
|
||||
rssPages: rss,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// processState expands the kernel's single-letter state so the UI does not have to.
|
||||
func processState(value string) string {
|
||||
if value == "" {
|
||||
return "unknown"
|
||||
}
|
||||
switch value[0] {
|
||||
case 'R':
|
||||
return "running"
|
||||
case 'S':
|
||||
return "sleeping"
|
||||
case 'D':
|
||||
return "uninterruptible"
|
||||
case 'Z':
|
||||
return "zombie"
|
||||
case 'T':
|
||||
return "stopped"
|
||||
case 't':
|
||||
return "tracing-stop"
|
||||
case 'X', 'x':
|
||||
return "dead"
|
||||
case 'I':
|
||||
return "idle"
|
||||
case 'K':
|
||||
return "wakekill"
|
||||
case 'W':
|
||||
return "waking"
|
||||
case 'P':
|
||||
return "parked"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// Processes reads the process inventory.
|
||||
//
|
||||
// A process that exits between readdir and open is the normal case, not an error: the
|
||||
// scan skips it and keeps going. Only a failure that invalidates the whole scan (no
|
||||
// uptime, unreadable procfs root) returns an error, because a half-empty inventory
|
||||
// published as complete would be worse than no inventory at all.
|
||||
func (c *Collector) Processes(ctx context.Context) (process.RawSnapshot, error) {
|
||||
if c == nil {
|
||||
return process.RawSnapshot{}, errors.New("hostcollect: collector is nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return process.RawSnapshot{}, err
|
||||
}
|
||||
now := c.options.Now().UTC()
|
||||
uptimeSeconds, err := c.uptimeSeconds()
|
||||
if err != nil {
|
||||
return process.RawSnapshot{}, err
|
||||
}
|
||||
entries, err := os.ReadDir(c.options.ProcRoot)
|
||||
if err != nil {
|
||||
return process.RawSnapshot{}, fmt.Errorf("hostcollect: list processes: %w", err)
|
||||
}
|
||||
|
||||
stats := make([]processStat, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return process.RawSnapshot{}, err
|
||||
}
|
||||
pid, ok := processID(entry.Name())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
data, readErr := readLimited(c.procPath(entry.Name(), "stat"), maxProcessFileBytes)
|
||||
if readErr != nil {
|
||||
continue
|
||||
}
|
||||
stat, parseErr := parseProcessStat(data, pid)
|
||||
if parseErr != nil {
|
||||
continue
|
||||
}
|
||||
stats = append(stats, stat)
|
||||
}
|
||||
|
||||
percentages := c.cpuPercentages(now, stats)
|
||||
rows := c.rank(stats, percentages)
|
||||
|
||||
processes := make([]process.RawProcess, 0, len(rows))
|
||||
for _, stat := range rows {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return process.RawSnapshot{}, err
|
||||
}
|
||||
row, ok := c.describe(stat, uptimeSeconds, percentages[stat.key()])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
processes = append(processes, row)
|
||||
}
|
||||
sort.Slice(processes, func(i, j int) bool { return processes[i].PID < processes[j].PID })
|
||||
|
||||
return process.RawSnapshot{
|
||||
Source: process.Source{
|
||||
ID: "process",
|
||||
Type: "agent",
|
||||
CapabilityVersion: process.ContractVersion,
|
||||
ObservedAt: now,
|
||||
},
|
||||
Processes: processes,
|
||||
ObservedAt: now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// cpuPercentages turns accumulated CPU ticks into a utilisation percentage relative to
|
||||
// the previous scan and replaces the retained state with the current one. The map is
|
||||
// rebuilt from the live process list every scan, so it cannot grow without bound as
|
||||
// processes come and go.
|
||||
func (c *Collector) cpuPercentages(now time.Time, stats []processStat) map[processKey]float64 {
|
||||
percentages := make(map[processKey]float64, len(stats))
|
||||
current := make(map[processKey]processCPUSample, len(stats))
|
||||
|
||||
c.mu.Lock()
|
||||
previous := c.processes
|
||||
for _, stat := range stats {
|
||||
key := stat.key()
|
||||
current[key] = processCPUSample{at: now, ticks: stat.ticks}
|
||||
sample, seen := previous[key]
|
||||
if !seen {
|
||||
continue
|
||||
}
|
||||
elapsed := now.Sub(sample.at).Seconds()
|
||||
if elapsed <= 0 || stat.ticks < sample.ticks {
|
||||
continue
|
||||
}
|
||||
percent := float64(stat.ticks-sample.ticks) / c.options.ClockTicks / elapsed * 100
|
||||
if percent < 0 {
|
||||
percent = 0
|
||||
}
|
||||
if percent > maxProcessCPUPercent {
|
||||
percent = maxProcessCPUPercent
|
||||
}
|
||||
percentages[key] = percent
|
||||
}
|
||||
c.processes = current
|
||||
c.mu.Unlock()
|
||||
|
||||
return percentages
|
||||
}
|
||||
|
||||
// rank keeps the busiest processes when the inventory exceeds MaxRows. Truncating by
|
||||
// cost rather than by PID order keeps the rows an operator actually needs.
|
||||
func (c *Collector) rank(stats []processStat, percentages map[processKey]float64) []processStat {
|
||||
ordered := append([]processStat(nil), stats...)
|
||||
sort.SliceStable(ordered, func(i, j int) bool {
|
||||
left, right := percentages[ordered[i].key()], percentages[ordered[j].key()]
|
||||
if left != right {
|
||||
return left > right
|
||||
}
|
||||
if ordered[i].rssPages != ordered[j].rssPages {
|
||||
return ordered[i].rssPages > ordered[j].rssPages
|
||||
}
|
||||
return ordered[i].pid < ordered[j].pid
|
||||
})
|
||||
if len(ordered) > c.options.ProcessLimits.MaxRows {
|
||||
ordered = ordered[:c.options.ProcessLimits.MaxRows]
|
||||
}
|
||||
return ordered
|
||||
}
|
||||
|
||||
// describe fills in the fields that need a second read, for the retained rows only.
|
||||
//
|
||||
// Only the program name is taken from the command line, never the arguments: argv
|
||||
// routinely carries tokens, passwords and connection strings, and the snapshot is
|
||||
// stored and rendered. The read itself is bounded, so an enormous argv costs one page,
|
||||
// not a copy of the whole command line.
|
||||
func (c *Collector) describe(stat processStat, uptimeSeconds, cpuPercent float64) (process.RawProcess, bool) {
|
||||
directory := strconv.Itoa(stat.pid)
|
||||
name := stat.comm
|
||||
if data, err := readLimited(c.procPath(directory, "cmdline"), maxCmdlineBytes); err == nil {
|
||||
if argv0 := commandName(data); argv0 != "" {
|
||||
name = argv0
|
||||
}
|
||||
}
|
||||
if name == "" {
|
||||
// Neither comm nor cmdline is usable — the task is gone or unreadable.
|
||||
return process.RawProcess{}, false
|
||||
}
|
||||
memoryBytes := stat.rssPages * uint64(c.options.PageSize) //nolint:gosec // PageSize is validated positive
|
||||
if rss, ok := c.residentBytes(directory); ok {
|
||||
memoryBytes = rss
|
||||
}
|
||||
runtimeSeconds := uptimeSeconds - float64(stat.startTime)/c.options.ClockTicks
|
||||
if runtimeSeconds < 0 || runtimeSeconds > maxUptimeSeconds {
|
||||
runtimeSeconds = 0
|
||||
}
|
||||
return process.RawProcess{
|
||||
PID: stat.pid,
|
||||
Name: truncate(name, maxNameBytes),
|
||||
State: stat.state,
|
||||
RuntimeSeconds: runtimeSeconds,
|
||||
CPUPercent: cpuPercent,
|
||||
MemoryBytes: memoryBytes,
|
||||
}, true
|
||||
}
|
||||
|
||||
// residentBytes prefers VmRSS from /proc/<pid>/status, which the kernel already
|
||||
// reports in kB and keeps consistent, over the page count in stat. Kernel threads have
|
||||
// no VmRSS at all, which is why the stat value remains the fallback.
|
||||
func (c *Collector) residentBytes(directory string) (uint64, bool) {
|
||||
data, err := readLimited(c.procPath(directory, "status"), maxProcessFileBytes)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
for _, line := range lines(data) {
|
||||
key, rest, found := strings.Cut(line, ":")
|
||||
if !found || key != "VmRSS" {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(rest)
|
||||
if len(fields) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
value, ok := parseUint(fields[0])
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
if len(fields) > 1 && strings.EqualFold(fields[1], "kB") {
|
||||
const kib = 1024
|
||||
if value > (1<<64-1)/kib {
|
||||
return 0, false
|
||||
}
|
||||
value *= kib
|
||||
}
|
||||
return value, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// commandName extracts the program name from a NUL separated command line.
|
||||
func commandName(data []byte) string {
|
||||
text := string(data)
|
||||
if index := strings.IndexByte(text, 0); index >= 0 {
|
||||
text = text[:index]
|
||||
}
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
return sanitizeName(filepath.Base(text))
|
||||
}
|
||||
|
||||
// sanitizeName drops control characters so a hostile process name cannot inject
|
||||
// terminal escapes or NUL bytes into logs and stored snapshots.
|
||||
func sanitizeName(value string) string {
|
||||
cleaned := strings.Map(func(r rune) rune {
|
||||
if r < 0x20 || r == 0x7f {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, value)
|
||||
return truncate(strings.TrimSpace(cleaned), maxNameBytes)
|
||||
}
|
||||
|
||||
func processID(name string) (int, bool) {
|
||||
pid, err := strconv.Atoi(name)
|
||||
if err != nil || pid < 1 {
|
||||
return 0, false
|
||||
}
|
||||
return pid, true
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/itworx/pulse/internal/process"
|
||||
)
|
||||
|
||||
func processByPID(t *testing.T, snapshot process.RawSnapshot, pid int) process.RawProcess {
|
||||
t.Helper()
|
||||
for _, item := range snapshot.Processes {
|
||||
if item.PID == pid {
|
||||
return item
|
||||
}
|
||||
}
|
||||
t.Fatalf("pid %d not found in %+v", pid, snapshot.Processes)
|
||||
return process.RawProcess{}
|
||||
}
|
||||
|
||||
func TestParseProcessStatHandlesParenthesesInTheProgramName(t *testing.T) {
|
||||
stat, err := parseProcessStat(readFixture(t, "proc-healthy", "1234", "stat"), 1234)
|
||||
if err != nil {
|
||||
t.Fatalf("parseProcessStat returned error: %v", err)
|
||||
}
|
||||
if stat.comm != "my app) (2)" {
|
||||
t.Fatalf("comm = %q; the parser must cut at the last closing parenthesis", stat.comm)
|
||||
}
|
||||
if stat.ticks != 100000+20000 {
|
||||
t.Fatalf("ticks = %d", stat.ticks)
|
||||
}
|
||||
if stat.startTime != 5000 || stat.rssPages != 250000 {
|
||||
t.Fatalf("unexpected stat: %+v", stat)
|
||||
}
|
||||
if stat.state != "sleeping" {
|
||||
t.Fatalf("state = %q", stat.state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseProcessStatRejectsMalformedInput(t *testing.T) {
|
||||
cases := map[string][]byte{
|
||||
"truncated": readFixture(t, "proc-healthy", "9999", "stat"),
|
||||
"no comm": []byte("1 systemd S 0 1\n"),
|
||||
"empty": {},
|
||||
"bad number": []byte("1 (x) S 0 0 0 0 0 0 0 0 0 0 nan nan 0 0 0 0 0 0 0 0 0 0\n"),
|
||||
}
|
||||
for name, data := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := parseProcessStat(data, 1); err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessesCollectsInventoryAndSkipsUnreadableTasks(t *testing.T) {
|
||||
collector, clock := newTestCollector(t, Options{PageSize: 4096})
|
||||
snapshot, err := collector.Processes(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Processes returned error: %v", err)
|
||||
}
|
||||
if len(snapshot.Processes) != 5 {
|
||||
t.Fatalf("processes = %d, want 5 (1, 2, 1234, 4567, 5555): %+v", len(snapshot.Processes), snapshot.Processes)
|
||||
}
|
||||
// 9999 has a truncated stat file; 3131 exists in the directory listing and even has
|
||||
// a cmdline, but its stat is already gone — the task exited between readdir and
|
||||
// open, which must cost one row and not the whole collection.
|
||||
for _, pid := range []int{9999, 3131} {
|
||||
for _, item := range snapshot.Processes {
|
||||
if item.PID == pid {
|
||||
t.Fatalf("pid %d must be skipped: malformed stat or a task that vanished mid-scan", pid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
init := processByPID(t, snapshot, 1)
|
||||
if init.Name != "init" {
|
||||
t.Fatalf("name = %q, want the command line's program name", init.Name)
|
||||
}
|
||||
if init.MemoryBytes != 12844*1024 {
|
||||
t.Fatalf("memory = %d, want VmRSS from status", init.MemoryBytes)
|
||||
}
|
||||
if init.RuntimeSeconds < 351281 || init.RuntimeSeconds > 351283 {
|
||||
t.Fatalf("runtime = %v", init.RuntimeSeconds)
|
||||
}
|
||||
|
||||
kernelThread := processByPID(t, snapshot, 2)
|
||||
if kernelThread.Name != "kthreadd" {
|
||||
t.Fatalf("a kernel thread with an empty cmdline must fall back to comm, got %q", kernelThread.Name)
|
||||
}
|
||||
if kernelThread.MemoryBytes != 0 {
|
||||
t.Fatalf("memory = %d, want 0 for a kernel thread", kernelThread.MemoryBytes)
|
||||
}
|
||||
|
||||
// 5555 has a stat file but no status or cmdline: it exited between the two reads.
|
||||
vanished := processByPID(t, snapshot, 5555)
|
||||
if vanished.Name != "short-lived" || vanished.MemoryBytes != 500*4096 {
|
||||
t.Fatalf("unexpected fallback for a task that vanished between reads: %+v", vanished)
|
||||
}
|
||||
if vanished.State != "zombie" {
|
||||
t.Fatalf("state = %q", vanished.State)
|
||||
}
|
||||
|
||||
if _, err := process.Normalize(snapshot, clock.now(), process.Limits{}); err != nil {
|
||||
t.Fatalf("process.Normalize rejected the collected snapshot: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessesNeverCopyCommandLineArguments(t *testing.T) {
|
||||
collector, _ := newTestCollector(t, Options{})
|
||||
snapshot, err := collector.Processes(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Processes returned error: %v", err)
|
||||
}
|
||||
payload, err := json.Marshal(snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
for _, secret := range []string{"SUPER-SECRET-VALUE", "hunter2", "--token", "--password", "virtio-net-pci"} {
|
||||
if strings.Contains(string(payload), secret) {
|
||||
t.Fatalf("snapshot leaked command line argument %q", secret)
|
||||
}
|
||||
}
|
||||
weird := processByPID(t, snapshot, 1234)
|
||||
if weird.Name != "weird app (2)" {
|
||||
t.Fatalf("name = %q, want only the program name", weird.Name)
|
||||
}
|
||||
// The 29 KiB command line of pid 4567 is read bounded and reduced to its base name.
|
||||
huge := processByPID(t, snapshot, 4567)
|
||||
if huge.Name != "qemu-system-x86_64" {
|
||||
t.Fatalf("name = %q", huge.Name)
|
||||
}
|
||||
if len(huge.Name) > maxNameBytes {
|
||||
t.Fatalf("name length = %d", len(huge.Name))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessesEnforceTheRowLimitByCost(t *testing.T) {
|
||||
collector, _ := newTestCollector(t, Options{ProcessLimits: process.Limits{MaxRows: 2, MaxPageSize: 10}})
|
||||
snapshot, err := collector.Processes(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Processes returned error: %v", err)
|
||||
}
|
||||
if len(snapshot.Processes) != 2 {
|
||||
t.Fatalf("processes = %d, want the limit of 2", len(snapshot.Processes))
|
||||
}
|
||||
// Without a previous sample every CPU percentage is zero, so the tie breaks on
|
||||
// resident pages: the two largest tasks survive.
|
||||
if snapshot.Processes[0].PID != 1234 || snapshot.Processes[1].PID != 4567 {
|
||||
t.Fatalf("unexpected survivors: %+v", snapshot.Processes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessCPUPercentIsADeltaAndSurvivesPIDReuse(t *testing.T) {
|
||||
root := copyTree(t, filepath.Join("testdata", "proc-healthy"))
|
||||
collector, clock := newTestCollector(t, Options{ProcRoot: root})
|
||||
|
||||
first, err := collector.Processes(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
if processByPID(t, first, 1234).CPUPercent != 0 {
|
||||
t.Fatal("a single sample cannot yield CPU utilisation")
|
||||
}
|
||||
|
||||
// 500 extra ticks over 10 seconds at 100 USER_HZ is exactly 50% of one core.
|
||||
writeFile(t, filepath.Join(root, "1234", "stat"),
|
||||
"1234 (my app) (2)) S 1 1234 1234 0 -1 4194304 55555 0 12 0 100400 20100 0 0 20 0 8 0 5000 987654321 250000 0 1 1 0 0 0 0 0 0 0 0 0 0 17 5 0 0 0 0 0\n")
|
||||
// pid 1 is replaced by a new task that reuses the PID: a different start time.
|
||||
writeFile(t, filepath.Join(root, "1", "stat"),
|
||||
"1 (systemd) S 0 1 1 0 -1 4194560 12345 678910 12 34 90456 789 1000 2000 20 0 1 0 99 172032000 3210 0 1 1 0 0 0 0 0 0 0 0 0 0 17 3 0 0 0 0 0\n")
|
||||
clock.advance(10 * time.Second)
|
||||
|
||||
second, err := collector.Processes(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("second scan: %v", err)
|
||||
}
|
||||
if got := processByPID(t, second, 1234).CPUPercent; got < 49.9 || got > 50.1 {
|
||||
t.Fatalf("cpu = %v, want 50", got)
|
||||
}
|
||||
if got := processByPID(t, second, 1).CPUPercent; got != 0 {
|
||||
t.Fatalf("cpu = %v; a reused PID must not inherit the previous task's ticks", got)
|
||||
}
|
||||
if _, err := process.Normalize(second, clock.now(), process.Limits{}); err != nil {
|
||||
t.Fatalf("process.Normalize rejected the snapshot: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessCPUPercentIgnoresBackwardsCounters(t *testing.T) {
|
||||
root := copyTree(t, filepath.Join("testdata", "proc-healthy"))
|
||||
collector, clock := newTestCollector(t, Options{ProcRoot: root})
|
||||
if _, err := collector.Processes(context.Background()); err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
writeFile(t, filepath.Join(root, "1234", "stat"),
|
||||
"1234 (my app) (2)) S 1 1234 1234 0 -1 4194304 55555 0 12 0 1 1 0 0 20 0 8 0 5000 987654321 250000 0 1 1 0 0 0 0 0 0 0 0 0 0 17 5 0 0 0 0 0\n")
|
||||
clock.advance(10 * time.Second)
|
||||
|
||||
second, err := collector.Processes(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("second scan: %v", err)
|
||||
}
|
||||
if got := processByPID(t, second, 1234).CPUPercent; got != 0 {
|
||||
t.Fatalf("cpu = %v, want 0 for a counter that went backwards", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessesRetainedStateStaysBounded(t *testing.T) {
|
||||
root := copyTree(t, filepath.Join("testdata", "proc-healthy"))
|
||||
collector, clock := newTestCollector(t, Options{ProcRoot: root})
|
||||
if _, err := collector.Processes(context.Background()); err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(root, "1234")); err != nil {
|
||||
t.Fatalf("remove: %v", err)
|
||||
}
|
||||
clock.advance(10 * time.Second)
|
||||
if _, err := collector.Processes(context.Background()); err != nil {
|
||||
t.Fatalf("second scan: %v", err)
|
||||
}
|
||||
collector.mu.Lock()
|
||||
retained := len(collector.processes)
|
||||
collector.mu.Unlock()
|
||||
if retained != 4 {
|
||||
t.Fatalf("retained samples = %d, want only the live tasks", retained)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessesFailWhenTheProcfsRootIsUnusable(t *testing.T) {
|
||||
collector, _ := newTestCollector(t, Options{ProcRoot: filepath.Join("testdata", "does-not-exist")})
|
||||
if _, err := collector.Processes(context.Background()); err == nil {
|
||||
t.Fatal("expected an error when the procfs root cannot be read")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessesHonourContextCancellation(t *testing.T) {
|
||||
collector, _ := newTestCollector(t, Options{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := collector.Processes(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("error = %v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeNameDropsControlCharacters(t *testing.T) {
|
||||
if got := sanitizeName("bad\x00name\x1b[31m"); got != "badname[31m" {
|
||||
t.Fatalf("sanitizeName = %q", got)
|
||||
}
|
||||
if got := commandName([]byte("/usr/bin/env\x00FOO=bar\x00")); got != "env" {
|
||||
t.Fatalf("commandName = %q", got)
|
||||
}
|
||||
if got := commandName(nil); got != "" {
|
||||
t.Fatalf("commandName = %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// readLimited reads at most limit bytes from path. Every /proc file the collector
|
||||
// touches is small, but /proc is a kernel interface and a bounded read keeps a
|
||||
// pathological or hostile file from becoming an unbounded allocation.
|
||||
func readLimited(path string, limit int64) ([]byte, error) {
|
||||
file, err := os.Open(path) //nolint:gosec // paths are derived from the configured procfs root
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
data, err := io.ReadAll(io.LimitReader(file, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func readTrimmed(path string, limit int64) (string, error) {
|
||||
data, err := readLimited(path, limit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(data)), nil
|
||||
}
|
||||
|
||||
// parseUint accepts the decimal unsigned values procfs uses and reports failure
|
||||
// instead of guessing, so a malformed line degrades one field rather than producing a
|
||||
// plausible-looking zero.
|
||||
func parseUint(value string) (uint64, bool) {
|
||||
parsed, err := strconv.ParseUint(strings.TrimSpace(value), 10, 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return parsed, true
|
||||
}
|
||||
|
||||
func parseFloat(value string) (float64, bool) {
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return parsed, true
|
||||
}
|
||||
|
||||
func truncate(value string, max int) string {
|
||||
if len(value) > max {
|
||||
return value[:max]
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// lines splits procfs content without allocating a scanner per file.
|
||||
func lines(data []byte) []string {
|
||||
text := strings.TrimRight(string(data), "\n")
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(text, "\n")
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
//go:build linux
|
||||
|
||||
package hostcollect
|
||||
|
||||
import "syscall"
|
||||
|
||||
// statFS reads capacity and inode usage for one mount point. statfs(2) is a read-only
|
||||
// syscall that needs no capability, which keeps the agent inside cap_drop: [ALL].
|
||||
//
|
||||
// "Used" follows df: total blocks minus free blocks, so the filesystem's reserved
|
||||
// blocks count as used rather than as available headroom the operator does not have.
|
||||
func statFS(path string) (FilesystemUsage, error) {
|
||||
var stat syscall.Statfs_t
|
||||
if err := syscall.Statfs(path, &stat); err != nil {
|
||||
return FilesystemUsage{}, err
|
||||
}
|
||||
blockSize := uint64(stat.Bsize) //nolint:gosec,unconvert // Bsize is int64 on some arches, uint32 on others
|
||||
if blockSize == 0 {
|
||||
return FilesystemUsage{}, nil
|
||||
}
|
||||
blocks := stat.Blocks
|
||||
free := stat.Bfree
|
||||
if free > blocks {
|
||||
free = blocks
|
||||
}
|
||||
usage := FilesystemUsage{
|
||||
CapacityBytes: blocks * blockSize,
|
||||
UsedBytes: (blocks - free) * blockSize,
|
||||
InodesTotal: stat.Files,
|
||||
}
|
||||
if stat.Files > 0 {
|
||||
freeInodes := stat.Ffree
|
||||
if freeInodes > stat.Files {
|
||||
freeInodes = stat.Files
|
||||
}
|
||||
usage.InodesUsed = stat.Files - freeInodes
|
||||
}
|
||||
return usage, nil
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
//go:build !linux
|
||||
|
||||
package hostcollect
|
||||
|
||||
import "errors"
|
||||
|
||||
// statFS is unavailable off Linux. The package still builds and its parsers stay
|
||||
// testable on a developer machine; filesystem usage simply degrades to a warning.
|
||||
func statFS(string) (FilesystemUsage, error) {
|
||||
return FilesystemUsage{}, errors.New("hostcollect: statfs is only available on linux")
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
1 (systemd) S 0 1 1 0 -1 4194560 12345 678910 12 34 456 789 1000 2000 20 0 1 0 42 172032000 3210 18446744073709551615 1 1 0 0 0 0 671173123 4096 1260 0 0 0 17 3 0 0 0 0 0
|
||||
@@ -0,0 +1,11 @@
|
||||
Name: systemd
|
||||
Umask: 0022
|
||||
State: S (sleeping)
|
||||
Tgid: 1
|
||||
Pid: 1
|
||||
PPid: 0
|
||||
VmPeak: 180000 kB
|
||||
VmSize: 168000 kB
|
||||
VmRSS: 12844 kB
|
||||
RssAnon: 4000 kB
|
||||
Threads: 1
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
1234 (my app) (2)) S 1 1234 1234 0 -1 4194304 55555 0 12 0 100000 20000 0 0 20 0 8 0 5000 987654321 250000 18446744073709551615 1 1 0 0 0 0 0 0 0 0 0 0 17 5 0 0 0 0 0
|
||||
@@ -0,0 +1,5 @@
|
||||
Name: my app (2)
|
||||
State: S (sleeping)
|
||||
Pid: 1234
|
||||
VmRSS: 1024000 kB
|
||||
Threads: 8
|
||||
@@ -0,0 +1 @@
|
||||
2 (kthreadd) S 0 0 0 0 -1 2129984 0 0 0 0 0 12 0 0 20 0 1 0 43 0 0 18446744073709551615 0 0 0 0 0 0 0 2147483647 0 0 0 0 17 0 0 0 0 0 0
|
||||
@@ -0,0 +1,6 @@
|
||||
Name: kthreadd
|
||||
State: S (sleeping)
|
||||
Tgid: 2
|
||||
Pid: 2
|
||||
PPid: 0
|
||||
Threads: 1
|
||||
Binary file not shown.
@@ -0,0 +1,2 @@
|
||||
Name: gone
|
||||
State: R (running)
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
4567 (qemu-system-x86) R 1 4567 4567 0 -1 4194304 999999 0 100 0 900000 100000 0 0 20 0 12 0 6000 9876543210 1000000 18446744073709551615 1 1 0 0 0 0 0 0 0 0 0 0 17 2 0 0 0 0 0
|
||||
@@ -0,0 +1,5 @@
|
||||
Name: qemu-system-x86
|
||||
State: R (running)
|
||||
Pid: 4567
|
||||
VmRSS: 16777216 kB
|
||||
Threads: 12
|
||||
@@ -0,0 +1 @@
|
||||
5555 (short-lived) Z 1 5555 5555 0 -1 4194304 10 0 0 0 10 20 0 0 20 0 1 0 7000 0 500 18446744073709551615 1 1 0 0 0 0 0 0 0 0 0 0 17 1 0 0 0 0 0
|
||||
@@ -0,0 +1 @@
|
||||
9999 (broken) S 1 9999
|
||||
@@ -0,0 +1 @@
|
||||
1.52 2.08 2.35 3/1187 28941
|
||||
@@ -0,0 +1,16 @@
|
||||
MemTotal: 32819484 kB
|
||||
MemFree: 1234560 kB
|
||||
MemAvailable: 24680240 kB
|
||||
Buffers: 123456 kB
|
||||
Cached: 20000000 kB
|
||||
SwapCached: 0 kB
|
||||
Active: 10000000 kB
|
||||
Inactive: 15000000 kB
|
||||
SwapTotal: 8388604 kB
|
||||
SwapFree: 8000000 kB
|
||||
Dirty: 1234 kB
|
||||
SReclaimable: 1000000 kB
|
||||
HugePages_Total: 0
|
||||
HugePages_Free: 0
|
||||
Hugepagesize: 2048 kB
|
||||
DirectMap1G: 33554432 kB
|
||||
@@ -0,0 +1,12 @@
|
||||
rootfs / rootfs rw,size=16384000k 0 0
|
||||
proc /proc proc rw,relatime 0 0
|
||||
sysfs /sys sysfs rw,relatime 0 0
|
||||
tmpfs /run tmpfs rw,nosuid,nodev 0 0
|
||||
cgroup2 /sys/fs/cgroup cgroup2 rw,nosuid,nodev,noexec,relatime 0 0
|
||||
/dev/sda1 /boot vfat rw,noatime,fmask=0177 0 0
|
||||
/dev/md1 /mnt/disk1 xfs rw,noatime,attr2,inode64 0 0
|
||||
/dev/md2 /mnt/disk2 xfs rw,noatime,attr2,inode64 0 0
|
||||
/dev/nvme0n1p1 /mnt/cache btrfs rw,noatime,space_cache=v2 0 0
|
||||
shfs /mnt/user fuse.shfs rw,nosuid,nodev,noatime,allow_other 0 0
|
||||
/dev/sdc1 /mnt/disks/Media\040Backup btrfs rw,noatime 0 0
|
||||
/dev/md1 /mnt/disk1 xfs rw,noatime,attr2,inode64 0 0
|
||||
@@ -0,0 +1,7 @@
|
||||
Inter-| Receive | Transmit
|
||||
face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed
|
||||
lo: 12345678 98765 0 0 0 0 0 0 12345678 98765 0 0 0 0 0 0
|
||||
eth0:18446744073709551615 9876543 12 3 0 0 0 45678 987654321 5432109 1 7 0 0 0 0
|
||||
br0: 55555555 444444 0 0 0 0 0 0 66666666 333333 0 0 0 0 0 0
|
||||
wlan0: garbage 444444 0 0 0 0 0 0 66666666 333333 0 0 0 0 0 0
|
||||
tap0: 1 2 3
|
||||
@@ -0,0 +1 @@
|
||||
1 (init) S 0 1 1 0 -1 4194560 1 1 1 1 1 1 1 1 20 0 1 0 42 100 10 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
cpu 1234567 8901 234567 45678901 12345 0 6789 1234 5000 100
|
||||
cpu0 617283 4450 117283 22839450 6172 0 3394 617 2500 50
|
||||
cpu1 617284 4451 117284 22839451 6173 0 3395 617 2500 50
|
||||
intr 987654321 12 0 0 0 0 0 0 0 1 0 0
|
||||
ctxt 1234567890
|
||||
btime 1754300000
|
||||
processes 987654
|
||||
procs_running 3
|
||||
procs_blocked 0
|
||||
softirq 123456789 1234 2345 3456 4567 5678 0 6789 7890 0 8901
|
||||
@@ -0,0 +1 @@
|
||||
tower
|
||||
@@ -0,0 +1 @@
|
||||
6.1.79-Unraid
|
||||
@@ -0,0 +1 @@
|
||||
351282.31 2609913.42
|
||||
@@ -0,0 +1 @@
|
||||
not-a-load
|
||||
@@ -0,0 +1,10 @@
|
||||
MemTotal: 8123456 kB
|
||||
MemFree: 123456 kB
|
||||
Buffers: 23456 kB
|
||||
Cached: 2000000 kB
|
||||
SReclaimable: 100000 kB
|
||||
SwapTotal: 0 kB
|
||||
SwapFree: 0 kB
|
||||
Bogus line without colon
|
||||
Broken: not-a-number kB
|
||||
HugePages_Total: 0
|
||||
@@ -0,0 +1,4 @@
|
||||
proc /proc proc rw,relatime 0 0
|
||||
sysfs /sys sysfs rw,relatime 0 0
|
||||
tmpfs /run tmpfs rw 0 0
|
||||
short line
|
||||
@@ -0,0 +1,2 @@
|
||||
Inter-| Receive | Transmit
|
||||
face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed
|
||||
@@ -0,0 +1,6 @@
|
||||
cpu 1000 200 300 4000
|
||||
cpu0 500 100 150 2000
|
||||
cpu2 500 100 150 2000
|
||||
cpu-bogus 1 2 3 4
|
||||
cpu9 not a number here
|
||||
intr 1 2 3
|
||||
@@ -0,0 +1 @@
|
||||
container-id-not-the-host
|
||||
@@ -0,0 +1 @@
|
||||
nonsense
|
||||
@@ -0,0 +1 @@
|
||||
unknown
|
||||
@@ -0,0 +1 @@
|
||||
up
|
||||
@@ -0,0 +1,34 @@
|
||||
package hostcollect
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// maxUptimeSeconds is the bound internal/host enforces (100 years). Rejecting here
|
||||
// keeps an implausible reading from failing normalization later, where the cause is
|
||||
// harder to see.
|
||||
const maxUptimeSeconds = 100 * 365 * 24 * 60 * 60
|
||||
|
||||
// parseUptime reads the first field of /proc/uptime, the seconds since boot. The
|
||||
// second field (idle time summed over cores) is intentionally unused.
|
||||
func parseUptime(data []byte) (float64, error) {
|
||||
fields := strings.Fields(string(data))
|
||||
if len(fields) == 0 {
|
||||
return 0, errBadUptime
|
||||
}
|
||||
seconds, ok := parseFloat(fields[0])
|
||||
if !ok || seconds < 0 || seconds > maxUptimeSeconds || math.IsNaN(seconds) || math.IsInf(seconds, 0) {
|
||||
return 0, errBadUptime
|
||||
}
|
||||
return seconds, nil
|
||||
}
|
||||
|
||||
func (c *Collector) uptimeSeconds() (float64, error) {
|
||||
data, err := readLimited(c.procPath("uptime"), maxProcessFileBytes)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("hostcollect: read uptime: %w", err)
|
||||
}
|
||||
return parseUptime(data)
|
||||
}
|
||||
Reference in New Issue
Block a user