mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 18:45:53 +00:00
3a4a3fd62b
Expose confined, identity-bound filesystem observations through the shared resource pipeline so investigations can distinguish an exhausted container mount from unrelated host capacity. Keep unavailable measurements explicit. Isolate alert-history reads from durable writes and reuse one chronological fold across polling. Catch up through bounded durable event IDs so simultaneous readers do not replay every retained snapshot. Retain expired actions when investigation outcomes move back to needs attention, and keep attached Assistant context focused. Record live storage diagnosis, healthy and dependency controls, approved and rejected Docker outcomes, source-bound browser proof and exact test limits. Missing-access continuity, VM dispatch completion and remaining Assistant orchestration defects stay open in the redesign plan.
1444 lines
44 KiB
Go
1444 lines
44 KiB
Go
package dockeragent
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
"math/big"
|
|
"net/netip"
|
|
"net/url"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
containertypes "github.com/moby/moby/api/types/container"
|
|
"github.com/moby/moby/api/types/image"
|
|
networktypes "github.com/moby/moby/api/types/network"
|
|
systemtypes "github.com/moby/moby/api/types/system"
|
|
"github.com/moby/moby/api/types/volume"
|
|
"github.com/moby/moby/client"
|
|
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
|
|
)
|
|
|
|
// buildReport gathers all system and container metrics into a single report
|
|
func (a *Agent) buildReport(ctx context.Context) (agentsdocker.Report, error) {
|
|
// Cycle-level containment: per-call deadlines below should bound every
|
|
// docker call, but one live exercise saw a call park for minutes with its
|
|
// deadline never firing, so the whole cycle gets its own ceiling plus an
|
|
// independent watchdog that dumps goroutines if even this deadline fails
|
|
// to abort the cycle.
|
|
ctx, cancel := context.WithTimeout(ctx, dockerCollectCycleTimeout)
|
|
defer cancel()
|
|
stopWatchdog := startCollectCycleWatchdog(a.logger, dockerCollectCycleTimeout+dockerCollectWatchdogGrace)
|
|
defer stopWatchdog()
|
|
|
|
info, err := dockerCallWithRetry(ctx, dockerInfoCallTimeout, func(callCtx context.Context) (systemtypes.Info, error) {
|
|
return a.docker.Info(callCtx)
|
|
})
|
|
if err != nil {
|
|
return agentsdocker.Report{}, fmt.Errorf("failed to query docker info: %w", annotateDockerConnectionError(err))
|
|
}
|
|
if a.cfg.HelperInventory != nil {
|
|
if err := validateCollectorDirectRuntime(a.docker, info); err != nil {
|
|
return agentsdocker.Report{}, fmt.Errorf("%w: %v", errCollectorRuntimeBoundaryChanged, err)
|
|
}
|
|
}
|
|
|
|
a.runtimeVer = info.ServerVersion
|
|
if a.daemonHost == "" {
|
|
a.daemonHost = a.docker.DaemonHost()
|
|
}
|
|
|
|
// Use current runtime as preference to avoid spurious switching.
|
|
// This preserves user's --docker-runtime choice (stored in a.runtime at init).
|
|
newRuntime := detectRuntime(info, a.daemonHost, a.runtime)
|
|
if newRuntime != a.runtime {
|
|
if a.runtime != "" {
|
|
a.logger.Info().
|
|
Str("runtime_previous", string(a.runtime)).
|
|
Str("runtime_current", string(newRuntime)).
|
|
Msg("Detected container runtime change")
|
|
}
|
|
a.runtime = newRuntime
|
|
a.supportsSwarm = newRuntime == RuntimeDocker
|
|
if newRuntime == RuntimePodman {
|
|
if a.cfg.IncludeServices {
|
|
a.logger.Warn().Msg("Podman runtime detected during report; disabling Swarm service collection")
|
|
}
|
|
if a.cfg.IncludeTasks {
|
|
a.logger.Warn().Msg("Podman runtime detected during report; disabling Swarm task collection")
|
|
}
|
|
a.cfg.IncludeServices = false
|
|
a.cfg.IncludeTasks = false
|
|
}
|
|
a.cfg.Runtime = string(newRuntime)
|
|
}
|
|
|
|
a.cpuCount = info.NCPU
|
|
|
|
agentID := a.cfg.AgentID
|
|
if agentID == "" {
|
|
// In unified mode, use the EXACT same fallback chain as hostagent:
|
|
// machineID -> hostname. Never use daemonID in unified mode because
|
|
// hostagent doesn't use it, and using different IDs causes token
|
|
// binding conflicts on the server (reported in #985, #986).
|
|
if a.cfg.AgentType == "unified" {
|
|
agentID = a.machineID
|
|
if agentID == "" {
|
|
agentID = a.hostName
|
|
}
|
|
} else {
|
|
// Standalone mode: prefer daemonID for backward compatibility,
|
|
// then fall back to machineID -> hostname.
|
|
// Use cached daemon ID from init rather than info.ID from current call.
|
|
// Podman can return different/empty IDs across calls, causing token
|
|
// binding conflicts on the server.
|
|
agentID = a.daemonID
|
|
if agentID == "" {
|
|
agentID = a.machineID
|
|
}
|
|
if agentID == "" {
|
|
agentID = a.hostName
|
|
}
|
|
}
|
|
}
|
|
a.hostID = agentID
|
|
|
|
hostName := a.hostName
|
|
if hostName == "" {
|
|
hostName = info.Name
|
|
}
|
|
|
|
uptime := readSystemUptime()
|
|
|
|
metricsCtx, metricsCancel := context.WithTimeout(ctx, 10*time.Second)
|
|
snapshot, err := hostmetricsCollectWithDiskFilters(metricsCtx, a.cfg.DiskExclude, a.cfg.DiskInclude)
|
|
metricsCancel()
|
|
if err != nil {
|
|
return agentsdocker.Report{}, fmt.Errorf("collect host metrics: %w", err)
|
|
}
|
|
|
|
collectContainers := a.cfg.IncludeContainers
|
|
if !collectContainers && (a.cfg.IncludeServices || a.cfg.IncludeTasks) && !info.Swarm.ControlAvailable {
|
|
collectContainers = true
|
|
}
|
|
|
|
var containers []agentsdocker.Container
|
|
if collectContainers {
|
|
var err error
|
|
containers, err = a.collectContainers(ctx)
|
|
if err != nil {
|
|
return agentsdocker.Report{}, err
|
|
}
|
|
}
|
|
|
|
services, tasks, nodes, secrets, configs, swarmInfo := a.collectSwarmData(ctx, info, containers)
|
|
diskUsageResult, storageUsage, err := a.collectStorageUsage(ctx)
|
|
if err != nil {
|
|
a.logger.Warn().Err(err).Msg("failed to collect Docker storage usage")
|
|
}
|
|
images, err := a.collectImages(ctx, diskUsageResult.Images.Items)
|
|
if err != nil {
|
|
a.logger.Warn().Err(err).Msg("failed to collect Docker images")
|
|
}
|
|
volumes, err := a.collectVolumes(ctx, diskUsageResult.Volumes.Items)
|
|
if err != nil {
|
|
a.logger.Warn().Err(err).Msg("failed to collect Docker volumes")
|
|
}
|
|
networks, err := a.collectNetworks(ctx)
|
|
if err != nil {
|
|
a.logger.Warn().Err(err).Msg("failed to collect Docker networks")
|
|
}
|
|
|
|
// Use Docker's MemTotal, but fall back to gopsutil's reading if Docker returns 0.
|
|
// This can happen in Docker-in-LXC setups where Docker daemon can't read host memory.
|
|
totalMemory := info.MemTotal
|
|
if totalMemory <= 0 && snapshot.Memory.TotalBytes > 0 {
|
|
totalMemory = snapshot.Memory.TotalBytes
|
|
}
|
|
|
|
report := agentsdocker.Report{
|
|
Agent: agentsdocker.AgentInfo{
|
|
ID: agentID,
|
|
Version: a.agentVersion,
|
|
Type: a.cfg.AgentType,
|
|
IntervalSeconds: int(a.cfg.Interval / time.Second),
|
|
Modules: a.helperOperationModuleStatuses(),
|
|
},
|
|
Host: agentsdocker.HostInfo{
|
|
Hostname: hostName,
|
|
Name: info.Name,
|
|
MachineID: a.machineID,
|
|
OS: info.OperatingSystem,
|
|
Runtime: string(a.runtime),
|
|
RuntimeVersion: a.runtimeVer,
|
|
KernelVersion: info.KernelVersion,
|
|
Architecture: info.Architecture,
|
|
DockerVersion: info.ServerVersion,
|
|
TotalCPU: info.NCPU,
|
|
TotalMemoryBytes: totalMemory,
|
|
UptimeSeconds: uptime,
|
|
CPUUsagePercent: safeFloat(snapshot.CPUUsagePercent),
|
|
LoadAverage: append([]float64(nil), snapshot.LoadAverage...),
|
|
Memory: snapshot.Memory,
|
|
Disks: append([]agentsdocker.Disk(nil), snapshot.Disks...),
|
|
Network: append([]agentsdocker.NetworkInterface(nil), snapshot.Network...),
|
|
Security: buildHostSecurityInfo(info),
|
|
},
|
|
SequenceID: a.nextReportSequenceID(),
|
|
Timestamp: time.Now().UTC(),
|
|
}
|
|
|
|
if swarmInfo != nil {
|
|
report.Host.Swarm = swarmInfo
|
|
}
|
|
|
|
if a.cfg.IncludeContainers {
|
|
report.Containers = containers
|
|
}
|
|
if len(images) > 0 {
|
|
report.Images = images
|
|
}
|
|
if len(volumes) > 0 {
|
|
report.Volumes = volumes
|
|
}
|
|
if len(networks) > 0 {
|
|
report.Networks = networks
|
|
}
|
|
if a.cfg.IncludeServices && len(services) > 0 {
|
|
report.Services = services
|
|
}
|
|
if a.cfg.IncludeTasks && len(tasks) > 0 {
|
|
report.Tasks = tasks
|
|
}
|
|
if len(nodes) > 0 {
|
|
report.Nodes = nodes
|
|
}
|
|
if len(secrets) > 0 {
|
|
report.Secrets = secrets
|
|
}
|
|
if len(configs) > 0 {
|
|
report.Configs = configs
|
|
}
|
|
if storageUsage != nil {
|
|
report.StorageUsage = storageUsage
|
|
}
|
|
|
|
if report.Agent.IntervalSeconds <= 0 {
|
|
report.Agent.IntervalSeconds = int(30 * time.Second / time.Second)
|
|
}
|
|
|
|
return report, nil
|
|
}
|
|
|
|
func buildHostSecurityInfo(info systemtypes.Info) *agentsdocker.HostSecurityInfo {
|
|
authzPlugins := normalizedNonEmptyStrings(info.Plugins.Authorization)
|
|
if len(authzPlugins) == 0 {
|
|
return nil
|
|
}
|
|
return &agentsdocker.HostSecurityInfo{
|
|
AuthorizationPlugins: authzPlugins,
|
|
}
|
|
}
|
|
|
|
func normalizedNonEmptyStrings(values []string) []string {
|
|
if len(values) == 0 {
|
|
return nil
|
|
}
|
|
|
|
normalized := make([]string, 0, len(values))
|
|
seen := make(map[string]struct{}, len(values))
|
|
for _, value := range values {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[value]; ok {
|
|
continue
|
|
}
|
|
seen[value] = struct{}{}
|
|
normalized = append(normalized, value)
|
|
}
|
|
if len(normalized) == 0 {
|
|
return nil
|
|
}
|
|
return normalized
|
|
}
|
|
|
|
func cloneStringMap(src map[string]string) map[string]string {
|
|
if len(src) == 0 {
|
|
return nil
|
|
}
|
|
out := make(map[string]string, len(src))
|
|
for k, v := range src {
|
|
out[k] = v
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (a *Agent) collectContainers(ctx context.Context) ([]agentsdocker.Container, error) {
|
|
options := dockerContainerListOptions{All: true}
|
|
if len(a.stateFilters) > 0 {
|
|
filterArgs := newDockerFilters()
|
|
for _, state := range a.stateFilters {
|
|
filterArgs.Add("status", state)
|
|
}
|
|
options.Filters = filterArgs
|
|
}
|
|
|
|
list, err := dockerCallWithRetry(ctx, dockerContainerListCallTimeout, func(callCtx context.Context) ([]containertypes.Summary, error) {
|
|
return a.docker.ContainerList(callCtx, options)
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to list containers: %w", annotateDockerConnectionError(err))
|
|
}
|
|
|
|
containers := make([]agentsdocker.Container, 0, len(list))
|
|
active := make(map[string]struct{}, len(list))
|
|
for _, summary := range list {
|
|
if len(a.allowedStates) > 0 {
|
|
if _, ok := a.allowedStates[strings.ToLower(string(summary.State))]; !ok {
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Skip backup containers created during updates - they're temporary
|
|
if isBackupContainer(summary.Names) {
|
|
continue
|
|
}
|
|
|
|
active[summary.ID] = struct{}{}
|
|
|
|
container, err := a.collectContainer(ctx, summary)
|
|
if err != nil {
|
|
a.logger.Warn().Str("container", strings.Join(summary.Names, ",")).Err(err).Msg("failed to collect container stats")
|
|
continue
|
|
}
|
|
containers = append(containers, container)
|
|
}
|
|
a.pruneStaleCPUSamples(active)
|
|
a.pruneInactiveContainerInspectCache(active)
|
|
return containers, nil
|
|
}
|
|
|
|
func (a *Agent) collectImages(ctx context.Context, usageImages []image.Summary) ([]agentsdocker.Image, error) {
|
|
usageByID := make(map[string]image.Summary, len(usageImages))
|
|
for _, summary := range usageImages {
|
|
if id := strings.TrimSpace(summary.ID); id != "" {
|
|
usageByID[id] = summary
|
|
}
|
|
}
|
|
|
|
list, err := dockerCallWithRetry(ctx, dockerInventoryCallTimeout, func(callCtx context.Context) ([]image.Summary, error) {
|
|
// Keep image identity and tags on the live telemetry cadence without
|
|
// asking the daemon to recompute shared layer sizes every 30 seconds.
|
|
// The throttled /system/df snapshot above already owns that expensive
|
|
// calculation and supplies its cached values here (#1729).
|
|
return a.docker.ImageList(callCtx, dockerImageListOptions{All: true})
|
|
})
|
|
if err != nil {
|
|
return nil, annotateDockerConnectionError(err)
|
|
}
|
|
|
|
images := make([]agentsdocker.Image, 0, len(list))
|
|
for _, summary := range list {
|
|
sharedSize := summary.SharedSize
|
|
containers := summary.Containers
|
|
if usage, ok := usageByID[strings.TrimSpace(summary.ID)]; ok {
|
|
sharedSize = usage.SharedSize
|
|
containers = usage.Containers
|
|
}
|
|
// Docker uses -1 when these optional calculations were not requested.
|
|
// Do not project that sentinel as a real negative byte/container count.
|
|
sharedSize = max(sharedSize, 0)
|
|
containers = max(containers, 0)
|
|
images = append(images, agentsdocker.Image{
|
|
ID: strings.TrimSpace(summary.ID),
|
|
RepoTags: normalizedNonEmptyStrings(summary.RepoTags),
|
|
RepoDigests: normalizedNonEmptyStrings(summary.RepoDigests),
|
|
SizeBytes: summary.Size,
|
|
SharedSizeBytes: sharedSize,
|
|
Containers: containers,
|
|
CreatedAt: dockerUnixTimestamp(summary.Created),
|
|
Labels: cloneStringMap(summary.Labels),
|
|
})
|
|
}
|
|
return images, nil
|
|
}
|
|
|
|
func (a *Agent) collectVolumes(ctx context.Context, usageVolumes []volume.Volume) ([]agentsdocker.Volume, error) {
|
|
usageByName := make(map[string]volume.UsageData, len(usageVolumes))
|
|
for _, volume := range usageVolumes {
|
|
name := strings.TrimSpace(volume.Name)
|
|
if name == "" || volume.UsageData == nil {
|
|
continue
|
|
}
|
|
usageByName[name] = *volume.UsageData
|
|
}
|
|
|
|
list, err := dockerCallWithRetry(ctx, dockerInventoryCallTimeout, func(callCtx context.Context) ([]volume.Volume, error) {
|
|
return a.docker.VolumeList(callCtx, dockerVolumeListOptions{})
|
|
})
|
|
if err != nil {
|
|
return nil, annotateDockerConnectionError(err)
|
|
}
|
|
|
|
volumes := make([]agentsdocker.Volume, 0, len(list))
|
|
for _, v := range list {
|
|
sizeBytes := int64(0)
|
|
refCount := int64(0)
|
|
if v.UsageData != nil {
|
|
sizeBytes = v.UsageData.Size
|
|
refCount = v.UsageData.RefCount
|
|
} else if usage, ok := usageByName[strings.TrimSpace(v.Name)]; ok {
|
|
sizeBytes = usage.Size
|
|
refCount = usage.RefCount
|
|
}
|
|
volumes = append(volumes, agentsdocker.Volume{
|
|
Name: strings.TrimSpace(v.Name),
|
|
Driver: strings.TrimSpace(v.Driver),
|
|
Mountpoint: strings.TrimSpace(v.Mountpoint),
|
|
Scope: strings.TrimSpace(v.Scope),
|
|
CreatedAt: strings.TrimSpace(v.CreatedAt),
|
|
SizeBytes: sizeBytes,
|
|
RefCount: refCount,
|
|
Labels: cloneStringMap(v.Labels),
|
|
Options: cloneStringMap(v.Options),
|
|
})
|
|
}
|
|
return volumes, nil
|
|
}
|
|
|
|
func (a *Agent) collectNetworks(ctx context.Context) ([]agentsdocker.Network, error) {
|
|
list, err := dockerCallWithRetry(ctx, dockerInventoryCallTimeout, func(callCtx context.Context) ([]networktypes.Summary, error) {
|
|
return a.docker.NetworkList(callCtx, dockerNetworkListOptions{})
|
|
})
|
|
if err != nil {
|
|
return nil, annotateDockerConnectionError(err)
|
|
}
|
|
|
|
networks := make([]agentsdocker.Network, 0, len(list))
|
|
for _, n := range list {
|
|
subnets := make([]agentsdocker.NetworkSubnet, 0, len(n.IPAM.Config))
|
|
for _, config := range n.IPAM.Config {
|
|
subnet := ""
|
|
if config.Subnet.IsValid() {
|
|
subnet = config.Subnet.String()
|
|
}
|
|
gateway := ""
|
|
if config.Gateway.IsValid() {
|
|
gateway = config.Gateway.String()
|
|
}
|
|
subnets = append(subnets, agentsdocker.NetworkSubnet{
|
|
Subnet: subnet,
|
|
Gateway: gateway,
|
|
})
|
|
}
|
|
networks = append(networks, agentsdocker.Network{
|
|
ID: strings.TrimSpace(n.ID),
|
|
Name: strings.TrimSpace(n.Name),
|
|
Driver: strings.TrimSpace(n.Driver),
|
|
Scope: strings.TrimSpace(n.Scope),
|
|
CreatedAt: n.Created,
|
|
EnableIPv4: n.EnableIPv4,
|
|
EnableIPv6: n.EnableIPv6,
|
|
Internal: n.Internal,
|
|
Attachable: n.Attachable,
|
|
Ingress: n.Ingress,
|
|
ConfigOnly: n.ConfigOnly,
|
|
Subnets: subnets,
|
|
Labels: cloneStringMap(n.Labels),
|
|
Options: cloneStringMap(n.Options),
|
|
})
|
|
}
|
|
return networks, nil
|
|
}
|
|
|
|
func (a *Agent) collectStorageUsage(ctx context.Context) (client.DiskUsageResult, *agentsdocker.StorageUsage, error) {
|
|
a.storageUsageMu.Lock()
|
|
defer a.storageUsageMu.Unlock()
|
|
|
|
now := time.Now()
|
|
if !a.storageUsageCache.nextRefresh.IsZero() && now.Before(a.storageUsageCache.nextRefresh) {
|
|
if a.storageUsageCache.valid {
|
|
return a.storageUsageCache.result, cloneDockerStorageUsage(a.storageUsageCache.usage), nil
|
|
}
|
|
// The last refresh attempt already surfaced its error. Keep the
|
|
// expensive scan suppressed until the bounded refresh window expires.
|
|
return client.DiskUsageResult{}, nil, nil
|
|
}
|
|
a.storageUsageCache.nextRefresh = now.Add(dockerStorageUsageRefreshInterval)
|
|
|
|
// A full daemon storage walk must not be retried immediately. If it times
|
|
// out on an appliance daemon, the normal retry policy would double the load
|
|
// and the next live telemetry tick would begin another walk.
|
|
result, err := dockerCallWithRetryAttempts(ctx, dockerInventoryCallTimeout, 1, func(callCtx context.Context) (client.DiskUsageResult, error) {
|
|
return a.docker.DiskUsage(callCtx, dockerDiskUsageOptions{
|
|
Containers: true,
|
|
Images: true,
|
|
Volumes: true,
|
|
BuildCache: true,
|
|
Verbose: true,
|
|
})
|
|
})
|
|
if err != nil {
|
|
annotatedErr := annotateDockerConnectionError(err)
|
|
if a.storageUsageCache.valid {
|
|
return a.storageUsageCache.result, cloneDockerStorageUsage(a.storageUsageCache.usage), annotatedErr
|
|
}
|
|
return client.DiskUsageResult{}, nil, annotatedErr
|
|
}
|
|
|
|
usage := &agentsdocker.StorageUsage{
|
|
Images: agentsdocker.StorageUsageBucket{
|
|
TotalCount: result.Images.TotalCount,
|
|
ActiveCount: result.Images.ActiveCount,
|
|
TotalSizeBytes: result.Images.TotalSize,
|
|
ReclaimableBytes: result.Images.Reclaimable,
|
|
},
|
|
Containers: agentsdocker.StorageUsageBucket{
|
|
TotalCount: result.Containers.TotalCount,
|
|
ActiveCount: result.Containers.ActiveCount,
|
|
TotalSizeBytes: result.Containers.TotalSize,
|
|
ReclaimableBytes: result.Containers.Reclaimable,
|
|
},
|
|
Volumes: agentsdocker.StorageUsageBucket{
|
|
TotalCount: result.Volumes.TotalCount,
|
|
ActiveCount: result.Volumes.ActiveCount,
|
|
TotalSizeBytes: result.Volumes.TotalSize,
|
|
ReclaimableBytes: result.Volumes.Reclaimable,
|
|
},
|
|
BuildCache: agentsdocker.StorageUsageBucket{
|
|
TotalCount: result.BuildCache.TotalCount,
|
|
ActiveCount: result.BuildCache.ActiveCount,
|
|
TotalSizeBytes: result.BuildCache.TotalSize,
|
|
ReclaimableBytes: result.BuildCache.Reclaimable,
|
|
},
|
|
}
|
|
a.storageUsageCache.result = result
|
|
a.storageUsageCache.usage = cloneDockerStorageUsage(usage)
|
|
a.storageUsageCache.valid = true
|
|
a.storageUsageCache.nextRefresh = time.Now().Add(dockerStorageUsageRefreshInterval)
|
|
return result, usage, nil
|
|
}
|
|
|
|
func cloneDockerStorageUsage(usage *agentsdocker.StorageUsage) *agentsdocker.StorageUsage {
|
|
if usage == nil {
|
|
return nil
|
|
}
|
|
cloned := *usage
|
|
return &cloned
|
|
}
|
|
|
|
func dockerUnixTimestamp(seconds int64) time.Time {
|
|
if seconds <= 0 {
|
|
return time.Time{}
|
|
}
|
|
return time.Unix(seconds, 0).UTC()
|
|
}
|
|
|
|
func (a *Agent) pruneStaleCPUSamples(active map[string]struct{}) {
|
|
a.cpuMu.Lock()
|
|
defer a.cpuMu.Unlock()
|
|
|
|
if len(a.prevContainerCPU) == 0 {
|
|
return
|
|
}
|
|
|
|
for containerID := range a.prevContainerCPU {
|
|
if _, ok := active[containerID]; !ok {
|
|
delete(a.prevContainerCPU, containerID)
|
|
// Reset stats failure counter when containers are removed,
|
|
// though it's global per agent so not strictly necessary but good hygiene
|
|
}
|
|
}
|
|
}
|
|
|
|
func (a *Agent) collectContainer(ctx context.Context, summary containertypes.Summary) (agentsdocker.Container, error) {
|
|
const perContainerTimeout = 15 * time.Second
|
|
|
|
containerCtx, cancel := context.WithTimeout(ctx, perContainerTimeout)
|
|
defer cancel()
|
|
|
|
requestSize := a.cfg.CollectDiskMetrics
|
|
inspect, err := a.inspectContainer(containerCtx, summary, requestSize)
|
|
if err != nil {
|
|
return agentsdocker.Container{}, fmt.Errorf("inspect: %w", err)
|
|
}
|
|
|
|
var (
|
|
cpuPercent float64
|
|
memUsage int64
|
|
memLimit int64
|
|
memPercent float64
|
|
blockIO *agentsdocker.ContainerBlockIO
|
|
networkRX uint64
|
|
networkTX uint64
|
|
)
|
|
|
|
if inspect.State.Running || inspect.State.Paused {
|
|
statsResp, err := a.docker.ContainerStatsOneShot(containerCtx, summary.ID)
|
|
if err != nil {
|
|
return agentsdocker.Container{}, fmt.Errorf("stats: %w", err)
|
|
}
|
|
defer func() {
|
|
if closeErr := statsResp.Body.Close(); closeErr != nil {
|
|
a.logger.Warn().Err(closeErr).Str("container", summary.ID).Msg("Failed to close container stats response body")
|
|
}
|
|
}()
|
|
|
|
payload, err := readBodyWithLimit(statsResp.Body, maxContainerStatsBodyBytes)
|
|
if err != nil {
|
|
return agentsdocker.Container{}, fmt.Errorf("read stats: %w", err)
|
|
}
|
|
|
|
stats, podmanCPUPercent, err := decodeContainerStatsPayload(payload)
|
|
if err != nil {
|
|
return agentsdocker.Container{}, fmt.Errorf("decode stats: %w", err)
|
|
}
|
|
|
|
if a.runtime == RuntimePodman && podmanCPUPercent != nil {
|
|
cpuPercent = safeFloat(*podmanCPUPercent)
|
|
} else {
|
|
cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats)
|
|
}
|
|
memUsage, memLimit, memPercent = calculateMemoryUsage(stats)
|
|
blockIO = summarizeBlockIO(stats)
|
|
networkRX, networkTX = summarizeNetworkIO(stats)
|
|
} else {
|
|
a.cpuMu.Lock()
|
|
delete(a.prevContainerCPU, summary.ID)
|
|
a.cpuMu.Unlock()
|
|
}
|
|
|
|
createdAt := time.Unix(summary.Created, 0)
|
|
|
|
startedAt := parseTime(inspect.State.StartedAt)
|
|
finishedAt := parseTime(inspect.State.FinishedAt)
|
|
|
|
uptimeSeconds := int64(0)
|
|
if !startedAt.IsZero() && inspect.State.Running {
|
|
uptimeSeconds = int64(time.Since(startedAt).Seconds())
|
|
if uptimeSeconds < 0 {
|
|
uptimeSeconds = 0
|
|
}
|
|
}
|
|
|
|
health := ""
|
|
if inspect.State.Health != nil {
|
|
health = string(inspect.State.Health.Status)
|
|
}
|
|
healthcheckTargets := []string(nil)
|
|
if inspect.Config != nil && inspect.Config.Healthcheck != nil {
|
|
healthcheckTargets = extractHealthcheckTargets(inspect.Config.Healthcheck.Test)
|
|
}
|
|
|
|
ports := make([]agentsdocker.ContainerPort, len(summary.Ports))
|
|
for i, port := range summary.Ports {
|
|
ports[i] = agentsdocker.ContainerPort{
|
|
PrivatePort: int(port.PrivatePort),
|
|
PublicPort: int(port.PublicPort),
|
|
Protocol: port.Type,
|
|
IP: addrString(port.IP),
|
|
}
|
|
}
|
|
|
|
labels := make(map[string]string, len(summary.Labels))
|
|
for k, v := range summary.Labels {
|
|
labels[k] = v
|
|
}
|
|
|
|
networks := make([]agentsdocker.ContainerNetwork, 0)
|
|
if inspect.NetworkSettings != nil {
|
|
for name, cfg := range inspect.NetworkSettings.Networks {
|
|
networks = append(networks, agentsdocker.ContainerNetwork{
|
|
Name: name,
|
|
IPv4: addrString(cfg.IPAddress),
|
|
IPv6: addrString(cfg.GlobalIPv6Address),
|
|
})
|
|
}
|
|
}
|
|
|
|
var startedPtr, finishedPtr *time.Time
|
|
if !startedAt.IsZero() {
|
|
started := startedAt
|
|
startedPtr = &started
|
|
}
|
|
if !finishedAt.IsZero() && !inspect.State.Running {
|
|
finished := finishedAt
|
|
finishedPtr = &finished
|
|
}
|
|
|
|
var writableLayerBytes int64
|
|
if inspect.SizeRw != nil {
|
|
writableLayerBytes = *inspect.SizeRw
|
|
}
|
|
|
|
var rootFsBytes int64
|
|
if inspect.SizeRootFs != nil {
|
|
rootFsBytes = *inspect.SizeRootFs
|
|
}
|
|
|
|
var mounts []agentsdocker.ContainerMount
|
|
if len(inspect.Mounts) > 0 {
|
|
mounts = make([]agentsdocker.ContainerMount, 0, len(inspect.Mounts))
|
|
for _, mount := range inspect.Mounts {
|
|
mounts = append(mounts, agentsdocker.ContainerMount{
|
|
Type: string(mount.Type),
|
|
Source: mount.Source,
|
|
Destination: mount.Destination,
|
|
Mode: mount.Mode,
|
|
RW: mount.RW,
|
|
Propagation: string(mount.Propagation),
|
|
Name: mount.Name,
|
|
Driver: mount.Driver,
|
|
})
|
|
}
|
|
}
|
|
// Docker's --tmpfs mounts can exist only in HostConfig.Tmpfs. Preserve
|
|
// their configuration alongside inspected mounts without inventing usage.
|
|
if inspect.HostConfig != nil && len(inspect.HostConfig.Tmpfs) > 0 {
|
|
reported := make(map[string]bool, len(mounts))
|
|
for _, mount := range mounts {
|
|
reported[mount.Destination] = true
|
|
}
|
|
destinations := make([]string, 0, len(inspect.HostConfig.Tmpfs))
|
|
for destination := range inspect.HostConfig.Tmpfs {
|
|
if !reported[destination] {
|
|
destinations = append(destinations, destination)
|
|
}
|
|
}
|
|
sort.Strings(destinations)
|
|
for _, destination := range destinations {
|
|
options := inspect.HostConfig.Tmpfs[destination]
|
|
writable := true
|
|
for _, option := range strings.Split(options, ",") {
|
|
switch strings.TrimSpace(option) {
|
|
case "ro":
|
|
writable = false
|
|
case "rw":
|
|
writable = true
|
|
}
|
|
}
|
|
mounts = append(mounts, agentsdocker.ContainerMount{
|
|
Type: "tmpfs", Destination: destination, Mode: options, RW: writable,
|
|
})
|
|
}
|
|
}
|
|
|
|
oomKilled := inspect.State.OOMKilled
|
|
container := agentsdocker.Container{
|
|
ID: summary.ID,
|
|
Name: trimLeadingSlash(summary.Names),
|
|
Image: summary.Image,
|
|
ImageDigest: summary.ImageID, // sha256:... digest of the image
|
|
CreatedAt: createdAt,
|
|
State: string(summary.State),
|
|
Status: summary.Status,
|
|
Health: health,
|
|
HealthcheckTargets: healthcheckTargets,
|
|
CPUPercent: cpuPercent,
|
|
MemoryUsageBytes: memUsage,
|
|
MemoryLimitBytes: memLimit,
|
|
MemoryPercent: memPercent,
|
|
UptimeSeconds: uptimeSeconds,
|
|
RestartCount: inspect.RestartCount,
|
|
ExitCode: inspect.State.ExitCode,
|
|
OOMKilled: &oomKilled,
|
|
StartedAt: startedPtr,
|
|
FinishedAt: finishedPtr,
|
|
Ports: ports,
|
|
Labels: labels,
|
|
Env: maskSensitiveEnvVars(inspect.Config.Env),
|
|
Networks: networks,
|
|
NetworkRXBytes: networkRX,
|
|
NetworkTXBytes: networkTX,
|
|
WritableLayerBytes: writableLayerBytes,
|
|
RootFilesystemBytes: rootFsBytes,
|
|
BlockIO: blockIO,
|
|
Mounts: mounts,
|
|
Filesystems: a.collectContainerFilesystems(containerCtx, summary.ID, inspect, mounts),
|
|
}
|
|
|
|
if a.runtime == RuntimePodman {
|
|
if meta := extractPodmanMetadata(labels); meta != nil {
|
|
container.Podman = meta
|
|
}
|
|
}
|
|
|
|
// Check for image updates if registry checker is enabled
|
|
if a.registryChecker != nil && a.registryChecker.Enabled() {
|
|
// Get the actual manifest digest (RepoDigest) from the image for accurate comparison.
|
|
// The ImageID is a local content-addressable ID that differs from the registry manifest digest.
|
|
// We also get the architecture details to correctly resolve manifest lists from the registry.
|
|
digestForComparison, arch, os, variant := a.getImageRepoDigest(containerCtx, summary.ImageID, summary.Image)
|
|
|
|
var imageToCheck string
|
|
// Always prefer the image name from inspect config as it's the authoritative source
|
|
// and avoids issues with short IDs or digests in summary.
|
|
// HOWEVER, if the config image IS a digest (starts with sha256:), fall back to container.Image
|
|
// which usually contains the human-readable tag/name.
|
|
imageToCheck = container.Image
|
|
if inspect.Config != nil && inspect.Config.Image != "" {
|
|
if !strings.HasPrefix(inspect.Config.Image, "sha256:") {
|
|
imageToCheck = inspect.Config.Image
|
|
}
|
|
}
|
|
|
|
// Additional safety: if imageToCheck is still a SHA, we can't check it
|
|
if strings.HasPrefix(imageToCheck, "sha256:") {
|
|
container.UpdateStatus = &agentsdocker.UpdateStatus{
|
|
UpdateAvailable: false,
|
|
CurrentDigest: digestForComparison,
|
|
LastChecked: time.Now(),
|
|
Error: "digest-pinned image",
|
|
}
|
|
// Skip to end of update check block - don't call registry
|
|
} else {
|
|
a.logger.Debug().
|
|
Str("container", container.Name).
|
|
Str("image", imageToCheck).
|
|
Str("compareDigest", digestForComparison).
|
|
Str("arch", arch).
|
|
Str("os", os).
|
|
Str("variant", variant).
|
|
Msg("Checking update for container")
|
|
|
|
result := a.registryChecker.CheckImageUpdate(ctx, imageToCheck, digestForComparison, arch, os, variant)
|
|
if result != nil {
|
|
container.UpdateStatus = &agentsdocker.UpdateStatus{
|
|
UpdateAvailable: result.UpdateAvailable,
|
|
CurrentDigest: result.CurrentDigest,
|
|
LatestDigest: result.LatestDigest,
|
|
LastChecked: result.CheckedAt,
|
|
Error: result.Error,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if requestSize {
|
|
a.logger.Debug().
|
|
Str("container", container.Name).
|
|
Int64("writableLayerBytes", writableLayerBytes).
|
|
Int64("rootFilesystemBytes", rootFsBytes).
|
|
Int("mountCount", len(mounts)).
|
|
Msg("Collected container disk metrics")
|
|
}
|
|
|
|
return container, nil
|
|
}
|
|
|
|
func (a *Agent) inspectContainer(ctx context.Context, summary containertypes.Summary, requestSize bool) (containertypes.InspectResponse, error) {
|
|
// Empty IDs/statuses do not provide enough evidence to detect a lifecycle
|
|
// change between reports (some Podman compatibility endpoints omit Status),
|
|
// so keep those on the live path rather than accepting bounded stale state.
|
|
cacheable := strings.TrimSpace(summary.ID) != "" && strings.TrimSpace(summary.Status) != "" && isInactiveContainerState(summary.State)
|
|
fingerprint := inactiveContainerInspectFingerprint(summary)
|
|
if cacheable {
|
|
a.inactiveInspectMu.Lock()
|
|
entry, ok := a.inactiveInspects[summary.ID]
|
|
if ok && entry.withSize == requestSize && entry.fingerprint == fingerprint && time.Now().Before(entry.expiresAt) {
|
|
inspect := entry.inspect
|
|
a.inactiveInspectMu.Unlock()
|
|
return inspect, nil
|
|
}
|
|
a.inactiveInspectMu.Unlock()
|
|
}
|
|
|
|
inspect, _, err := a.docker.ContainerInspectWithRaw(ctx, summary.ID, requestSize)
|
|
if err != nil {
|
|
return containertypes.InspectResponse{}, err
|
|
}
|
|
|
|
a.inactiveInspectMu.Lock()
|
|
defer a.inactiveInspectMu.Unlock()
|
|
if !cacheable || inspect.State == nil || inspect.State.Running || inspect.State.Paused || inspect.State.Restarting {
|
|
delete(a.inactiveInspects, summary.ID)
|
|
return inspect, nil
|
|
}
|
|
if a.inactiveInspects == nil {
|
|
a.inactiveInspects = make(map[string]inactiveContainerInspectCacheEntry)
|
|
}
|
|
a.inactiveInspects[summary.ID] = inactiveContainerInspectCacheEntry{
|
|
inspect: inspect,
|
|
fingerprint: fingerprint,
|
|
withSize: requestSize,
|
|
expiresAt: time.Now().Add(dockerInactiveInspectRefreshInterval),
|
|
}
|
|
return inspect, nil
|
|
}
|
|
|
|
func isInactiveContainerState(state containertypes.ContainerState) bool {
|
|
switch strings.ToLower(strings.TrimSpace(string(state))) {
|
|
case "created", "exited", "dead":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func inactiveContainerInspectFingerprint(summary containertypes.Summary) string {
|
|
// Status includes the latest exit result/time, so an inactive container
|
|
// that ran and stopped entirely between two reports cannot reuse its old
|
|
// exit code, restart count, or lifecycle timestamps.
|
|
return strings.ToLower(strings.TrimSpace(string(summary.State))) + "\x00" + strings.TrimSpace(summary.Status)
|
|
}
|
|
|
|
func (a *Agent) pruneInactiveContainerInspectCache(active map[string]struct{}) {
|
|
a.inactiveInspectMu.Lock()
|
|
defer a.inactiveInspectMu.Unlock()
|
|
for containerID := range a.inactiveInspects {
|
|
if _, ok := active[containerID]; !ok {
|
|
delete(a.inactiveInspects, containerID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (a *Agent) clearDockerCollectionCaches() {
|
|
a.storageUsageMu.Lock()
|
|
a.storageUsageCache = dockerStorageUsageCache{}
|
|
a.storageUsageMu.Unlock()
|
|
|
|
a.inactiveInspectMu.Lock()
|
|
a.inactiveInspects = nil
|
|
a.inactiveInspectMu.Unlock()
|
|
}
|
|
|
|
var (
|
|
healthcheckURLPattern = regexp.MustCompile(`(?i)https?://[^\s'"<>]+`)
|
|
healthcheckTargetPattern = regexp.MustCompile(`(?i)^[a-z0-9][a-z0-9._-]{0,252}$`)
|
|
)
|
|
|
|
// extractHealthcheckTargets derives only secret-free URL hostnames from a
|
|
// container health check. Raw health-check command text can contain paths,
|
|
// query strings, credentials, or environment expansion and must never cross
|
|
// the agent reporting boundary.
|
|
func extractHealthcheckTargets(test []string) []string {
|
|
const maxTargets = 16
|
|
seen := make(map[string]struct{})
|
|
targets := make([]string, 0)
|
|
for _, part := range test {
|
|
for _, candidate := range healthcheckURLPattern.FindAllString(part, -1) {
|
|
if len(candidate) > 2048 {
|
|
continue
|
|
}
|
|
parsed, err := url.Parse(candidate)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
target := strings.ToLower(strings.TrimSpace(parsed.Hostname()))
|
|
if address, err := netip.ParseAddr(target); err == nil {
|
|
target = address.String()
|
|
} else if !healthcheckTargetPattern.MatchString(target) {
|
|
continue
|
|
}
|
|
if _, exists := seen[target]; exists {
|
|
continue
|
|
}
|
|
seen[target] = struct{}{}
|
|
targets = append(targets, target)
|
|
if len(targets) == maxTargets {
|
|
return targets
|
|
}
|
|
}
|
|
}
|
|
if len(targets) == 0 {
|
|
return nil
|
|
}
|
|
return targets
|
|
}
|
|
|
|
// getImageRepoDigest retrieves the RepoDigest for an image and its platform details.
|
|
// It returns the digest, architecture, OS, and variant.
|
|
func (a *Agent) getImageRepoDigest(ctx context.Context, imageID, imageName string) (string, string, string, string) {
|
|
imageInspect, _, err := a.docker.ImageInspectWithRaw(ctx, imageID)
|
|
if err != nil {
|
|
a.logger.Debug().
|
|
Err(err).
|
|
Str("imageID", imageID).
|
|
Str("imageName", imageName).
|
|
Msg("Failed to inspect image for RepoDigest")
|
|
return "", "", "", ""
|
|
}
|
|
|
|
arch := imageInspect.Architecture
|
|
os := imageInspect.Os
|
|
variant := imageInspect.Variant
|
|
|
|
if len(imageInspect.RepoDigests) == 0 {
|
|
// Locally built images won't have RepoDigests
|
|
return "", arch, os, variant
|
|
}
|
|
|
|
// Try to find a RepoDigest that matches the image reference
|
|
// RepoDigests format: "registry/repo@sha256:..."
|
|
for _, repoDigest := range imageInspect.RepoDigests {
|
|
// Extract just the digest part (after @)
|
|
if idx := strings.LastIndex(repoDigest, "@"); idx >= 0 {
|
|
repoRef := repoDigest[:idx] // e.g., "docker.io/library/nginx"
|
|
digest := repoDigest[idx+1:] // e.g., "sha256:abc..."
|
|
|
|
// Check if this RepoDigest matches our image reference
|
|
// Normalize both for comparison
|
|
if matchesImageReference(imageName, repoRef) {
|
|
return digest, arch, os, variant
|
|
}
|
|
}
|
|
}
|
|
|
|
// If no exact match, return the first RepoDigest's digest
|
|
// This handles cases where the image was pulled with a different tag
|
|
if idx := strings.LastIndex(imageInspect.RepoDigests[0], "@"); idx >= 0 {
|
|
return imageInspect.RepoDigests[0][idx+1:], arch, os, variant
|
|
}
|
|
|
|
return "", arch, os, variant
|
|
}
|
|
|
|
func addrString(addr netip.Addr) string {
|
|
if !addr.IsValid() {
|
|
return ""
|
|
}
|
|
return addr.String()
|
|
}
|
|
|
|
// matchesImageReference checks if a RepoDigest repository matches an image reference.
|
|
// It handles Docker Hub's various naming conventions.
|
|
func matchesImageReference(imageName, repoRef string) bool {
|
|
// Normalize image name by removing tag
|
|
if idx := strings.LastIndex(imageName, ":"); idx >= 0 {
|
|
// Make sure it's a tag, not a port (check if there's a / after it)
|
|
if !strings.Contains(imageName[idx:], "/") {
|
|
imageName = imageName[:idx]
|
|
}
|
|
}
|
|
|
|
// Direct match
|
|
if imageName == repoRef {
|
|
return true
|
|
}
|
|
|
|
// Docker Hub library images: "nginx" == "docker.io/library/nginx"
|
|
if repoRef == "docker.io/library/"+imageName {
|
|
return true
|
|
}
|
|
|
|
// Docker Hub with namespace: "myuser/myapp" == "docker.io/myuser/myapp"
|
|
if repoRef == "docker.io/"+imageName {
|
|
return true
|
|
}
|
|
|
|
// Registry prefix matching (e.g., "ghcr.io/user/repo" matches "ghcr.io/user/repo")
|
|
// Already handled by direct match above
|
|
|
|
return false
|
|
}
|
|
|
|
func extractPodmanMetadata(labels map[string]string) *agentsdocker.PodmanContainer {
|
|
if len(labels) == 0 {
|
|
return nil
|
|
}
|
|
|
|
meta := &agentsdocker.PodmanContainer{}
|
|
|
|
if v := strings.TrimSpace(labels["io.podman.annotations.pod.name"]); v != "" {
|
|
meta.PodName = v
|
|
}
|
|
|
|
if v := strings.TrimSpace(labels["io.podman.annotations.pod.id"]); v != "" {
|
|
meta.PodID = v
|
|
}
|
|
|
|
if v := strings.TrimSpace(labels["io.podman.annotations.pod.infra"]); v != "" {
|
|
if parsed, err := strconv.ParseBool(v); err == nil {
|
|
meta.Infra = parsed
|
|
} else if strings.EqualFold(v, "yes") || strings.EqualFold(v, "true") {
|
|
meta.Infra = true
|
|
}
|
|
}
|
|
|
|
if v := strings.TrimSpace(labels["io.podman.compose.project"]); v != "" {
|
|
meta.ComposeProject = v
|
|
}
|
|
|
|
if v := strings.TrimSpace(labels["io.podman.compose.service"]); v != "" {
|
|
meta.ComposeService = v
|
|
}
|
|
|
|
if v := strings.TrimSpace(labels["io.podman.compose.working_dir"]); v != "" {
|
|
meta.ComposeWorkdir = v
|
|
}
|
|
|
|
if v := strings.TrimSpace(labels["io.podman.compose.config-hash"]); v != "" {
|
|
meta.ComposeConfig = v
|
|
}
|
|
|
|
if v := strings.TrimSpace(labels["io.containers.autoupdate"]); v != "" {
|
|
meta.AutoUpdatePolicy = v
|
|
}
|
|
|
|
if v := strings.TrimSpace(labels["io.containers.autoupdate.restart"]); v != "" {
|
|
meta.AutoUpdateRestart = v
|
|
}
|
|
|
|
if v := strings.TrimSpace(labels["io.podman.annotations.userns"]); v != "" {
|
|
meta.UserNS = v
|
|
} else if v := strings.TrimSpace(labels["io.containers.userns"]); v != "" {
|
|
meta.UserNS = v
|
|
}
|
|
|
|
if meta.PodName == "" && meta.PodID == "" && meta.ComposeProject == "" && meta.AutoUpdatePolicy == "" && meta.UserNS == "" && !meta.Infra {
|
|
return nil
|
|
}
|
|
|
|
return meta
|
|
}
|
|
|
|
type podmanCompatStatsProbe struct {
|
|
CPUStats struct {
|
|
CPU *float64 `json:"cpu"`
|
|
} `json:"cpu_stats"`
|
|
}
|
|
|
|
func decodeContainerStatsPayload(payload []byte) (containertypes.StatsResponse, *float64, error) {
|
|
var stats containertypes.StatsResponse
|
|
if err := json.Unmarshal(payload, &stats); err != nil {
|
|
return containertypes.StatsResponse{}, nil, err
|
|
}
|
|
|
|
var probe podmanCompatStatsProbe
|
|
if err := json.Unmarshal(payload, &probe); err != nil {
|
|
return stats, nil, nil
|
|
}
|
|
|
|
return stats, probe.CPUStats.CPU, nil
|
|
}
|
|
|
|
func (a *Agent) calculateContainerCPUPercent(containerID string, stats containertypes.StatsResponse) float64 {
|
|
a.cpuMu.Lock()
|
|
defer a.cpuMu.Unlock()
|
|
|
|
current := cpuSample{
|
|
totalUsage: stats.CPUStats.CPUUsage.TotalUsage,
|
|
systemUsage: stats.CPUStats.SystemUsage,
|
|
onlineCPUs: stats.CPUStats.OnlineCPUs,
|
|
read: stats.Read,
|
|
}
|
|
|
|
// Always use manual delta tracking. Docker's PreCPUStats is unreliable for
|
|
// one-shot stats because many Docker versions don't update their internal
|
|
// cache between non-streaming reads, causing PreCPUStats to remain stale
|
|
// (from container start) and producing a constant lifetime-average CPU%
|
|
// instead of a current value.
|
|
prev, ok := a.prevContainerCPU[containerID]
|
|
if !ok {
|
|
// First time seeing this container - store current sample and return 0
|
|
// On next collection cycle we'll have a previous sample to compare against
|
|
a.prevContainerCPU[containerID] = current
|
|
a.logger.Debug().
|
|
Str("container_id", containerID[:12]).
|
|
Uint64("total_usage", current.totalUsage).
|
|
Uint64("system_usage", current.systemUsage).
|
|
Msg("First CPU sample collected, no previous data for delta calculation")
|
|
return 0
|
|
}
|
|
|
|
// We have a previous sample - update it after calculation
|
|
a.prevContainerCPU[containerID] = current
|
|
|
|
var totalDelta float64
|
|
if current.totalUsage >= prev.totalUsage {
|
|
totalDelta = float64(current.totalUsage - prev.totalUsage)
|
|
} else {
|
|
// Counter likely reset (container restart); fall back to current reading.
|
|
totalDelta = float64(current.totalUsage)
|
|
}
|
|
|
|
if totalDelta <= 0 {
|
|
return 0
|
|
}
|
|
|
|
if a.runtime == RuntimePodman {
|
|
if !prev.read.IsZero() && !current.read.IsZero() {
|
|
elapsed := current.read.Sub(prev.read).Seconds()
|
|
if elapsed > 0 {
|
|
denominator := elapsed * 1e9
|
|
if denominator > 0 {
|
|
cpuPercent := (totalDelta / denominator) * 100.0
|
|
result := safeFloat(cpuPercent)
|
|
a.logger.Debug().
|
|
Str("container_id", containerID[:12]).
|
|
Float64("cpu_percent", result).
|
|
Float64("total_delta", totalDelta).
|
|
Float64("elapsed_seconds", elapsed).
|
|
Msg("CPU calculated from Podman wall-clock delta")
|
|
return result
|
|
}
|
|
}
|
|
}
|
|
|
|
a.logger.Debug().
|
|
Str("container_id", containerID[:12]).
|
|
Float64("total_delta", totalDelta).
|
|
Bool("prev_read_zero", prev.read.IsZero()).
|
|
Bool("current_read_zero", current.read.IsZero()).
|
|
Msg("Podman CPU calculation failed: no valid wall-clock delta available")
|
|
return 0
|
|
}
|
|
|
|
onlineCPUs := current.onlineCPUs
|
|
if onlineCPUs == 0 {
|
|
onlineCPUs = prev.onlineCPUs
|
|
}
|
|
if onlineCPUs == 0 && a.cpuCount > 0 {
|
|
onlineCPUs = uint32(a.cpuCount)
|
|
}
|
|
if onlineCPUs == 0 {
|
|
return 0
|
|
}
|
|
|
|
var systemDelta float64
|
|
if current.systemUsage >= prev.systemUsage {
|
|
systemDelta = float64(current.systemUsage - prev.systemUsage)
|
|
}
|
|
// If systemUsage went backward (counter reset), leave systemDelta as 0
|
|
// to fall through to time-based calculation below
|
|
|
|
if systemDelta > 0 {
|
|
cpuPercent := safeFloat((totalDelta / systemDelta) * float64(onlineCPUs) * 100.0)
|
|
a.logger.Debug().
|
|
Str("container_id", containerID[:12]).
|
|
Float64("cpu_percent", cpuPercent).
|
|
Float64("total_delta", totalDelta).
|
|
Float64("system_delta", systemDelta).
|
|
Uint32("online_cpus", onlineCPUs).
|
|
Msg("CPU calculated from system delta")
|
|
return cpuPercent
|
|
}
|
|
|
|
// Fall back to time-based calculation
|
|
if !prev.read.IsZero() && !current.read.IsZero() {
|
|
elapsed := current.read.Sub(prev.read).Seconds()
|
|
if elapsed > 0 {
|
|
denominator := elapsed * float64(onlineCPUs) * 1e9
|
|
if denominator > 0 {
|
|
cpuPercent := (totalDelta / denominator) * 100.0
|
|
result := safeFloat(cpuPercent)
|
|
a.logger.Debug().
|
|
Str("container_id", containerID[:12]).
|
|
Float64("cpu_percent", result).
|
|
Float64("total_delta", totalDelta).
|
|
Float64("elapsed_seconds", elapsed).
|
|
Uint32("online_cpus", onlineCPUs).
|
|
Msg("CPU calculated from time-based delta")
|
|
return result
|
|
}
|
|
}
|
|
}
|
|
|
|
a.logger.Debug().
|
|
Str("container_id", containerID[:12]).
|
|
Float64("total_delta", totalDelta).
|
|
Float64("system_delta", systemDelta).
|
|
Bool("prev_read_zero", prev.read.IsZero()).
|
|
Bool("current_read_zero", current.read.IsZero()).
|
|
Msg("CPU calculation failed: no valid delta method available")
|
|
return 0
|
|
}
|
|
|
|
func calculateCPUPercent(stats containertypes.StatsResponse, hostCPUs int) float64 {
|
|
totalDelta := float64(stats.CPUStats.CPUUsage.TotalUsage - stats.PreCPUStats.CPUUsage.TotalUsage)
|
|
systemDelta := float64(stats.CPUStats.SystemUsage - stats.PreCPUStats.SystemUsage)
|
|
|
|
if totalDelta <= 0 || systemDelta <= 0 {
|
|
return 0
|
|
}
|
|
|
|
onlineCPUs := stats.CPUStats.OnlineCPUs
|
|
if onlineCPUs == 0 {
|
|
onlineCPUs = uint32(len(stats.CPUStats.CPUUsage.PercpuUsage))
|
|
}
|
|
if onlineCPUs == 0 && hostCPUs > 0 {
|
|
onlineCPUs = uint32(hostCPUs)
|
|
}
|
|
|
|
if onlineCPUs == 0 {
|
|
return 0
|
|
}
|
|
|
|
return safeFloat((totalDelta / systemDelta) * float64(onlineCPUs) * 100.0)
|
|
}
|
|
|
|
func calculateMemoryUsage(stats containertypes.StatsResponse) (usage int64, limit int64, percent float64) {
|
|
usage = int64(stats.MemoryStats.Usage)
|
|
|
|
// Subtract reclaimable cache from usage to match `docker stats` behavior.
|
|
// Docker subtracts cache/file to show "actual" memory usage rather than
|
|
// memory.current which includes reclaimable filesystem cache.
|
|
//
|
|
// cgroup v1: "cache" stat contains the reclaimable cache
|
|
// cgroup v2: "cache" doesn't exist, use "inactive_file" (preferred) or "file"
|
|
var cacheBytes uint64
|
|
if cache, ok := stats.MemoryStats.Stats["cache"]; ok {
|
|
// cgroup v1
|
|
cacheBytes = cache
|
|
} else if inactiveFile, ok := stats.MemoryStats.Stats["inactive_file"]; ok {
|
|
// cgroup v2: inactive_file is the reclaimable portion of file cache
|
|
// This matches what docker CLI does internally
|
|
cacheBytes = inactiveFile
|
|
}
|
|
|
|
if cacheBytes > 0 && int64(cacheBytes) < usage {
|
|
usage -= int64(cacheBytes)
|
|
}
|
|
|
|
limit = int64(stats.MemoryStats.Limit)
|
|
if limit > 0 {
|
|
percent = (float64(usage) / float64(limit)) * 100.0
|
|
}
|
|
|
|
return usage, limit, safeFloat(percent)
|
|
}
|
|
|
|
func safeFloat(val float64) float64 {
|
|
if math.IsNaN(val) || math.IsInf(val, 0) {
|
|
return 0
|
|
}
|
|
return val
|
|
}
|
|
|
|
func parseTime(value string) time.Time {
|
|
if value == "" || value == "0001-01-01T00:00:00Z" {
|
|
return time.Time{}
|
|
}
|
|
if strings.Contains(value, ".") {
|
|
if t, err := time.Parse(time.RFC3339Nano, value); err == nil {
|
|
return t
|
|
}
|
|
} else {
|
|
if t, err := time.Parse(time.RFC3339, value); err == nil {
|
|
return t
|
|
}
|
|
}
|
|
return time.Time{}
|
|
}
|
|
|
|
func trimLeadingSlash(names []string) string {
|
|
if len(names) == 0 {
|
|
return ""
|
|
}
|
|
name := names[0]
|
|
return strings.TrimPrefix(name, "/")
|
|
}
|
|
|
|
func randomDuration(max time.Duration) time.Duration {
|
|
if max <= 0 {
|
|
return 0
|
|
}
|
|
|
|
n, err := randIntFn(rand.Reader, big.NewInt(int64(max)))
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
|
|
return time.Duration(n.Int64())
|
|
}
|
|
|
|
func summarizeBlockIO(stats containertypes.StatsResponse) *agentsdocker.ContainerBlockIO {
|
|
if len(stats.BlkioStats.IoServiceBytesRecursive) == 0 {
|
|
return nil
|
|
}
|
|
|
|
var readBytes, writeBytes uint64
|
|
var readPresent, writePresent bool
|
|
|
|
for _, entry := range stats.BlkioStats.IoServiceBytesRecursive {
|
|
op := strings.ToLower(entry.Op)
|
|
switch op {
|
|
case "read":
|
|
readPresent = true
|
|
readBytes += entry.Value
|
|
case "write":
|
|
writePresent = true
|
|
writeBytes += entry.Value
|
|
}
|
|
}
|
|
|
|
if !readPresent && !writePresent {
|
|
return nil
|
|
}
|
|
|
|
return &agentsdocker.ContainerBlockIO{
|
|
ReadBytes: readBytes,
|
|
WriteBytes: writeBytes,
|
|
ReadBytesPresent: &readPresent,
|
|
WriteBytesPresent: &writePresent,
|
|
}
|
|
}
|
|
|
|
func summarizeNetworkIO(stats containertypes.StatsResponse) (uint64, uint64) {
|
|
if len(stats.Networks) == 0 {
|
|
return 0, 0
|
|
}
|
|
|
|
var rxBytes uint64
|
|
var txBytes uint64
|
|
for _, network := range stats.Networks {
|
|
rxBytes += network.RxBytes
|
|
txBytes += network.TxBytes
|
|
}
|
|
|
|
return rxBytes, txBytes
|
|
}
|
|
|
|
// sensitiveEnvPatterns are substrings that, when found in an env var name (case-insensitive),
|
|
// indicate the value should be masked for security.
|
|
var sensitiveEnvPatterns = []string{
|
|
"password", "passwd", "secret", "key", "token", "credential", "auth",
|
|
"api_key", "apikey", "private", "access_token", "refresh_token",
|
|
"database_url", "connection_string", "encryption",
|
|
}
|
|
|
|
// maskSensitiveEnvVars returns a copy of the environment variables with sensitive values masked.
|
|
// Environment variables whose names contain sensitive keywords will have their values replaced with "***".
|
|
func maskSensitiveEnvVars(envVars []string) []string {
|
|
if len(envVars) == 0 {
|
|
return nil
|
|
}
|
|
|
|
result := make([]string, 0, len(envVars))
|
|
for _, env := range envVars {
|
|
parts := strings.SplitN(env, "=", 2)
|
|
if len(parts) != 2 {
|
|
result = append(result, env)
|
|
continue
|
|
}
|
|
|
|
name := parts[0]
|
|
value := parts[1]
|
|
|
|
// Check if the environment variable name contains a sensitive pattern
|
|
lowerName := strings.ToLower(name)
|
|
isSensitive := false
|
|
for _, pattern := range sensitiveEnvPatterns {
|
|
if strings.Contains(lowerName, pattern) {
|
|
isSensitive = true
|
|
break
|
|
}
|
|
}
|
|
|
|
if isSensitive && value != "" {
|
|
result = append(result, name+"=***")
|
|
} else {
|
|
result = append(result, env)
|
|
}
|
|
}
|
|
|
|
return result
|
|
}
|