Files
rcourtman 3a4a3fd62b Preserve native filesystem evidence and Patrol action history
Expose confined, identity-bound filesystem observations through the shared
resource pipeline so investigations can distinguish an exhausted container
mount from unrelated host capacity. Keep unavailable measurements explicit.

Isolate alert-history reads from durable writes and reuse one chronological
fold across polling. Catch up through bounded durable event IDs so simultaneous
readers do not replay every retained snapshot. Retain expired actions when
investigation outcomes move back to needs attention, and keep attached
Assistant context focused.

Record live storage diagnosis, healthy and dependency controls, approved and
rejected Docker outcomes, source-bound browser proof and exact test limits.
Missing-access continuity, VM dispatch completion and remaining Assistant
orchestration defects stay open in the redesign plan.
2026-09-07 09:45:31 +01:00

1444 lines
44 KiB
Go

package dockeragent
import (
"context"
"crypto/rand"
"encoding/json"
"fmt"
"math"
"math/big"
"net/netip"
"net/url"
"regexp"
"sort"
"strconv"
"strings"
"time"
containertypes "github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/image"
networktypes "github.com/moby/moby/api/types/network"
systemtypes "github.com/moby/moby/api/types/system"
"github.com/moby/moby/api/types/volume"
"github.com/moby/moby/client"
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
)
// buildReport gathers all system and container metrics into a single report
func (a *Agent) buildReport(ctx context.Context) (agentsdocker.Report, error) {
// Cycle-level containment: per-call deadlines below should bound every
// docker call, but one live exercise saw a call park for minutes with its
// deadline never firing, so the whole cycle gets its own ceiling plus an
// independent watchdog that dumps goroutines if even this deadline fails
// to abort the cycle.
ctx, cancel := context.WithTimeout(ctx, dockerCollectCycleTimeout)
defer cancel()
stopWatchdog := startCollectCycleWatchdog(a.logger, dockerCollectCycleTimeout+dockerCollectWatchdogGrace)
defer stopWatchdog()
info, err := dockerCallWithRetry(ctx, dockerInfoCallTimeout, func(callCtx context.Context) (systemtypes.Info, error) {
return a.docker.Info(callCtx)
})
if err != nil {
return agentsdocker.Report{}, fmt.Errorf("failed to query docker info: %w", annotateDockerConnectionError(err))
}
if a.cfg.HelperInventory != nil {
if err := validateCollectorDirectRuntime(a.docker, info); err != nil {
return agentsdocker.Report{}, fmt.Errorf("%w: %v", errCollectorRuntimeBoundaryChanged, err)
}
}
a.runtimeVer = info.ServerVersion
if a.daemonHost == "" {
a.daemonHost = a.docker.DaemonHost()
}
// Use current runtime as preference to avoid spurious switching.
// This preserves user's --docker-runtime choice (stored in a.runtime at init).
newRuntime := detectRuntime(info, a.daemonHost, a.runtime)
if newRuntime != a.runtime {
if a.runtime != "" {
a.logger.Info().
Str("runtime_previous", string(a.runtime)).
Str("runtime_current", string(newRuntime)).
Msg("Detected container runtime change")
}
a.runtime = newRuntime
a.supportsSwarm = newRuntime == RuntimeDocker
if newRuntime == RuntimePodman {
if a.cfg.IncludeServices {
a.logger.Warn().Msg("Podman runtime detected during report; disabling Swarm service collection")
}
if a.cfg.IncludeTasks {
a.logger.Warn().Msg("Podman runtime detected during report; disabling Swarm task collection")
}
a.cfg.IncludeServices = false
a.cfg.IncludeTasks = false
}
a.cfg.Runtime = string(newRuntime)
}
a.cpuCount = info.NCPU
agentID := a.cfg.AgentID
if agentID == "" {
// In unified mode, use the EXACT same fallback chain as hostagent:
// machineID -> hostname. Never use daemonID in unified mode because
// hostagent doesn't use it, and using different IDs causes token
// binding conflicts on the server (reported in #985, #986).
if a.cfg.AgentType == "unified" {
agentID = a.machineID
if agentID == "" {
agentID = a.hostName
}
} else {
// Standalone mode: prefer daemonID for backward compatibility,
// then fall back to machineID -> hostname.
// Use cached daemon ID from init rather than info.ID from current call.
// Podman can return different/empty IDs across calls, causing token
// binding conflicts on the server.
agentID = a.daemonID
if agentID == "" {
agentID = a.machineID
}
if agentID == "" {
agentID = a.hostName
}
}
}
a.hostID = agentID
hostName := a.hostName
if hostName == "" {
hostName = info.Name
}
uptime := readSystemUptime()
metricsCtx, metricsCancel := context.WithTimeout(ctx, 10*time.Second)
snapshot, err := hostmetricsCollectWithDiskFilters(metricsCtx, a.cfg.DiskExclude, a.cfg.DiskInclude)
metricsCancel()
if err != nil {
return agentsdocker.Report{}, fmt.Errorf("collect host metrics: %w", err)
}
collectContainers := a.cfg.IncludeContainers
if !collectContainers && (a.cfg.IncludeServices || a.cfg.IncludeTasks) && !info.Swarm.ControlAvailable {
collectContainers = true
}
var containers []agentsdocker.Container
if collectContainers {
var err error
containers, err = a.collectContainers(ctx)
if err != nil {
return agentsdocker.Report{}, err
}
}
services, tasks, nodes, secrets, configs, swarmInfo := a.collectSwarmData(ctx, info, containers)
diskUsageResult, storageUsage, err := a.collectStorageUsage(ctx)
if err != nil {
a.logger.Warn().Err(err).Msg("failed to collect Docker storage usage")
}
images, err := a.collectImages(ctx, diskUsageResult.Images.Items)
if err != nil {
a.logger.Warn().Err(err).Msg("failed to collect Docker images")
}
volumes, err := a.collectVolumes(ctx, diskUsageResult.Volumes.Items)
if err != nil {
a.logger.Warn().Err(err).Msg("failed to collect Docker volumes")
}
networks, err := a.collectNetworks(ctx)
if err != nil {
a.logger.Warn().Err(err).Msg("failed to collect Docker networks")
}
// Use Docker's MemTotal, but fall back to gopsutil's reading if Docker returns 0.
// This can happen in Docker-in-LXC setups where Docker daemon can't read host memory.
totalMemory := info.MemTotal
if totalMemory <= 0 && snapshot.Memory.TotalBytes > 0 {
totalMemory = snapshot.Memory.TotalBytes
}
report := agentsdocker.Report{
Agent: agentsdocker.AgentInfo{
ID: agentID,
Version: a.agentVersion,
Type: a.cfg.AgentType,
IntervalSeconds: int(a.cfg.Interval / time.Second),
Modules: a.helperOperationModuleStatuses(),
},
Host: agentsdocker.HostInfo{
Hostname: hostName,
Name: info.Name,
MachineID: a.machineID,
OS: info.OperatingSystem,
Runtime: string(a.runtime),
RuntimeVersion: a.runtimeVer,
KernelVersion: info.KernelVersion,
Architecture: info.Architecture,
DockerVersion: info.ServerVersion,
TotalCPU: info.NCPU,
TotalMemoryBytes: totalMemory,
UptimeSeconds: uptime,
CPUUsagePercent: safeFloat(snapshot.CPUUsagePercent),
LoadAverage: append([]float64(nil), snapshot.LoadAverage...),
Memory: snapshot.Memory,
Disks: append([]agentsdocker.Disk(nil), snapshot.Disks...),
Network: append([]agentsdocker.NetworkInterface(nil), snapshot.Network...),
Security: buildHostSecurityInfo(info),
},
SequenceID: a.nextReportSequenceID(),
Timestamp: time.Now().UTC(),
}
if swarmInfo != nil {
report.Host.Swarm = swarmInfo
}
if a.cfg.IncludeContainers {
report.Containers = containers
}
if len(images) > 0 {
report.Images = images
}
if len(volumes) > 0 {
report.Volumes = volumes
}
if len(networks) > 0 {
report.Networks = networks
}
if a.cfg.IncludeServices && len(services) > 0 {
report.Services = services
}
if a.cfg.IncludeTasks && len(tasks) > 0 {
report.Tasks = tasks
}
if len(nodes) > 0 {
report.Nodes = nodes
}
if len(secrets) > 0 {
report.Secrets = secrets
}
if len(configs) > 0 {
report.Configs = configs
}
if storageUsage != nil {
report.StorageUsage = storageUsage
}
if report.Agent.IntervalSeconds <= 0 {
report.Agent.IntervalSeconds = int(30 * time.Second / time.Second)
}
return report, nil
}
func buildHostSecurityInfo(info systemtypes.Info) *agentsdocker.HostSecurityInfo {
authzPlugins := normalizedNonEmptyStrings(info.Plugins.Authorization)
if len(authzPlugins) == 0 {
return nil
}
return &agentsdocker.HostSecurityInfo{
AuthorizationPlugins: authzPlugins,
}
}
func normalizedNonEmptyStrings(values []string) []string {
if len(values) == 0 {
return nil
}
normalized := make([]string, 0, len(values))
seen := make(map[string]struct{}, len(values))
for _, value := range values {
value = strings.TrimSpace(value)
if value == "" {
continue
}
if _, ok := seen[value]; ok {
continue
}
seen[value] = struct{}{}
normalized = append(normalized, value)
}
if len(normalized) == 0 {
return nil
}
return normalized
}
func cloneStringMap(src map[string]string) map[string]string {
if len(src) == 0 {
return nil
}
out := make(map[string]string, len(src))
for k, v := range src {
out[k] = v
}
return out
}
func (a *Agent) collectContainers(ctx context.Context) ([]agentsdocker.Container, error) {
options := dockerContainerListOptions{All: true}
if len(a.stateFilters) > 0 {
filterArgs := newDockerFilters()
for _, state := range a.stateFilters {
filterArgs.Add("status", state)
}
options.Filters = filterArgs
}
list, err := dockerCallWithRetry(ctx, dockerContainerListCallTimeout, func(callCtx context.Context) ([]containertypes.Summary, error) {
return a.docker.ContainerList(callCtx, options)
})
if err != nil {
return nil, fmt.Errorf("failed to list containers: %w", annotateDockerConnectionError(err))
}
containers := make([]agentsdocker.Container, 0, len(list))
active := make(map[string]struct{}, len(list))
for _, summary := range list {
if len(a.allowedStates) > 0 {
if _, ok := a.allowedStates[strings.ToLower(string(summary.State))]; !ok {
continue
}
}
// Skip backup containers created during updates - they're temporary
if isBackupContainer(summary.Names) {
continue
}
active[summary.ID] = struct{}{}
container, err := a.collectContainer(ctx, summary)
if err != nil {
a.logger.Warn().Str("container", strings.Join(summary.Names, ",")).Err(err).Msg("failed to collect container stats")
continue
}
containers = append(containers, container)
}
a.pruneStaleCPUSamples(active)
a.pruneInactiveContainerInspectCache(active)
return containers, nil
}
func (a *Agent) collectImages(ctx context.Context, usageImages []image.Summary) ([]agentsdocker.Image, error) {
usageByID := make(map[string]image.Summary, len(usageImages))
for _, summary := range usageImages {
if id := strings.TrimSpace(summary.ID); id != "" {
usageByID[id] = summary
}
}
list, err := dockerCallWithRetry(ctx, dockerInventoryCallTimeout, func(callCtx context.Context) ([]image.Summary, error) {
// Keep image identity and tags on the live telemetry cadence without
// asking the daemon to recompute shared layer sizes every 30 seconds.
// The throttled /system/df snapshot above already owns that expensive
// calculation and supplies its cached values here (#1729).
return a.docker.ImageList(callCtx, dockerImageListOptions{All: true})
})
if err != nil {
return nil, annotateDockerConnectionError(err)
}
images := make([]agentsdocker.Image, 0, len(list))
for _, summary := range list {
sharedSize := summary.SharedSize
containers := summary.Containers
if usage, ok := usageByID[strings.TrimSpace(summary.ID)]; ok {
sharedSize = usage.SharedSize
containers = usage.Containers
}
// Docker uses -1 when these optional calculations were not requested.
// Do not project that sentinel as a real negative byte/container count.
sharedSize = max(sharedSize, 0)
containers = max(containers, 0)
images = append(images, agentsdocker.Image{
ID: strings.TrimSpace(summary.ID),
RepoTags: normalizedNonEmptyStrings(summary.RepoTags),
RepoDigests: normalizedNonEmptyStrings(summary.RepoDigests),
SizeBytes: summary.Size,
SharedSizeBytes: sharedSize,
Containers: containers,
CreatedAt: dockerUnixTimestamp(summary.Created),
Labels: cloneStringMap(summary.Labels),
})
}
return images, nil
}
func (a *Agent) collectVolumes(ctx context.Context, usageVolumes []volume.Volume) ([]agentsdocker.Volume, error) {
usageByName := make(map[string]volume.UsageData, len(usageVolumes))
for _, volume := range usageVolumes {
name := strings.TrimSpace(volume.Name)
if name == "" || volume.UsageData == nil {
continue
}
usageByName[name] = *volume.UsageData
}
list, err := dockerCallWithRetry(ctx, dockerInventoryCallTimeout, func(callCtx context.Context) ([]volume.Volume, error) {
return a.docker.VolumeList(callCtx, dockerVolumeListOptions{})
})
if err != nil {
return nil, annotateDockerConnectionError(err)
}
volumes := make([]agentsdocker.Volume, 0, len(list))
for _, v := range list {
sizeBytes := int64(0)
refCount := int64(0)
if v.UsageData != nil {
sizeBytes = v.UsageData.Size
refCount = v.UsageData.RefCount
} else if usage, ok := usageByName[strings.TrimSpace(v.Name)]; ok {
sizeBytes = usage.Size
refCount = usage.RefCount
}
volumes = append(volumes, agentsdocker.Volume{
Name: strings.TrimSpace(v.Name),
Driver: strings.TrimSpace(v.Driver),
Mountpoint: strings.TrimSpace(v.Mountpoint),
Scope: strings.TrimSpace(v.Scope),
CreatedAt: strings.TrimSpace(v.CreatedAt),
SizeBytes: sizeBytes,
RefCount: refCount,
Labels: cloneStringMap(v.Labels),
Options: cloneStringMap(v.Options),
})
}
return volumes, nil
}
func (a *Agent) collectNetworks(ctx context.Context) ([]agentsdocker.Network, error) {
list, err := dockerCallWithRetry(ctx, dockerInventoryCallTimeout, func(callCtx context.Context) ([]networktypes.Summary, error) {
return a.docker.NetworkList(callCtx, dockerNetworkListOptions{})
})
if err != nil {
return nil, annotateDockerConnectionError(err)
}
networks := make([]agentsdocker.Network, 0, len(list))
for _, n := range list {
subnets := make([]agentsdocker.NetworkSubnet, 0, len(n.IPAM.Config))
for _, config := range n.IPAM.Config {
subnet := ""
if config.Subnet.IsValid() {
subnet = config.Subnet.String()
}
gateway := ""
if config.Gateway.IsValid() {
gateway = config.Gateway.String()
}
subnets = append(subnets, agentsdocker.NetworkSubnet{
Subnet: subnet,
Gateway: gateway,
})
}
networks = append(networks, agentsdocker.Network{
ID: strings.TrimSpace(n.ID),
Name: strings.TrimSpace(n.Name),
Driver: strings.TrimSpace(n.Driver),
Scope: strings.TrimSpace(n.Scope),
CreatedAt: n.Created,
EnableIPv4: n.EnableIPv4,
EnableIPv6: n.EnableIPv6,
Internal: n.Internal,
Attachable: n.Attachable,
Ingress: n.Ingress,
ConfigOnly: n.ConfigOnly,
Subnets: subnets,
Labels: cloneStringMap(n.Labels),
Options: cloneStringMap(n.Options),
})
}
return networks, nil
}
func (a *Agent) collectStorageUsage(ctx context.Context) (client.DiskUsageResult, *agentsdocker.StorageUsage, error) {
a.storageUsageMu.Lock()
defer a.storageUsageMu.Unlock()
now := time.Now()
if !a.storageUsageCache.nextRefresh.IsZero() && now.Before(a.storageUsageCache.nextRefresh) {
if a.storageUsageCache.valid {
return a.storageUsageCache.result, cloneDockerStorageUsage(a.storageUsageCache.usage), nil
}
// The last refresh attempt already surfaced its error. Keep the
// expensive scan suppressed until the bounded refresh window expires.
return client.DiskUsageResult{}, nil, nil
}
a.storageUsageCache.nextRefresh = now.Add(dockerStorageUsageRefreshInterval)
// A full daemon storage walk must not be retried immediately. If it times
// out on an appliance daemon, the normal retry policy would double the load
// and the next live telemetry tick would begin another walk.
result, err := dockerCallWithRetryAttempts(ctx, dockerInventoryCallTimeout, 1, func(callCtx context.Context) (client.DiskUsageResult, error) {
return a.docker.DiskUsage(callCtx, dockerDiskUsageOptions{
Containers: true,
Images: true,
Volumes: true,
BuildCache: true,
Verbose: true,
})
})
if err != nil {
annotatedErr := annotateDockerConnectionError(err)
if a.storageUsageCache.valid {
return a.storageUsageCache.result, cloneDockerStorageUsage(a.storageUsageCache.usage), annotatedErr
}
return client.DiskUsageResult{}, nil, annotatedErr
}
usage := &agentsdocker.StorageUsage{
Images: agentsdocker.StorageUsageBucket{
TotalCount: result.Images.TotalCount,
ActiveCount: result.Images.ActiveCount,
TotalSizeBytes: result.Images.TotalSize,
ReclaimableBytes: result.Images.Reclaimable,
},
Containers: agentsdocker.StorageUsageBucket{
TotalCount: result.Containers.TotalCount,
ActiveCount: result.Containers.ActiveCount,
TotalSizeBytes: result.Containers.TotalSize,
ReclaimableBytes: result.Containers.Reclaimable,
},
Volumes: agentsdocker.StorageUsageBucket{
TotalCount: result.Volumes.TotalCount,
ActiveCount: result.Volumes.ActiveCount,
TotalSizeBytes: result.Volumes.TotalSize,
ReclaimableBytes: result.Volumes.Reclaimable,
},
BuildCache: agentsdocker.StorageUsageBucket{
TotalCount: result.BuildCache.TotalCount,
ActiveCount: result.BuildCache.ActiveCount,
TotalSizeBytes: result.BuildCache.TotalSize,
ReclaimableBytes: result.BuildCache.Reclaimable,
},
}
a.storageUsageCache.result = result
a.storageUsageCache.usage = cloneDockerStorageUsage(usage)
a.storageUsageCache.valid = true
a.storageUsageCache.nextRefresh = time.Now().Add(dockerStorageUsageRefreshInterval)
return result, usage, nil
}
func cloneDockerStorageUsage(usage *agentsdocker.StorageUsage) *agentsdocker.StorageUsage {
if usage == nil {
return nil
}
cloned := *usage
return &cloned
}
func dockerUnixTimestamp(seconds int64) time.Time {
if seconds <= 0 {
return time.Time{}
}
return time.Unix(seconds, 0).UTC()
}
func (a *Agent) pruneStaleCPUSamples(active map[string]struct{}) {
a.cpuMu.Lock()
defer a.cpuMu.Unlock()
if len(a.prevContainerCPU) == 0 {
return
}
for containerID := range a.prevContainerCPU {
if _, ok := active[containerID]; !ok {
delete(a.prevContainerCPU, containerID)
// Reset stats failure counter when containers are removed,
// though it's global per agent so not strictly necessary but good hygiene
}
}
}
func (a *Agent) collectContainer(ctx context.Context, summary containertypes.Summary) (agentsdocker.Container, error) {
const perContainerTimeout = 15 * time.Second
containerCtx, cancel := context.WithTimeout(ctx, perContainerTimeout)
defer cancel()
requestSize := a.cfg.CollectDiskMetrics
inspect, err := a.inspectContainer(containerCtx, summary, requestSize)
if err != nil {
return agentsdocker.Container{}, fmt.Errorf("inspect: %w", err)
}
var (
cpuPercent float64
memUsage int64
memLimit int64
memPercent float64
blockIO *agentsdocker.ContainerBlockIO
networkRX uint64
networkTX uint64
)
if inspect.State.Running || inspect.State.Paused {
statsResp, err := a.docker.ContainerStatsOneShot(containerCtx, summary.ID)
if err != nil {
return agentsdocker.Container{}, fmt.Errorf("stats: %w", err)
}
defer func() {
if closeErr := statsResp.Body.Close(); closeErr != nil {
a.logger.Warn().Err(closeErr).Str("container", summary.ID).Msg("Failed to close container stats response body")
}
}()
payload, err := readBodyWithLimit(statsResp.Body, maxContainerStatsBodyBytes)
if err != nil {
return agentsdocker.Container{}, fmt.Errorf("read stats: %w", err)
}
stats, podmanCPUPercent, err := decodeContainerStatsPayload(payload)
if err != nil {
return agentsdocker.Container{}, fmt.Errorf("decode stats: %w", err)
}
if a.runtime == RuntimePodman && podmanCPUPercent != nil {
cpuPercent = safeFloat(*podmanCPUPercent)
} else {
cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats)
}
memUsage, memLimit, memPercent = calculateMemoryUsage(stats)
blockIO = summarizeBlockIO(stats)
networkRX, networkTX = summarizeNetworkIO(stats)
} else {
a.cpuMu.Lock()
delete(a.prevContainerCPU, summary.ID)
a.cpuMu.Unlock()
}
createdAt := time.Unix(summary.Created, 0)
startedAt := parseTime(inspect.State.StartedAt)
finishedAt := parseTime(inspect.State.FinishedAt)
uptimeSeconds := int64(0)
if !startedAt.IsZero() && inspect.State.Running {
uptimeSeconds = int64(time.Since(startedAt).Seconds())
if uptimeSeconds < 0 {
uptimeSeconds = 0
}
}
health := ""
if inspect.State.Health != nil {
health = string(inspect.State.Health.Status)
}
healthcheckTargets := []string(nil)
if inspect.Config != nil && inspect.Config.Healthcheck != nil {
healthcheckTargets = extractHealthcheckTargets(inspect.Config.Healthcheck.Test)
}
ports := make([]agentsdocker.ContainerPort, len(summary.Ports))
for i, port := range summary.Ports {
ports[i] = agentsdocker.ContainerPort{
PrivatePort: int(port.PrivatePort),
PublicPort: int(port.PublicPort),
Protocol: port.Type,
IP: addrString(port.IP),
}
}
labels := make(map[string]string, len(summary.Labels))
for k, v := range summary.Labels {
labels[k] = v
}
networks := make([]agentsdocker.ContainerNetwork, 0)
if inspect.NetworkSettings != nil {
for name, cfg := range inspect.NetworkSettings.Networks {
networks = append(networks, agentsdocker.ContainerNetwork{
Name: name,
IPv4: addrString(cfg.IPAddress),
IPv6: addrString(cfg.GlobalIPv6Address),
})
}
}
var startedPtr, finishedPtr *time.Time
if !startedAt.IsZero() {
started := startedAt
startedPtr = &started
}
if !finishedAt.IsZero() && !inspect.State.Running {
finished := finishedAt
finishedPtr = &finished
}
var writableLayerBytes int64
if inspect.SizeRw != nil {
writableLayerBytes = *inspect.SizeRw
}
var rootFsBytes int64
if inspect.SizeRootFs != nil {
rootFsBytes = *inspect.SizeRootFs
}
var mounts []agentsdocker.ContainerMount
if len(inspect.Mounts) > 0 {
mounts = make([]agentsdocker.ContainerMount, 0, len(inspect.Mounts))
for _, mount := range inspect.Mounts {
mounts = append(mounts, agentsdocker.ContainerMount{
Type: string(mount.Type),
Source: mount.Source,
Destination: mount.Destination,
Mode: mount.Mode,
RW: mount.RW,
Propagation: string(mount.Propagation),
Name: mount.Name,
Driver: mount.Driver,
})
}
}
// Docker's --tmpfs mounts can exist only in HostConfig.Tmpfs. Preserve
// their configuration alongside inspected mounts without inventing usage.
if inspect.HostConfig != nil && len(inspect.HostConfig.Tmpfs) > 0 {
reported := make(map[string]bool, len(mounts))
for _, mount := range mounts {
reported[mount.Destination] = true
}
destinations := make([]string, 0, len(inspect.HostConfig.Tmpfs))
for destination := range inspect.HostConfig.Tmpfs {
if !reported[destination] {
destinations = append(destinations, destination)
}
}
sort.Strings(destinations)
for _, destination := range destinations {
options := inspect.HostConfig.Tmpfs[destination]
writable := true
for _, option := range strings.Split(options, ",") {
switch strings.TrimSpace(option) {
case "ro":
writable = false
case "rw":
writable = true
}
}
mounts = append(mounts, agentsdocker.ContainerMount{
Type: "tmpfs", Destination: destination, Mode: options, RW: writable,
})
}
}
oomKilled := inspect.State.OOMKilled
container := agentsdocker.Container{
ID: summary.ID,
Name: trimLeadingSlash(summary.Names),
Image: summary.Image,
ImageDigest: summary.ImageID, // sha256:... digest of the image
CreatedAt: createdAt,
State: string(summary.State),
Status: summary.Status,
Health: health,
HealthcheckTargets: healthcheckTargets,
CPUPercent: cpuPercent,
MemoryUsageBytes: memUsage,
MemoryLimitBytes: memLimit,
MemoryPercent: memPercent,
UptimeSeconds: uptimeSeconds,
RestartCount: inspect.RestartCount,
ExitCode: inspect.State.ExitCode,
OOMKilled: &oomKilled,
StartedAt: startedPtr,
FinishedAt: finishedPtr,
Ports: ports,
Labels: labels,
Env: maskSensitiveEnvVars(inspect.Config.Env),
Networks: networks,
NetworkRXBytes: networkRX,
NetworkTXBytes: networkTX,
WritableLayerBytes: writableLayerBytes,
RootFilesystemBytes: rootFsBytes,
BlockIO: blockIO,
Mounts: mounts,
Filesystems: a.collectContainerFilesystems(containerCtx, summary.ID, inspect, mounts),
}
if a.runtime == RuntimePodman {
if meta := extractPodmanMetadata(labels); meta != nil {
container.Podman = meta
}
}
// Check for image updates if registry checker is enabled
if a.registryChecker != nil && a.registryChecker.Enabled() {
// Get the actual manifest digest (RepoDigest) from the image for accurate comparison.
// The ImageID is a local content-addressable ID that differs from the registry manifest digest.
// We also get the architecture details to correctly resolve manifest lists from the registry.
digestForComparison, arch, os, variant := a.getImageRepoDigest(containerCtx, summary.ImageID, summary.Image)
var imageToCheck string
// Always prefer the image name from inspect config as it's the authoritative source
// and avoids issues with short IDs or digests in summary.
// HOWEVER, if the config image IS a digest (starts with sha256:), fall back to container.Image
// which usually contains the human-readable tag/name.
imageToCheck = container.Image
if inspect.Config != nil && inspect.Config.Image != "" {
if !strings.HasPrefix(inspect.Config.Image, "sha256:") {
imageToCheck = inspect.Config.Image
}
}
// Additional safety: if imageToCheck is still a SHA, we can't check it
if strings.HasPrefix(imageToCheck, "sha256:") {
container.UpdateStatus = &agentsdocker.UpdateStatus{
UpdateAvailable: false,
CurrentDigest: digestForComparison,
LastChecked: time.Now(),
Error: "digest-pinned image",
}
// Skip to end of update check block - don't call registry
} else {
a.logger.Debug().
Str("container", container.Name).
Str("image", imageToCheck).
Str("compareDigest", digestForComparison).
Str("arch", arch).
Str("os", os).
Str("variant", variant).
Msg("Checking update for container")
result := a.registryChecker.CheckImageUpdate(ctx, imageToCheck, digestForComparison, arch, os, variant)
if result != nil {
container.UpdateStatus = &agentsdocker.UpdateStatus{
UpdateAvailable: result.UpdateAvailable,
CurrentDigest: result.CurrentDigest,
LatestDigest: result.LatestDigest,
LastChecked: result.CheckedAt,
Error: result.Error,
}
}
}
}
if requestSize {
a.logger.Debug().
Str("container", container.Name).
Int64("writableLayerBytes", writableLayerBytes).
Int64("rootFilesystemBytes", rootFsBytes).
Int("mountCount", len(mounts)).
Msg("Collected container disk metrics")
}
return container, nil
}
func (a *Agent) inspectContainer(ctx context.Context, summary containertypes.Summary, requestSize bool) (containertypes.InspectResponse, error) {
// Empty IDs/statuses do not provide enough evidence to detect a lifecycle
// change between reports (some Podman compatibility endpoints omit Status),
// so keep those on the live path rather than accepting bounded stale state.
cacheable := strings.TrimSpace(summary.ID) != "" && strings.TrimSpace(summary.Status) != "" && isInactiveContainerState(summary.State)
fingerprint := inactiveContainerInspectFingerprint(summary)
if cacheable {
a.inactiveInspectMu.Lock()
entry, ok := a.inactiveInspects[summary.ID]
if ok && entry.withSize == requestSize && entry.fingerprint == fingerprint && time.Now().Before(entry.expiresAt) {
inspect := entry.inspect
a.inactiveInspectMu.Unlock()
return inspect, nil
}
a.inactiveInspectMu.Unlock()
}
inspect, _, err := a.docker.ContainerInspectWithRaw(ctx, summary.ID, requestSize)
if err != nil {
return containertypes.InspectResponse{}, err
}
a.inactiveInspectMu.Lock()
defer a.inactiveInspectMu.Unlock()
if !cacheable || inspect.State == nil || inspect.State.Running || inspect.State.Paused || inspect.State.Restarting {
delete(a.inactiveInspects, summary.ID)
return inspect, nil
}
if a.inactiveInspects == nil {
a.inactiveInspects = make(map[string]inactiveContainerInspectCacheEntry)
}
a.inactiveInspects[summary.ID] = inactiveContainerInspectCacheEntry{
inspect: inspect,
fingerprint: fingerprint,
withSize: requestSize,
expiresAt: time.Now().Add(dockerInactiveInspectRefreshInterval),
}
return inspect, nil
}
func isInactiveContainerState(state containertypes.ContainerState) bool {
switch strings.ToLower(strings.TrimSpace(string(state))) {
case "created", "exited", "dead":
return true
default:
return false
}
}
func inactiveContainerInspectFingerprint(summary containertypes.Summary) string {
// Status includes the latest exit result/time, so an inactive container
// that ran and stopped entirely between two reports cannot reuse its old
// exit code, restart count, or lifecycle timestamps.
return strings.ToLower(strings.TrimSpace(string(summary.State))) + "\x00" + strings.TrimSpace(summary.Status)
}
func (a *Agent) pruneInactiveContainerInspectCache(active map[string]struct{}) {
a.inactiveInspectMu.Lock()
defer a.inactiveInspectMu.Unlock()
for containerID := range a.inactiveInspects {
if _, ok := active[containerID]; !ok {
delete(a.inactiveInspects, containerID)
}
}
}
func (a *Agent) clearDockerCollectionCaches() {
a.storageUsageMu.Lock()
a.storageUsageCache = dockerStorageUsageCache{}
a.storageUsageMu.Unlock()
a.inactiveInspectMu.Lock()
a.inactiveInspects = nil
a.inactiveInspectMu.Unlock()
}
var (
healthcheckURLPattern = regexp.MustCompile(`(?i)https?://[^\s'"<>]+`)
healthcheckTargetPattern = regexp.MustCompile(`(?i)^[a-z0-9][a-z0-9._-]{0,252}$`)
)
// extractHealthcheckTargets derives only secret-free URL hostnames from a
// container health check. Raw health-check command text can contain paths,
// query strings, credentials, or environment expansion and must never cross
// the agent reporting boundary.
func extractHealthcheckTargets(test []string) []string {
const maxTargets = 16
seen := make(map[string]struct{})
targets := make([]string, 0)
for _, part := range test {
for _, candidate := range healthcheckURLPattern.FindAllString(part, -1) {
if len(candidate) > 2048 {
continue
}
parsed, err := url.Parse(candidate)
if err != nil {
continue
}
target := strings.ToLower(strings.TrimSpace(parsed.Hostname()))
if address, err := netip.ParseAddr(target); err == nil {
target = address.String()
} else if !healthcheckTargetPattern.MatchString(target) {
continue
}
if _, exists := seen[target]; exists {
continue
}
seen[target] = struct{}{}
targets = append(targets, target)
if len(targets) == maxTargets {
return targets
}
}
}
if len(targets) == 0 {
return nil
}
return targets
}
// getImageRepoDigest retrieves the RepoDigest for an image and its platform details.
// It returns the digest, architecture, OS, and variant.
func (a *Agent) getImageRepoDigest(ctx context.Context, imageID, imageName string) (string, string, string, string) {
imageInspect, _, err := a.docker.ImageInspectWithRaw(ctx, imageID)
if err != nil {
a.logger.Debug().
Err(err).
Str("imageID", imageID).
Str("imageName", imageName).
Msg("Failed to inspect image for RepoDigest")
return "", "", "", ""
}
arch := imageInspect.Architecture
os := imageInspect.Os
variant := imageInspect.Variant
if len(imageInspect.RepoDigests) == 0 {
// Locally built images won't have RepoDigests
return "", arch, os, variant
}
// Try to find a RepoDigest that matches the image reference
// RepoDigests format: "registry/repo@sha256:..."
for _, repoDigest := range imageInspect.RepoDigests {
// Extract just the digest part (after @)
if idx := strings.LastIndex(repoDigest, "@"); idx >= 0 {
repoRef := repoDigest[:idx] // e.g., "docker.io/library/nginx"
digest := repoDigest[idx+1:] // e.g., "sha256:abc..."
// Check if this RepoDigest matches our image reference
// Normalize both for comparison
if matchesImageReference(imageName, repoRef) {
return digest, arch, os, variant
}
}
}
// If no exact match, return the first RepoDigest's digest
// This handles cases where the image was pulled with a different tag
if idx := strings.LastIndex(imageInspect.RepoDigests[0], "@"); idx >= 0 {
return imageInspect.RepoDigests[0][idx+1:], arch, os, variant
}
return "", arch, os, variant
}
func addrString(addr netip.Addr) string {
if !addr.IsValid() {
return ""
}
return addr.String()
}
// matchesImageReference checks if a RepoDigest repository matches an image reference.
// It handles Docker Hub's various naming conventions.
func matchesImageReference(imageName, repoRef string) bool {
// Normalize image name by removing tag
if idx := strings.LastIndex(imageName, ":"); idx >= 0 {
// Make sure it's a tag, not a port (check if there's a / after it)
if !strings.Contains(imageName[idx:], "/") {
imageName = imageName[:idx]
}
}
// Direct match
if imageName == repoRef {
return true
}
// Docker Hub library images: "nginx" == "docker.io/library/nginx"
if repoRef == "docker.io/library/"+imageName {
return true
}
// Docker Hub with namespace: "myuser/myapp" == "docker.io/myuser/myapp"
if repoRef == "docker.io/"+imageName {
return true
}
// Registry prefix matching (e.g., "ghcr.io/user/repo" matches "ghcr.io/user/repo")
// Already handled by direct match above
return false
}
func extractPodmanMetadata(labels map[string]string) *agentsdocker.PodmanContainer {
if len(labels) == 0 {
return nil
}
meta := &agentsdocker.PodmanContainer{}
if v := strings.TrimSpace(labels["io.podman.annotations.pod.name"]); v != "" {
meta.PodName = v
}
if v := strings.TrimSpace(labels["io.podman.annotations.pod.id"]); v != "" {
meta.PodID = v
}
if v := strings.TrimSpace(labels["io.podman.annotations.pod.infra"]); v != "" {
if parsed, err := strconv.ParseBool(v); err == nil {
meta.Infra = parsed
} else if strings.EqualFold(v, "yes") || strings.EqualFold(v, "true") {
meta.Infra = true
}
}
if v := strings.TrimSpace(labels["io.podman.compose.project"]); v != "" {
meta.ComposeProject = v
}
if v := strings.TrimSpace(labels["io.podman.compose.service"]); v != "" {
meta.ComposeService = v
}
if v := strings.TrimSpace(labels["io.podman.compose.working_dir"]); v != "" {
meta.ComposeWorkdir = v
}
if v := strings.TrimSpace(labels["io.podman.compose.config-hash"]); v != "" {
meta.ComposeConfig = v
}
if v := strings.TrimSpace(labels["io.containers.autoupdate"]); v != "" {
meta.AutoUpdatePolicy = v
}
if v := strings.TrimSpace(labels["io.containers.autoupdate.restart"]); v != "" {
meta.AutoUpdateRestart = v
}
if v := strings.TrimSpace(labels["io.podman.annotations.userns"]); v != "" {
meta.UserNS = v
} else if v := strings.TrimSpace(labels["io.containers.userns"]); v != "" {
meta.UserNS = v
}
if meta.PodName == "" && meta.PodID == "" && meta.ComposeProject == "" && meta.AutoUpdatePolicy == "" && meta.UserNS == "" && !meta.Infra {
return nil
}
return meta
}
type podmanCompatStatsProbe struct {
CPUStats struct {
CPU *float64 `json:"cpu"`
} `json:"cpu_stats"`
}
func decodeContainerStatsPayload(payload []byte) (containertypes.StatsResponse, *float64, error) {
var stats containertypes.StatsResponse
if err := json.Unmarshal(payload, &stats); err != nil {
return containertypes.StatsResponse{}, nil, err
}
var probe podmanCompatStatsProbe
if err := json.Unmarshal(payload, &probe); err != nil {
return stats, nil, nil
}
return stats, probe.CPUStats.CPU, nil
}
func (a *Agent) calculateContainerCPUPercent(containerID string, stats containertypes.StatsResponse) float64 {
a.cpuMu.Lock()
defer a.cpuMu.Unlock()
current := cpuSample{
totalUsage: stats.CPUStats.CPUUsage.TotalUsage,
systemUsage: stats.CPUStats.SystemUsage,
onlineCPUs: stats.CPUStats.OnlineCPUs,
read: stats.Read,
}
// Always use manual delta tracking. Docker's PreCPUStats is unreliable for
// one-shot stats because many Docker versions don't update their internal
// cache between non-streaming reads, causing PreCPUStats to remain stale
// (from container start) and producing a constant lifetime-average CPU%
// instead of a current value.
prev, ok := a.prevContainerCPU[containerID]
if !ok {
// First time seeing this container - store current sample and return 0
// On next collection cycle we'll have a previous sample to compare against
a.prevContainerCPU[containerID] = current
a.logger.Debug().
Str("container_id", containerID[:12]).
Uint64("total_usage", current.totalUsage).
Uint64("system_usage", current.systemUsage).
Msg("First CPU sample collected, no previous data for delta calculation")
return 0
}
// We have a previous sample - update it after calculation
a.prevContainerCPU[containerID] = current
var totalDelta float64
if current.totalUsage >= prev.totalUsage {
totalDelta = float64(current.totalUsage - prev.totalUsage)
} else {
// Counter likely reset (container restart); fall back to current reading.
totalDelta = float64(current.totalUsage)
}
if totalDelta <= 0 {
return 0
}
if a.runtime == RuntimePodman {
if !prev.read.IsZero() && !current.read.IsZero() {
elapsed := current.read.Sub(prev.read).Seconds()
if elapsed > 0 {
denominator := elapsed * 1e9
if denominator > 0 {
cpuPercent := (totalDelta / denominator) * 100.0
result := safeFloat(cpuPercent)
a.logger.Debug().
Str("container_id", containerID[:12]).
Float64("cpu_percent", result).
Float64("total_delta", totalDelta).
Float64("elapsed_seconds", elapsed).
Msg("CPU calculated from Podman wall-clock delta")
return result
}
}
}
a.logger.Debug().
Str("container_id", containerID[:12]).
Float64("total_delta", totalDelta).
Bool("prev_read_zero", prev.read.IsZero()).
Bool("current_read_zero", current.read.IsZero()).
Msg("Podman CPU calculation failed: no valid wall-clock delta available")
return 0
}
onlineCPUs := current.onlineCPUs
if onlineCPUs == 0 {
onlineCPUs = prev.onlineCPUs
}
if onlineCPUs == 0 && a.cpuCount > 0 {
onlineCPUs = uint32(a.cpuCount)
}
if onlineCPUs == 0 {
return 0
}
var systemDelta float64
if current.systemUsage >= prev.systemUsage {
systemDelta = float64(current.systemUsage - prev.systemUsage)
}
// If systemUsage went backward (counter reset), leave systemDelta as 0
// to fall through to time-based calculation below
if systemDelta > 0 {
cpuPercent := safeFloat((totalDelta / systemDelta) * float64(onlineCPUs) * 100.0)
a.logger.Debug().
Str("container_id", containerID[:12]).
Float64("cpu_percent", cpuPercent).
Float64("total_delta", totalDelta).
Float64("system_delta", systemDelta).
Uint32("online_cpus", onlineCPUs).
Msg("CPU calculated from system delta")
return cpuPercent
}
// Fall back to time-based calculation
if !prev.read.IsZero() && !current.read.IsZero() {
elapsed := current.read.Sub(prev.read).Seconds()
if elapsed > 0 {
denominator := elapsed * float64(onlineCPUs) * 1e9
if denominator > 0 {
cpuPercent := (totalDelta / denominator) * 100.0
result := safeFloat(cpuPercent)
a.logger.Debug().
Str("container_id", containerID[:12]).
Float64("cpu_percent", result).
Float64("total_delta", totalDelta).
Float64("elapsed_seconds", elapsed).
Uint32("online_cpus", onlineCPUs).
Msg("CPU calculated from time-based delta")
return result
}
}
}
a.logger.Debug().
Str("container_id", containerID[:12]).
Float64("total_delta", totalDelta).
Float64("system_delta", systemDelta).
Bool("prev_read_zero", prev.read.IsZero()).
Bool("current_read_zero", current.read.IsZero()).
Msg("CPU calculation failed: no valid delta method available")
return 0
}
func calculateCPUPercent(stats containertypes.StatsResponse, hostCPUs int) float64 {
totalDelta := float64(stats.CPUStats.CPUUsage.TotalUsage - stats.PreCPUStats.CPUUsage.TotalUsage)
systemDelta := float64(stats.CPUStats.SystemUsage - stats.PreCPUStats.SystemUsage)
if totalDelta <= 0 || systemDelta <= 0 {
return 0
}
onlineCPUs := stats.CPUStats.OnlineCPUs
if onlineCPUs == 0 {
onlineCPUs = uint32(len(stats.CPUStats.CPUUsage.PercpuUsage))
}
if onlineCPUs == 0 && hostCPUs > 0 {
onlineCPUs = uint32(hostCPUs)
}
if onlineCPUs == 0 {
return 0
}
return safeFloat((totalDelta / systemDelta) * float64(onlineCPUs) * 100.0)
}
func calculateMemoryUsage(stats containertypes.StatsResponse) (usage int64, limit int64, percent float64) {
usage = int64(stats.MemoryStats.Usage)
// Subtract reclaimable cache from usage to match `docker stats` behavior.
// Docker subtracts cache/file to show "actual" memory usage rather than
// memory.current which includes reclaimable filesystem cache.
//
// cgroup v1: "cache" stat contains the reclaimable cache
// cgroup v2: "cache" doesn't exist, use "inactive_file" (preferred) or "file"
var cacheBytes uint64
if cache, ok := stats.MemoryStats.Stats["cache"]; ok {
// cgroup v1
cacheBytes = cache
} else if inactiveFile, ok := stats.MemoryStats.Stats["inactive_file"]; ok {
// cgroup v2: inactive_file is the reclaimable portion of file cache
// This matches what docker CLI does internally
cacheBytes = inactiveFile
}
if cacheBytes > 0 && int64(cacheBytes) < usage {
usage -= int64(cacheBytes)
}
limit = int64(stats.MemoryStats.Limit)
if limit > 0 {
percent = (float64(usage) / float64(limit)) * 100.0
}
return usage, limit, safeFloat(percent)
}
func safeFloat(val float64) float64 {
if math.IsNaN(val) || math.IsInf(val, 0) {
return 0
}
return val
}
func parseTime(value string) time.Time {
if value == "" || value == "0001-01-01T00:00:00Z" {
return time.Time{}
}
if strings.Contains(value, ".") {
if t, err := time.Parse(time.RFC3339Nano, value); err == nil {
return t
}
} else {
if t, err := time.Parse(time.RFC3339, value); err == nil {
return t
}
}
return time.Time{}
}
func trimLeadingSlash(names []string) string {
if len(names) == 0 {
return ""
}
name := names[0]
return strings.TrimPrefix(name, "/")
}
func randomDuration(max time.Duration) time.Duration {
if max <= 0 {
return 0
}
n, err := randIntFn(rand.Reader, big.NewInt(int64(max)))
if err != nil {
return 0
}
return time.Duration(n.Int64())
}
func summarizeBlockIO(stats containertypes.StatsResponse) *agentsdocker.ContainerBlockIO {
if len(stats.BlkioStats.IoServiceBytesRecursive) == 0 {
return nil
}
var readBytes, writeBytes uint64
var readPresent, writePresent bool
for _, entry := range stats.BlkioStats.IoServiceBytesRecursive {
op := strings.ToLower(entry.Op)
switch op {
case "read":
readPresent = true
readBytes += entry.Value
case "write":
writePresent = true
writeBytes += entry.Value
}
}
if !readPresent && !writePresent {
return nil
}
return &agentsdocker.ContainerBlockIO{
ReadBytes: readBytes,
WriteBytes: writeBytes,
ReadBytesPresent: &readPresent,
WriteBytesPresent: &writePresent,
}
}
func summarizeNetworkIO(stats containertypes.StatsResponse) (uint64, uint64) {
if len(stats.Networks) == 0 {
return 0, 0
}
var rxBytes uint64
var txBytes uint64
for _, network := range stats.Networks {
rxBytes += network.RxBytes
txBytes += network.TxBytes
}
return rxBytes, txBytes
}
// sensitiveEnvPatterns are substrings that, when found in an env var name (case-insensitive),
// indicate the value should be masked for security.
var sensitiveEnvPatterns = []string{
"password", "passwd", "secret", "key", "token", "credential", "auth",
"api_key", "apikey", "private", "access_token", "refresh_token",
"database_url", "connection_string", "encryption",
}
// maskSensitiveEnvVars returns a copy of the environment variables with sensitive values masked.
// Environment variables whose names contain sensitive keywords will have their values replaced with "***".
func maskSensitiveEnvVars(envVars []string) []string {
if len(envVars) == 0 {
return nil
}
result := make([]string, 0, len(envVars))
for _, env := range envVars {
parts := strings.SplitN(env, "=", 2)
if len(parts) != 2 {
result = append(result, env)
continue
}
name := parts[0]
value := parts[1]
// Check if the environment variable name contains a sensitive pattern
lowerName := strings.ToLower(name)
isSensitive := false
for _, pattern := range sensitiveEnvPatterns {
if strings.Contains(lowerName, pattern) {
isSensitive = true
break
}
}
if isSensitive && value != "" {
result = append(result, name+"=***")
} else {
result = append(result, env)
}
}
return result
}