diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 4754a53d0d..f7aba26ab8 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -15,6 +15,21 @@ ## Purpose +### Podman interval CPU evidence — issue #2447 + +Every one-shot container stats response retains a cumulative CPU baseline, +even when a positive Podman compatibility percentage takes precedence. Zero, +absent or invalid percentages use the valid nanosecond/wall-clock interval, +without Docker's host-core multiplier. First samples, counter resets and changes +of container start time rebaseline rather than presenting lifetime CPU as current +usage. Repeated or older read timestamps cannot move the baseline backwards. +Positive native percentages keep the units established by #1391; a genuine idle +counter interval remains zero. Report shapes, enrollment, runtime transport trust +and command authority are unchanged. `TestCollectContainerPodmanCPUIntervals` +in `internal/dockeragent/agent_internal_test.go` exercises decoding through the +real collector, idle/advancing counters, start changes and Docker isolation. +Synthetic controls do not establish a native Podman payload or reporter recovery. + ### MD RAID required members and spares — issue #2369 Host RAID reports carry optional `requiredDevices`: the configured member count diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 26ce03a228..38bca29d81 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -17,6 +17,20 @@ ## Purpose +### Podman zero-percentage fallback — issue #2447 + +One-shot compatibility stats with `cpu:0` do not suppress advancing cumulative +CPU samples. The collector records every interval and uses Podman's CPU-time +nanoseconds over elapsed wall-clock nanoseconds, independent of host core count. +A positive finite reported percentage remains authoritative, preserving #1391. +Counter resets or a changed container start establish a new baseline; duplicate +and out-of-order reads cannot create subsequent spikes. No timestamp means no +measurable counter interval. The existing report CPU field, graph/alert units, +Docker system-counter calculation, memory and permission scopes are unchanged. +`TestCollectContainerPodmanCPUIntervals` in `agent_internal_test.go` covers the +actual JSON/collector boundary, including changing percentage availability. +Native Podman shape and graph/alert acceptance remain separate evidence. + ### Guest-agent coordination around backups — issue #2439 All PVE QEMU guest-agent reads, including disk diagnostics and legacy meminfo diff --git a/internal/dockeragent/agent.go b/internal/dockeragent/agent.go index 0c268a91a3..4872bdc4fe 100644 --- a/internal/dockeragent/agent.go +++ b/internal/dockeragent/agent.go @@ -229,6 +229,7 @@ type cpuSample struct { systemUsage uint64 onlineCPUs uint32 read time.Time + startedAt time.Time } // New creates a new Docker / Podman module instance. diff --git a/internal/dockeragent/agent_cpu_test.go b/internal/dockeragent/agent_cpu_test.go index f719bff24c..3bbfefee98 100644 --- a/internal/dockeragent/agent_cpu_test.go +++ b/internal/dockeragent/agent_cpu_test.go @@ -28,7 +28,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) { OnlineCPUs: 2, }, } - got := agent.calculateContainerCPUPercent("container-123456", stats1) + got := agent.calculateContainerCPUPercent("container-123456", stats1, time.Time{}) if got != 0 { t.Fatalf("first call: expected 0, got %f", got) } @@ -45,7 +45,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) { OnlineCPUs: 2, }, } - got = agent.calculateContainerCPUPercent("container-123456", stats2) + got = agent.calculateContainerCPUPercent("container-123456", stats2, time.Time{}) if got <= 0 { t.Fatalf("second call: expected percent > 0, got %f", got) } @@ -68,7 +68,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) { }, } - got := agent.calculateContainerCPUPercent("container-123456", stats) + got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}) if got != 0 { t.Fatalf("expected 0, got %f", got) } @@ -103,7 +103,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) { }, } - got := agent.calculateContainerCPUPercent("container-123456", stats) + got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}) if got <= 0 { t.Fatalf("expected percent > 0, got %f", got) } @@ -136,7 +136,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) { }, } - got := agent.calculateContainerCPUPercent("container-123456", stats) + got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}) if got <= 0 { t.Fatalf("expected percent > 0, got %f", got) } @@ -168,7 +168,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) { }, } - got := agent.calculateContainerCPUPercent("container-123456", stats) + got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}) if got <= 0 { t.Fatalf("expected percent > 0, got %f", got) } @@ -200,7 +200,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) { }, } - if got := agent.calculateContainerCPUPercent("container-123456", stats); got != 0 { + if got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}); got != 0 { t.Fatalf("expected 0, got %f", got) } }) @@ -232,7 +232,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) { }, } - if got := agent.calculateContainerCPUPercent("container-123456", stats); got != 0 { + if got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}); got != 0 { t.Fatalf("expected 0, got %f", got) } }) @@ -263,7 +263,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) { }, } - if got := agent.calculateContainerCPUPercent("container-123456", stats); got != 0 { + if got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}); got != 0 { t.Fatalf("expected 0, got %f", got) } }) @@ -291,7 +291,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) { PreCPUStats: containertypes.CPUStats{}, } - got := agent.calculateContainerCPUPercent("container-123456", stats) + got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}) if got != 0 { t.Fatalf("expected 0, got %f", got) } @@ -319,7 +319,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) { }, } - got := agent.calculateContainerCPUPercent("container-123456", stats) + got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}) if got != 0 { t.Fatalf("expected 0 on first call (manual tracking), got %f", got) } @@ -349,7 +349,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) { }, } - got := agent.calculateContainerCPUPercent("container-123456", stats) + got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}) if math.Abs(got-25.0) > 0.01 { t.Fatalf("expected podman cpu percent near 25.0, got %f", got) } diff --git a/internal/dockeragent/agent_internal_test.go b/internal/dockeragent/agent_internal_test.go index 193caaf5bb..c4bffdc700 100644 --- a/internal/dockeragent/agent_internal_test.go +++ b/internal/dockeragent/agent_internal_test.go @@ -1840,3 +1840,137 @@ func TestCollectorRuntimeBoundaryLossIsDaemonUnavailable(t *testing.T) { } } } + +// Exercise the compatibility JSON through the actual container collector, not +// just the delta helper. These are synthetic inputs, not a native Podman trace. +func TestCollectContainerPodmanCPUIntervals(t *testing.T) { + type sample struct { + second int + total uint64 + percent string // Empty means absent; other values are literal JSON. + start int + stopped bool + want float64 + } + tests := []struct { + name string + runtime RuntimeKind + cpus int + samples []sample + }{ + {"zero with advancing and idle counters", RuntimePodman, 16, []sample{ + {second: 0, total: 100_000_000, percent: "0"}, + {second: 1, total: 130_000_000, percent: "0", want: 3}, + {second: 2, total: 130_000_000, percent: "0"}, + {second: 3, total: 1_630_000_000, percent: "0", want: 150}, + }}, + {"absent percentage uses nanosecond wall clock units", RuntimePodman, 1, []sample{ + {second: 0, total: 100_000_000}, + {second: 1, total: 130_000_000, want: 3}, + }}, + {"null percentage", RuntimePodman, 16, []sample{ + {second: 0, total: 100_000_000, percent: "null"}, + {second: 1, total: 130_000_000, percent: "null", want: 3}, + }}, + {"positive percentages keep units and retain the baseline", RuntimePodman, 16, []sample{ + {second: 0, total: 100_000_000, percent: "0.32", want: 0.32}, + {second: 1, total: 130_000_000, percent: "0.64", want: 0.64}, + {second: 2, total: 160_000_000, percent: "0", want: 3}, + {second: 3, total: 190_000_000, want: 3}, + }}, + {"absent then reported then zero", RuntimePodman, 16, []sample{ + {second: 0, total: 100_000_000}, + {second: 1, total: 130_000_000, percent: "0.32", want: 0.32}, + {second: 2, total: 160_000_000, percent: "0", want: 3}, + }}, + {"invalid negative percentage", RuntimePodman, 16, []sample{ + {second: 0, total: 100_000_000, percent: "-1"}, + {second: 1, total: 130_000_000, percent: "-1", want: 3}, + }}, + {"unparseable optional percentage", RuntimePodman, 16, []sample{ + {second: 0, total: 100_000_000, percent: `"unknown"`}, + {second: 1, total: 130_000_000, percent: `"unknown"`, want: 3}, + }}, + {"counter reset rebaselines rather than using new lifetime", RuntimePodman, 16, []sample{ + {second: 0, total: 100_000_000, percent: "0"}, + {second: 1, total: 10_000_000, percent: "0"}, + {second: 2, total: 40_000_000, percent: "0", want: 3}, + }}, + {"restart with an already larger counter", RuntimePodman, 16, []sample{ + {second: 0, total: 100_000_000, percent: "0"}, + {second: 1, total: 260_000_000, percent: "0", start: 1}, + {second: 2, total: 290_000_000, percent: "0", start: 1, want: 3}, + }}, + {"repeated and old timestamps do not move the baseline", RuntimePodman, 16, []sample{ + {second: 0, total: 100_000_000, percent: "0"}, + {second: 1, total: 130_000_000, percent: "0", want: 3}, + {second: 1, total: 150_000_000, percent: "0"}, + {second: 0, total: 170_000_000, percent: "0"}, + {second: 2, total: 160_000_000, percent: "0", want: 3}, + }}, + {"missing timestamp cannot invent an interval", RuntimePodman, 16, []sample{ + {second: -1, total: 100_000_000, percent: "0"}, + {second: 1, total: 130_000_000, percent: "0"}, + {second: 2, total: 160_000_000, percent: "0", want: 3}, + }}, + {"stop clears the baseline", RuntimePodman, 16, []sample{ + {second: 0, total: 100_000_000, percent: "0.32", want: 0.32}, + {second: 1, stopped: true}, + {second: 2, total: 260_000_000, percent: "0"}, + {second: 3, total: 290_000_000, percent: "0", want: 3}, + }}, + {"docker ignores compat percentage and retains system delta", RuntimeDocker, 16, []sample{ + {second: 0, total: 100_000_000, percent: "0.32"}, + {second: 1, total: 130_000_000, percent: "0.32", want: 48}, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + epoch := time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC) + var current sample + agent := &Agent{ + runtime: tt.runtime, + logger: zerolog.Nop(), + cpuCount: tt.cpus, + docker: &fakeDockerClient{ + containerInspectWithRawFn: func(context.Context, string, bool) (containertypes.InspectResponse, []byte, error) { + inspect := baseInspect() + inspect.State = &containertypes.State{ + Running: !current.stopped, + StartedAt: epoch.Add(time.Duration(current.start-60) * time.Second).Format(time.RFC3339Nano), + } + return inspect, nil, nil + }, + containerStatsOneShotFn: func(context.Context, string) (dockerStatsResponseReader, error) { + read := "null" + if current.second >= 0 { + read = fmt.Sprintf("%q", epoch.Add(time.Duration(current.second)*time.Second).Format(time.RFC3339Nano)) + } + percent := "" + if current.percent != "" { + percent = `,"cpu":` + current.percent + } + payload := fmt.Sprintf(`{"read":%s,"cpu_stats":{"cpu_usage":{"total_usage":%d},"system_cpu_usage":%d,"online_cpus":%d%s},"memory_stats":{"usage":1000000,"limit":4000000}}`, + read, current.total, uint64(current.second+2)*1_000_000_000, tt.cpus, percent) + return dockerStatsResponseReader{Body: io.NopCloser(strings.NewReader(payload))}, nil + }, + }, + } + for i, s := range tt.samples { + current = s + container, err := agent.collectContainer(context.Background(), containertypes.Summary{ + ID: "container-123456", Names: []string{"/cpu-fixture"}, Image: "fixture:local", + }) + if err != nil { + t.Fatalf("sample %d: %v", i, err) + } + if math.Abs(container.CPUPercent-s.want) > 1e-9 { + t.Errorf("sample %d: CPU = %g%%, want %g%%", i, container.CPUPercent, s.want) + } + if !s.stopped && (container.MemoryUsageBytes != 1_000_000 || container.MemoryLimitBytes != 4_000_000) { + t.Fatalf("sample %d: memory collection changed: %+v", i, container) + } + } + }) + } +} diff --git a/internal/dockeragent/collect.go b/internal/dockeragent/collect.go index 28c8b92874..9ca434c9f8 100644 --- a/internal/dockeragent/collect.go +++ b/internal/dockeragent/collect.go @@ -601,10 +601,14 @@ func (a *Agent) collectContainer(ctx context.Context, summary containertypes.Sum return agentsdocker.Container{}, fmt.Errorf("decode stats: %w", err) } - if a.runtime == RuntimePodman && podmanCPUPercent != nil { - cpuPercent = safeFloat(*podmanCPUPercent) - } else { - cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats) + // Keep a baseline even when Podman supplies a percentage: one-shot + // compatibility responses can report cpu:0 while total_usage advances. + // Preserve positive native percentages (and their units, #1391), but + // use our interval delta for an absent, zero or invalid percentage. + cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats, parseTime(inspect.State.StartedAt)) + if a.runtime == RuntimePodman && podmanCPUPercent != nil && + *podmanCPUPercent > 0 && !math.IsInf(*podmanCPUPercent, 0) { + cpuPercent = *podmanCPUPercent } memUsage, memLimit, memPercent = calculateMemoryUsage(stats) blockIO = summarizeBlockIO(stats) @@ -1138,7 +1142,7 @@ func decodeContainerStatsPayload(payload []byte) (containertypes.StatsResponse, return stats, probe.CPUStats.CPU, nil } -func (a *Agent) calculateContainerCPUPercent(containerID string, stats containertypes.StatsResponse) float64 { +func (a *Agent) calculateContainerCPUPercent(containerID string, stats containertypes.StatsResponse, startedAt time.Time) float64 { a.cpuMu.Lock() defer a.cpuMu.Unlock() @@ -1147,6 +1151,7 @@ func (a *Agent) calculateContainerCPUPercent(containerID string, stats container systemUsage: stats.CPUStats.SystemUsage, onlineCPUs: stats.CPUStats.OnlineCPUs, read: stats.Read, + startedAt: startedAt, } // Always use manual delta tracking. Docker's PreCPUStats is unreliable for @@ -1155,6 +1160,14 @@ func (a *Agent) calculateContainerCPUPercent(containerID string, stats container // (from container start) and producing a constant lifetime-average CPU% // instead of a current value. prev, ok := a.prevContainerCPU[containerID] + if a.runtime == RuntimePodman && ok && !prev.read.IsZero() && !current.read.IsZero() && !current.read.After(prev.read) { + // Repeated or out-of-order responses must not move the baseline back + // and manufacture usage in the next interval. + return 0 + } + if a.prevContainerCPU == nil { + a.prevContainerCPU = make(map[string]cpuSample) + } if !ok { // First time seeing this container - store current sample and return 0 // On next collection cycle we'll have a previous sample to compare against @@ -1169,6 +1182,12 @@ func (a *Agent) calculateContainerCPUPercent(containerID string, stats container // We have a previous sample - update it after calculation a.prevContainerCPU[containerID] = current + if a.runtime == RuntimePodman && (current.totalUsage < prev.totalUsage || + (!prev.startedAt.IsZero() && !current.startedAt.IsZero() && !current.startedAt.Equal(prev.startedAt))) { + // A restart is not an interval sample, even if the new lifetime counter + // has already exceeded the old one. Start again from this baseline. + return 0 + } var totalDelta float64 if current.totalUsage >= prev.totalUsage {