gsadmin/pulse
1
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-10-24 04:33:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Keep Podman CPU interval evidence when one-shot percentages are zero

Browse Source 
Retain a baseline on every response and fall back to interval counters for zero, absent or invalid compatibility percentages. Preserve positive native units and Docker calculations, rebaseline on restarts and reject out-of-order intervals. Add connected collector controls and the matching monitoring and agent contracts. Addresses the source gap reported in #2447; native payload and reporter acceptance remain unverified.

Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
2026-10-03 23:57:05 +01:00
parent a30758c33a
commit ab978b6cfc
6 changed files with 200 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+15
View File
@@ -15,6 +15,21 @@
## Purpose
### Podman interval CPU evidence — issue #2447
Every one-shot container stats response retains a cumulative CPU baseline,
even when a positive Podman compatibility percentage takes precedence. Zero,
absent or invalid percentages use the valid nanosecond/wall-clock interval,
without Docker's host-core multiplier. First samples, counter resets and changes
of container start time rebaseline rather than presenting lifetime CPU as current
usage. Repeated or older read timestamps cannot move the baseline backwards.
Positive native percentages keep the units established by #1391; a genuine idle
counter interval remains zero. Report shapes, enrollment, runtime transport trust
and command authority are unchanged. `TestCollectContainerPodmanCPUIntervals`
in `internal/dockeragent/agent_internal_test.go` exercises decoding through the
real collector, idle/advancing counters, start changes and Docker isolation.
Synthetic controls do not establish a native Podman payload or reporter recovery.
### MD RAID required members and spares — issue #2369
Host RAID reports carry optional `requiredDevices`: the configured member count
docs/release-control/v6/internal/subsystems/monitoring.md
+14
View File
@@ -17,6 +17,20 @@
## Purpose
### Podman zero-percentage fallback — issue #2447
One-shot compatibility stats with `cpu:0` do not suppress advancing cumulative
CPU samples. The collector records every interval and uses Podman's CPU-time
nanoseconds over elapsed wall-clock nanoseconds, independent of host core count.
A positive finite reported percentage remains authoritative, preserving #1391.
Counter resets or a changed container start establish a new baseline; duplicate
and out-of-order reads cannot create subsequent spikes. No timestamp means no
measurable counter interval. The existing report CPU field, graph/alert units,
Docker system-counter calculation, memory and permission scopes are unchanged.
`TestCollectContainerPodmanCPUIntervals` in `agent_internal_test.go` covers the
actual JSON/collector boundary, including changing percentage availability.
Native Podman shape and graph/alert acceptance remain separate evidence.
### Guest-agent coordination around backups — issue #2439
All PVE QEMU guest-agent reads, including disk diagnostics and legacy meminfo
internal/dockeragent/agent.go
+1
View File
@@ -229,6 +229,7 @@ type cpuSample struct {
systemUsage uint64
onlineCPUs uint32
read time.Time
startedAt time.Time
}
// New creates a new Docker / Podman module instance.
internal/dockeragent/agent_cpu_test.go
+12 -12
View File
@@ -28,7 +28,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
OnlineCPUs: 2,
},
}
got := agent.calculateContainerCPUPercent("container-123456", stats1)
got := agent.calculateContainerCPUPercent("container-123456", stats1, time.Time{})
if got != 0 {
t.Fatalf("first call: expected 0, got %f", got)
}
@@ -45,7 +45,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
OnlineCPUs: 2,
},
}
got = agent.calculateContainerCPUPercent("container-123456", stats2)
got = agent.calculateContainerCPUPercent("container-123456", stats2, time.Time{})
if got <= 0 {
t.Fatalf("second call: expected percent > 0, got %f", got)
}
@@ -68,7 +68,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
got := agent.calculateContainerCPUPercent("container-123456", stats)
got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if got != 0 {
t.Fatalf("expected 0, got %f", got)
}
@@ -103,7 +103,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
got := agent.calculateContainerCPUPercent("container-123456", stats)
got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if got <= 0 {
t.Fatalf("expected percent > 0, got %f", got)
}
@@ -136,7 +136,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
got := agent.calculateContainerCPUPercent("container-123456", stats)
got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if got <= 0 {
t.Fatalf("expected percent > 0, got %f", got)
}
@@ -168,7 +168,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
got := agent.calculateContainerCPUPercent("container-123456", stats)
got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if got <= 0 {
t.Fatalf("expected percent > 0, got %f", got)
}
@@ -200,7 +200,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
if got := agent.calculateContainerCPUPercent("container-123456", stats); got != 0 {
if got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}); got != 0 {
t.Fatalf("expected 0, got %f", got)
}
})
@@ -232,7 +232,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
if got := agent.calculateContainerCPUPercent("container-123456", stats); got != 0 {
if got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}); got != 0 {
t.Fatalf("expected 0, got %f", got)
}
})
@@ -263,7 +263,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
if got := agent.calculateContainerCPUPercent("container-123456", stats); got != 0 {
if got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}); got != 0 {
t.Fatalf("expected 0, got %f", got)
}
})
@@ -291,7 +291,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
PreCPUStats: containertypes.CPUStats{},
}
got := agent.calculateContainerCPUPercent("container-123456", stats)
got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if got != 0 {
t.Fatalf("expected 0, got %f", got)
}
@@ -319,7 +319,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
got := agent.calculateContainerCPUPercent("container-123456", stats)
got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if got != 0 {
t.Fatalf("expected 0 on first call (manual tracking), got %f", got)
}
@@ -349,7 +349,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
got := agent.calculateContainerCPUPercent("container-123456", stats)
got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if math.Abs(got-25.0) > 0.01 {
t.Fatalf("expected podman cpu percent near 25.0, got %f", got)
}
internal/dockeragent/agent_internal_test.go
+134
View File
@@ -1840,3 +1840,137 @@ func TestCollectorRuntimeBoundaryLossIsDaemonUnavailable(t *testing.T) {
}
}
}
// Exercise the compatibility JSON through the actual container collector, not
// just the delta helper. These are synthetic inputs, not a native Podman trace.
func TestCollectContainerPodmanCPUIntervals(t *testing.T) {
type sample struct {
second int
total uint64
percent string // Empty means absent; other values are literal JSON.
start int
stopped bool
want float64
}
tests := []struct {
name string
runtime RuntimeKind
cpus int
samples []sample
}{
{"zero with advancing and idle counters", RuntimePodman, 16, []sample{
{second: 0, total: 100_000_000, percent: "0"},
{second: 1, total: 130_000_000, percent: "0", want: 3},
{second: 2, total: 130_000_000, percent: "0"},
{second: 3, total: 1_630_000_000, percent: "0", want: 150},
}},
{"absent percentage uses nanosecond wall clock units", RuntimePodman, 1, []sample{
{second: 0, total: 100_000_000},
{second: 1, total: 130_000_000, want: 3},
}},
{"null percentage", RuntimePodman, 16, []sample{
{second: 0, total: 100_000_000, percent: "null"},
{second: 1, total: 130_000_000, percent: "null", want: 3},
}},
{"positive percentages keep units and retain the baseline", RuntimePodman, 16, []sample{
{second: 0, total: 100_000_000, percent: "0.32", want: 0.32},
{second: 1, total: 130_000_000, percent: "0.64", want: 0.64},
{second: 2, total: 160_000_000, percent: "0", want: 3},
{second: 3, total: 190_000_000, want: 3},
}},
{"absent then reported then zero", RuntimePodman, 16, []sample{
{second: 0, total: 100_000_000},
{second: 1, total: 130_000_000, percent: "0.32", want: 0.32},
{second: 2, total: 160_000_000, percent: "0", want: 3},
}},
{"invalid negative percentage", RuntimePodman, 16, []sample{
{second: 0, total: 100_000_000, percent: "-1"},
{second: 1, total: 130_000_000, percent: "-1", want: 3},
}},
{"unparseable optional percentage", RuntimePodman, 16, []sample{
{second: 0, total: 100_000_000, percent: `"unknown"`},
{second: 1, total: 130_000_000, percent: `"unknown"`, want: 3},
}},
{"counter reset rebaselines rather than using new lifetime", RuntimePodman, 16, []sample{
{second: 0, total: 100_000_000, percent: "0"},
{second: 1, total: 10_000_000, percent: "0"},
{second: 2, total: 40_000_000, percent: "0", want: 3},
}},
{"restart with an already larger counter", RuntimePodman, 16, []sample{
{second: 0, total: 100_000_000, percent: "0"},
{second: 1, total: 260_000_000, percent: "0", start: 1},
{second: 2, total: 290_000_000, percent: "0", start: 1, want: 3},
}},
{"repeated and old timestamps do not move the baseline", RuntimePodman, 16, []sample{
{second: 0, total: 100_000_000, percent: "0"},
{second: 1, total: 130_000_000, percent: "0", want: 3},
{second: 1, total: 150_000_000, percent: "0"},
{second: 0, total: 170_000_000, percent: "0"},
{second: 2, total: 160_000_000, percent: "0", want: 3},
}},
{"missing timestamp cannot invent an interval", RuntimePodman, 16, []sample{
{second: -1, total: 100_000_000, percent: "0"},
{second: 1, total: 130_000_000, percent: "0"},
{second: 2, total: 160_000_000, percent: "0", want: 3},
}},
{"stop clears the baseline", RuntimePodman, 16, []sample{
{second: 0, total: 100_000_000, percent: "0.32", want: 0.32},
{second: 1, stopped: true},
{second: 2, total: 260_000_000, percent: "0"},
{second: 3, total: 290_000_000, percent: "0", want: 3},
}},
{"docker ignores compat percentage and retains system delta", RuntimeDocker, 16, []sample{
{second: 0, total: 100_000_000, percent: "0.32"},
{second: 1, total: 130_000_000, percent: "0.32", want: 48},
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
epoch := time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC)
var current sample
agent := &Agent{
runtime: tt.runtime,
logger: zerolog.Nop(),
cpuCount: tt.cpus,
docker: &fakeDockerClient{
containerInspectWithRawFn: func(context.Context, string, bool) (containertypes.InspectResponse, []byte, error) {
inspect := baseInspect()
inspect.State = &containertypes.State{
Running: !current.stopped,
StartedAt: epoch.Add(time.Duration(current.start-60) * time.Second).Format(time.RFC3339Nano),
}
return inspect, nil, nil
},
containerStatsOneShotFn: func(context.Context, string) (dockerStatsResponseReader, error) {
read := "null"
if current.second >= 0 {
read = fmt.Sprintf("%q", epoch.Add(time.Duration(current.second)*time.Second).Format(time.RFC3339Nano))
}
percent := ""
if current.percent != "" {
percent = `,"cpu":` + current.percent
}
payload := fmt.Sprintf(`{"read":%s,"cpu_stats":{"cpu_usage":{"total_usage":%d},"system_cpu_usage":%d,"online_cpus":%d%s},"memory_stats":{"usage":1000000,"limit":4000000}}`,
read, current.total, uint64(current.second+2)*1_000_000_000, tt.cpus, percent)
return dockerStatsResponseReader{Body: io.NopCloser(strings.NewReader(payload))}, nil
},
},
}
for i, s := range tt.samples {
current = s
container, err := agent.collectContainer(context.Background(), containertypes.Summary{
ID: "container-123456", Names: []string{"/cpu-fixture"}, Image: "fixture:local",
})
if err != nil {
t.Fatalf("sample %d: %v", i, err)
}
if math.Abs(container.CPUPercent-s.want) > 1e-9 {
t.Errorf("sample %d: CPU = %g%%, want %g%%", i, container.CPUPercent, s.want)
}
if !s.stopped && (container.MemoryUsageBytes != 1_000_000 || container.MemoryLimitBytes != 4_000_000) {
t.Fatalf("sample %d: memory collection changed: %+v", i, container)
}
}
})
}
}
internal/dockeragent/collect.go
+24 -5
View File
@@ -601,10 +601,14 @@ func (a *Agent) collectContainer(ctx context.Context, summary containertypes.Sum
return agentsdocker.Container{}, fmt.Errorf("decode stats: %w", err)
}
if a.runtime == RuntimePodman && podmanCPUPercent != nil {
cpuPercent = safeFloat(*podmanCPUPercent)
} else {
cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats)
// Keep a baseline even when Podman supplies a percentage: one-shot
// compatibility responses can report cpu:0 while total_usage advances.
// Preserve positive native percentages (and their units, #1391), but
// use our interval delta for an absent, zero or invalid percentage.
cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats, parseTime(inspect.State.StartedAt))
if a.runtime == RuntimePodman && podmanCPUPercent != nil &&
*podmanCPUPercent > 0 && !math.IsInf(*podmanCPUPercent, 0) {
cpuPercent = *podmanCPUPercent
}
memUsage, memLimit, memPercent = calculateMemoryUsage(stats)
blockIO = summarizeBlockIO(stats)
@@ -1138,7 +1142,7 @@ func decodeContainerStatsPayload(payload []byte) (containertypes.StatsResponse,
return stats, probe.CPUStats.CPU, nil
}
func (a *Agent) calculateContainerCPUPercent(containerID string, stats containertypes.StatsResponse) float64 {
func (a *Agent) calculateContainerCPUPercent(containerID string, stats containertypes.StatsResponse, startedAt time.Time) float64 {
a.cpuMu.Lock()
defer a.cpuMu.Unlock()
@@ -1147,6 +1151,7 @@ func (a *Agent) calculateContainerCPUPercent(containerID string, stats container
systemUsage: stats.CPUStats.SystemUsage,
onlineCPUs: stats.CPUStats.OnlineCPUs,
read: stats.Read,
startedAt: startedAt,
}
// Always use manual delta tracking. Docker's PreCPUStats is unreliable for
@@ -1155,6 +1160,14 @@ func (a *Agent) calculateContainerCPUPercent(containerID string, stats container
// (from container start) and producing a constant lifetime-average CPU%
// instead of a current value.
prev, ok := a.prevContainerCPU[containerID]
if a.runtime == RuntimePodman && ok && !prev.read.IsZero() && !current.read.IsZero() && !current.read.After(prev.read) {
// Repeated or out-of-order responses must not move the baseline back
// and manufacture usage in the next interval.
return 0
}
if a.prevContainerCPU == nil {
a.prevContainerCPU = make(map[string]cpuSample)
}
if !ok {
// First time seeing this container - store current sample and return 0
// On next collection cycle we'll have a previous sample to compare against
@@ -1169,6 +1182,12 @@ func (a *Agent) calculateContainerCPUPercent(containerID string, stats container
// We have a previous sample - update it after calculation
a.prevContainerCPU[containerID] = current
if a.runtime == RuntimePodman && (current.totalUsage < prev.totalUsage ||
(!prev.startedAt.IsZero() && !current.startedAt.IsZero() && !current.startedAt.Equal(prev.startedAt))) {
// A restart is not an interval sample, even if the new lifetime counter
// has already exceeded the old one. Start again from this baseline.
return 0
}
var totalDelta float64
if current.totalUsage >= prev.totalUsage {