Integrate reviewed truthful guest filesystem counters

Retain exact reviewed Core commit 6dcff07343. Its dedicated regression tests, both parent-production controls, full Proxmox race result and focused monitoring race/vet receipts support the existing counter invariant repair. The broad monitoring timeout remains adverse evidence, not whole-suite acceptance; native QEMU-only recovery and containing publication remain separate.

Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
2026-10-07 14:26:04 +01:00
6 changed files with 402 additions and 19 deletions
@@ -230,6 +230,25 @@ establish the reporter's native payload, installed cause or containing release.
### Filesystem collection precedes optional guest reads — discussion #2538
Filesystem byte observations do not default absent, null, negative or
fractional usage to a healthy zero. Positive capacity requires an explicit
integral used-byte count within capacity; counters and the deduplicated guest
aggregate must fit the signed disk model. Numeric strings, integral decimal
and exponent forms, privileged-total fallback, genuine zero/full readings and
Windows mount/device metadata retain their existing support. Exact integer
bytes do not round through a JSON float, including object-style replies.
Ambiguous duplicate/case-conflicting byte fields are not usage evidence.
Malformed array rows remain isolated from usable peers. If no usable reading
survives, the existing unavailable/previous-reading path remains in force.
An unrepresentable aggregate is unavailable, not a wrapped or silently partial
total; invalid counters are not labelled as special mounts. These completed
replies do not alter command admission, locks, retries, cooldown or backup
behaviour. `guest_filesystem_bytes_test.go` in both Proxmox and monitoring
covers exact counters, wire/object/peer admission, signed arithmetic and an
ordinary later poll with an explicit healthy zero. This establishes source
reading integrity, not native QEMU-only recovery or independent thaw.
Both the detailed-status and recent-evidence/no-status QEMU builders collect
filesystem usage before optional Linux `/proc/meminfo` and guest metadata.
Windows/Android guests may answer filesystem queries without supporting every
@@ -0,0 +1,110 @@
package monitoring
import (
"context"
"fmt"
"math"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
func TestVMFilesystemByteSummaryRejectsInvalidCounters(t *testing.T) {
for name, bad := range map[string]proxmox.VMFileSystem{
"overused": {TotalBytes: 1000, UsedBytes: 1001},
"signed-overflow": {TotalBytes: uint64(math.MaxInt64) + 1, UsedBytes: 400},
"unsigned-limit": {TotalBytes: math.MaxUint64, UsedBytes: math.MaxUint64},
} {
t.Run(name, func(t *testing.T) {
bad.Mountpoint = "/bad"
bad.Type = "ext4"
bad.Disk = "bad-device"
good := proxmox.VMFileSystem{Mountpoint: "/", Type: "ext4", Disk: "good-device", TotalBytes: 1000, UsedBytes: 400}
summary := (&Monitor{}).summarizeVMFSInfo("fixture", proxmox.ClusterResource{}, []proxmox.VMFileSystem{bad, good})
if summary.totalBytes != 1000 || summary.usedBytes != 400 || len(summary.individualDisks) != 1 || summary.individualDisks[0].Free != 600 || summary.individualDisks[0].Mountpoint != "/" {
t.Fatalf("invalid counter contaminated summary or valid peer: %+v", summary)
}
})
}
}
func TestVMFilesystemByteSummaryCannotOverflowSignedModel(t *testing.T) {
for name, totals := range map[string][]uint64{
"signed-sum": {math.MaxInt64, 1},
"unsigned-sum": {math.MaxInt64, math.MaxInt64, 2},
} {
t.Run(name, func(t *testing.T) {
var fs []proxmox.VMFileSystem
for i, total := range totals {
fs = append(fs, proxmox.VMFileSystem{Mountpoint: fmt.Sprintf("/disk%d", i), Type: "ext4", Disk: fmt.Sprint(i), TotalBytes: total, UsedBytes: 0})
}
summary := (&Monitor{}).summarizeVMFSInfo("fixture", proxmox.ClusterResource{}, fs)
if summary.totalBytes != 0 || summary.usedBytes != 0 || len(summary.individualDisks) != 0 {
t.Fatalf("overflow became a partial/wrapped aggregate: %+v", summary)
}
})
}
// The exact signed boundary remains usable; a duplicate mount is counted
// only once, and distinct capacity is not clamped into this boundary.
fs := []proxmox.VMFileSystem{
{Mountpoint: "/a", Type: "ext4", Disk: "a", TotalBytes: math.MaxInt64 - 1, UsedBytes: math.MaxInt64 - 2},
{Mountpoint: "/bind", Type: "ext4", Disk: "a", TotalBytes: math.MaxInt64 - 1, UsedBytes: math.MaxInt64 - 2},
{Mountpoint: "/b", Type: "ext4", Disk: "b", TotalBytes: 1, UsedBytes: 1},
}
summary := (&Monitor{}).summarizeVMFSInfo("fixture", proxmox.ClusterResource{}, fs)
if summary.totalBytes != math.MaxInt64 || summary.usedBytes != math.MaxInt64-1 || len(summary.individualDisks) != 3 {
t.Fatalf("valid signed boundary/dedup changed: %+v", summary)
}
}
func TestVMFilesystemInvalidWireReadingDoesNotBecomeHealthyZero(t *testing.T) {
for name, bad := range map[string]string{
"missing-used": `[{"mountpoint":"/","type":"ext4","total-bytes":1000}]`,
"negative-used": `[{"mountpoint":"/","type":"ext4","total-bytes":1000,"used-bytes":-1}]`,
"overused": `[{"mountpoint":"/","type":"ext4","total-bytes":1000,"used-bytes":1001}]`,
"aggregate-overflow": `[{"mountpoint":"/a","type":"ext4","total-bytes":9223372036854775807,"used-bytes":1},{"mountpoint":"/b","type":"ext4","total-bytes":1,"used-bytes":0}]`,
} {
t.Run(name, func(t *testing.T) {
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/config") {
fmt.Fprint(w, `{"data":{}}`)
return
}
if !strings.HasSuffix(r.URL.Path, "/agent/get-fsinfo") {
t.Errorf("unexpected request: %s", r.URL.Path)
http.NotFound(w, r)
return
}
if calls.Add(1) == 1 {
fmt.Fprintf(w, `{"data":{"result":%s}}`, bad)
return
}
fmt.Fprint(w, `{"data":{"result":[{"mountpoint":"/","type":"ext4","total-bytes":1000,"used-bytes":0}]}}`)
}))
defer server.Close()
c, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture"})
if err != nil {
t.Fatal(err)
}
m := &Monitor{}
res := proxmox.ClusterResource{Node: "node", VMID: 105, MaxDisk: 1000}
total, used, free, usage, disks, fromAgent, reason := m.updateVMDisksFromGuestAgentFSInfo(context.Background(), "fixture", res, c, 1000, 0, 0)
if total != 1000 || used != 0 || free != 1000 || usage != -1 || len(disks) != 0 || fromAgent || reason == "" {
t.Fatalf("invalid wire reading was published as usage: %d/%d/%d %v %+v %v %q", total, used, free, usage, disks, fromAgent, reason)
}
if name == "aggregate-overflow" && reason != "agent-error" {
t.Fatalf("invalid aggregate was misreported as special mounts: %q", reason)
}
// A later ordinary poll with an explicit zero is genuinely current usage.
total, used, free, usage, disks, fromAgent, reason = m.updateVMDisksFromGuestAgentFSInfo(context.Background(), "fixture", res, c, 1000, 0, 0)
if total != 1000 || used != 0 || free != 1000 || usage != 0 || len(disks) != 1 || !fromAgent || reason != "" || calls.Load() != 2 {
t.Fatalf("ordinary corrected reading did not recover: %d/%d/%d %v %+v %v %q calls=%d", total, used, free, usage, disks, fromAgent, reason, calls.Load())
}
})
}
}
@@ -3,6 +3,7 @@ package monitoring
import (
"context"
"fmt"
"math"
"strings"
"time"
@@ -573,6 +574,7 @@ type vmFSInfoSummary struct {
individualDisks []models.Disk
skippedFS []string
includedFS []string
invalidBytes bool
}
func (m *Monitor) fetchVMFSInfo(ctx context.Context, instanceName string, res proxmox.ClusterResource, client PVEClientInterface) ([]proxmox.VMFileSystem, string, bool) {
@@ -645,6 +647,14 @@ func (m *Monitor) summarizeVMFSInfo(instanceName string, res proxmox.ClusterReso
Msg("Processing filesystems from guest agent")
for _, fs := range fsInfo {
// Other PVEClientInterface implementations can supply records without
// the wire decoder. Never subtract/cast contradictory or unrepresentable
// counters into the signed disk model, even in an individual mount row.
if fs.UsedBytes > fs.TotalBytes || fs.TotalBytes > math.MaxInt64 {
summary.invalidBytes = true
summary.skippedFS = append(summary.skippedFS, fs.Mountpoint+"(invalid-byte-counts)")
continue
}
// Skip special filesystems and mounts
shouldSkip, reasons := fsfilters.ShouldSkipFilesystem(fs.Type, fs.Mountpoint, fs.TotalBytes, fs.UsedBytes)
if shouldSkip {
@@ -716,6 +726,11 @@ func (m *Monitor) summarizeVMFSInfo(instanceName string, res proxmox.ClusterReso
}
if countThisFS {
if fs.TotalBytes > math.MaxInt64-summary.totalBytes {
// A partial aggregate would look like complete guest usage.
// Leave it unavailable rather than wrap or silently drop a disk.
return vmFSInfoSummary{invalidBytes: true}
}
summary.totalBytes += fs.TotalBytes
summary.usedBytes += fs.UsedBytes
}
@@ -854,7 +869,8 @@ func (m *Monitor) updateVMDisksFromGuestAgentFSInfo(
return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, summary.individualDisks, true, ""
}
// Only special filesystems found - show allocated disk size instead
// No usable aggregate - show allocated disk size instead. Invalid counters
// are a reading failure, not evidence that every filesystem is special.
if diskTotal > 0 {
diskUsage = -1 // Show as allocated size
}
@@ -862,8 +878,12 @@ func (m *Monitor) updateVMDisksFromGuestAgentFSInfo(
Str("instance", instanceName).
Str("vm", res.Name).
Int("filesystems_found", len(fsInfo)).
Msg("Guest agent provided filesystem info but no usable filesystems found (all were special mounts)")
Bool("invalid_byte_counts", summary.invalidBytes).
Msg("Guest agent provided filesystem info but no usable disk aggregate was available")
if summary.invalidBytes {
return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, nil, false, "agent-error"
}
return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, nil, false, "special-filesystems-only"
}
+29 -17
View File
@@ -1702,14 +1702,25 @@ type VMFileSystem struct {
}
func (fs *VMFileSystem) UnmarshalJSON(data []byte) error {
fields, valid := guestAgentResponseFields(data)
if !valid {
return fmt.Errorf("guest filesystem record is ambiguous or malformed")
}
for name := range fields {
for _, canonical := range []string{"total-bytes", "total-bytes-privileged", "used-bytes"} {
if name != canonical && strings.EqualFold(name, canonical) {
return fmt.Errorf("guest filesystem byte count has a noncanonical field name")
}
}
}
type rawVMFileSystem struct {
Name string `json:"name"`
Type string `json:"type"`
Mountpoint string `json:"mountpoint"`
TotalBytes interface{} `json:"total-bytes"`
TotalBytesPrivileged interface{} `json:"total-bytes-privileged"`
UsedBytes interface{} `json:"used-bytes"`
DiskRaw interface{} `json:"disk"`
Name string `json:"name"`
Type string `json:"type"`
Mountpoint string `json:"mountpoint"`
TotalBytes json.RawMessage `json:"total-bytes"`
TotalBytesPrivileged json.RawMessage `json:"total-bytes-privileged"`
UsedBytes json.RawMessage `json:"used-bytes"`
DiskRaw interface{} `json:"disk"`
}
var raw rawVMFileSystem
@@ -1717,21 +1728,24 @@ func (fs *VMFileSystem) UnmarshalJSON(data []byte) error {
return err
}
total, err := parseUint64Flexible(raw.TotalBytes)
total, err := parseVMFilesystemBytes(raw.TotalBytes, false)
if err != nil {
return err
}
totalPrivileged, err := parseUint64Flexible(raw.TotalBytesPrivileged)
totalPrivileged, err := parseVMFilesystemBytes(raw.TotalBytesPrivileged, false)
if err != nil {
return err
}
used, err := parseUint64Flexible(raw.UsedBytes)
used, err := parseVMFilesystemBytes(raw.UsedBytes, total > 0 || totalPrivileged > 0)
if err != nil {
return err
}
if total == 0 && totalPrivileged > 0 {
total = totalPrivileged
}
if used > total {
return fmt.Errorf("guest filesystem used bytes exceed total bytes")
}
fs.Name = raw.Name
fs.Type = raw.Type
@@ -2060,11 +2074,12 @@ func (c *Client) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]VMFi
// If that fails, try as an object (might be an error response or different format)
var objectResult struct {
Data struct {
Result interface{} `json:"result"`
Result json.RawMessage `json:"result"`
} `json:"data"`
}
if err := json.Unmarshal(bodyBytes, &objectResult); err == nil {
if objectResult.Data.Result == nil {
rawFS := bytes.TrimSpace(objectResult.Data.Result)
if len(rawFS) == 0 || bytes.Equal(rawFS, []byte("null")) {
log.Debug().
Str("node", node).
Int("vmid", vmid).
@@ -2072,11 +2087,8 @@ func (c *Client) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]VMFi
return []VMFileSystem{}, nil
}
if fsMap, ok := objectResult.Data.Result.(map[string]interface{}); ok && looksLikeVMFilesystemResult(fsMap) {
rawFS, marshalErr := json.Marshal(fsMap)
if marshalErr != nil {
return nil, fmt.Errorf("failed to marshal object-style guest filesystem result: %w", marshalErr)
}
var fsMap map[string]interface{}
if json.Unmarshal(rawFS, &fsMap) == nil && looksLikeVMFilesystemResult(fsMap) {
var fs VMFileSystem
if unmarshalErr := json.Unmarshal(rawFS, &fs); unmarshalErr != nil {
return nil, fmt.Errorf("failed to parse object-style guest filesystem result: %w", unmarshalErr)
+83
View File
@@ -0,0 +1,83 @@
package proxmox
import (
"bytes"
"encoding/json"
"fmt"
"strconv"
"strings"
)
// Filesystem usage is an observation, not a defaultable configuration value.
// Preserve numeric/string compatibility without rounding, truncating negative
// or fractional readings, or converting absent usage into a healthy zero.
// The signed limit matches models.Disk, where these counters are published.
func parseVMFilesystemBytes(raw json.RawMessage, required bool) (uint64, error) {
raw = bytes.TrimSpace(raw)
if len(raw) == 0 || bytes.Equal(raw, []byte("null")) {
if !required {
return 0, nil
}
return 0, fmt.Errorf("guest filesystem byte count is missing")
}
s := string(raw)
if raw[0] == '"' {
if err := json.Unmarshal(raw, &s); err != nil {
return 0, err
}
s = strings.TrimSpace(s)
}
invalid := fmt.Errorf("guest filesystem byte count is not a non-negative signed-range integer")
if s == "" || strings.HasPrefix(s, "-") {
return 0, invalid
}
if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") {
return strconv.ParseUint(s[2:], 16, 63)
}
s = strings.TrimPrefix(s, "+")
var exponent int64
if i := strings.IndexAny(s, "eE"); i >= 0 {
var err error
exponent, err = strconv.ParseInt(s[i+1:], 10, 32)
if err != nil {
return 0, invalid
}
s = s[:i]
}
var fractionalDigits int64
if i := strings.IndexByte(s, '.'); i >= 0 {
fractionalDigits = int64(len(s) - i - 1)
s = s[:i] + s[i+1:]
}
if s == "" {
return 0, invalid
}
for _, digit := range s {
if digit < '0' || digit > '9' {
return 0, invalid
}
}
s = strings.TrimLeft(s, "0")
if s == "" {
return 0, nil // An explicitly reported zero, including 0.0 or 0e3.
}
shift := exponent - fractionalDigits
if shift < 0 {
// Only an integral decimal is a byte count; do not truncate fractions.
if -shift >= int64(len(s)) {
return 0, invalid
}
end := int64(len(s)) + shift
if strings.Trim(s[end:], "0") != "" {
return 0, invalid
}
s = s[:end]
} else {
// Bound expansion before allocating, even for an enormous exponent.
if int64(len(s))+shift > 19 {
return 0, invalid
}
s += strings.Repeat("0", int(shift))
}
return strconv.ParseUint(s, 10, 63)
}
+139
View File
@@ -0,0 +1,139 @@
package proxmox
import (
"context"
"encoding/json"
"fmt"
"math"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
)
func TestVMFilesystemBytesRejectUnknownOrContradictoryUsage(t *testing.T) {
for name, counters := range map[string]string{
"missing-used": `"total-bytes":1000`,
"null-used": `"total-bytes":1000,"used-bytes":null`,
"empty-used": `"total-bytes":1000,"used-bytes":""`,
"negative-used": `"total-bytes":1000,"used-bytes":-1`,
"negative-string": `"total-bytes":1000,"used-bytes":"-1.0"`,
"fractional-used": `"total-bytes":1000,"used-bytes":0.5`,
"fractional-string": `"total-bytes":1000,"used-bytes":"1e-1"`,
"used-exceeds-total": `"total-bytes":1000,"used-bytes":1001`,
"signed-overflow": `"total-bytes":9223372036854775808,"used-bytes":1`,
"uint-overflow": `"total-bytes":18446744073709551616,"used-bytes":1`,
"exponent-overflow": `"total-bytes":"1e1000000","used-bytes":1`,
"invalid-zero-exponent": `"total-bytes":1000,"used-bytes":"0e9999999999999999"`,
"negative-total": `"total-bytes":-1000,"used-bytes":0`,
"fractional-total": `"total-bytes":"1000.5","used-bytes":0`,
"invalid-privileged": `"total-bytes":1000,"total-bytes-privileged":-1,"used-bytes":0`,
"missing-privileged-used": `"total-bytes-privileged":1000`,
"null-privileged-used": `"total-bytes":0,"total-bytes-privileged":1000,"used-bytes":null`,
"missing-total-nonzero-used": `"used-bytes":10`,
"duplicate-used": `"total-bytes":1000,"used-bytes":1001,"used-bytes":0`,
"escaped-duplicate": `"total-bytes":1000,"used-bytes":1001,"\u0075sed-bytes":0`,
"case-conflict": `"total-bytes":1000,"used-bytes":1001,"Used-Bytes":0`,
"case-only": `"total-bytes":1000,"Used-Bytes":0`,
"duplicate-total": `"total-bytes":1,"total-bytes":1000,"used-bytes":100`,
"case-total": `"Total-Bytes":1000,"used-bytes":0`,
"bool-used": `"total-bytes":1000,"used-bytes":false`,
"nan-used": `"total-bytes":1000,"used-bytes":"NaN"`,
} {
t.Run(name, func(t *testing.T) {
var fs VMFileSystem
if err := json.Unmarshal([]byte(`{"mountpoint":"/","type":"ext4",`+counters+`}`), &fs); err == nil {
t.Fatalf("untrustworthy byte counters became a filesystem reading: %+v", fs)
}
})
}
}
func TestVMFilesystemBytesPreserveExactCompatibleReadings(t *testing.T) {
for name, tc := range map[string]struct {
counters string
total, used uint64
}{
"integer": {`"total-bytes":1000,"used-bytes":400`, 1000, 400},
"explicit-zero": {`"total-bytes":1000,"used-bytes":0`, 1000, 0},
"full": {`"total-bytes":1000,"used-bytes":1000`, 1000, 1000},
"numeric-strings": {`"total-bytes":" 1000 ","used-bytes":"400"`, 1000, 400},
"decimal": {`"total-bytes":1000.0,"used-bytes":"400.00"`, 1000, 400},
"scientific": {`"total-bytes":1e3,"used-bytes":"4.00e2"`, 1000, 400},
"small-exponent": {`"total-bytes":"100000e-2","used-bytes":"4000e-1"`, 1000, 400},
"hexadecimal": {`"total-bytes":"0x3e8","used-bytes":"0X190"`, 1000, 400},
"large-exact-integer": {`"total-bytes":9007199254740993,"used-bytes":9007199254740991`, 9007199254740993, 9007199254740991},
"large-exact-decimal": {`"total-bytes":"9007199254740993.0","used-bytes":"9007199254740991.0"`, 9007199254740993, 9007199254740991},
"signed-boundary": {`"total-bytes":9223372036854775807,"used-bytes":9223372036854775806`, math.MaxInt64, math.MaxInt64 - 1},
"privileged-fallback": {`"total-bytes":null,"total-bytes-privileged":1000,"used-bytes":400`, 1000, 400},
"zero-total-fallback": {`"total-bytes":0,"total-bytes-privileged":"1000.0","used-bytes":400`, 1000, 400},
"unformatted": {`"total-bytes":0`, 0, 0},
"metadata-only": {`"disk":[{"dev":"/dev/vda"}]`, 0, 0},
} {
t.Run(name, func(t *testing.T) {
var fs VMFileSystem
if err := json.Unmarshal([]byte(`{"mountpoint":"/","type":"ext4",`+tc.counters+`}`), &fs); err != nil {
t.Fatal(err)
}
if fs.TotalBytes != tc.total || fs.UsedBytes != tc.used {
t.Fatalf("bytes = %d/%d, want %d/%d", fs.TotalBytes, fs.UsedBytes, tc.total, tc.used)
}
})
}
}
func TestVMFilesystemBytesWireAdmissionKeepsValidPeersAndObjectPrecision(t *testing.T) {
for name, tc := range map[string]struct {
result string
total, used uint64
wantError bool
}{
"array-peers": {`[{"mountpoint":"/bad","type":"ext4","total-bytes":1000},{"mountpoint":"/negative","type":"ext4","total-bytes":1000,"used-bytes":-1},{"mountpoint":"/overfull","type":"ext4","total-bytes":1000,"used-bytes":1001},{"mountpoint":"/","type":"ext4","total-bytes":1000,"used-bytes":0}]`, 1000, 0, false},
"object-precision": {`{"mountpoint":"C:\\","type":"ntfs","total-bytes":9007199254740993,"used-bytes":9007199254740991,"disk":{"bus-type":"scsi","target":2}}`, 9007199254740993, 9007199254740991, false},
"object-duplicate": {`{"mountpoint":"C:\\","type":"ntfs","total-bytes":1000,"used-bytes":1001,"used-bytes":0}`, 0, 0, true},
"object-missing": {`{"mountpoint":"C:\\","type":"ntfs","total-bytes":1000}`, 0, 0, true},
} {
t.Run(name, func(t *testing.T) {
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/config") {
fmt.Fprint(w, `{"data":{}}`)
return
}
calls.Add(1)
if strings.HasSuffix(r.URL.Path, "get-fsinfo") {
fmt.Fprintf(w, `{"data":{"result":%s}}`, tc.result)
return
}
backupAgentPayload(w, r)
}))
defer server.Close()
c := backupTestClient(t, server.URL)
readings, err := c.GetVMFSInfo(context.Background(), "node", 105)
if tc.wantError {
if err == nil || len(readings) != 0 {
t.Fatalf("invalid object admitted: %+v %v", readings, err)
}
} else {
if err != nil || len(readings) != 1 {
t.Fatalf("valid readings lost: %+v %v", readings, err)
}
if readings[0].TotalBytes != tc.total || readings[0].UsedBytes != tc.used {
t.Fatalf("wire bytes rounded or defaulted: %+v", readings[0])
}
if name == "object-precision" && readings[0].Disk != "scsi-2" {
t.Fatalf("disk metadata compatibility changed: %+v", readings[0])
}
}
// Bad counters are a completed reply with unusable readings. They do not
// change serial-command admission, replay the request or start a cooldown.
if _, err := c.GetVMAgentVersion(context.Background(), "node", 105); err != nil {
t.Fatal(err)
}
if calls.Load() != 2 {
t.Fatalf("wire requests = %d, want two single attempts", calls.Load())
}
})
}
}