mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-26 21:47:20 +00:00
Keep QGA fallback eligibility bound to its original observation
Retained guest identity can survive a cancelled detail cycle or failed metadata refresh. Keep its original eligibility time through canonical snapshots instead of borrowing each VM poll timestamp, and preserve ordinary current-read recovery and the existing guest-command safety fences. Change-source: pulse-maintainer
This commit is contained in:
@@ -15,6 +15,16 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
### QGA query-eligibility provenance stays internal
|
||||
|
||||
The shared VM model's `GuestAgentEvidence` is monitor-owned source-time
|
||||
coordination, not agent input or a public field. It prevents retained guest
|
||||
identity from renewing fallback QGA eligibility. Agent enrollment, token and
|
||||
machine identity, reporting and linked Pulse-agent readings are unchanged.
|
||||
`TestGuestAgentEvidenceIsInternalSnapshotEvidence` checks snapshot isolation
|
||||
and omission from both model and frontend JSON.
|
||||
|
||||
|
||||
### QGA disk provenance stays outside the agent report contract
|
||||
|
||||
The shared VM model carries an internal filesystem source/time observation for
|
||||
|
||||
@@ -17,6 +17,25 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
### QGA fallback eligibility keeps its original observation age
|
||||
|
||||
A successful unlocked VM-status availability, usable filesystem reading or
|
||||
useful guest metadata can establish fallback query eligibility. A later poll
|
||||
that preserves optional IP/OS/version strings does not renew that evidence.
|
||||
Internal `GuestAgentEvidence` crosses model, canonical and previous-state
|
||||
snapshots, including an explicit missing origin. Legacy direct snapshots import
|
||||
their receipt once; expired, missing and future origins cannot admit a fallback.
|
||||
The existing ten-minute boundary, authoritative unavailable status, metadata
|
||||
backoff and every-command backup-lock/shared uncertainty fences remain intact.
|
||||
Cached display identity remains available, but cannot upgrade a failed/empty
|
||||
refresh to current availability. Linked Pulse-agent disks remain independent.
|
||||
`TestGuestAgentAdmissionEvidenceContract` checks the original-age boundaries,
|
||||
repeated canonical replacement and both cluster/node collectors through a
|
||||
cancelled detail cycle, unavailable status, missing status and ordinary recovery
|
||||
with measured-zero disk usage. This is synthetic source proof, not native thaw,
|
||||
QEMU-only Windows acceptance or reporter resolution.
|
||||
|
||||
|
||||
### QEMU disk fallbacks retain their original filesystem source time
|
||||
|
||||
A successful QGA filesystem read owns the disk observation's source and original
|
||||
|
||||
@@ -21,6 +21,19 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
### Retained guest identity cannot renew QGA eligibility
|
||||
|
||||
The shared VM's internal `GuestAgentEvidence` bounds eligibility when a fresh
|
||||
VM-status read is unavailable. Snapshot copy and canonical replacement keep
|
||||
its original time, including explicit missing evidence, without changing disk
|
||||
capacity, usage, History storage or linked-agent precedence. No JSON field,
|
||||
credential or command path is added.
|
||||
`TestGuestAgentEvidenceIsInternalSnapshotEvidence` checks the shared-model
|
||||
copy/wire boundary; `TestGuestAgentAdmissionEvidenceContract` covers expiry,
|
||||
unknown/future origins and ordinary zero-reading recovery through both collectors.
|
||||
Synthetic source proof does not establish native recovery or published availability.
|
||||
|
||||
|
||||
### Shared guest disk evidence is not recovery-point identity
|
||||
|
||||
The canonical Proxmox facet carries internal QGA filesystem source/time evidence
|
||||
|
||||
@@ -23,6 +23,18 @@ and sort the complete canonical change table while startup and ingestion wait.
|
||||
|
||||
## Purpose
|
||||
|
||||
### Internal QGA query-eligibility evidence
|
||||
|
||||
The native Proxmox guest facet carries `GuestAgentEvidence` internally. A full
|
||||
guest observation replaces this value, including explicit absence; a partial
|
||||
non-guest facet does not own it. Cloning and the VM view preserve the original
|
||||
time for the monitor's previous-state input. Resource/VM JSON, identity,
|
||||
correlation and linked Pulse-agent precedence do not change.
|
||||
`TestGuestAgentEvidenceDoesNotChangeWireOrIdentity` and
|
||||
`TestGuestAgentEvidenceMergeAndCloneKeepsOriginalAge` check wire equality,
|
||||
identity equality, clone isolation and loss/recovery replacement.
|
||||
|
||||
|
||||
### Internal guest disk observation provenance
|
||||
|
||||
A Proxmox VM facet preserves the successful filesystem read's original source
|
||||
|
||||
@@ -30,6 +30,27 @@ func TestGuestDiskObservationIsInternalSnapshotEvidence(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuestAgentEvidenceIsInternalSnapshotEvidence(t *testing.T) {
|
||||
origin := GuestAgentEvidence{Explicit: true, ObservedAt: time.Now().Add(-time.Minute)}
|
||||
vm := VM{ID: "pve:node:105", VMID: 105, Type: "qemu", GuestAgentEvidence: origin}
|
||||
copy := cloneVMs([]VM{vm})[0]
|
||||
if copy.GuestAgentEvidence != origin {
|
||||
t.Fatal("snapshot cloning lost original eligibility evidence")
|
||||
}
|
||||
copy.GuestAgentEvidence.ObservedAt = time.Now()
|
||||
if vm.GuestAgentEvidence != origin {
|
||||
t.Fatal("snapshot mutation changed source-owned evidence")
|
||||
}
|
||||
for name, value := range map[string]any{"model": vm, "frontend": vm.ToFrontend()} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
wire, err := json.Marshal(value)
|
||||
if err != nil || strings.Contains(string(wire), "Evidence") || strings.Contains(string(wire), "ObservedAt") {
|
||||
t.Fatalf("internal evidence leaked into the public/agent contract: %s / %v", wire, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuestMemoryObservationWireContract(t *testing.T) {
|
||||
legacy := Memory{Total: 100, Used: 25, Free: 75, Usage: 25}
|
||||
payload, err := json.Marshal(legacy)
|
||||
|
||||
@@ -262,6 +262,7 @@ type VM struct {
|
||||
Disks []Disk `json:"disks,omitempty"`
|
||||
DiskStatusReason string `json:"diskStatusReason,omitempty"` // Why disk stats are unavailable
|
||||
DiskObservation GuestDiskObservation `json:"-"` // Source-owned carry-forward evidence, not the latest poll time.
|
||||
GuestAgentEvidence GuestAgentEvidence `json:"-"` // Original eligibility evidence, independent of retained identity.
|
||||
GuestAgentStatus string `json:"guestAgentStatus,omitempty"` // QEMU guest-agent runtime status
|
||||
GuestAgentExpected bool `json:"guestAgentExpected,omitempty"`
|
||||
IPAddresses []string `json:"ipAddresses,omitempty"`
|
||||
@@ -3236,6 +3237,15 @@ type GuestDiskObservation struct {
|
||||
ObservedAt time.Time
|
||||
}
|
||||
|
||||
// GuestAgentEvidence bounds fallback query eligibility when VM status is
|
||||
// unavailable. Explicit distinguishes an observed lack of evidence from a
|
||||
// legacy snapshot; an ordinary VM poll must not renew retained metadata.
|
||||
// This is internal coordination, not agent input or public wire data.
|
||||
type GuestAgentEvidence struct {
|
||||
Explicit bool
|
||||
ObservedAt time.Time
|
||||
}
|
||||
|
||||
// Disk represents disk usage
|
||||
type Disk struct {
|
||||
Total int64 `json:"total"`
|
||||
|
||||
@@ -1131,6 +1131,12 @@ func TestGuestDiskObservationContract(t *testing.T) {
|
||||
t.Run("ordinary-deferral-expiry-recovery", testGuestDiskOrdinaryDeferralExpiryAndRecovery)
|
||||
}
|
||||
|
||||
func TestGuestAgentAdmissionEvidenceContract(t *testing.T) {
|
||||
t.Run("retained-identity-cannot-renew", testGuestAgentRetainedIdentityCannotRenewAdmission)
|
||||
t.Run("original-age", testGuestAgentEvidenceOriginalAge)
|
||||
t.Run("canonical-continuity", testGuestAgentEvidenceCanonicalContinuity)
|
||||
}
|
||||
|
||||
func TestProxmoxGuestDiskInventoryPrefersCanonicalLinkedHostAgentSource(t *testing.T) {
|
||||
requiredSnippets := map[string][]string{
|
||||
"guest_host_agent_fallback.go": {
|
||||
|
||||
@@ -11,30 +11,45 @@ import (
|
||||
const recentGuestAgentEvidenceMaxAge = 10 * time.Minute
|
||||
|
||||
func hasRecentGuestAgentEvidence(prev *models.VM, now time.Time) bool {
|
||||
// Previous unified read-state snapshots normalize guest status to resource health
|
||||
// (for example "online"), so continuity depends on recent guest-agent evidence
|
||||
// rather than on replaying an exact prior runtime status string.
|
||||
if prev == nil || prev.Type != "qemu" {
|
||||
return false
|
||||
}
|
||||
if prev.LastSeen.IsZero() || now.Sub(prev.LastSeen) > recentGuestAgentEvidenceMaxAge {
|
||||
return false
|
||||
}
|
||||
observedAt := guestAgentEvidenceTime(prev)
|
||||
return !observedAt.IsZero() && !observedAt.After(now) && now.Sub(observedAt) <= recentGuestAgentEvidenceMaxAge
|
||||
}
|
||||
|
||||
func guestAgentEvidenceTime(prev *models.VM) time.Time {
|
||||
if prev == nil || prev.Type != "qemu" {
|
||||
return time.Time{}
|
||||
}
|
||||
if prev.GuestAgentEvidence.Explicit || !prev.GuestAgentEvidence.ObservedAt.IsZero() {
|
||||
return prev.GuestAgentEvidence.ObservedAt
|
||||
}
|
||||
// Import a legacy direct snapshot's receipt once. Modern producers mark
|
||||
// evidence explicitly, including its absence, before preserving identity.
|
||||
// Its subsequent LastSeen can therefore never refresh this original time.
|
||||
if prev.AgentVersion != "" ||
|
||||
prev.GuestAgentStatus == "available" ||
|
||||
len(prev.IPAddresses) > 0 ||
|
||||
len(prev.NetworkInterfaces) > 0 ||
|
||||
prev.OSName != "" ||
|
||||
prev.OSVersion != "" {
|
||||
return true
|
||||
return prev.LastSeen
|
||||
}
|
||||
|
||||
if len(prev.Disks) > 0 && !strings.HasPrefix(prev.DiskStatusReason, "prev-") {
|
||||
return true
|
||||
return prev.LastSeen
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
return false
|
||||
func retainGuestAgentEvidence(prev *models.VM, now time.Time) models.GuestAgentEvidence {
|
||||
evidence := models.GuestAgentEvidence{Explicit: true}
|
||||
renewGuestAgentEvidence(&evidence, guestAgentEvidenceTime(prev), now)
|
||||
return evidence
|
||||
}
|
||||
|
||||
func renewGuestAgentEvidence(evidence *models.GuestAgentEvidence, observedAt, now time.Time) {
|
||||
if !observedAt.IsZero() && !observedAt.After(now) && observedAt.After(evidence.ObservedAt) {
|
||||
evidence.ObservedAt = observedAt
|
||||
}
|
||||
}
|
||||
|
||||
func shouldQueryGuestAgent(vmStatus *proxmox.VMStatus, prev *models.VM, now time.Time) bool {
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
|
||||
)
|
||||
|
||||
func testGuestAgentEvidenceOriginalAge(t *testing.T) {
|
||||
now := time.Date(2026, time.October, 9, 15, 0, 0, 0, time.UTC)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
evidence models.GuestAgentEvidence
|
||||
seen time.Time
|
||||
want bool
|
||||
}{
|
||||
{"recent", models.GuestAgentEvidence{Explicit: true, ObservedAt: now.Add(-time.Minute)}, now, true},
|
||||
{"inclusive-boundary", models.GuestAgentEvidence{Explicit: true, ObservedAt: now.Add(-recentGuestAgentEvidenceMaxAge)}, now, true},
|
||||
{"expired-with-fresh-poll", models.GuestAgentEvidence{Explicit: true, ObservedAt: now.Add(-recentGuestAgentEvidenceMaxAge - time.Nanosecond)}, now, false},
|
||||
{"explicitly-missing", models.GuestAgentEvidence{Explicit: true}, now, false},
|
||||
{"future-origin", models.GuestAgentEvidence{Explicit: true, ObservedAt: now.Add(time.Second)}, now, false},
|
||||
{"legacy-recent", models.GuestAgentEvidence{}, now.Add(-time.Minute), true},
|
||||
{"legacy-expired", models.GuestAgentEvidence{}, now.Add(-20 * time.Minute), false},
|
||||
{"legacy-future", models.GuestAgentEvidence{}, now.Add(time.Second), false},
|
||||
{"legacy-missing", models.GuestAgentEvidence{}, time.Time{}, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
prev := models.VM{Type: "qemu", LastSeen: tc.seen, AgentVersion: "cached", IPAddresses: []string{"192.0.2.10"}, GuestAgentStatus: "available", GuestAgentEvidence: tc.evidence}
|
||||
if got := hasRecentGuestAgentEvidence(&prev, now); got != tc.want {
|
||||
t.Errorf("eligibility=%t want=%t", got, tc.want)
|
||||
}
|
||||
retained := retainGuestAgentEvidence(&prev, now)
|
||||
prev.LastSeen = now
|
||||
prev.GuestAgentEvidence = retained
|
||||
if got := hasRecentGuestAgentEvidence(&prev, now); got != tc.want {
|
||||
t.Errorf("new poll changed eligibility=%t want=%t", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
key := guestMetadataCacheKey("pve", "node", 105)
|
||||
m := &Monitor{guestMetadataCache: map[string]guestMetadataCacheEntry{key: metadataObservationFixture(now.Add(time.Second))}}
|
||||
if m.hasRecentGuestMetadataEvidence("pve", "node", 105, now) {
|
||||
t.Fatal("future metadata admitted a guest query")
|
||||
}
|
||||
}
|
||||
|
||||
func testGuestAgentEvidenceCanonicalContinuity(t *testing.T) {
|
||||
origin := time.Date(2026, time.October, 9, 15, 0, 0, 0, time.UTC)
|
||||
vm := models.VM{ID: "pve:node:105", Instance: "pve", Node: "node", VMID: 105, Type: "qemu", Status: "running", LastSeen: origin, AgentVersion: "cached", GuestAgentEvidence: models.GuestAgentEvidence{Explicit: true, ObservedAt: origin}}
|
||||
registry := unifiedresources.NewRegistry(nil)
|
||||
for minute := 1; minute <= 12; minute++ {
|
||||
now := origin.Add(time.Duration(minute) * time.Minute)
|
||||
vm.LastSeen = now
|
||||
registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
|
||||
vm = previousVMFromView(registry.VMs()[0])
|
||||
if vm.GuestAgentEvidence.ObservedAt != origin || hasRecentGuestAgentEvidence(&vm, now) != (minute <= 10) {
|
||||
t.Fatalf("minute %d renewed evidence or lost inclusive expiry: %+v", minute, vm.GuestAgentEvidence)
|
||||
}
|
||||
vm.GuestAgentEvidence = retainGuestAgentEvidence(&vm, now)
|
||||
}
|
||||
vm.GuestAgentEvidence = models.GuestAgentEvidence{Explicit: true}
|
||||
registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
|
||||
vm = previousVMFromView(registry.VMs()[0])
|
||||
if hasRecentGuestAgentEvidence(&vm, vm.LastSeen) {
|
||||
t.Fatal("canonical replacement resurrected unknown evidence")
|
||||
}
|
||||
// Fresh QGA/PVE evidence restores eligibility; linked Pulse-agent disks
|
||||
// alone cannot turn this evidence into an observation of QGA availability.
|
||||
now := origin.Add(13 * time.Minute)
|
||||
renewGuestAgentEvidence(&vm.GuestAgentEvidence, now, now)
|
||||
registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
|
||||
vm = previousVMFromView(registry.VMs()[0])
|
||||
if !hasRecentGuestAgentEvidence(&vm, now) {
|
||||
t.Fatal("new ordinary availability did not restore eligibility")
|
||||
}
|
||||
client := &linkedDiskPollClient{}
|
||||
m := &Monitor{rateTracker: NewRateTracker()}
|
||||
id := makeGuestID("pve", "node", 106)
|
||||
linked, _, _, _, _, ok := m.buildVMFromClusterResource(context.Background(), "pve", proxmox.ClusterResource{Type: "qemu", Node: "node", VMID: 106, Status: "running", MaxMem: 8000, MaxDisk: 1000}, client, id, map[string]models.Host{id: {Memory: models.Memory{Total: 8000, Used: 2000}, Disks: []models.Disk{{Total: 1000, Used: 250, Free: 750, Usage: 25}}, LastSeen: time.Now()}}, nil)
|
||||
if !ok || linked.DiskStatusReason != "" || linked.Disk.Usage != 25 || hasRecentGuestAgentEvidence(&linked, time.Now()) || client.fsCalls != 0 {
|
||||
t.Fatalf("linked agent changed QGA admission or lost independent disks: %+v", linked)
|
||||
}
|
||||
}
|
||||
|
||||
// This body also runs on the exact parent: optional identity preservation is
|
||||
// not a new guest-agent observation. All HTTP endpoints here are synthetic.
|
||||
func testGuestAgentRetainedIdentityCannotRenewAdmission(t *testing.T) {
|
||||
for _, collector := range []string{"cluster", "node"} {
|
||||
t.Run(collector, func(t *testing.T) {
|
||||
var phase, guestCalls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/config"):
|
||||
fmt.Fprint(w, `{"data":{}}`)
|
||||
case strings.HasSuffix(r.URL.Path, "/status/current"):
|
||||
switch phase.Load() {
|
||||
case 0:
|
||||
fmt.Fprint(w, `{"data":{"status":"running","agent":{"enabled":1,"available":0},"maxmem":8000,"meminfo":{"available":6000}}}`)
|
||||
case 1:
|
||||
http.Error(w, "permission denied", http.StatusForbidden)
|
||||
default:
|
||||
fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":8000,"meminfo":{"available":6000}}}`)
|
||||
}
|
||||
case strings.Contains(r.URL.Path, "/agent/"):
|
||||
guestCalls.Add(1)
|
||||
if phase.Load() != 2 {
|
||||
http.Error(w, "permission denied", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "get-fsinfo"):
|
||||
fmt.Fprint(w, `{"data":{"result":[{"mountpoint":"C:\\","type":"ntfs","total-bytes":1000,"used-bytes":0}]}}`)
|
||||
case strings.HasSuffix(r.URL.Path, "network-get-interfaces"):
|
||||
fmt.Fprint(w, `{"data":{"result":[{"name":"Ethernet","ip-addresses":[{"ip-address":"192.0.2.11","ip-address-type":"ipv4","prefix":24}]}]}}`)
|
||||
case strings.HasSuffix(r.URL.Path, "get-osinfo"):
|
||||
fmt.Fprint(w, `{"data":{"name":"Windows","version":"fixture"}}`)
|
||||
case strings.HasSuffix(r.URL.Path, "info"):
|
||||
fmt.Fprint(w, `{"data":{"result":{"version":"2.0"}}}`)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := &Monitor{config: &config.Config{}, state: models.NewState(), rateTracker: NewRateTracker(), metricsHistory: NewMetricsHistory(32, time.Hour), guestMetadataLimiter: make(map[string]time.Time)}
|
||||
m.alertManager = alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
|
||||
t.Cleanup(m.alertManager.Stop)
|
||||
m.guestMetadataRetryBackoff = time.Nanosecond
|
||||
id := makeGuestID("evidence", "node", 105)
|
||||
prev := models.VM{ID: id, Instance: "evidence", Node: "node", VMID: 105, Type: "qemu", Status: "running", LastSeen: time.Now().Add(-20 * time.Minute), IPAddresses: []string{"192.0.2.10"}, OSName: "Windows", AgentVersion: "1.0"}
|
||||
res := proxmox.ClusterResource{Type: "qemu", Node: "node", VMID: 105, Name: "windows", Status: "running", MaxMem: 8000, Mem: 2000, MaxDisk: 1000, CPU: .2}
|
||||
registry := unifiedresources.NewRegistry(nil)
|
||||
build := func() models.VM {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if phase.Load() == -1 {
|
||||
canceled, cancel := context.WithCancel(ctx)
|
||||
cancel()
|
||||
ctx = canceled
|
||||
}
|
||||
var vms []models.VM
|
||||
if collector == "cluster" {
|
||||
vms = m.collectClusterVMResources(ctx, "evidence", []indexedClusterResource{{resource: res, guestID: id}}, client, map[string]models.VM{id: prev}, nil)
|
||||
} else {
|
||||
vms, _ = m.pollNodeVMsWithClusterResourceBuilder(ctx, "evidence", "node", []proxmox.VM{{VMID: 105, Name: "windows", Status: "running", MaxMem: 8000, Mem: 2000, MaxDisk: 1000, CPU: .2}}, client, map[string]models.VM{id: prev}, nil)
|
||||
}
|
||||
if len(vms) != 1 {
|
||||
t.Fatal("guest disappeared")
|
||||
}
|
||||
vm := vms[0]
|
||||
m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
|
||||
registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
|
||||
prev = previousVMFromView(registry.VMs()[0])
|
||||
if vm.ID != id || vm.CPU != .2 {
|
||||
t.Error("independent reading or identity lost")
|
||||
}
|
||||
return vm
|
||||
}
|
||||
phase.Store(-1)
|
||||
skipped := build()
|
||||
if len(skipped.IPAddresses) != 1 || skipped.OSName != "Windows" || skipped.AgentVersion != "1.0" {
|
||||
t.Fatal("cancelled detail did not preserve optional identity")
|
||||
}
|
||||
phase.Store(0)
|
||||
for poll := 0; poll < 3; poll++ {
|
||||
vm := build()
|
||||
if vm.GuestAgentStatus != "not-running" || vm.GuestAgentExpected {
|
||||
t.Errorf("poll %d: retained identity became current availability: %s/%t", poll, vm.GuestAgentStatus, vm.GuestAgentExpected)
|
||||
}
|
||||
}
|
||||
phase.Store(1)
|
||||
for poll := 0; poll < 3; poll++ {
|
||||
build()
|
||||
}
|
||||
if guestCalls.Load() != 0 {
|
||||
t.Errorf("expired retained identity admitted %d guest commands after a missing status", guestCalls.Load())
|
||||
}
|
||||
if got := m.metricsHistory.GetGuestMetrics(id, "cpu", time.Hour); len(got) != 7 {
|
||||
t.Errorf("independent CPU History lost: %d", len(got))
|
||||
}
|
||||
phase.Store(2)
|
||||
recovered := build()
|
||||
if recovered.GuestAgentStatus != "available" || guestCalls.Load() != 4 || recovered.AgentVersion != "2.0" || recovered.DiskStatusReason != "" || recovered.Disk.Usage != 0 {
|
||||
t.Errorf("ordinary fresh availability/disk-zero recovery failed: status=%s calls=%d version=%s disk=%+v reason=%s", recovered.GuestAgentStatus, guestCalls.Load(), recovered.AgentVersion, recovered.Disk, recovered.DiskStatusReason)
|
||||
}
|
||||
if !hasRecentGuestAgentEvidence(&prev, time.Now()) {
|
||||
t.Error("fresh metadata-only canonical evidence lost")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -87,7 +87,18 @@ func (m *Monitor) hasRecentGuestMetadataEvidence(instanceName, nodeName string,
|
||||
if !ok || entry.fetchedAt.IsZero() || !guestMetadataCacheHasUsefulData(entry) {
|
||||
return false
|
||||
}
|
||||
return now.Sub(entry.fetchedAt) < guestMetadataCacheEntryTTL(entry)
|
||||
return !entry.fetchedAt.After(now) && now.Sub(entry.fetchedAt) < guestMetadataCacheEntryTTL(entry)
|
||||
}
|
||||
|
||||
func (m *Monitor) guestMetadataEvidenceTime(instanceName, nodeName string, vmid int) time.Time {
|
||||
key := guestMetadataCacheKey(instanceName, nodeName, vmid)
|
||||
m.guestMetadataMu.RLock()
|
||||
entry := m.guestMetadataCache[key]
|
||||
m.guestMetadataMu.RUnlock()
|
||||
if !guestMetadataCacheHasUsefulData(entry) {
|
||||
return time.Time{}
|
||||
}
|
||||
return entry.fetchedAt
|
||||
}
|
||||
|
||||
func (m *Monitor) tryReserveGuestMetadataFetch(key string, now time.Time) bool {
|
||||
|
||||
@@ -147,24 +147,25 @@ func previousVMFromView(vm *unifiedresources.VMView) models.VM {
|
||||
}
|
||||
}
|
||||
return models.VM{
|
||||
ID: makeGuestID(instance, node, vmid),
|
||||
Instance: instance,
|
||||
Node: node,
|
||||
VMID: vmid,
|
||||
Name: vm.Name(),
|
||||
Type: "qemu",
|
||||
Status: vm.RuntimeStatus(),
|
||||
IPAddresses: vm.IPAddresses(),
|
||||
OSName: vm.OSName(),
|
||||
OSVersion: vm.OSVersion(),
|
||||
AgentVersion: vm.AgentVersion(),
|
||||
Lock: vm.Lock(),
|
||||
Disk: disk,
|
||||
NetworkInterfaces: guestNetworkInterfacesFromReadStateView(vm.NetworkInterfaces()),
|
||||
Disks: guestDisksFromReadStateView(vm.Disks()),
|
||||
DiskStatusReason: vm.DiskStatusReason(),
|
||||
DiskObservation: vm.DiskObservation(),
|
||||
LastSeen: vm.LastSeen(),
|
||||
ID: makeGuestID(instance, node, vmid),
|
||||
Instance: instance,
|
||||
Node: node,
|
||||
VMID: vmid,
|
||||
Name: vm.Name(),
|
||||
Type: "qemu",
|
||||
Status: vm.RuntimeStatus(),
|
||||
IPAddresses: vm.IPAddresses(),
|
||||
OSName: vm.OSName(),
|
||||
OSVersion: vm.OSVersion(),
|
||||
AgentVersion: vm.AgentVersion(),
|
||||
Lock: vm.Lock(),
|
||||
Disk: disk,
|
||||
NetworkInterfaces: guestNetworkInterfacesFromReadStateView(vm.NetworkInterfaces()),
|
||||
Disks: guestDisksFromReadStateView(vm.Disks()),
|
||||
DiskStatusReason: vm.DiskStatusReason(),
|
||||
DiskObservation: vm.DiskObservation(),
|
||||
GuestAgentEvidence: vm.GuestAgentEvidence(),
|
||||
LastSeen: vm.LastSeen(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ type vmBuildState struct {
|
||||
diskObservation models.GuestDiskObservation
|
||||
guestAgentStatus string
|
||||
guestAgentExpected bool
|
||||
guestAgentEvidence models.GuestAgentEvidence
|
||||
individualDisks []models.Disk
|
||||
ipAddresses []string
|
||||
networkInterfaces []models.GuestNetworkInterface
|
||||
@@ -68,6 +69,10 @@ func (m *Monitor) applyVMStatusDetails(
|
||||
res.Lock = status.Lock
|
||||
state.detailedStatus = status
|
||||
state.guestAgentStatus, state.guestAgentExpected = vmGuestAgentRuntimeState(status, recentGuestAgentEvidence)
|
||||
if status.Lock == "" && status.Agent.IsAvailable() {
|
||||
now := time.Now()
|
||||
renewGuestAgentEvidence(&state.guestAgentEvidence, observedAtOr(status.ObservedAt, now), now)
|
||||
}
|
||||
// Filesystem usage is cross-platform. Read it before optional Linux memory
|
||||
// and metadata commands can start a shared uncertainty pause. Every command
|
||||
// still verifies the same operation lock and is admitted only once.
|
||||
@@ -146,6 +151,7 @@ func (m *Monitor) applyVMGuestAgentFSInfo(ctx context.Context, instanceName stri
|
||||
}
|
||||
if state.diskFromAgent {
|
||||
state.diskObservation.ObservedAt = time.Now()
|
||||
renewGuestAgentEvidence(&state.guestAgentEvidence, state.diskObservation.ObservedAt, state.diskObservation.ObservedAt)
|
||||
}
|
||||
if guestAgentDiskDeferred(state.diskStatusReason) {
|
||||
state.guestAgentStatus = "deferred"
|
||||
@@ -230,6 +236,7 @@ func (m *Monitor) buildVMFromClusterResource(
|
||||
m.hasRecentGuestMetadataEvidence(instanceName, res.Node, res.VMID, prePollTime)
|
||||
|
||||
state := vmBuildState{
|
||||
guestAgentEvidence: retainGuestAgentEvidence(prevVM, prePollTime),
|
||||
diskObservation: models.GuestDiskObservation{Source: "guest-agent"},
|
||||
memTotal: res.MaxMem,
|
||||
memUsed: res.Mem,
|
||||
@@ -327,7 +334,8 @@ func (m *Monitor) buildVMFromClusterResource(
|
||||
}
|
||||
|
||||
state.guestAgentExpected = true
|
||||
if state.guestAgentStatus != "deferred" && (len(guestIPs) > 0 || len(guestIfaces) > 0 || guestOSName != "" || guestOSVersion != "" || guestAgentVersion != "" || state.diskFromAgent) {
|
||||
// Cached strings are display continuity, not a new successful read.
|
||||
if state.guestAgentStatus != "deferred" && (state.diskFromAgent || m.hasRecentGuestMetadataEvidence(instanceName, res.Node, res.VMID, time.Now())) {
|
||||
state.guestAgentStatus = "available"
|
||||
}
|
||||
}
|
||||
@@ -372,6 +380,12 @@ func (m *Monitor) buildVMFromClusterResource(
|
||||
}
|
||||
|
||||
sampleTime := time.Now()
|
||||
if res.Status == "running" {
|
||||
renewGuestAgentEvidence(&state.guestAgentEvidence, m.guestMetadataEvidenceTime(instanceName, res.Node, res.VMID), sampleTime)
|
||||
} else {
|
||||
// A stopped guest supplies no current fallback eligibility evidence.
|
||||
state.guestAgentEvidence = models.GuestAgentEvidence{Explicit: true}
|
||||
}
|
||||
state.diskTotal, state.diskUsed, state.diskFree, state.diskUsage, state.individualDisks, state.diskStatusReason = stabilizeGuestLowTrustDisk(
|
||||
prevVM,
|
||||
res.Status,
|
||||
@@ -506,6 +520,7 @@ func (m *Monitor) buildVMFromClusterResource(
|
||||
GuestAgentStatus: state.guestAgentStatus,
|
||||
Lock: res.Lock,
|
||||
GuestAgentExpected: state.guestAgentExpected,
|
||||
GuestAgentEvidence: state.guestAgentEvidence,
|
||||
IPAddresses: state.ipAddresses,
|
||||
OSName: state.osName,
|
||||
OSVersion: state.osVersion,
|
||||
|
||||
@@ -1844,6 +1844,7 @@ func resourceFromVM(vm models.VM) (Resource, ResourceIdentity) {
|
||||
BackupInProgress: vm.BackupInProgress,
|
||||
DiskStatusReason: vm.DiskStatusReason,
|
||||
DiskObservation: vm.DiskObservation,
|
||||
GuestAgentEvidence: vm.GuestAgentEvidence,
|
||||
GuestAgentStatus: vm.GuestAgentStatus,
|
||||
GuestAgentExpected: vm.GuestAgentExpected,
|
||||
OSName: vm.OSName,
|
||||
|
||||
@@ -27,6 +27,22 @@ func TestGuestDiskObservationDoesNotChangeWireOrIdentity(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuestAgentEvidenceDoesNotChangeWireOrIdentity(t *testing.T) {
|
||||
vm := models.VM{ID: "pve:node:105", VMID: 105, Name: "guest", Instance: "pve", Node: "node", Type: "qemu", Status: "running", LastSeen: time.Now(),
|
||||
GuestAgentEvidence: models.GuestAgentEvidence{Explicit: true, ObservedAt: time.Now().Add(-time.Minute)}}
|
||||
resource, identity := resourceFromVM(vm)
|
||||
wire, err := json.Marshal(resource)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vm.GuestAgentEvidence = models.GuestAgentEvidence{}
|
||||
legacyResource, legacyIdentity := resourceFromVM(vm)
|
||||
legacyWire, err := json.Marshal(legacyResource)
|
||||
if err != nil || string(wire) != string(legacyWire) || !reflect.DeepEqual(identity, legacyIdentity) {
|
||||
t.Fatal("internal admission evidence changed guest identity or public bytes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshCanonicalIdentityPrefersTargetsAndCanonicalHostData(t *testing.T) {
|
||||
resource := Resource{
|
||||
ID: "agent-1",
|
||||
|
||||
@@ -4245,6 +4245,7 @@ func mergeProxmoxData(existing *ProxmoxData, incoming *ProxmoxData) *ProxmoxData
|
||||
// A completed guest observation also owns loss/recovery of its original
|
||||
// filesystem evidence. Missing time must not inherit an older origin.
|
||||
merged.DiskObservation = incoming.DiskObservation
|
||||
merged.GuestAgentEvidence = incoming.GuestAgentEvidence
|
||||
merged.GuestAgentStatus = incoming.GuestAgentStatus
|
||||
merged.GuestAgentExpected = incoming.GuestAgentExpected
|
||||
}
|
||||
|
||||
@@ -41,6 +41,26 @@ func TestGuestDiskObservationMergeReplacesMissingAndDifferentSource(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuestAgentEvidenceMergeAndCloneKeepsOriginalAge(t *testing.T) {
|
||||
origin := models.GuestAgentEvidence{Explicit: true, ObservedAt: time.Now().Add(-time.Minute)}
|
||||
existing := &ProxmoxData{VMID: 105, GuestAgentEvidence: origin}
|
||||
for _, incoming := range []ProxmoxData{
|
||||
{VMID: 105, GuestAgentEvidence: origin},
|
||||
{VMID: 105, GuestAgentEvidence: models.GuestAgentEvidence{Explicit: true}},
|
||||
{VMID: 105},
|
||||
{NodeDisplayName: "partial"},
|
||||
} {
|
||||
merged := mergeProxmoxData(existing, &incoming)
|
||||
want := incoming.GuestAgentEvidence
|
||||
if incoming.VMID == 0 {
|
||||
want = origin
|
||||
}
|
||||
if merged.GuestAgentEvidence != want || existing.GuestAgentEvidence != origin {
|
||||
t.Fatal("merge renewed/resurrected evidence or mutated its source")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLinkedMergeAllowsOneSidedNodeHostLinkWhenHostnameCorroborates(t *testing.T) {
|
||||
registry := NewRegistry(NewMemoryStore())
|
||||
|
||||
|
||||
@@ -426,7 +426,8 @@ type ProxmoxData struct {
|
||||
|
||||
// Original QGA filesystem evidence survives internal clone/merge/previous
|
||||
// state, never JSON. Source identity or UpdatedAt cannot renew it.
|
||||
DiskObservation models.GuestDiskObservation `json:"-"`
|
||||
DiskObservation models.GuestDiskObservation `json:"-"`
|
||||
GuestAgentEvidence models.GuestAgentEvidence `json:"-"`
|
||||
|
||||
Lock string `json:"lock,omitempty"` // Proxmox lock state (e.g. "backup", "migrate", "snapshot")
|
||||
// SensorSetupOutdated is derived by the registry, never ingested: the
|
||||
|
||||
@@ -267,6 +267,13 @@ func (v VMView) DiskObservation() models.GuestDiskObservation {
|
||||
return v.r.Proxmox.DiskObservation
|
||||
}
|
||||
|
||||
func (v VMView) GuestAgentEvidence() models.GuestAgentEvidence {
|
||||
if v.r == nil || v.r.Proxmox == nil {
|
||||
return models.GuestAgentEvidence{}
|
||||
}
|
||||
return v.r.Proxmox.GuestAgentEvidence
|
||||
}
|
||||
|
||||
func (v VMView) OSName() string {
|
||||
if v.r == nil || v.r.Proxmox == nil {
|
||||
return ""
|
||||
|
||||
@@ -29,6 +29,26 @@ func TestGuestDiskObservationViewUsesOnlyItsOwnSource(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuestAgentEvidenceViewKeepsOnlyItsOriginalObservation(t *testing.T) {
|
||||
origin := models.GuestAgentEvidence{Explicit: true, ObservedAt: time.Now().Add(-time.Minute)}
|
||||
vm := models.VM{VMID: 105, Type: "qemu", Status: "running", LastSeen: time.Now(), AgentVersion: "retained", GuestAgentEvidence: origin}
|
||||
resource, _ := resourceFromVM(vm)
|
||||
view := NewVMView(&resource)
|
||||
if view.GuestAgentEvidence() != origin {
|
||||
t.Fatal("adapter/view replaced original eligibility age with the poll time")
|
||||
}
|
||||
copy := cloneResource(&resource)
|
||||
copy.Proxmox.GuestAgentEvidence.ObservedAt = time.Now()
|
||||
if view.GuestAgentEvidence() != origin {
|
||||
t.Fatal("cloned resource aliases source-owned eligibility evidence")
|
||||
}
|
||||
for _, empty := range []VMView{NewVMView(nil), NewVMView(&Resource{})} {
|
||||
if empty.GuestAgentEvidence() != (models.GuestAgentEvidence{}) {
|
||||
t.Fatal("missing guest facet invented eligibility evidence")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ptrInt64(v int64) *int64 { return &v }
|
||||
func ptrInt(v int) *int { return &v }
|
||||
|
||||
|
||||
Reference in New Issue
Block a user