Merge pull request #1969 from rcourtman/maintainer/20260907T231621Z

Strengthen recurring incident and guest-memory regression coverage

docs/release-control/v6/internal/subsystems/ai-runtime.md

@@ -54,6 +54,10 @@ remain outside this guarantee. Legacy shell lookup retains its time tolerance;
 events preceding the selected shell's exact opening are not projected into it.
 `TestIncidentStore_CanonicalProjectionOccurrenceBounds` verifies both boundaries,
 subsecond starts, unordered successors and unrelated alert/resource isolation.
+It also checkpoints all retained shells to JSON, reconstructs the incident store
+and replays a firing: projected identity/state stays identical and the unchanged
+checkpoint is not replaced. The canonical timeline stays in memory; this is not
+a durable event-store restart or installed-process write-rate measurement.
 
 ### Unchanged incident JSON checkpoints
 

internal/ai/memory/incidents_test.go

@@ -1205,6 +1205,47 @@ func TestIncidentStore_CanonicalProjectionOccurrenceBounds(t *testing.T) {
 			if projected.Status != IncidentStatusResolved || projected.ClosedAt == nil || !projected.ClosedAt.Equal(end) || projected.Acknowledged {
 				t.Fatalf("another occurrence changed historical state: %+v", projected)
 			}
+
+			// Checkpoint all retained boundaries, not just an isolated closed
+			// shell. The canonical timeline remains in memory: this exercises
+			// JSON recovery and projection, not a durable event-store restart.
+			before, err := json.Marshal(projected)
+			if err != nil {
+				t.Fatal(err)
+			}
+			store.dataDir = t.TempDir()
+			store.filePath = store.dataDir + "/ai_incidents.json"
+			if err := store.saveToDisk(); err != nil {
+				t.Fatal(err)
+			}
+			reloaded := NewIncidentStore(IncidentStoreConfig{})
+			reloaded.filePath = store.filePath
+			if err := reloaded.loadFromDisk(); err != nil {
+				t.Fatal(err)
+			}
+			reloaded.SetResourceTimelineStore(canonical)
+			for i := 0; i < 10; i++ {
+				reloaded.RecordAlertFired(old)
+			}
+			if len(reloaded.incidents) != len(store.incidents) {
+				t.Fatal("checkpoint replay duplicated a retained occurrence")
+			}
+			after, err := json.Marshal(reloaded.GetTimelineByAlertAt(old.ID, start))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if string(before) != string(after) {
+				t.Fatalf("checkpoint replay changed projected identity or timeline:\nbefore %s\nafter %s", before, after)
+			}
+			// Keep persistence synchronous so a successful comparison proves
+			// no checkpoint replacement, rather than a scheduling observation.
+			reloaded.dataDir = store.dataDir
+			if err := reloaded.saveToDisk(); err != nil {
+				t.Fatal(err)
+			}
+			if got := reloaded.savesCompleted.Load(); got != 0 {
+				t.Fatalf("unchanged checkpoint replay replaced JSON %d times", got)
+			}
 		})
 	}
 }

internal/monitoring/guest_memory_agent_test.go

@@ -174,6 +174,68 @@ func TestGetVMAgentMemAvailableCachesResults(t *testing.T) {
 	})
 }
 
+func TestGetVMAgentMemoryAvailabilityExpiresFailureAndRecovers(t *testing.T) {
+	t.Parallel()
+	for _, failure := range []struct {
+		name string
+		err  error
+	}{
+		{name: "request error", err: errors.New("guest unavailable")},
+		{name: "missing memory source"},
+	} {
+		t.Run(failure.name, func(t *testing.T) {
+			mon := &Monitor{}
+			initial := proxmox.LinuxMemoryAvailability{
+				Total: 8192, Available: 4096, EffectiveAvailable: 4096,
+				Source: "meminfo-available",
+			}
+			client := &guestMemoryAgentTestClient{stubPVEClient: &stubPVEClient{}, memInfo: &initial}
+			key := guestMemoryCacheKey("pve-a", "node1", 100)
+			read := func() (proxmox.LinuxMemoryAvailability, error) {
+				return mon.getVMAgentMemoryAvailability(context.Background(), client, "pve-a", "node1", 100)
+			}
+			// Age entries directly: no sleeps or wall-clock boundary races.
+			expire := func(ttl time.Duration) {
+				t.Helper()
+				entry := mon.vmAgentMemCache[key]
+				entry.fetchedAt = time.Now().Add(-ttl - time.Second)
+				mon.vmAgentMemCache[key] = entry
+			}
+			if got, err := read(); err != nil || got != initial {
+				t.Fatalf("initial read = %+v, %v", got, err)
+			}
+			client.memInfo = &proxmox.LinuxMemoryAvailability{}
+			client.memErr = failure.err
+			if got, err := read(); err != nil || got != initial || client.memCalls != 1 {
+				t.Fatalf("unexpired cache = %+v, %v; calls = %d", got, err, client.memCalls)
+			}
+			expire(vmAgentMemCacheTTL)
+			if got, err := read(); err == nil || got != (proxmox.LinuxMemoryAvailability{}) || client.memCalls != 2 {
+				t.Fatalf("expired cache must not serve stale success: %+v, %v; calls = %d", got, err, client.memCalls)
+			}
+			if entry := mon.vmAgentMemCache[key]; !entry.negative || entry.info.Source != "" || entry.available != 0 {
+				t.Fatalf("failure retained positive memory: %+v", entry)
+			}
+			// Explicit zero available is a valid full-pressure observation, not
+			// another missing sample; it must replace the expired negative entry.
+			recovered := proxmox.LinuxMemoryAvailability{Total: 8192, Source: "meminfo-available"}
+			client.memInfo, client.memErr = &recovered, nil
+			if got, err := read(); err == nil || got.Source != "" || client.memCalls != 2 {
+				t.Fatalf("negative backoff = %+v, %v; calls = %d", got, err, client.memCalls)
+			}
+			expire(vmAgentMemNegativeTTL)
+			for i := 0; i < 2; i++ {
+				if got, err := read(); err != nil || got != recovered || client.memCalls != 3 {
+					t.Fatalf("recovery read %d = %+v, %v; calls = %d", i, got, err, client.memCalls)
+				}
+			}
+			if mon.vmAgentMemCache[key].negative {
+				t.Fatal("valid zero-available recovery remained negative")
+			}
+		})
+	}
+}
+
 func TestGetVMAgentMemAvailableRetriesKnownNonWindowsGuestSoonerAfterNegativeCache(t *testing.T) {
 	t.Parallel()