Fix cache-aware Linux memory fallbacks

This commit is contained in:
rcourtman
2026-07-24 00:19:09 +01:00
parent b81ba7dd06
commit bf67ba9201
77 changed files with 2684 additions and 575 deletions
@@ -3716,6 +3716,13 @@ for lifecycle-adjacent diagnostics surfaces, legacy aliases and empty
fallback-reason fields must still normalize onto the governed canonical reason
contract instead of depending on monitor-owned snapshot accessors to have run
first.
Unified Linux agents carry memory evidence across this lifecycle boundary, but
they do not own memory-source inference. A report with only total and free
bytes must remain explicitly usage-unavailable because Linux `MemFree` excludes
reclaimable page cache; a complete used/cache/free split or an explicit
cache-aware usage remains trusted. Lifecycle transport, reconnect, and JSON
reload must preserve `UsageUnavailable`, including known total capacity,
without turning it into zero usage or an enrollment/freshness failure.
That same shared diagnostics dependency now also assumes local commercial and
onboarding analytics stay out of user diagnostics entirely: lifecycle-adjacent
admin surfaces may consume operational diagnostics, but they must not restore
@@ -37,6 +37,12 @@ Availability incident and alert identity belongs to the source-owned
machine, but it must not copy the check incident onto that machine or create a
second alert lifecycle. Failure, recovery, history, acknowledgement, and
notification routing therefore remain stable under relinking and restart.
Linux memory thresholds consume only canonical cache-aware usage. An explicit
usage-unavailable sample, a non-finite percentage, or contradictory
used/free/total evidence must not open or clear a memory alert. If such a
sample follows an active alert, the alert remains active with its last trusted
value until a subsequent trusted sample proves recovery; missing evidence is
not evidence that pressure disappeared.
## Canonical Files
@@ -5901,6 +5901,13 @@ That same diagnostics boundary must also backfill canonical fallback reasons
when a raw snapshot reaches the API layer without one, so
`buildMemorySourceDiagnostics` stays self-consistent even if a caller bypasses
`GetDiagnosticSnapshots()` and hands diagnostics a legacy alias directly.
The canonical `unavailable` memory source is serialized with trust
`unavailable`, fallback reason `cache-aware-memory-unavailable`, and the raw
source evidence that was actually observed. Node diagnostics keep status and
RRD available/used/total fields distinct; guest diagnostics keep status
meminfo, RRD, guest-agent free/cache/reclaimable/shmem, and linked-agent fields
distinct. Diagnostics must not relabel RRD evidence as meminfo or turn an
unknown usage into zero merely because total capacity is known.
PVE and PBS node diagnostics must also classify connection failures into
machine-readable `errorKind` values with operator guidance in
`troubleshooting`. Missing stored credentials, Proxmox auth rejection, TLS
@@ -3486,6 +3486,13 @@ Standalone, TrueNAS, and vSphere platform tables and their table-model helpers
must compose those helpers instead of declaring local `metricFallback` /
`finiteMetric` helpers or inlining centered muted dash fallback markup in
metric cells.
Canonical Linux memory usage-unavailable is a first-class metric fallback,
not a numeric zero. Shared workload bars, platform tables, drawers, and live
history labels must render `N/A` (while retaining known capacity where useful)
and must not synthesize a zero-width used segment or a healthy-looking `0%`.
When unified-resource metadata carries an unavailable raw Proxmox, agent, or
Docker memory facet alongside a trusted metric from another source, the
trusted merged metric wins; the raw facet remains diagnostic evidence only.
Platform load-failure states are registry-backed as well.
`PlatformErrorState` owns the repeated table-card error shell, warning icon,
and Refresh action for platform page and table load failures; platform
@@ -1619,12 +1619,43 @@ when Proxmox reports a saturated or materially inconsistent used figure, so
Windows and ballooned guests do not get pinned to false 100% usage samples.
That same guest-memory boundary also owns fallback order and cache scoping for
Proxmox VMs when `MemInfo` is absent. Monitoring must try instance-scoped RRD
`memavailable`, then guest-agent `/proc/meminfo` via the shared Proxmox
client, and only then linked host-agent memory as the final fallback. Both RRD
and guest-agent fallback caches must key on `(instance, node, vmid)` instead
of raw `node/vmid`, so separate Proxmox instances cannot leak stale or foreign
memory evidence into each other just because they reuse the same node name and
VMID.
`memavailable`, RRD `memused`, guest-agent `/proc/meminfo` via the shared
Proxmox client, and only then linked host-agent memory. Saturated Linux VM
status is the narrow exception: a queryable guest agent's cache-aware
`/proc/meminfo` evidence is tried before RRD because the saturated status is
the known Proxmox failure mode. Both RRD and guest-agent fallback caches must
key on `(instance, node, vmid)` instead of raw `node/vmid`, so separate
Proxmox instances cannot leak stale or foreign memory evidence into each other
just because they reuse the same node name and VMID.
Linux memory availability must never be inferred from `MemTotal-MemFree`.
Nodes and guests prefer a valid explicit `MemAvailable`/`available` field,
then a complete reclaimable-component estimate, then valid RRD availability or
used evidence. The conservative old-kernel guest-agent estimate is
`MemFree + Buffers + Cached + SReclaimable - Shmem`; it is valid without swap
but not from truncated or total/free-only meminfo. A material `total-used`
gap may remain a lower-trust estimate only when it supplies independent
evidence that the reported used value already excludes cache. Invalid,
overflowed, non-finite, over-total, or conflicting candidates must be rejected
before the next source is considered; an explicitly present zero RRD used or
zero available value remains a valid idle or full-pressure sample rather than
being mistaken for an absent field.
Node RRD fallback caches must key on `(instance, node)`, just as guest RRD and
guest-agent caches key on `(instance, node, vmid)`, so identically named nodes
in different Proxmox instances cannot exchange memory evidence.
Running LXC cluster-resource memory is cache-inclusive and therefore cannot be
used as live usage when both RRD `memavailable` and `memused` are absent.
Unified Linux and Docker agent ingest likewise must not repair a missing used
value from total minus free alone; it may use an explicit used/percentage or
complete free-plus-cache evidence. In every collector, known capacity with no
cache-aware usage is represented by `models.Memory.UsageUnavailable`, the
canonical memory source `unavailable`, and fallback reason
`cache-aware-memory-unavailable`. A recent trusted node or guest snapshot may
be carried across a transient reconnect under the existing bounded
`previous-snapshot` rule; otherwise the unknown state must remain honest.
Unknown memory samples must not append zeroes to in-memory or persistent
history, project a canonical unified-resource memory metric, start or clear a
threshold alert, or render as 0% in product surfaces. Existing active alerts
remain fail-safe until a later trusted sample crosses the clear threshold.
That same guest-memory boundary also owns stabilization when Proxmox falls
back to low-trust VM full-usage readings. The shared VM polling paths must use
the previous guest diagnostic snapshot, not the resource model, to decide when
@@ -1574,7 +1574,12 @@ resize-observer plus tooltip lifecycle live in
`frontend-modern/src/components/Workloads/useStackedMemoryBarState.ts`.
Future memory-bar runtime changes must extend through those owners instead of
reintroducing mixed resize state, balloon branching, and tooltip shaping into
the shell.
the shell. The model must also short-circuit explicit usage-unavailable input:
it may retain total capacity for explanatory copy, but it emits no numeric
memory segments, percentage, or fit calculation that could make unknown usage
look like zero. Platform tables and resource-detail mappers consume the same
canonical unavailable state instead of recomputing `total - free` in render
hot paths.
The dashboard metric bar now follows that same pattern: the shell stays in
`frontend-modern/src/components/Workloads/MetricBar.tsx`, while width,
show-label, sublabel-fit, and threshold-color derivation live in
@@ -1547,6 +1547,14 @@ recovery scope, or a storage/recovery-owned secret source.
contract. `internal/unifiedresources/types.go` carries the reclaimable
page-cache split (`AgentMemoryMeta.cache`, holding used + cache + free
within the reported total) as host RAM description for machine surfaces.
It also carries `AgentMemoryMeta.usageUnavailable`, and the parallel
Proxmox and Docker raw memory facets, so a collector can preserve known
capacity while declining to invent Linux usage from `total - MemFree`.
A newer unavailable report clears a stale canonical memory metric only
when both observations come from the same source; a trusted linked source
remains eligible to supply the merged metric. Storage and recovery
consumers must preserve that distinction and must not reinterpret unknown
host RAM usage as zero, full pressure, or a storage-health signal.
Storage and recovery must not reinterpret that reclaimable RAM figure as
disk cache, ZFS ARC sizing, storage-tier health, or capacity-planning
evidence; disk and pool truth stays on the canonical storage and
@@ -2255,6 +2255,17 @@ parallel payload, mirroring the `proxmox.memoryCache` transport nodes and
guests use. The field is additive and omitted when an agent does not report
it; consumers must treat missing cache as zero rather than inferring it from
free space.
`AgentMemoryMeta.usageUnavailable`, `DockerData.memory.usageUnavailable`, and
`ProxmoxData.memory.usageUnavailable` carry the separate case where capacity is
known but cache-aware live usage is not. Adapters must omit
`ResourceMetrics.memory` for that source rather than
publishing a synthetic 0% value. Registry and presentation merges must retain
a trusted memory metric from another source when the current source is
unavailable, clear an older metric from the same unavailable source, and keep
the raw source facet available for diagnostics and capacity display. Frontend
adapters apply the unavailable marker only when no trusted merged metric
exists, so cross-source recovery cannot be masked by an unknown Proxmox or
agent facet.
Frontend resource identity presenters may append a runtime version to a
displayed system badge only when that version is sourced from the same canonical
platform or host-profile identity, such as PVE `ResourceProxmoxMeta.pveVersion`
@@ -60,6 +60,38 @@ describe('buildMemory', () => {
});
});
it('preserves explicit unavailable memory through agent detail mapping', () => {
const fallback: Partial<MemoryType> = {
total: 200,
used: 0,
free: 0,
usage: 0,
usageUnavailable: true,
};
expect(buildMemory(undefined, fallback)).toEqual({
total: 200,
used: 0,
free: 0,
usage: 0,
usageUnavailable: true,
});
});
it('lets a canonical cross-source metric override an unavailable fallback', () => {
const fallback: Partial<MemoryType> = {
total: 200,
usageUnavailable: true,
};
expect(buildMemory({ current: 0.25, total: 200, used: 50 }, fallback)).toEqual({
total: 200,
used: 50,
free: 150,
usage: 0.25,
});
});
it('computes usage from total>0 and ignores fallback.usage', () => {
const fallback: Partial<MemoryType> = { usage: 0.99 };
@@ -273,6 +305,28 @@ describe('toNodeFromProxmox', () => {
});
});
it('preserves unavailable Proxmox capacity when no canonical metric exists', () => {
const node = toNodeFromProxmox(
baseProxmoxResource({
memory: undefined,
platformData: {
proxmox: {
nodeName: 'pve-node-1',
memory: { total: 8192, usageUnavailable: true },
},
},
}),
);
expect(node?.memory).toEqual({
total: 8192,
used: 0,
free: 0,
usage: 0,
usageUnavailable: true,
});
});
it('formats lastSeen as an ISO string of the finite timestamp', () => {
const node = toNodeFromProxmox(baseProxmoxResource({ lastSeen: 1_700_000_000_000 }));
expect(node?.lastSeen).toBe(new Date(1_700_000_000_000).toISOString());
@@ -28,6 +28,7 @@ export type ProxmoxPlatformData = {
nodeName?: string;
clusterName?: string;
vmid?: number;
memory?: Partial<Memory>;
pveVersion?: string;
kernelVersion?: string;
uptime?: number;
@@ -189,15 +190,17 @@ const getPreferredHostLabel = (resource: Resource): string =>
resource.id;
export const buildMemory = (metric?: ResourceMetric, fallback?: Partial<Memory>): Memory => {
const usageUnavailable = metric == null && fallback?.usageUnavailable === true;
const total = metric?.total ?? fallback?.total ?? 0;
const used = metric?.used ?? fallback?.used ?? 0;
const free = metric?.free ?? fallback?.free ?? Math.max(total - used, 0);
const usage = total > 0 ? used / total : (fallback?.usage ?? 0);
const free = metric?.free ?? fallback?.free ?? (usageUnavailable ? 0 : Math.max(total - used, 0));
const usage = usageUnavailable ? 0 : total > 0 ? used / total : (fallback?.usage ?? 0);
return {
total,
used,
free,
usage,
...(usageUnavailable ? { usageUnavailable: true } : {}),
};
};
@@ -241,7 +244,7 @@ export const toNodeFromProxmox = (resource: Resource): Node | null => {
const proxmox = platformData?.proxmox;
if (!proxmox) return null;
const memory = buildMemory(resource.memory);
const memory = buildMemory(resource.memory, proxmox.memory);
const disk = buildDisk(resource.disk);
const lastSeen = Number.isFinite(resource.lastSeen)
? new Date(resource.lastSeen).toISOString()
@@ -368,6 +368,7 @@ export function GuestRow(props: GuestRowProps) {
<StackedMemoryBar
used={props.guest.memory?.used || 0}
total={props.guest.memory?.total || 0}
unavailable={props.guest.memory?.usageUnavailable === true}
percentOnly={memoryPercentOnly()}
cache={props.guest.memory?.cache || 0}
cacheInclusiveLabel="Shown in Proxmox"
@@ -383,13 +384,15 @@ export function GuestRow(props: GuestRowProps) {
>
{renderMetricSparkline(
'memory',
formatMetricPercent(
usagePercent(
props.guest.memory?.used,
props.guest.memory?.total,
props.guest.memory?.usage,
),
),
props.guest.memory?.usageUnavailable
? 'N/A'
: formatMetricPercent(
usagePercent(
props.guest.memory?.used,
props.guest.memory?.total,
props.guest.memory?.usage,
),
),
`${props.guest.name} memory history`,
'%',
'inline',
@@ -250,9 +250,12 @@ export function NodeDrawerOverview(props: NodeDrawerOverviewProps) {
const memoryRows = (): NodeOverviewRow[] => [
{
label: 'Usage',
value: `${getUsedPercent(props.node.memory?.used, props.node.memory?.total)} · ${formatBytes(
props.node.memory?.used || 0,
)}`,
value:
props.node.memory?.usageUnavailable === true
? 'Unavailable'
: `${getUsedPercent(props.node.memory?.used, props.node.memory?.total)} · ${formatBytes(
props.node.memory?.used || 0,
)}`,
},
{ label: 'Total', value: formatBytes(props.node.memory?.total || 0) },
...(props.node.memory?.cache
@@ -263,7 +266,9 @@ export function NodeDrawerOverview(props: NodeDrawerOverviewProps) {
} satisfies NodeOverviewRow,
]
: []),
{ label: 'Free', value: formatBytes(props.node.memory?.free || 0) },
...(props.node.memory?.usageUnavailable === true
? []
: [{ label: 'Free', value: formatBytes(props.node.memory?.free || 0) }]),
...(props.node.memory?.swapTotal
? [
{
@@ -44,6 +44,15 @@ describe('StackedMemoryBar', () => {
expect(screen.getByText('50%')).toBeInTheDocument();
});
it('renders N/A with no utilization segment when usage is unavailable', () => {
const { container } = render(() => (
<StackedMemoryBar used={0} total={8 * 1024 ** 3} unavailable />
));
expect(screen.getByText('N/A')).toBeInTheDocument();
expect(screen.queryByText('0%')).not.toBeInTheDocument();
expect(getSegments(container)).toHaveLength(0);
});
it('renders 0% when both used and total are 0', () => {
render(() => <StackedMemoryBar used={0} total={0} />);
expect(screen.getByText('0%')).toBeInTheDocument();
@@ -72,7 +72,12 @@ export function StackedMemoryBar(props: StackedMemoryBarProps) {
<span class="absolute inset-0 flex items-center justify-center text-[10px] font-semibold text-base-content leading-none pointer-events-none min-w-0 overflow-hidden">
<span class="max-w-full min-w-0 whitespace-nowrap overflow-hidden text-ellipsis px-0.5 text-center">
<span>
<AnimatedNumber value={presentation().displayPercentValue} format={formatPercent} />
<Show
when={!presentation().unavailable}
fallback={<span class="text-muted">N/A</span>}
>
<AnimatedNumber value={presentation().displayPercentValue} format={formatPercent} />
</Show>
</span>
<Show when={presentation().showSublabel}>
<span class="metric-sublabel font-normal text-muted">
@@ -237,7 +237,9 @@ export function WorkloadPanel(props: WorkloadPanelProps) {
return renderGroupNodeSparkline(
node,
'memory',
formatMetricPercent(getUsedPercent(node.memory?.used, node.memory?.total)),
node.memory?.usageUnavailable
? 'N/A'
: formatMetricPercent(getUsedPercent(node.memory?.used, node.memory?.total)),
`${node.name} memory history`,
'%',
'inline',
@@ -249,6 +251,7 @@ export function WorkloadPanel(props: WorkloadPanelProps) {
<StackedMemoryBar
used={node.memory?.used || 0}
total={node.memory?.total || 0}
unavailable={node.memory?.usageUnavailable === true}
balloon={node.memory?.balloon || 0}
swapUsed={node.memory?.swapUsed || 0}
swapTotal={node.memory?.swapTotal || 0}
@@ -129,6 +129,21 @@ describe('guestDrawerModel (branch coverage)', () => {
expect(result.disk).toBeUndefined();
});
it('does not synthesize a history point from unavailable live memory', () => {
const result = getGuestDrawerHistoryFallbackMetrics(
makeGuest({
memory: {
total: 8192,
used: 0,
free: 0,
usage: 0,
usageUnavailable: true,
},
}),
);
expect(result.memory).toBeUndefined();
});
it('drops a non-finite network value via the finite() guard', () => {
const result = getGuestDrawerHistoryFallbackMetrics(
makeGuest({ networkIn: Number.POSITIVE_INFINITY }),
@@ -452,6 +467,25 @@ describe('guestDrawerModel (branch coverage)', () => {
).toEqual([]);
});
it('shows unavailable usage and known capacity without showing zero usage', () => {
expect(
getGuestDrawerMemoryRows(
makeGuest({
memory: {
total: 8192,
used: 0,
free: 0,
usage: 0,
usageUnavailable: true,
},
}),
),
).toEqual([
{ label: 'Usage', value: 'Unavailable' },
{ label: 'Total', value: '8.00 KB' },
]);
});
it('emits only Usage and Total when total>0 and no cache/free/balloon/swap', () => {
expect(
getGuestDrawerMemoryRows(
@@ -5,6 +5,32 @@ import { buildStackedMemoryBarPresentation } from '../stackedMemoryBarModel';
const GiB = 1024 ** 3;
describe('buildStackedMemoryBarPresentation', () => {
it('renders unavailable usage without inventing a zero-percent segment', () => {
const presentation = buildStackedMemoryBarPresentation(
{ used: 0, total: 8 * GiB, unavailable: true },
400,
);
expect(presentation.unavailable).toBe(true);
expect(presentation.segments).toEqual([]);
expect(presentation.displaySublabel).toBe('');
expect(presentation.showSublabel).toBe(false);
expect(presentation.tooltipRows).toEqual([
{
borderTop: false,
label: 'Usage',
labelClass: 'text-slate-400',
value: 'Unavailable',
},
{
borderTop: true,
label: 'Total',
labelClass: 'text-slate-400',
value: '8.00 GB',
},
]);
});
it('renders the used | reclaimable cache split with a source-neutral reconciliation row', () => {
const presentation = buildStackedMemoryBarPresentation(
{ used: 4 * GiB, total: 16 * GiB, cache: 6 * GiB },
@@ -72,7 +72,7 @@ export const getGuestDrawerHistoryFallbackMetrics = (
guest: Guest,
): Record<string, number | undefined> => {
const cpuPercent = getWorkloadCPUPercent(guest.cpu);
const memUsage = guest.memory?.usage;
const memUsage = guest.memory?.usageUnavailable ? undefined : guest.memory?.usage;
const diskUsage = guest.disk?.usage;
const finite = (value: number | undefined): number | undefined =>
typeof value === 'number' && Number.isFinite(value) ? value : undefined;
@@ -302,7 +302,12 @@ export const getGuestDrawerMemoryRows = (guest: Guest): GuestDrawerMemoryRow[] =
const used = memory.used ?? 0;
const cache = memory.cache ?? 0;
if (total > 0) {
if (memory.usageUnavailable) {
rows.push({ label: 'Usage', value: 'Unavailable' });
if (total > 0) {
rows.push({ label: 'Total', value: formatBytes(total) });
}
} else if (total > 0) {
rows.push({
label: 'Usage',
value: `${formatPercent((used / total) * 100)} · ${formatBytes(used)}`,
@@ -12,6 +12,7 @@ import type { MetricDisplayThresholds, MetricSeverity } from '@/utils/metricThre
export interface StackedMemoryBarProps {
used: number;
total: number;
unavailable?: boolean;
percentOnly?: number;
/** Reclaimable buff/cache (available - truly free); used + cache + free ≈ total. */
cache?: number;
@@ -51,6 +52,7 @@ export interface StackedMemoryBarPresentation {
swapBarPercent: number;
tooltipRows: StackedMemoryTooltipRow[];
tooltipTitle: string;
unavailable: boolean;
}
// Tooltip legend for the used segment tracks the same severity that colors
@@ -79,6 +81,7 @@ function getEffectiveCache(props: StackedMemoryBarProps): number {
}
function getUtilizationPercent(props: StackedMemoryBarProps): number {
if (props.unavailable) return 0;
if (props.total > 0) {
return (props.used / props.total) * 100;
}
@@ -92,6 +95,9 @@ function getSegments(
props: StackedMemoryBarProps,
utilizationPercent: number,
): StackedMemorySegment[] {
if (props.unavailable) {
return [];
}
if (props.total <= 0) {
if (utilizationPercent <= 0) {
return [];
@@ -161,7 +167,22 @@ function getTooltipRows(
const hasActiveBallooning = props.total > 0 && balloon > 0 && balloon < props.total;
const hasSwap = (props.swapTotal || 0) > 0;
if (props.total > 0) {
if (props.unavailable) {
rows.push({
borderTop: false,
label: 'Usage',
labelClass: 'text-slate-400',
value: 'Unavailable',
});
if (props.total > 0) {
rows.push({
borderTop: true,
label: 'Total',
labelClass: 'text-slate-400',
value: formatBytes(props.total),
});
}
} else if (props.total > 0) {
const usedPercent = (props.used / props.total) * 100;
rows.push({
borderTop: false,
@@ -236,7 +257,9 @@ export function buildStackedMemoryBarPresentation(
const utilizationPercent = getUtilizationPercent(props);
const displayLabel = formatPercent(utilizationPercent);
const displaySublabel =
props.total > 0 ? `${formatBytes(props.used)}/${formatBytes(props.total)}` : '';
!props.unavailable && props.total > 0
? `${formatBytes(props.used)}/${formatBytes(props.total)}`
: '';
const showSublabel =
displaySublabel.length > 0 &&
containerWidth >= estimateTextWidth(`${displayLabel} (${displaySublabel})`);
@@ -259,5 +282,6 @@ export function buildStackedMemoryBarPresentation(
: 0,
tooltipRows: getTooltipRows(props, displayLabel),
tooltipTitle: 'Memory Composition',
unavailable: props.unavailable === true,
};
}
@@ -65,6 +65,12 @@ const getNumericField = (value: unknown, field: string): number | undefined => {
return typeof fieldValue === 'number' ? fieldValue : undefined;
};
const getBooleanField = (value: unknown, field: string): boolean | undefined => {
if (!value || typeof value !== 'object') return undefined;
const fieldValue = (value as Record<string, unknown>)[field];
return typeof fieldValue === 'boolean' ? fieldValue : undefined;
};
const formatLastSeenRow = (value: string | number | null | undefined): DockerOverviewRow | null => {
if (value == null) return null;
const parsed = typeof value === 'number' ? new Date(value) : new Date(String(value));
@@ -142,7 +148,7 @@ export function DockerHostDrawerOverview(props: DockerHostDrawerOverviewProps) {
return 0;
};
const memorySource = () => props.host.memory ?? agent()?.memory;
const memorySource = () => props.host.memory ?? agent()?.memory ?? docker()?.memory;
const diskSource = () => props.host.disk;
const systemRows = (): DockerOverviewRow[] => [
@@ -261,7 +267,12 @@ export function DockerHostDrawerOverview(props: DockerHostDrawerOverviewProps) {
const memory = memorySource();
if (!memory) return [];
const rows: DockerOverviewRow[] = [];
if (typeof memory.total === 'number' && memory.total > 0) {
if (getBooleanField(memory, 'usageUnavailable') === true) {
rows.push({ label: 'Usage', value: 'Unavailable' });
if (typeof memory.total === 'number' && memory.total > 0) {
rows.push({ label: 'Total', value: formatBytes(memory.total) });
}
} else if (typeof memory.total === 'number' && memory.total > 0) {
rows.push({
label: 'Usage',
value: `${getUsedPercent(memory.used, memory.total)} · ${formatBytes(memory.used || 0)}`,
@@ -53,21 +53,28 @@ const percentFromMetric = (metric: Resource['cpu'] | undefined): number | undefi
const memoryTotalFor = (host: Resource): number =>
getPlatformTableFiniteMetric(host.memory?.total) ??
getPlatformTableFiniteMetric(host.agent?.memory?.total) ??
getPlatformTableFiniteMetric(host.docker?.memory?.total) ??
0;
const memoryUsedFor = (host: Resource): number =>
getPlatformTableFiniteMetric(host.memory?.used) ??
getPlatformTableFiniteMetric(host.agent?.memory?.used) ??
getPlatformTableFiniteMetric(host.docker?.memory?.used) ??
0;
const memoryPercentOnlyFor = (host: Resource): number | undefined => {
if (memoryTotalFor(host) > 0) return undefined;
return (
getPlatformTableFiniteMetric(host.memory?.current) ??
getPlatformTableFiniteMetric(host.agent?.memory?.usage)
getPlatformTableFiniteMetric(host.agent?.memory?.usage) ??
getPlatformTableFiniteMetric(host.docker?.memory?.usage)
);
};
const memoryUnavailableFor = (host: Resource): boolean =>
!host.memory &&
(host.agent?.memory?.usageUnavailable === true || host.docker?.memory?.usageUnavailable === true);
// Host telemetry the Docker agent reports beyond the typed Resource docker
// block. One cast site shared by the row renderer and the sort accessor.
type DockerHostDockerMeta = NonNullable<Resource['docker']> & {
@@ -124,6 +131,7 @@ const getDockerHostSortValue = (host: Resource, key: DockerHostSortKey): Platfor
case 'cpu':
return percentFromMetric(host.cpu) ?? null;
case 'memory': {
if (memoryUnavailableFor(host)) return null;
const total = memoryTotalFor(host);
if (total > 0) return (memoryUsedFor(host) / total) * 100;
return memoryPercentOnlyFor(host) ?? null;
@@ -452,6 +460,7 @@ export const DockerHostsTable: Component<{
<StackedMemoryBar
used={memoryUsed()}
total={memoryTotal()}
unavailable={memoryUnavailableFor(host)}
percentOnly={memoryPercentOnly()}
/>
</Show>
@@ -26,11 +26,17 @@ vi.mock('@/stores/alertsActivation', () => ({
}));
vi.mock('@/components/Workloads/StackedMemoryBar', () => ({
StackedMemoryBar: (props: { used: number; total: number; percentOnly?: number }) => (
StackedMemoryBar: (props: {
used: number;
total: number;
unavailable?: boolean;
percentOnly?: number;
}) => (
<div
data-testid="stacked-memory-bar"
data-used={String(props.used)}
data-total={String(props.total)}
data-unavailable={String(props.unavailable === true)}
data-percent-only={String(props.percentOnly ?? '')}
/>
),
@@ -356,4 +362,35 @@ describe('DockerHostsTable', () => {
expect(screen.getByTestId('stacked-disk-bar')).toHaveAttribute('data-disks', '0');
expect(screen.getByTestId('stacked-disk-bar')).toHaveAttribute('data-aggregate-usage', '71');
});
it('renders unavailable Docker memory honestly while retaining known capacity', () => {
render(() => (
<DockerHostsTable
resources={[
makeDockerHost({
status: 'online',
memory: undefined,
agent: undefined,
docker: {
runtimeVersion: '27.5.1',
containerCount: 12,
memory: { total: 8_000, usageUnavailable: true },
},
}),
]}
emptyIcon={<span />}
emptyTitle="No Docker hosts"
emptyDescription="No hosts"
showToolbar={false}
/>
));
expect(screen.getByTestId('stacked-memory-bar')).toHaveAttribute('data-total', '8000');
expect(screen.getByTestId('stacked-memory-bar')).toHaveAttribute('data-unavailable', 'true');
fireEvent.click(screen.getByText('docker-01').closest('tr')!);
const drawer = screen.getByTestId('docker-host-drawer');
expect(within(drawer).getByText('Unavailable')).toBeInTheDocument();
expect(within(drawer).getByText('7.81 KB')).toBeInTheDocument();
});
});
@@ -437,6 +437,7 @@ export const ProxmoxNodesTable: Component<{
<StackedMemoryBar
used={memoryUsed()}
total={memoryTotal()}
unavailable={drawerNode()?.memory?.usageUnavailable === true}
percentOnly={memoryPercentOnly()}
cache={drawerNode()?.memory?.cache || 0}
cacheInclusiveLabel="Shown in Proxmox"
@@ -448,10 +449,14 @@ export const ProxmoxNodesTable: Component<{
>
<MetricMiniSparkline
series={memorySeries()}
valueLabel={formatPlatformTablePercentValue(memoryPercent(), {
normalizeRatio: true,
clamp: true,
})}
valueLabel={
drawerNode()?.memory?.usageUnavailable
? 'N/A'
: formatPlatformTablePercentValue(memoryPercent(), {
normalizeRatio: true,
clamp: true,
})
}
title={`${name()} memory history`}
/>
</Show>
@@ -868,6 +868,9 @@ const memoryPercentOnlyFor = (machine: Resource): number | undefined => {
);
};
const memoryUnavailableFor = (machine: Resource): boolean =>
!machine.memory && machine.agent?.memory?.usageUnavailable === true;
const memoryBalloonFor = (machine: Resource): number | undefined =>
getPlatformTableFiniteMetric(machine.agent?.memory?.balloon);
@@ -1656,6 +1659,7 @@ export const AgentsMachinesTable: Component<{
<StackedMemoryBar
used={memoryUsed()}
total={memoryTotal()}
unavailable={memoryUnavailableFor(machine)}
percentOnly={memoryPercentOnly()}
cache={memoryCache()}
balloon={memoryBalloon()}
@@ -77,6 +77,31 @@ describe('agentMachineTableModel coverage', () => {
it('returns undefined when no memory source is available', () => {
expect(getAgentMachineMemoryPercent(resource({}))).toBeUndefined();
});
it('does not turn an unavailable raw agent capacity into zero percent', () => {
expect(
getAgentMachineMemoryPercent(
resource({
agent: {
memory: { total: 16_000, used: 0, usageUnavailable: true },
},
}),
),
).toBeUndefined();
});
it('keeps a trusted merged metric when the agent facet is unavailable', () => {
expect(
getAgentMachineMemoryPercent(
resource({
memory: { current: 25, total: 16_000, used: 4_000 },
agent: {
memory: { total: 16_000, used: 0, usageUnavailable: true },
},
}),
),
).toBe(25);
});
});
describe('timestampMillisFrom', () => {
@@ -309,6 +309,9 @@ export const getAgentMachineCpuPercent = (machine: Resource): number | undefined
getMetricPercent(machine.cpu);
export const getAgentMachineMemoryPercent = (machine: Resource): number | undefined => {
if (!machine.memory && machine.agent?.memory?.usageUnavailable === true) {
return undefined;
}
const total = getPlatformTableFiniteMetric(machine.memory?.total);
const used = getPlatformTableFiniteMetric(machine.memory?.used);
if (total && total > 0 && typeof used === 'number') {
+2
View File
@@ -1087,6 +1087,8 @@ export interface Memory {
free: number;
/** Reclaimable buff/cache (available - truly free); used + cache + free ≈ total. */
cache?: number;
/** Total is known, but no cache-aware source can establish current usage. */
usageUnavailable?: boolean;
usage: number;
balloon?: number;
swapUsed?: number;
+3
View File
@@ -665,6 +665,8 @@ export interface ResourceProxmoxMeta {
balloon?: number;
/** Reclaimable buff/cache split out of the memory metric's free bytes. */
memoryCache?: number;
/** Raw capacity and explicit availability state when no canonical metric is projected. */
memory?: Partial<Memory>;
pendingUpdates?: number;
isOci?: boolean;
osTemplate?: string;
@@ -698,6 +700,7 @@ export interface ResourceDockerMeta {
runtime?: string;
runtimeVersion?: string;
dockerVersion?: string;
memory?: Partial<Memory>;
os?: string;
kernelVersion?: string;
architecture?: string;
@@ -679,6 +679,54 @@ describe('buildMemory (via nodeFromResource)', () => {
expect(node?.memory.balloon).toBe(30);
});
it('preserves explicit unavailable usage from the fallback memory contract', () => {
const node = nodeFromResource(
agentNode({
memory: undefined,
proxmox: {
nodeName: 'n1',
memory: {
total: 8192,
used: 0,
free: 0,
usage: 0,
usageUnavailable: true,
},
} as unknown as Resource['proxmox'],
}),
);
expect(node?.memory).toMatchObject({
total: 8192,
used: 0,
usage: 0,
usageUnavailable: true,
});
});
it('lets a trusted merged metric override an unavailable Proxmox fallback', () => {
const node = nodeFromResource(
agentNode({
memory: { current: 50, total: 8192, used: 4096, free: 4096 },
proxmox: {
nodeName: 'n1',
memory: {
total: 8192,
used: 0,
free: 0,
usage: 0,
usageUnavailable: true,
},
} as unknown as Resource['proxmox'],
}),
);
expect(node?.memory).toMatchObject({
total: 8192,
used: 4096,
usage: 50,
usageUnavailable: false,
});
});
it('prefers proxmoxMeta.memoryCache over fallback.cache', () => {
const node = nodeFromResource(
agentNode({
@@ -1009,3 +1009,46 @@ describe('resourceStateAdapters nodeFromResource', () => {
});
});
});
describe('resourceStateAdapters unavailable memory contract', () => {
const unavailableProxmoxMemory = {
nodeName: 'n1',
memory: {
total: 8192,
used: 0,
free: 0,
usage: 0,
usageUnavailable: true,
},
} as unknown as Resource['proxmox'];
it('preserves explicit unavailable usage when no trusted metric exists', () => {
const node = nodeFromResource({
...createNodeResource({}),
memory: undefined,
proxmox: unavailableProxmoxMemory,
});
expect(node?.memory).toMatchObject({
total: 8192,
used: 0,
usage: 0,
usageUnavailable: true,
});
});
it('lets a trusted merged metric override unavailable raw evidence', () => {
const node = nodeFromResource({
...createNodeResource({}),
memory: { current: 50, total: 8192, used: 4096, free: 4096 },
proxmox: unavailableProxmoxMemory,
});
expect(node?.memory).toMatchObject({
total: 8192,
used: 4096,
usage: 50,
usageUnavailable: false,
});
});
});
@@ -836,18 +836,24 @@ const buildMemory = (
proxmoxMeta?: Record<string, unknown>,
): Memory => {
const total = metric?.total ?? asNumber(fallback?.total) ?? 0;
const usageUnavailable = metric == null && asBoolean(fallback?.usageUnavailable) === true;
const used = metric?.used ?? asNumber(fallback?.used) ?? 0;
const cache = asNumber(proxmoxMeta?.memoryCache) ?? asNumber(fallback?.cache) ?? 0;
// The metric ships no free bytes for PVE payloads; total-used is the
// reclaimable-inclusive available, so carve the cache back out when known.
const free = metric?.free ?? asNumber(fallback?.free) ?? Math.max(total - used - cache, 0);
const free =
metric?.free ??
asNumber(fallback?.free) ??
(usageUnavailable ? 0 : Math.max(total - used - cache, 0));
const usage =
metric?.current ?? (total > 0 ? (used / total) * 100 : (asNumber(fallback?.usage) ?? 0));
metric?.current ??
(usageUnavailable ? 0 : total > 0 ? (used / total) * 100 : (asNumber(fallback?.usage) ?? 0));
return {
total,
used,
free,
usage,
usageUnavailable,
cache: cache > 0 ? cache : undefined,
swapUsed: asNumber(proxmoxMeta?.swapUsed) ?? asNumber(fallback?.swapUsed),
swapTotal: asNumber(proxmoxMeta?.swapTotal) ?? asNumber(fallback?.swapTotal),
+5 -1
View File
@@ -261,6 +261,10 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
if monitorOnly {
evalOpts = &metricOptions{MonitorOnly: true}
}
var memoryMetric *UnifiedResourceMetric
if !snapshot.MemoryUnavailable {
memoryMetric = &UnifiedResourceMetric{Percent: memUsage}
}
m.evaluateUnifiedMetrics(&UnifiedResourceInput{
ID: guestID,
Type: snapshot.resourceType(),
@@ -268,7 +272,7 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
Node: node,
Instance: instanceName,
CPU: &UnifiedResourceMetric{Percent: cpu},
Memory: &UnifiedResourceMetric{Percent: memUsage},
Memory: memoryMetric,
Disk: &UnifiedResourceMetric{Percent: diskUsage},
DiskRead: &UnifiedResourceMetric{Value: float64(diskRead) / 1024 / 1024},
DiskWrite: &UnifiedResourceMetric{Value: float64(diskWrite) / 1024 / 1024},
+46 -41
View File
@@ -28,11 +28,14 @@ type guestSnapshot struct {
CPUPercent float64
MemUsage float64
DiskUsage float64
DiskRead int64
DiskWrite int64
NetworkIn int64
NetworkOut int64
// MemoryUnavailable prevents a missing cache-aware sample from clearing a
// real alert or starting a false one.
MemoryUnavailable bool
DiskUsage float64
DiskRead int64
DiskWrite int64
NetworkIn int64
NetworkOut int64
Disks []models.Disk
Tags []string
@@ -93,47 +96,49 @@ func (g guestSnapshot) metrics() guestMetrics {
func guestSnapshotFromVM(vm models.VM) guestSnapshot {
return guestSnapshot{
Kind: guestKindVM,
ID: vm.ID,
VMID: vm.VMID,
Name: vm.Name,
Node: vm.Node,
Instance: vm.Instance,
Status: vm.Status,
Lock: vm.Lock,
CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(vm.CPU),
MemUsage: vm.Memory.Usage,
DiskUsage: vm.Disk.Usage,
DiskRead: vm.DiskRead,
DiskWrite: vm.DiskWrite,
NetworkIn: vm.NetworkIn,
NetworkOut: vm.NetworkOut,
Disks: append([]models.Disk(nil), vm.Disks...),
Tags: append([]string(nil), vm.Tags...),
OnBoot: vm.OnBoot,
Kind: guestKindVM,
ID: vm.ID,
VMID: vm.VMID,
Name: vm.Name,
Node: vm.Node,
Instance: vm.Instance,
Status: vm.Status,
Lock: vm.Lock,
CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(vm.CPU),
MemUsage: vm.Memory.Usage,
MemoryUnavailable: vm.Memory.UsageUnavailable,
DiskUsage: vm.Disk.Usage,
DiskRead: vm.DiskRead,
DiskWrite: vm.DiskWrite,
NetworkIn: vm.NetworkIn,
NetworkOut: vm.NetworkOut,
Disks: append([]models.Disk(nil), vm.Disks...),
Tags: append([]string(nil), vm.Tags...),
OnBoot: vm.OnBoot,
}.normalizeCollections()
}
func guestSnapshotFromContainer(container models.Container) guestSnapshot {
return guestSnapshot{
Kind: guestKindContainer,
ID: container.ID,
VMID: container.VMID,
Name: container.Name,
Node: container.Node,
Instance: container.Instance,
Status: container.Status,
Lock: container.Lock,
CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(container.CPU),
MemUsage: container.Memory.Usage,
DiskUsage: container.Disk.Usage,
DiskRead: container.DiskRead,
DiskWrite: container.DiskWrite,
NetworkIn: container.NetworkIn,
NetworkOut: container.NetworkOut,
Disks: append([]models.Disk(nil), container.Disks...),
Tags: append([]string(nil), container.Tags...),
OnBoot: container.OnBoot,
Kind: guestKindContainer,
ID: container.ID,
VMID: container.VMID,
Name: container.Name,
Node: container.Node,
Instance: container.Instance,
Status: container.Status,
Lock: container.Lock,
CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(container.CPU),
MemUsage: container.Memory.Usage,
MemoryUnavailable: container.Memory.UsageUnavailable,
DiskUsage: container.Disk.Usage,
DiskRead: container.DiskRead,
DiskWrite: container.DiskWrite,
NetworkIn: container.NetworkIn,
NetworkOut: container.NetworkOut,
Disks: append([]models.Disk(nil), container.Disks...),
Tags: append([]string(nil), container.Tags...),
OnBoot: container.OnBoot,
}.normalizeCollections()
}
+2 -2
View File
@@ -307,7 +307,7 @@ func (m *Manager) CheckHost(host models.Host) {
m.clearHostMetricAlerts(host.ID, "cpu")
}
if thresholds.Memory != nil {
if thresholds.Memory != nil && host.Memory.HasKnownUsage() {
memMetadata := cloneMetadata(baseMetadata)
memMetadata["metric"] = "memory"
memMetadata["memoryUsagePercent"] = host.Memory.Usage
@@ -326,7 +326,7 @@ func (m *Manager) CheckHost(host models.Host) {
} else {
m.checkMetricWithCanonicalSpec(spec, resourceName, nodeName, instanceName, "agent", host.Memory.Usage, thresholds.Memory, &metricOptions{Metadata: memMetadata})
}
} else {
} else if thresholds.Memory == nil {
m.clearHostMetricAlerts(host.ID, "memory")
}
+5 -1
View File
@@ -100,6 +100,10 @@ func (m *Manager) CheckNode(node models.Node) {
Str("node", node.Name).
Msg("Skipping node metric alerts - host agent is monitoring this machine")
} else {
var memoryMetric *UnifiedResourceMetric
if node.Memory.HasKnownUsage() {
memoryMetric = &UnifiedResourceMetric{Percent: node.Memory.Usage}
}
m.evaluateUnifiedMetrics(&UnifiedResourceInput{
ID: node.ID,
Type: "node",
@@ -107,7 +111,7 @@ func (m *Manager) CheckNode(node models.Node) {
Node: node.Name,
Instance: node.Instance,
CPU: &UnifiedResourceMetric{Percent: node.CPU * 100},
Memory: &UnifiedResourceMetric{Percent: node.Memory.Usage},
Memory: memoryMetric,
Disk: &UnifiedResourceMetric{Percent: node.Disk.Usage},
}, thresholds, nil)
@@ -6,6 +6,7 @@ import (
"time"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
@@ -566,3 +567,105 @@ func TestEnsureOperationalContractKeepsExplicitUncertainStateAheadOfLegacyAcknow
t.Fatalf("LastObservedAt = %s, want %s", got, observedAt)
}
}
func TestMemoryUnavailablePreservesActiveAlertsUntilTrustedRecovery(t *testing.T) {
tests := []struct {
name string
setup func(*Manager)
check func(*Manager, models.Memory)
}{
{
name: "guest",
setup: func(manager *Manager) {
manager.UpdateConfig(AlertConfig{
Enabled: true,
GuestDefaults: ThresholdConfig{
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
},
})
},
check: func(manager *Manager, memory models.Memory) {
manager.CheckGuest(models.VM{
ID: "qemu/1501",
Name: "linux-guest",
Node: "pve-1",
Instance: "test",
Status: "running",
Memory: memory,
}, "test")
},
},
{
name: "node",
setup: func(manager *Manager) {
manager.UpdateConfig(AlertConfig{
Enabled: true,
NodeDefaults: ThresholdConfig{
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
},
})
},
check: func(manager *Manager, memory models.Memory) {
manager.CheckNode(models.Node{
ID: "pve-1",
Name: "pve-1",
Instance: "test",
Status: "online",
Memory: memory,
})
},
},
{
name: "agent",
setup: func(manager *Manager) {
manager.UpdateConfig(AlertConfig{
Enabled: true,
AgentDefaults: ThresholdConfig{
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
},
})
},
check: func(manager *Manager, memory models.Memory) {
manager.CheckHost(models.Host{
ID: "agent-1501",
Hostname: "linux-host",
Status: "online",
Memory: memory,
})
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
manager := newUnifiedEvalParityManager(t)
tt.setup(manager)
disableTestTimeThresholds(manager)
tt.check(manager, models.Memory{
Total: 8 << 30,
Used: 7 << 30,
Usage: 90,
})
active := manager.GetActiveAlerts()
if len(active) != 1 || active[0].Type != "memory" {
t.Fatalf("active alerts after trusted high sample = %#v, want one memory alert", active)
}
tt.check(manager, models.UnavailableMemory(8<<30))
active = manager.GetActiveAlerts()
if len(active) != 1 || active[0].Type != "memory" || active[0].Value != 90 {
t.Fatalf("active alerts after unavailable sample = %#v, want prior memory alert", active)
}
tt.check(manager, models.Memory{
Total: 8 << 30,
Used: 5 << 30,
Usage: 70,
})
if active = manager.GetActiveAlerts(); len(active) != 0 {
t.Fatalf("active alerts after trusted recovery = %#v, want cleared", active)
}
})
}
}
+7
View File
@@ -1,6 +1,7 @@
package alerts
import (
"math"
"strings"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
@@ -255,6 +256,12 @@ func buildUnifiedMetricCandidates(input *UnifiedResourceInput, thresholds Thresh
if metric == nil {
return
}
if math.IsNaN(value) || math.IsInf(value, 0) {
return
}
if isPercentageMetric(metricType) && (value < 0 || value > 100) {
return
}
spec, err := buildCanonicalMetricSpec(input.ID, input.Name, resourceType, metricType, threshold)
if err != nil {
log.Warn().
+49
View File
@@ -33,6 +33,7 @@ func TestClassifyMemorySourceTrust(t *testing.T) {
{source: "listing", want: "fallback"},
{source: "powered-off", want: "fallback"},
{source: "previous-snapshot", want: "fallback"},
{source: "unavailable", want: "unavailable"},
{source: "", want: "fallback"},
}
@@ -45,6 +46,54 @@ func TestClassifyMemorySourceTrust(t *testing.T) {
}
}
func TestBuildMemorySourceDiagnosticsReportsUnavailableCacheAwareEvidence(t *testing.T) {
base := time.Date(2026, 7, 23, 12, 0, 0, 0, time.UTC)
snapshots := monitoring.DiagnosticSnapshotSet{
Nodes: []monitoring.NodeMemorySnapshot{{
Instance: "pve-a",
Node: "node-1",
RetrievedAt: base,
MemorySource: "unavailable",
FallbackReason: "cache-aware-memory-unavailable",
Raw: monitoring.NodeMemoryRaw{
Total: 8 << 30,
Used: 15 << 29,
Free: 1 << 29,
ProxmoxMemorySource: "cache-aware-unavailable",
},
}},
Guests: []monitoring.GuestMemorySnapshot{{
Instance: "pve-a",
GuestType: "qemu",
Node: "node-1",
VMID: 1501,
RetrievedAt: base,
MemorySource: "unavailable",
FallbackReason: "cache-aware-memory-unavailable",
Raw: monitoring.VMMemoryRaw{
MemInfoTotal: 8 << 30,
MemInfoFree: 1 << 29,
},
}},
}
nodeStats, breakdown := buildMemorySourceDiagnostics(snapshots)
if len(nodeStats) != 1 || nodeStats[0].Source != "unavailable" || nodeStats[0].Trust != "unavailable" {
t.Fatalf("node memory source stats = %#v, want explicit unavailable source", nodeStats)
}
if len(breakdown) != 2 {
t.Fatalf("memory source breakdown = %#v, want node and guest entries", breakdown)
}
for _, entry := range breakdown {
if entry.Source != "unavailable" || entry.Trust != "unavailable" || !entry.Fallback {
t.Fatalf("unavailable breakdown entry = %#v", entry)
}
if len(entry.FallbackReasons) != 1 || entry.FallbackReasons[0] != "cache-aware-memory-unavailable" {
t.Fatalf("fallback reasons = %#v, want cache-aware-memory-unavailable", entry.FallbackReasons)
}
}
}
func TestBuildMemorySourceDiagnostics(t *testing.T) {
base := time.Date(2026, 3, 14, 16, 30, 0, 0, time.UTC)
snapshots := monitoring.DiagnosticSnapshotSet{
@@ -0,0 +1,51 @@
package models
import (
"encoding/json"
"math"
"testing"
)
func TestUnavailableMemorySurvivesJSONReload(t *testing.T) {
before := UnavailableMemory(8 << 30)
payload, err := json.Marshal(before)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var after Memory
if err := json.Unmarshal(payload, &after); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if !after.UsageUnavailable {
t.Fatalf("UsageUnavailable = false after reload; payload=%s", payload)
}
if after.Total != before.Total {
t.Fatalf("Total = %d, want %d", after.Total, before.Total)
}
if after.HasKnownUsage() {
t.Fatalf("HasKnownUsage() = true after unavailable reload: %+v", after)
}
}
func TestMemoryHasKnownUsageAcceptsTrustedZeroUsage(t *testing.T) {
memory := Memory{Total: 8 << 30, Free: 8 << 30}
if !memory.HasKnownUsage() {
t.Fatalf("HasKnownUsage() = false for a valid zero-usage sample: %+v", memory)
}
}
func TestMemoryHasKnownUsageRejectsNonFiniteAndContradictoryValues(t *testing.T) {
for name, memory := range map[string]Memory{
"nan": {Total: 8 << 30, Used: 4 << 30, Free: 4 << 30, Usage: math.NaN()},
"infinity": {Total: 8 << 30, Used: 4 << 30, Free: 4 << 30, Usage: math.Inf(1)},
"free overflow": {Total: 8 << 30, Free: 9 << 30},
} {
t.Run(name, func(t *testing.T) {
if memory.HasKnownUsage() {
t.Fatalf("HasKnownUsage() = true for invalid memory: %+v", memory)
}
})
}
}
+38 -5
View File
@@ -3,6 +3,7 @@ package models
import (
"errors"
"fmt"
"math"
"net"
"net/url"
"sort"
@@ -2803,11 +2804,43 @@ type Memory struct {
Used int64 `json:"used"`
Free int64 `json:"free"`
// Reclaimable buff/cache (available - truly free); used + cache + free ≈ total.
Cache int64 `json:"cache,omitempty"`
Usage float64 `json:"usage"`
Balloon int64 `json:"balloon,omitempty"`
SwapUsed int64 `json:"swapUsed,omitempty"`
SwapTotal int64 `json:"swapTotal,omitempty"`
Cache int64 `json:"cache,omitempty"`
// UsageUnavailable is true when total capacity is known but no cache-aware
// source can establish current usage. Consumers must not interpret the
// zero-valued Used, Free, or Usage fields as a measurement in that state.
UsageUnavailable bool `json:"usageUnavailable,omitempty"`
Usage float64 `json:"usage"`
Balloon int64 `json:"balloon,omitempty"`
SwapUsed int64 `json:"swapUsed,omitempty"`
SwapTotal int64 `json:"swapTotal,omitempty"`
}
// HasKnownUsage reports whether this memory value is safe for live
// projections, history, and threshold evaluation.
func (m Memory) HasKnownUsage() bool {
if m.UsageUnavailable || math.IsNaN(m.Usage) || math.IsInf(m.Usage, 0) || m.Usage < 0 || m.Usage > 100 {
return false
}
if m.Total == 0 {
return m.Usage > 0
}
return m.Total > 0 &&
m.Used >= 0 &&
m.Used <= m.Total &&
m.Free >= 0 &&
m.Free <= m.Total
}
// UnavailableMemory preserves a known capacity without inventing usage from
// total-free when cache-aware evidence is missing.
func UnavailableMemory(total int64) Memory {
if total < 0 {
total = 0
}
return Memory{
Total: total,
UsageUnavailable: true,
}
}
type GuestNetworkInterface struct {
@@ -866,14 +866,17 @@ func TestProxmoxGuestMemoryFallbackUsesInstanceScopedCachesAndAgentMeminfo(t *te
"guest_memory_sources.go": {
"func shouldPreferGuestAgentMemAvailable(status *proxmox.VMStatus, memTotal uint64) bool {",
"func (m *Monitor) tryGuestAgentMemAvailable(",
"if memAvailable == 0 && shouldPreferGuestAgentMemAvailable(status, memTotal) {",
"if rrdAvailable, rrdErr := m.getVMRRDMetrics(ctx, client, instanceName, node, vmid); rrdErr == nil && rrdAvailable > 0 {",
"if agentAvailable, ok := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw); ok {",
`memorySource = "guest-agent-meminfo"`,
"if !hasMemAvailable && shouldPreferGuestAgentMemAvailable(status, memTotal) {",
"if rrdMemory, rrdErr := m.getVMRRDMemory(ctx, client, instanceName, node, vmid); rrdErr == nil {",
"case rrdMemory.hasUsed && rrdMemory.used <= memTotal:",
"if agentAvailable, agentSource, ok := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw); ok {",
"memorySource = agentSource",
"guestRaw.GuestAgentMemAvailable = agentAvailable",
`memorySource = "unavailable"`,
},
"monitor.go": {
"func (m *Monitor) getVMRRDMetrics(ctx context.Context, client PVEClientInterface, instanceName, node string, vmid int) (uint64, error) {",
"func (m *Monitor) getVMRRDMemory(ctx context.Context, client PVEClientInterface, instanceName, node string, vmid int) (rrdMemCacheEntry, error) {",
"cacheKey := guestMemoryCacheKey(instanceName, node, vmid)",
"vmAgentMemCache map[string]agentMemCacheEntry",
},
@@ -1999,14 +2002,14 @@ func TestMockNativePollersDeferToCanonicalMockSampler(t *testing.T) {
file: "monitor_polling_vm.go",
snippets: []string{
"if !shouldSkipNativeMockStateMetricWrites() {",
`m.recordGuestMetric("vm", vm.ID, unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), vm.Memory.Usage, vm.Disk.Usage, -1, -1, -1, -1, now)`,
`m.recordGuestMetric("vm", vm.ID, unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), historyMemoryUsage(vm.Memory), vm.Disk.Usage, -1, -1, -1, -1, now)`,
},
},
{
file: "monitor_polling_containers.go",
snippets: []string{
"if !shouldSkipNativeMockStateMetricWrites() {",
`m.recordGuestMetric("container", ct.ID, unifiedresources.ProxmoxGuestCPUPercent(ct.CPU), ct.Memory.Usage, ct.Disk.Usage, -1, -1, -1, -1, now)`,
`m.recordGuestMetric("container", ct.ID, unifiedresources.ProxmoxGuestCPUPercent(ct.CPU), historyMemoryUsage(ct.Memory), ct.Disk.Usage, -1, -1, -1, -1, now)`,
},
},
{
@@ -64,6 +64,15 @@ type VMMemoryRaw struct {
MemInfoShared uint64 `json:"meminfoShared,omitempty"`
MemInfoTotalMinusUsed uint64 `json:"meminfoTotalMinusUsed,omitempty"`
GuestAgentMemAvailable uint64 `json:"guestAgentMemAvailable,omitempty"`
GuestAgentMemFree uint64 `json:"guestAgentMemFree,omitempty"`
GuestAgentMemBuffers uint64 `json:"guestAgentMemBuffers,omitempty"`
GuestAgentMemCached uint64 `json:"guestAgentMemCached,omitempty"`
GuestAgentSReclaimable uint64 `json:"guestAgentSReclaimable,omitempty"`
GuestAgentShmem uint64 `json:"guestAgentShmem,omitempty"`
GuestAgentDerived bool `json:"guestAgentDerived,omitempty"`
RRDMemAvailable uint64 `json:"rrdMemAvailable,omitempty"`
RRDMemUsed uint64 `json:"rrdMemUsed,omitempty"`
RRDMaxMem uint64 `json:"rrdMaxMem,omitempty"`
HostAgentTotal uint64 `json:"hostAgentTotal,omitempty"`
HostAgentUsed uint64 `json:"hostAgentUsed,omitempty"`
Agent int `json:"agent,omitempty"`
@@ -256,6 +265,15 @@ func (m *Monitor) logGuestMemorySource(instance, guestType, node string, vmid in
if snapshot.Raw.GuestAgentMemAvailable > 0 {
evt = evt.Uint64("guestAgentMemAvailable", snapshot.Raw.GuestAgentMemAvailable)
}
if snapshot.Raw.GuestAgentDerived {
evt = evt.Bool("guestAgentDerived", true)
}
if snapshot.Raw.RRDMemAvailable > 0 {
evt = evt.Uint64("rrdMemAvailable", snapshot.Raw.RRDMemAvailable)
}
if snapshot.Raw.RRDMemUsed > 0 {
evt = evt.Uint64("rrdMemUsed", snapshot.Raw.RRDMemUsed)
}
if snapshot.Raw.MemInfoBuffers > 0 {
evt = evt.Uint64("memInfoBuffers", snapshot.Raw.MemInfoBuffers)
}
+48 -12
View File
@@ -5,6 +5,8 @@ import (
"fmt"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
const (
@@ -17,6 +19,7 @@ const (
type agentMemCacheEntry struct {
available uint64
info proxmox.LinuxMemoryAvailability
negative bool
fetchedAt time.Time
}
@@ -25,17 +28,22 @@ type guestAgentMemAvailableClient interface {
GetVMMemAvailableFromAgent(ctx context.Context, node string, vmid int) (uint64, error)
}
type guestAgentMemoryAvailabilityClient interface {
GetVMMemoryAvailabilityFromAgent(ctx context.Context, node string, vmid int) (proxmox.LinuxMemoryAvailability, error)
}
func guestMemoryCacheKey(instanceName, node string, vmid int) string {
return fmt.Sprintf("%s/%s/%d", instanceName, node, vmid)
}
func (m *Monitor) getVMAgentMemAvailable(ctx context.Context, client PVEClientInterface, instanceName, node string, vmid int) (uint64, error) {
memClient, ok := client.(guestAgentMemAvailableClient)
if !ok {
return 0, fmt.Errorf("guest agent meminfo fallback unsupported")
}
info, err := m.getVMAgentMemoryAvailability(ctx, client, instanceName, node, vmid)
return info.EffectiveAvailable, err
}
func (m *Monitor) getVMAgentMemoryAvailability(ctx context.Context, client PVEClientInterface, instanceName, node string, vmid int) (proxmox.LinuxMemoryAvailability, error) {
if node == "" || vmid <= 0 {
return 0, fmt.Errorf("invalid arguments for guest agent meminfo lookup")
return proxmox.LinuxMemoryAvailability{}, fmt.Errorf("invalid arguments for guest agent meminfo lookup")
}
cacheKey := guestMemoryCacheKey(instanceName, node, vmid)
@@ -50,9 +58,16 @@ func (m *Monitor) getVMAgentMemAvailable(ctx context.Context, client PVEClientIn
if now.Sub(entry.fetchedAt) < ttl {
m.rrdCacheMu.RUnlock()
if entry.negative {
return 0, fmt.Errorf("guest agent meminfo fallback unavailable")
return proxmox.LinuxMemoryAvailability{}, fmt.Errorf("guest agent meminfo fallback unavailable")
}
return entry.available, nil
if entry.info.Source != "" {
return entry.info, nil
}
return proxmox.LinuxMemoryAvailability{
Available: entry.available,
EffectiveAvailable: entry.available,
Source: "meminfo-available",
}, nil
}
}
m.rrdCacheMu.RUnlock()
@@ -60,22 +75,43 @@ func (m *Monitor) getVMAgentMemAvailable(ctx context.Context, client PVEClientIn
requestCtx, cancel := context.WithTimeout(ctx, vmAgentMemRequestTTL)
defer cancel()
available, err := memClient.GetVMMemAvailableFromAgent(requestCtx, node, vmid)
var info proxmox.LinuxMemoryAvailability
var err error
if memClient, ok := client.(guestAgentMemoryAvailabilityClient); ok {
info, err = memClient.GetVMMemoryAvailabilityFromAgent(requestCtx, node, vmid)
} else if memClient, ok := client.(guestAgentMemAvailableClient); ok {
var available uint64
available, err = memClient.GetVMMemAvailableFromAgent(requestCtx, node, vmid)
if available > 0 {
info = proxmox.LinuxMemoryAvailability{
Available: available,
EffectiveAvailable: available,
Source: "meminfo-available",
}
}
} else {
return proxmox.LinuxMemoryAvailability{}, fmt.Errorf("guest agent meminfo fallback unsupported")
}
m.rrdCacheMu.Lock()
defer m.rrdCacheMu.Unlock()
if m.vmAgentMemCache == nil {
m.vmAgentMemCache = make(map[string]agentMemCacheEntry)
}
if err != nil || available == 0 {
if err != nil || info.Source == "" {
m.vmAgentMemCache[cacheKey] = agentMemCacheEntry{negative: true, fetchedAt: now}
if err == nil {
err = fmt.Errorf("guest agent meminfo fallback unavailable")
}
return 0, err
return proxmox.LinuxMemoryAvailability{}, err
}
m.vmAgentMemCache[cacheKey] = agentMemCacheEntry{available: available, fetchedAt: now}
return available, nil
m.vmAgentMemCache[cacheKey] = agentMemCacheEntry{
available: info.EffectiveAvailable,
info: info,
fetchedAt: now,
}
return info, nil
}
func (m *Monitor) vmAgentMemNegativeCacheTTL(instanceName, node string, vmid int) time.Duration {
@@ -13,6 +13,7 @@ type guestMemoryAgentTestClient struct {
*stubPVEClient
vmRRDPoints []proxmox.GuestRRDPoint
memAvailable uint64
memInfo *proxmox.LinuxMemoryAvailability
memErr error
rrdCalls int
memCalls int
@@ -31,6 +32,24 @@ func (c *guestMemoryAgentTestClient) GetVMMemAvailableFromAgent(ctx context.Cont
return c.memAvailable, nil
}
func (c *guestMemoryAgentTestClient) GetVMMemoryAvailabilityFromAgent(ctx context.Context, node string, vmid int) (proxmox.LinuxMemoryAvailability, error) {
c.memCalls++
if c.memErr != nil {
return proxmox.LinuxMemoryAvailability{}, c.memErr
}
if c.memInfo != nil {
return *c.memInfo, nil
}
if c.memAvailable == 0 {
return proxmox.LinuxMemoryAvailability{}, nil
}
return proxmox.LinuxMemoryAvailability{
Available: c.memAvailable,
EffectiveAvailable: c.memAvailable,
Source: "meminfo-available",
}, nil
}
func TestGetVMRRDMetricsCacheKeyIncludesInstance(t *testing.T) {
t.Parallel()
@@ -65,6 +84,79 @@ func TestGetVMRRDMetricsCacheKeyIncludesInstance(t *testing.T) {
}
}
func TestResolveGuestStatusMemoryAcceptsExplicitZeroRRDUsed(t *testing.T) {
t.Parallel()
const gib = uint64(1024 * 1024 * 1024)
mon := &Monitor{vmRRDMemCache: make(map[string]rrdMemCacheEntry)}
client := &guestMemoryAgentTestClient{
stubPVEClient: &stubPVEClient{},
vmRRDPoints: []proxmox.GuestRRDPoint{{
MaxMem: floatPtr(float64(8 * gib)),
MemUsed: floatPtr(0),
}},
}
total, used, source := mon.resolveGuestStatusMemory(
context.Background(),
client,
"pve-a",
"idle-vm",
"node1",
100,
"pve-a:node1:100",
&proxmox.VMStatus{MaxMem: 8 * gib, Mem: 8 * gib},
nil,
8*gib,
"",
&VMMemoryRaw{},
)
if total != 8*gib || used != 0 || source != "rrd-memused" {
t.Fatalf("resolved memory = total %d used %d source %q, want idle RRD sample", total, used, source)
}
}
func TestResolveGuestStatusMemoryAcceptsExplicitZeroGuestAgentAvailable(t *testing.T) {
t.Parallel()
const gib = uint64(1024 * 1024 * 1024)
mon := &Monitor{
vmRRDMemCache: make(map[string]rrdMemCacheEntry),
vmAgentMemCache: make(map[string]agentMemCacheEntry),
}
client := &guestMemoryAgentTestClient{
stubPVEClient: &stubPVEClient{},
memInfo: &proxmox.LinuxMemoryAvailability{
Total: 8 * gib,
Available: 0,
EffectiveAvailable: 0,
Source: "meminfo-available",
},
}
total, used, source := mon.resolveGuestStatusMemory(
context.Background(),
client,
"pve-a",
"pressured-vm",
"node1",
101,
"pve-a:node1:101",
&proxmox.VMStatus{
MaxMem: 8 * gib,
Mem: 8 * gib,
Agent: proxmox.VMAgentField{Value: 1},
},
nil,
8*gib,
"",
&VMMemoryRaw{},
)
if total != 8*gib || used != 8*gib || source != "guest-agent-meminfo" {
t.Fatalf("resolved memory = total %d used %d source %q, want full-pressure guest-agent sample", total, used, source)
}
}
func TestGetVMAgentMemAvailableCachesResults(t *testing.T) {
t.Parallel()
+121 -62
View File
@@ -44,9 +44,7 @@ func deriveGuestMemInfoAvailable(memInfo *proxmox.VMMemInfo, guestRaw *VMMemoryR
componentAvailable += memInfo.Cached
}
}
if memInfo.Total > 0 && componentAvailable > memInfo.Total {
componentAvailable = memInfo.Total
}
componentsConflict := memInfo.Total > 0 && componentAvailable > memInfo.Total
availableFromUsed := uint64(0)
if memInfo.Total > 0 && memInfo.Used > 0 && memInfo.Total >= memInfo.Used {
@@ -61,7 +59,7 @@ func deriveGuestMemInfoAvailable(memInfo *proxmox.VMMemInfo, guestRaw *VMMemoryR
memInfo.Cached == 0
switch {
case memInfo.Available > 0:
case memInfo.Available > 0 && (memInfo.Total == 0 || memInfo.Available <= memInfo.Total):
return memInfo.Available, "available-field"
case memInfo.Free > 0 || memInfo.Buffers > 0 || memInfo.Cached > 0:
if availableFromUsed > 0 && missingCacheMetrics {
@@ -76,6 +74,9 @@ func deriveGuestMemInfoAvailable(memInfo *proxmox.VMMemInfo, guestRaw *VMMemoryR
if missingCacheMetrics {
return 0, ""
}
if componentsConflict {
return 0, ""
}
return componentAvailable, "derived-free-buffers-cached"
default:
if availableFromUsed > 0 && missingCacheMetrics {
@@ -206,13 +207,27 @@ func (m *Monitor) tryGuestAgentMemAvailable(
vmid int,
memTotal uint64,
guestRaw *VMMemoryRaw,
) (uint64, bool) {
agentAvailable, agentErr := m.getVMAgentMemAvailable(ctx, client, instanceName, node, vmid)
if agentErr != nil || agentAvailable == 0 {
return 0, false
) (uint64, string, bool) {
availability, agentErr := m.getVMAgentMemoryAvailability(ctx, client, instanceName, node, vmid)
if agentErr != nil || availability.Source == "" {
return 0, "", false
}
agentAvailable := availability.EffectiveAvailable
if guestRaw != nil {
guestRaw.GuestAgentMemAvailable = agentAvailable
guestRaw.GuestAgentMemFree = availability.Free
guestRaw.GuestAgentMemBuffers = availability.Buffers
guestRaw.GuestAgentMemCached = availability.Cached
guestRaw.GuestAgentSReclaimable = availability.SReclaimable
guestRaw.GuestAgentShmem = availability.Shmem
guestRaw.GuestAgentDerived = availability.Source == "meminfo-derived"
}
if memTotal == 0 || agentAvailable > memTotal {
return 0, "", false
}
source := "guest-agent-meminfo"
if availability.Source == "meminfo-derived" {
source = "guest-agent-meminfo-derived"
}
log.Debug().
Str("vm", guestName).
@@ -220,8 +235,9 @@ func (m *Monitor) tryGuestAgentMemAvailable(
Int("vmid", vmid).
Uint64("total", memTotal).
Uint64("available", agentAvailable).
Str("source", source).
Msg("QEMU memory: using guest agent /proc/meminfo fallback (excludes reclaimable cache)")
return agentAvailable, true
return agentAvailable, source, true
}
func (m *Monitor) resolveGuestStatusMemory(
@@ -260,43 +276,67 @@ func (m *Monitor) resolveGuestStatusMemory(
}
memAvailable := uint64(0)
hasMemAvailable := false
derivedTotalMinusUsedAvailable := uint64(0)
selectedUsed := uint64(0)
hasSelectedUsed := false
if status.MemInfo != nil {
memAvailable, memorySource = deriveGuestMemInfoAvailable(status.MemInfo, guestRaw)
if memAvailable > 0 && memorySource == "derived-total-minus-used" {
log.Debug().
Str("vm", guestName).
Str("node", node).
Int("vmid", vmid).
Uint64("total", memTotal).
Uint64("available", memAvailable).
Uint64("availableFromUsed", guestRaw.MemInfoTotalMinusUsed).
Msg("QEMU memory: deriving guest available from total-used gap when cache fields are missing")
hasMemAvailable = memorySource != ""
if memAvailable > memTotal {
memAvailable = 0
hasMemAvailable = false
memorySource = ""
}
if memorySource == "derived-total-minus-used" {
derivedTotalMinusUsedAvailable = memAvailable
memAvailable = 0
hasMemAvailable = false
memorySource = ""
}
}
triedGuestAgentMemAvailable := false
if memAvailable == 0 && shouldPreferGuestAgentMemAvailable(status, memTotal) {
if !hasMemAvailable && shouldPreferGuestAgentMemAvailable(status, memTotal) {
triedGuestAgentMemAvailable = true
if agentAvailable, ok := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw); ok {
if agentAvailable, agentSource, ok := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw); ok {
memAvailable = agentAvailable
memorySource = "guest-agent-meminfo"
hasMemAvailable = true
memorySource = agentSource
}
}
if memAvailable == 0 {
if rrdAvailable, rrdErr := m.getVMRRDMetrics(ctx, client, instanceName, node, vmid); rrdErr == nil && rrdAvailable > 0 {
memAvailable = rrdAvailable
memorySource = "rrd-memavailable"
if !hasMemAvailable {
if rrdMemory, rrdErr := m.getVMRRDMemory(ctx, client, instanceName, node, vmid); rrdErr == nil {
if guestRaw != nil {
guestRaw.MemInfoAvailable = memAvailable
guestRaw.RRDMemAvailable = rrdMemory.available
guestRaw.RRDMemUsed = rrdMemory.used
guestRaw.RRDMaxMem = rrdMemory.total
}
switch {
case rrdMemory.hasAvail && rrdMemory.available <= memTotal:
memAvailable = rrdMemory.available
hasMemAvailable = true
memorySource = "rrd-memavailable"
log.Debug().
Str("vm", guestName).
Str("node", node).
Int("vmid", vmid).
Uint64("total", memTotal).
Uint64("available", memAvailable).
Msg("QEMU memory: using RRD memavailable fallback")
case rrdMemory.hasUsed && rrdMemory.used <= memTotal:
selectedUsed = rrdMemory.used
hasSelectedUsed = true
memorySource = "rrd-memused"
log.Debug().
Str("vm", guestName).
Str("node", node).
Int("vmid", vmid).
Uint64("total", memTotal).
Uint64("used", rrdMemory.used).
Msg("QEMU memory: using RRD memused fallback")
}
log.Debug().
Str("vm", guestName).
Str("node", node).
Int("vmid", vmid).
Uint64("total", memTotal).
Uint64("available", memAvailable).
Msg("QEMU memory: using RRD memavailable fallback (excludes reclaimable cache)")
} else if rrdErr != nil {
log.Debug().
Err(rrdErr).
@@ -307,50 +347,69 @@ func (m *Monitor) resolveGuestStatusMemory(
}
}
if memAvailable == 0 && status.Agent.IsAvailable() && !triedGuestAgentMemAvailable {
if agentAvailable, ok := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw); ok {
if !hasMemAvailable && !hasSelectedUsed && status.Agent.IsAvailable() && !triedGuestAgentMemAvailable {
if agentAvailable, agentSource, ok := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw); ok {
memAvailable = agentAvailable
memorySource = "guest-agent-meminfo"
hasMemAvailable = true
memorySource = agentSource
}
}
if memAvailable == 0 {
if !hasMemAvailable && !hasSelectedUsed && derivedTotalMinusUsedAvailable > 0 {
memAvailable = derivedTotalMinusUsedAvailable
hasMemAvailable = true
memorySource = "derived-total-minus-used"
log.Debug().
Str("vm", guestName).
Str("node", node).
Int("vmid", vmid).
Uint64("total", memTotal).
Uint64("available", memAvailable).
Uint64("availableFromUsed", derivedTotalMinusUsedAvailable).
Msg("QEMU memory: deriving guest available from total-used gap after preferred fallbacks")
}
if !hasMemAvailable && !hasSelectedUsed {
if agentHost, ok := vmIDToHostAgent[guestID]; ok &&
agentHost.Memory.Total > 0 &&
agentHost.Memory.Used >= 0 &&
agentHost.Memory.HasKnownUsage() &&
agentHost.Memory.Used <= int64(memTotal) &&
agentHost.Memory.Total >= agentHost.Memory.Used {
agentAvailable := uint64(agentHost.Memory.Total - agentHost.Memory.Used)
if agentAvailable > 0 {
memAvailable = agentAvailable
memorySource = "agent"
if guestRaw != nil {
guestRaw.HostAgentTotal = uint64(agentHost.Memory.Total)
guestRaw.HostAgentUsed = uint64(agentHost.Memory.Used)
}
log.Debug().
Str("vm", guestName).
Str("node", node).
Int("vmid", vmid).
Uint64("total", memTotal).
Uint64("available", memAvailable).
Int64("agentTotal", agentHost.Memory.Total).
Int64("agentUsed", agentHost.Memory.Used).
Msg("QEMU memory: using linked Pulse host agent memory (excludes page cache)")
memAvailable = agentAvailable
hasMemAvailable = true
selectedUsed = uint64(agentHost.Memory.Used)
hasSelectedUsed = true
memorySource = "agent"
if guestRaw != nil {
guestRaw.HostAgentTotal = uint64(agentHost.Memory.Total)
guestRaw.HostAgentUsed = uint64(agentHost.Memory.Used)
}
log.Debug().
Str("vm", guestName).
Str("node", node).
Int("vmid", vmid).
Uint64("total", memTotal).
Uint64("available", memAvailable).
Int64("agentTotal", agentHost.Memory.Total).
Int64("agentUsed", agentHost.Memory.Used).
Msg("QEMU memory: using linked Pulse host agent memory (excludes page cache)")
}
}
memUsed := uint64(0)
switch {
case memAvailable > 0:
if memAvailable > memTotal {
memAvailable = memTotal
}
case hasSelectedUsed:
memUsed = selectedUsed
case hasMemAvailable:
memUsed = memTotal - memAvailable
default:
memUsed, memorySource = selectGuestLowTrustUsedMemory(memTotal, status)
if memorySource == "" {
memorySource = "status-unavailable"
if status.MemInfo != nil {
memorySource = "unavailable"
} else {
memUsed, memorySource = selectGuestLowTrustUsedMemory(memTotal, status)
if memorySource == "" {
memorySource = "unavailable"
}
}
}
if memUsed > memTotal {
@@ -20,11 +20,11 @@ const (
func guestMemorySourceReliability(source string) int {
switch CanonicalMemorySource(source) {
case "available-field", "derived-free-buffers-cached",
"guest-agent-meminfo", "rrd-memavailable", "rrd-memused", "agent":
"guest-agent-meminfo", "guest-agent-meminfo-derived", "rrd-memavailable", "rrd-memused", "agent":
return guestMemoryReliabilityTrusted
case "derived-total-minus-used", "previous-snapshot":
return guestMemoryReliabilityFallback
case "unknown", "cluster-resources", "status-mem", "status-freemem", "status-unavailable":
case "unknown", "unavailable", "cluster-resources", "status-mem", "status-freemem", "status-unavailable":
return guestMemoryReliabilityLow
default:
return guestMemoryReliabilityFallback
@@ -36,6 +36,34 @@ func TestStabilizeGuestLowTrustMemoryCarriesForwardTrustedSnapshot(t *testing.T)
}
}
func TestStabilizeGuestLowTrustMemoryCarriesForwardAcrossUnavailableReconnect(t *testing.T) {
const gib = uint64(1024 * 1024 * 1024)
now := time.Now()
prev := &GuestMemorySnapshot{
Status: "running",
RetrievedAt: now.Add(-30 * time.Second),
MemorySource: "guest-agent-meminfo",
Memory: models.Memory{
Total: 8 * int64(gib),
Used: 3 * int64(gib),
Free: 5 * int64(gib),
Usage: 37.5,
},
}
used, source, notes := stabilizeGuestLowTrustMemory(prev, "running", "unavailable", 8*gib, 0, now, false)
if used != 3*gib {
t.Fatalf("used = %d, want %d", used, 3*gib)
}
if source != "previous-snapshot" {
t.Fatalf("source = %q, want previous-snapshot", source)
}
if len(notes) != 1 {
t.Fatalf("notes = %#v, want carry-forward diagnostic", notes)
}
}
func TestStabilizeGuestLowTrustMemoryUsesHealthyGuestAgentEvidence(t *testing.T) {
const gib = uint64(1024 * 1024 * 1024)
@@ -12,6 +12,7 @@ type MemorySourceDescriptor struct {
var memorySourceCatalog = map[string]MemorySourceDescriptor{
"": {Canonical: "unknown", Trust: "fallback", Fallback: true},
"unknown": {Canonical: "unknown", Trust: "fallback", Fallback: true},
"unavailable": {Canonical: "unavailable", Trust: "unavailable", Fallback: true, DefaultFallbackReason: "cache-aware-memory-unavailable"},
"nodes-endpoint": {Canonical: "nodes-endpoint", Trust: "fallback", Fallback: true},
"node-status-used": {Canonical: "node-status-used", Trust: "fallback", Fallback: true},
"previous-snapshot": {Canonical: "previous-snapshot", Trust: "fallback", Fallback: true, DefaultFallbackReason: "preserved-previous-snapshot"},
@@ -25,6 +26,7 @@ var memorySourceCatalog = map[string]MemorySourceDescriptor{
"derived-total-minus-used": {Canonical: "derived-total-minus-used", Trust: "derived", Fallback: true, DefaultFallbackReason: "derived-total-minus-used"},
"meminfo-total-minus-used": {Canonical: "derived-total-minus-used", Trust: "derived", Fallback: true, DefaultFallbackReason: "derived-total-minus-used"},
"guest-agent-meminfo": {Canonical: "guest-agent-meminfo", Trust: "fallback", Fallback: true, DefaultFallbackReason: "guest-agent-meminfo"},
"guest-agent-meminfo-derived": {Canonical: "guest-agent-meminfo-derived", Trust: "derived", Fallback: true, DefaultFallbackReason: "guest-agent-meminfo-derived"},
"rrd-memavailable": {Canonical: "rrd-memavailable", Trust: "fallback", Fallback: true, DefaultFallbackReason: "rrd-memavailable"},
"rrd-available": {Canonical: "rrd-memavailable", Trust: "fallback", Fallback: true, DefaultFallbackReason: "rrd-memavailable"},
"rrd-memused": {Canonical: "rrd-memused", Trust: "fallback", Fallback: true, DefaultFallbackReason: "rrd-memused"},
@@ -97,6 +97,14 @@ func TestDescribeMemorySourceCanonicalizesAliases(t *testing.T) {
wantCanonical: "powered-off",
wantTrust: "fallback",
},
{
name: "unavailable is explicit and diagnosable",
source: "unavailable",
wantCanonical: "unavailable",
wantTrust: "unavailable",
wantFallback: true,
wantDefaultFallback: "cache-aware-memory-unavailable",
},
{
name: "unknown source falls back canonically",
source: "custom-source",
@@ -138,7 +146,7 @@ func TestMockVMPollingDefersMemoryHistoryToCanonicalSampler(t *testing.T) {
requiredSnippets := []string{
"if !shouldSkipNativeMockStateMetricWrites() {",
`m.recordGuestMetric("vm", vm.ID, unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), vm.Memory.Usage, vm.Disk.Usage, -1, -1, -1, -1, now)`,
`m.recordGuestMetric("vm", vm.ID, unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), historyMemoryUsage(vm.Memory), vm.Disk.Usage, -1, -1, -1, -1, now)`,
}
for _, snippet := range requiredSnippets {
if !strings.Contains(source, snippet) {
@@ -61,6 +61,8 @@ func TestPollPVENodeMemoryTrustCharacterization(t *testing.T) {
wantFallback string
wantUsed uint64
wantRawSource string
wantRRDUsed uint64
wantUnknown bool
}{
{
name: "missing MemAvailable derives from free+buffers+cached",
@@ -92,6 +94,67 @@ func TestPollPVENodeMemoryTrustCharacterization(t *testing.T) {
wantUsed: 107351023616,
wantRawSource: "node-status-total-minus-used",
},
{
name: "RRD memused wins over MemFree and cache-inclusive status used",
nodeStatus: &proxmox.NodeStatus{
Memory: &proxmox.MemoryStatus{
Total: 8 * gib,
Used: 15 * gib / 2,
Free: gib / 2,
},
},
rrdPoints: []proxmox.NodeRRDPoint{{MemUsed: floatPtr(float64(6 * gib))}},
wantSource: "rrd-memused",
wantFallback: "rrd-memused",
wantUsed: 6 * gib,
wantRawSource: "rrd-memused",
wantRRDUsed: 6 * gib,
},
{
name: "explicit available wins over conflicting RRD",
nodeStatus: &proxmox.NodeStatus{
Memory: &proxmox.MemoryStatus{
Total: 8 * gib,
Used: 15 * gib / 2,
Free: gib / 2,
Available: 2 * gib,
},
},
rrdPoints: []proxmox.NodeRRDPoint{{MemUsed: floatPtr(float64(7 * gib))}},
wantSource: "available-field",
wantUsed: 6 * gib,
wantRawSource: "node-status",
},
{
name: "MemFree without cache evidence stays unknown",
nodeStatus: &proxmox.NodeStatus{
Memory: &proxmox.MemoryStatus{
Total: 8 * gib,
Used: 15 * gib / 2,
Free: gib / 2,
},
},
wantSource: "unavailable",
wantFallback: "cache-aware-memory-unavailable",
wantRawSource: "cache-aware-unavailable",
wantUnknown: true,
},
{
name: "conflicting cache components above total stay unknown",
nodeStatus: &proxmox.NodeStatus{
Memory: &proxmox.MemoryStatus{
Total: 8 * gib,
Used: 4 * gib,
Free: 4 * gib,
Buffers: 3 * gib,
Cached: 3 * gib,
},
},
wantSource: "unavailable",
wantFallback: "cache-aware-memory-unavailable",
wantRawSource: "cache-aware-unavailable",
wantUnknown: true,
},
}
for _, tt := range tests {
@@ -116,6 +179,9 @@ func TestPollPVENodeMemoryTrustCharacterization(t *testing.T) {
if got := uint64(modelNode.Memory.Used); got != tt.wantUsed {
t.Fatalf("modelNode.Memory.Used = %d, want %d", got, tt.wantUsed)
}
if modelNode.Memory.UsageUnavailable != tt.wantUnknown {
t.Fatalf("modelNode.Memory.UsageUnavailable = %t, want %t", modelNode.Memory.UsageUnavailable, tt.wantUnknown)
}
snap := mon.nodeSnapshots[makeNodeSnapshotKey("test", "node1")]
if snap.MemorySource != tt.wantSource {
@@ -127,6 +193,9 @@ func TestPollPVENodeMemoryTrustCharacterization(t *testing.T) {
if snap.Raw.ProxmoxMemorySource != tt.wantRawSource {
t.Fatalf("snapshot.Raw.ProxmoxMemorySource = %q, want %q", snap.Raw.ProxmoxMemorySource, tt.wantRawSource)
}
if snap.Raw.RRDUsed != tt.wantRRDUsed {
t.Fatalf("snapshot.Raw.RRDUsed = %d, want %d", snap.Raw.RRDUsed, tt.wantRRDUsed)
}
})
}
}
@@ -194,6 +263,23 @@ func TestPollPVENodePreservesPreviousSnapshotDuringTransientFallback(t *testing.
if snap.Memory.Used != first.Memory.Used {
t.Fatalf("snapshot.Memory.Used = %d, want preserved %d", snap.Memory.Used, first.Memory.Used)
}
third, _, _, err := mon.pollPVENode(
context.Background(),
"test",
&mon.config.PVEInstances[0],
client,
node,
"healthy",
map[string]models.Memory{second.ID: second.Memory},
nil,
)
if err != nil {
t.Fatalf("third pollPVENode() error = %v", err)
}
if !third.Memory.UsageUnavailable {
t.Fatalf("third.Memory = %+v, want honest unavailable state after bounded carry-forward", third.Memory)
}
}
func TestGuestDiskTrustCharacterizationCarriesForwardRecentSnapshot(t *testing.T) {
@@ -244,13 +330,16 @@ func TestHandleClusterVMResourceMemoryTrustCharacterization(t *testing.T) {
const gib = uint64(1024 * 1024 * 1024)
tests := []struct {
name string
status *proxmox.VMStatus
rrdAvailable uint64
wantSource string
wantUsed uint64
wantAvailable uint64
wantGap uint64
name string
status *proxmox.VMStatus
rrdAvailable uint64
rrdUsed uint64
agentAvailable uint64
wantSource string
wantUsed uint64
wantAvailable uint64
wantGap uint64
wantUnknown bool
}{
{
name: "cache inflated Linux VM usage prefers RRD memavailable fallback",
@@ -302,6 +391,102 @@ func TestHandleClusterVMResourceMemoryTrustCharacterization(t *testing.T) {
wantUsed: 5 * gib,
wantGap: 3 * gib,
},
{
name: "partial Linux meminfo uses RRD memused",
status: &proxmox.VMStatus{
Status: "running",
MaxMem: 8 * gib,
Mem: 7 * gib,
MemInfo: &proxmox.VMMemInfo{
Total: 8 * gib,
Free: gib / 2,
},
Agent: proxmox.VMAgentField{Value: 1},
},
rrdUsed: 5 * gib,
wantSource: "rrd-memused",
wantUsed: 5 * gib,
},
{
name: "partial Linux meminfo without cache aware fallback stays unknown",
status: &proxmox.VMStatus{
Status: "running",
MaxMem: 8 * gib,
Mem: 15 * gib / 2,
MemInfo: &proxmox.VMMemInfo{
Total: 8 * gib,
Free: gib / 2,
},
Agent: proxmox.VMAgentField{Value: 1},
},
wantSource: "unavailable",
wantUnknown: true,
},
{
name: "MemAvailable conflicting with the selected capacity stays unknown",
status: &proxmox.VMStatus{
Status: "running",
MaxMem: 8 * gib,
Mem: 7 * gib,
MemInfo: &proxmox.VMMemInfo{
Total: 16 * gib,
Available: 12 * gib,
},
},
wantSource: "unavailable",
wantUnknown: true,
},
{
name: "reclaimable components above the selected capacity stay unknown",
status: &proxmox.VMStatus{
Status: "running",
MaxMem: 8 * gib,
Mem: 7 * gib,
MemInfo: &proxmox.VMMemInfo{
Total: 8 * gib,
Free: 2 * gib,
Buffers: 2 * gib,
Cached: 5 * gib,
},
},
wantSource: "unavailable",
wantUnknown: true,
},
{
name: "guest agent availability above the selected capacity stays unknown",
status: &proxmox.VMStatus{
Status: "running",
MaxMem: 8 * gib,
Mem: 7 * gib,
MemInfo: &proxmox.VMMemInfo{
Total: 8 * gib,
Free: gib / 2,
},
Agent: proxmox.VMAgentField{Value: 1},
},
agentAvailable: 9 * gib,
wantSource: "unavailable",
wantUnknown: true,
},
{
name: "RRD available wins over delayed total minus used estimate",
status: &proxmox.VMStatus{
Status: "running",
MaxMem: 8 * gib,
Mem: 7 * gib,
MemInfo: &proxmox.VMMemInfo{
Total: 8 * gib,
Used: 5 * gib,
Free: gib / 2,
},
Agent: proxmox.VMAgentField{Value: 1},
},
rrdAvailable: 4 * gib,
wantSource: "rrd-memavailable",
wantUsed: 4 * gib,
wantAvailable: 4 * gib,
wantGap: 3 * gib,
},
{
name: "materially inconsistent status memory prefers freemem fallback",
status: &proxmox.VMStatus{
@@ -348,11 +533,14 @@ func TestHandleClusterVMResourceMemoryTrustCharacterization(t *testing.T) {
defer mon.notificationMgr.Stop()
client := &vmMemoryTrustStubClient{
stubPVEClient: &stubPVEClient{},
vmStatus: tt.status,
stubPVEClient: &stubPVEClient{},
vmStatus: tt.status,
vmAgentMemAvailable: tt.agentAvailable,
}
if tt.rrdAvailable > 0 {
client.vmRRDPoints = []proxmox.GuestRRDPoint{{MemAvailable: floatPtr(float64(tt.rrdAvailable))}}
} else if tt.rrdUsed > 0 {
client.vmRRDPoints = []proxmox.GuestRRDPoint{{MemUsed: floatPtr(float64(tt.rrdUsed))}}
}
res := proxmox.ClusterResource{
@@ -374,6 +562,9 @@ func TestHandleClusterVMResourceMemoryTrustCharacterization(t *testing.T) {
if got := uint64(vm.Memory.Used); got != tt.wantUsed {
t.Fatalf("vm.Memory.Used = %d, want %d", got, tt.wantUsed)
}
if vm.Memory.UsageUnavailable != tt.wantUnknown {
t.Fatalf("vm.Memory.UsageUnavailable = %t, want %t", vm.Memory.UsageUnavailable, tt.wantUnknown)
}
snap := mon.guestSnapshots[makeGuestSnapshotKey("test", "qemu", "node1", 101)]
if snap.MemorySource != tt.wantSource {
@@ -382,8 +573,11 @@ func TestHandleClusterVMResourceMemoryTrustCharacterization(t *testing.T) {
if tt.wantSource == "derived-total-minus-used" && snap.FallbackReason != "derived-total-minus-used" {
t.Fatalf("snapshot.FallbackReason = %q, want derived-total-minus-used", snap.FallbackReason)
}
if tt.wantAvailable > 0 && snap.Raw.MemInfoAvailable != tt.wantAvailable {
t.Fatalf("snapshot.Raw.MemInfoAvailable = %d, want %d", snap.Raw.MemInfoAvailable, tt.wantAvailable)
if tt.wantAvailable > 0 && snap.Raw.RRDMemAvailable != tt.wantAvailable {
t.Fatalf("snapshot.Raw.RRDMemAvailable = %d, want %d", snap.Raw.RRDMemAvailable, tt.wantAvailable)
}
if tt.rrdUsed > 0 && snap.Raw.RRDMemUsed != tt.rrdUsed {
t.Fatalf("snapshot.Raw.RRDMemUsed = %d, want %d", snap.Raw.RRDMemUsed, tt.rrdUsed)
}
if tt.wantGap > 0 && snap.Raw.MemInfoTotalMinusUsed != tt.wantGap {
t.Fatalf("snapshot.Raw.MemInfoTotalMinusUsed = %d, want %d", snap.Raw.MemInfoTotalMinusUsed, tt.wantGap)
@@ -645,8 +839,8 @@ func TestPollVMsWithNodesMemoryTrustCharacterization(t *testing.T) {
if got := uint64(snap.Memory.Used); got != tt.wantUsed {
t.Fatalf("snapshot.Memory.Used = %d, want %d", got, tt.wantUsed)
}
if tt.wantAvailable > 0 && snap.Raw.MemInfoAvailable != tt.wantAvailable {
t.Fatalf("snapshot.Raw.MemInfoAvailable = %d, want %d", snap.Raw.MemInfoAvailable, tt.wantAvailable)
if tt.wantAvailable > 0 && snap.Raw.RRDMemAvailable != tt.wantAvailable {
t.Fatalf("snapshot.Raw.RRDMemAvailable = %d, want %d", snap.Raw.RRDMemAvailable, tt.wantAvailable)
}
if tt.wantSource == "derived-total-minus-used" && snap.FallbackReason != "derived-total-minus-used" {
t.Fatalf("snapshot.FallbackReason = %q, want derived-total-minus-used", snap.FallbackReason)
@@ -736,6 +930,26 @@ func TestRecordGuestMetric_SkipsNativeWritesInMockMode(t *testing.T) {
}
}
func TestRecordGuestMetricPreservesHistoryWhenLiveMemoryIsUnavailable(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
mon := newTestPVEMonitor("test")
defer mon.alertManager.Stop()
defer mon.notificationMgr.Stop()
now := time.Now().UTC()
mon.recordGuestMetric("vm", "test:node1:101", 20, 76, 40, -1, -1, -1, -1, now.Add(-time.Minute))
mon.recordGuestMetric("vm", "test:node1:101", 25, -1, 42, -1, -1, -1, -1, now)
points := mon.metricsHistory.GetGuestMetrics("test:node1:101", "memory", time.Hour)
if len(points) != 1 {
t.Fatalf("memory history points = %d, want 1 trusted point", len(points))
}
if points[0].Value != 76 {
t.Fatalf("memory history value = %.2f, want 76", points[0].Value)
}
}
func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
@@ -749,6 +963,7 @@ func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T)
wantUsed uint64
wantAvail uint64
wantStatus string
wantUnknown bool
}{
{
name: "cache inflated LXC usage prefers RRD memavailable fallback",
@@ -788,7 +1003,42 @@ func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T)
wantStatus: "running",
},
{
name: "stopped LXC keeps cluster resources source",
name: "explicit zero RRD memavailable is valid full usage",
res: proxmox.ClusterResource{
ID: "lxc/205",
Type: "lxc",
Node: "node1",
Name: "ct-205",
Status: "running",
VMID: 205,
MaxMem: 8 * gib,
Mem: 7 * gib,
MaxCPU: 4,
},
lxcRRDPoints: []proxmox.GuestRRDPoint{{MemAvailable: floatPtr(0)}},
wantSource: "rrd-memavailable",
wantUsed: 8 * gib,
wantStatus: "running",
},
{
name: "running LXC without RRD memory stays unknown",
res: proxmox.ClusterResource{
ID: "lxc/204",
Type: "lxc",
Node: "node1",
Name: "ct-204",
Status: "running",
VMID: 204,
MaxMem: 8 * gib,
Mem: 7 * gib,
MaxCPU: 4,
},
wantSource: "unavailable",
wantStatus: "running",
wantUnknown: true,
},
{
name: "stopped LXC reports powered off memory",
res: proxmox.ClusterResource{
ID: "lxc/203",
Type: "lxc",
@@ -800,8 +1050,8 @@ func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T)
Mem: 2 * gib,
MaxCPU: 4,
},
wantSource: "cluster-resources",
wantUsed: 2 * gib,
wantSource: "powered-off",
wantUsed: 0,
wantStatus: "stopped",
},
}
@@ -834,6 +1084,9 @@ func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T)
if got := uint64(container.Memory.Used); got != tt.wantUsed {
t.Fatalf("container.Memory.Used = %d, want %d", got, tt.wantUsed)
}
if container.Memory.UsageUnavailable != tt.wantUnknown {
t.Fatalf("container.Memory.UsageUnavailable = %t, want %t", container.Memory.UsageUnavailable, tt.wantUnknown)
}
key := makeGuestSnapshotKey("test", container.Type, "node1", tt.res.VMID)
snap, ok := mon.guestSnapshots[key]
@@ -846,8 +1099,8 @@ func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T)
if got := uint64(snap.Memory.Used); got != tt.wantUsed {
t.Fatalf("snapshot.Memory.Used = %d, want %d", got, tt.wantUsed)
}
if tt.wantAvail > 0 && snap.Raw.MemInfoAvailable != tt.wantAvail {
t.Fatalf("snapshot.Raw.MemInfoAvailable = %d, want %d", snap.Raw.MemInfoAvailable, tt.wantAvail)
if tt.wantAvail > 0 && snap.Raw.RRDMemAvailable != tt.wantAvail {
t.Fatalf("snapshot.Raw.RRDMemAvailable = %d, want %d", snap.Raw.RRDMemAvailable, tt.wantAvail)
}
})
}
@@ -917,7 +1170,7 @@ func TestPollContainersWithNodesMemoryTrustCharacterization(t *testing.T) {
wantAvail: 3 * gib,
},
{
name: "stopped LXC keeps cluster resources source in node polling",
name: "stopped LXC reports powered off memory in node polling",
container: proxmox.Container{
VMID: 302,
Name: "ct-302",
@@ -927,8 +1180,8 @@ func TestPollContainersWithNodesMemoryTrustCharacterization(t *testing.T) {
Mem: 2 * gib,
CPUs: 2,
},
wantSource: "cluster-resources",
wantUsed: 2 * gib,
wantSource: "powered-off",
wantUsed: 0,
},
}
@@ -959,8 +1212,8 @@ func TestPollContainersWithNodesMemoryTrustCharacterization(t *testing.T) {
if got := uint64(snap.Memory.Used); got != tt.wantUsed {
t.Fatalf("snapshot.Memory.Used = %d, want %d", got, tt.wantUsed)
}
if tt.wantAvail > 0 && snap.Raw.MemInfoAvailable != tt.wantAvail {
t.Fatalf("snapshot.Raw.MemInfoAvailable = %d, want %d", snap.Raw.MemInfoAvailable, tt.wantAvail)
if tt.wantAvail > 0 && snap.Raw.RRDMemAvailable != tt.wantAvail {
t.Fatalf("snapshot.Raw.RRDMemAvailable = %d, want %d", snap.Raw.RRDMemAvailable, tt.wantAvail)
}
})
}
+86 -22
View File
@@ -1217,6 +1217,9 @@ type rrdMemCacheEntry struct {
available uint64
used uint64
total uint64
hasAvail bool
hasUsed bool
hasTotal bool
netIn float64
netOut float64
hasNetIn bool
@@ -1301,15 +1304,20 @@ type taskOutcome struct {
recordedAt time.Time
}
func (m *Monitor) getNodeRRDMetrics(ctx context.Context, client PVEClientInterface, nodeName string) (rrdMemCacheEntry, error) {
func nodeRRDCacheKey(instanceName, nodeName string) string {
return instanceName + "/" + nodeName
}
func (m *Monitor) getNodeRRDMetrics(ctx context.Context, client PVEClientInterface, instanceName, nodeName string) (rrdMemCacheEntry, error) {
if client == nil || nodeName == "" {
return rrdMemCacheEntry{}, fmt.Errorf("invalid arguments for RRD lookup")
}
now := time.Now()
cacheKey := nodeRRDCacheKey(instanceName, nodeName)
m.rrdCacheMu.RLock()
if entry, ok := m.nodeRRDMemCache[nodeName]; ok && now.Sub(entry.fetchedAt) < nodeRRDCacheTTL {
if entry, ok := m.nodeRRDMemCache[cacheKey]; ok && now.Sub(entry.fetchedAt) < nodeRRDCacheTTL {
m.rrdCacheMu.RUnlock()
return entry, nil
}
@@ -1326,6 +1334,9 @@ func (m *Monitor) getNodeRRDMetrics(ctx context.Context, client PVEClientInterfa
var memAvailable uint64
var memUsed uint64
var memTotal uint64
var hasAvail bool
var hasUsed bool
var hasTotal bool
var netIn float64
var netOut float64
var hasNetIn bool
@@ -1334,23 +1345,26 @@ func (m *Monitor) getNodeRRDMetrics(ctx context.Context, client PVEClientInterfa
for i := len(points) - 1; i >= 0; i-- {
point := points[i]
if memTotal == 0 && point.MemTotal != nil && !math.IsNaN(*point.MemTotal) && *point.MemTotal > 0 {
if !hasTotal && point.MemTotal != nil && !math.IsNaN(*point.MemTotal) && !math.IsInf(*point.MemTotal, 0) && *point.MemTotal > 0 && *point.MemTotal <= math.MaxUint64 {
memTotal = uint64(math.Round(*point.MemTotal))
hasTotal = true
}
if memAvailable == 0 && point.MemAvailable != nil && !math.IsNaN(*point.MemAvailable) && *point.MemAvailable > 0 {
if !hasAvail && point.MemAvailable != nil && !math.IsNaN(*point.MemAvailable) && !math.IsInf(*point.MemAvailable, 0) && *point.MemAvailable >= 0 && *point.MemAvailable <= math.MaxUint64 {
memAvailable = uint64(math.Round(*point.MemAvailable))
hasAvail = true
}
if memUsed == 0 && point.MemUsed != nil && !math.IsNaN(*point.MemUsed) && *point.MemUsed > 0 {
if !hasUsed && point.MemUsed != nil && !math.IsNaN(*point.MemUsed) && !math.IsInf(*point.MemUsed, 0) && *point.MemUsed >= 0 && *point.MemUsed <= math.MaxUint64 {
memUsed = uint64(math.Round(*point.MemUsed))
hasUsed = true
}
if !hasNetIn && point.NetIn != nil && !math.IsNaN(*point.NetIn) {
if !hasNetIn && point.NetIn != nil && !math.IsNaN(*point.NetIn) && !math.IsInf(*point.NetIn, 0) {
netIn = *point.NetIn
hasNetIn = true
}
if !hasNetOut && point.NetOut != nil && !math.IsNaN(*point.NetOut) {
if !hasNetOut && point.NetOut != nil && !math.IsNaN(*point.NetOut) && !math.IsInf(*point.NetOut, 0) {
netOut = *point.NetOut
hasNetOut = true
}
@@ -1358,14 +1372,16 @@ func (m *Monitor) getNodeRRDMetrics(ctx context.Context, client PVEClientInterfa
if memTotal > 0 {
if memAvailable > memTotal {
memAvailable = memTotal
memAvailable = 0
hasAvail = false
}
if memUsed > memTotal {
memUsed = memTotal
memUsed = 0
hasUsed = false
}
}
if memAvailable == 0 && memUsed == 0 && !hasNetIn && !hasNetOut {
if !hasAvail && !hasUsed && !hasNetIn && !hasNetOut {
return rrdMemCacheEntry{}, fmt.Errorf("rrd node metrics not present")
}
@@ -1373,6 +1389,9 @@ func (m *Monitor) getNodeRRDMetrics(ctx context.Context, client PVEClientInterfa
available: memAvailable,
used: memUsed,
total: memTotal,
hasAvail: hasAvail,
hasUsed: hasUsed,
hasTotal: hasTotal,
netIn: netIn,
netOut: netOut,
hasNetIn: hasNetIn,
@@ -1381,7 +1400,10 @@ func (m *Monitor) getNodeRRDMetrics(ctx context.Context, client PVEClientInterfa
}
m.rrdCacheMu.Lock()
m.nodeRRDMemCache[nodeName] = entry
if m.nodeRRDMemCache == nil {
m.nodeRRDMemCache = make(map[string]rrdMemCacheEntry)
}
m.nodeRRDMemCache[cacheKey] = entry
m.rrdCacheMu.Unlock()
return entry, nil
@@ -1391,8 +1413,19 @@ func (m *Monitor) getNodeRRDMetrics(ctx context.Context, client PVEClientInterfa
// short-lived cache to avoid a live API call on every poll for VMs that
// consistently lack guest-agent memory data (e.g. Windows VMs).
func (m *Monitor) getVMRRDMetrics(ctx context.Context, client PVEClientInterface, instanceName, node string, vmid int) (uint64, error) {
entry, err := m.getVMRRDMemory(ctx, client, instanceName, node, vmid)
if err != nil {
return 0, err
}
if !entry.hasAvail {
return 0, fmt.Errorf("rrd memavailable not present for VM %s/%d", node, vmid)
}
return entry.available, nil
}
func (m *Monitor) getVMRRDMemory(ctx context.Context, client PVEClientInterface, instanceName, node string, vmid int) (rrdMemCacheEntry, error) {
if client == nil || node == "" || vmid <= 0 {
return 0, fmt.Errorf("invalid arguments for VM RRD lookup")
return rrdMemCacheEntry{}, fmt.Errorf("invalid arguments for VM RRD lookup")
}
cacheKey := guestMemoryCacheKey(instanceName, node, vmid)
@@ -1401,34 +1434,65 @@ func (m *Monitor) getVMRRDMetrics(ctx context.Context, client PVEClientInterface
m.rrdCacheMu.RLock()
if entry, ok := m.vmRRDMemCache[cacheKey]; ok && now.Sub(entry.fetchedAt) < nodeRRDCacheTTL {
m.rrdCacheMu.RUnlock()
return entry.available, nil
return entry, nil
}
m.rrdCacheMu.RUnlock()
requestCtx, cancel := context.WithTimeout(ctx, nodeRRDRequestTimeout)
defer cancel()
points, err := client.GetVMRRDData(requestCtx, node, vmid, "hour", "AVERAGE", []string{"memavailable"})
points, err := client.GetVMRRDData(requestCtx, node, vmid, "hour", "AVERAGE", []string{"memavailable", "memused", "maxmem"})
if err != nil {
return 0, err
return rrdMemCacheEntry{}, err
}
if len(points) == 0 {
return 0, fmt.Errorf("no RRD points for VM %s/%d", node, vmid)
return rrdMemCacheEntry{}, fmt.Errorf("no RRD points for VM %s/%d", node, vmid)
}
var memAvailable uint64
var memUsed uint64
var memTotal uint64
var hasAvail bool
var hasUsed bool
var hasTotal bool
for i := len(points) - 1; i >= 0; i-- {
p := points[i]
if p.MemAvailable != nil && !math.IsNaN(*p.MemAvailable) && *p.MemAvailable > 0 {
if !hasAvail && p.MemAvailable != nil && !math.IsNaN(*p.MemAvailable) && !math.IsInf(*p.MemAvailable, 0) && *p.MemAvailable >= 0 && *p.MemAvailable <= math.MaxUint64 {
memAvailable = uint64(math.Round(*p.MemAvailable))
break
hasAvail = true
}
if !hasUsed && p.MemUsed != nil && !math.IsNaN(*p.MemUsed) && !math.IsInf(*p.MemUsed, 0) && *p.MemUsed >= 0 && *p.MemUsed <= math.MaxUint64 {
memUsed = uint64(math.Round(*p.MemUsed))
hasUsed = true
}
if !hasTotal && p.MaxMem != nil && !math.IsNaN(*p.MaxMem) && !math.IsInf(*p.MaxMem, 0) && *p.MaxMem > 0 && *p.MaxMem <= math.MaxUint64 {
memTotal = uint64(math.Round(*p.MaxMem))
hasTotal = true
}
}
if memAvailable == 0 {
return 0, fmt.Errorf("rrd memavailable not present for VM %s/%d", node, vmid)
if hasTotal {
if hasAvail && memAvailable > memTotal {
hasAvail = false
memAvailable = 0
}
if hasUsed && memUsed > memTotal {
hasUsed = false
memUsed = 0
}
}
if !hasAvail && !hasUsed {
return rrdMemCacheEntry{}, fmt.Errorf("rrd memory fields not present for VM %s/%d", node, vmid)
}
entry := rrdMemCacheEntry{available: memAvailable, fetchedAt: now}
entry := rrdMemCacheEntry{
available: memAvailable,
used: memUsed,
total: memTotal,
hasAvail: hasAvail,
hasUsed: hasUsed,
hasTotal: hasTotal,
fetchedAt: now,
}
m.rrdCacheMu.Lock()
if m.vmRRDMemCache == nil {
m.vmRRDMemCache = make(map[string]rrdMemCacheEntry)
@@ -1436,7 +1500,7 @@ func (m *Monitor) getVMRRDMetrics(ctx context.Context, client PVEClientInterface
m.vmRRDMemCache[cacheKey] = entry
m.rrdCacheMu.Unlock()
return memAvailable, nil
return entry, nil
}
// RemoveDockerHost removes a docker host from the shared state and clears related alerts.
+87 -59
View File
@@ -28,6 +28,62 @@ const hostContinuityRetention = 72 * time.Hour
const maxRetiredHostReportStreams = 8
func normalizeAgentMemory(total, used, free, cache int64, usage float64, swapTotal, swapUsed int64) models.Memory {
if total <= 0 {
return models.Memory{}
}
hasReportedUsed := used > 0
unavailable := func() models.Memory {
memory := models.UnavailableMemory(total)
memory.SwapTotal = max(0, swapTotal)
memory.SwapUsed = max(0, swapUsed)
return memory
}
usage = safeFloat(usage)
if used < 0 || free < 0 || cache < 0 || used > total || usage < 0 || usage > 100 {
return unavailable()
}
if used == 0 {
switch {
case usage > 0:
used = int64(float64(total) * usage / 100)
case cache > 0 && cache <= total && free <= total-cache:
used = total - free - cache
case free == total:
// A completely idle host is a valid zero-usage measurement.
default:
return unavailable()
}
}
if cache > total-used {
cache = total - used
}
if free > total-used-cache {
free = total - used - cache
}
// Byte counters are canonical when the agent reported them. Recompute the
// percentage so a stale percentage cannot diverge display, history, and
// threshold evaluation. Preserve a percentage-only report to avoid losing
// precision when deriving its byte approximation.
if hasReportedUsed || usage == 0 {
usage = safePercentage(float64(used), float64(total))
}
return models.Memory{
Total: total,
Used: used,
Free: free,
Cache: cache,
Usage: usage,
SwapTotal: max(0, swapTotal),
SwapUsed: max(0, swapUsed),
}
}
type hostReportOrder struct {
ObservedAt time.Time
LastReceivedAt time.Time
@@ -1847,37 +1903,21 @@ func (m *Monitor) ApplyDockerReport(report agentsdocker.Report, tokenRecord *con
loadAverage = append(loadAverage, report.Host.LoadAverage...)
}
var memory models.Memory
if report.Host.Memory.TotalBytes > 0 || report.Host.Memory.UsedBytes > 0 {
memory = models.Memory{
Total: report.Host.Memory.TotalBytes,
Used: report.Host.Memory.UsedBytes,
Free: report.Host.Memory.FreeBytes,
Usage: safeFloat(report.Host.Memory.Usage),
SwapTotal: report.Host.Memory.SwapTotal,
SwapUsed: report.Host.Memory.SwapUsed,
}
}
memory := normalizeAgentMemory(
report.Host.Memory.TotalBytes,
report.Host.Memory.UsedBytes,
report.Host.Memory.FreeBytes,
report.Host.Memory.CacheBytes,
report.Host.Memory.Usage,
report.Host.Memory.SwapTotal,
report.Host.Memory.SwapUsed,
)
// Fallback: if gopsutil's memory reading failed but Docker's TotalMemoryBytes
// is valid (possibly already a fallback from the agent), use that for Total.
// This handles Docker-in-LXC scenarios where both Docker and gopsutil may
// fail to read memory stats, but the agent fix provides a valid fallback.
if memory.Total <= 0 && report.Host.TotalMemoryBytes > 0 {
memory.Total = report.Host.TotalMemoryBytes
}
// Additional fallback for Docker-in-LXC: gopsutil may read Total and Free
// correctly from cgroup limits but return 0 for Used. Calculate Used from
// Total - Free when this happens. This fixes the "0B / 7GB" display issue.
if memory.Used <= 0 && memory.Total > 0 && memory.Free > 0 {
memory.Used = memory.Total - memory.Free
if memory.Used < 0 {
memory.Used = 0
}
// Recalculate usage percentage
if memory.Total > 0 {
memory.Usage = safePercentage(float64(memory.Used), float64(memory.Total))
}
memory = models.UnavailableMemory(report.Host.TotalMemoryBytes)
}
disks := make([]models.Disk, 0, len(report.Host.Disks))
@@ -2041,13 +2081,17 @@ func (m *Monitor) ApplyDockerReport(report agentsdocker.Report, tokenRecord *con
if m.metricsHistory != nil {
m.metricsHistory.AddGuestMetric(hostMetricKey, "cpu", host.CPUUsage, now)
m.metricsHistory.AddGuestMetric(hostMetricKey, "memory", host.Memory.Usage, now)
if host.Memory.HasKnownUsage() {
m.metricsHistory.AddGuestMetric(hostMetricKey, "memory", host.Memory.Usage, now)
}
m.metricsHistory.AddGuestMetric(hostMetricKey, "disk", hostDiskPercent, now)
}
if m.metricsStore != nil {
m.metricsStore.Write("dockerHost", host.ID, "cpu", host.CPUUsage, now)
m.metricsStore.Write("dockerHost", host.ID, "memory", host.Memory.Usage, now)
if host.Memory.HasKnownUsage() {
m.metricsStore.Write("dockerHost", host.ID, "memory", host.Memory.Usage, now)
}
m.metricsStore.Write("dockerHost", host.ID, "disk", hostDiskPercent, now)
}
@@ -2379,35 +2423,15 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config.
displayName = hostname
}
memory := models.Memory{
Total: report.Metrics.Memory.TotalBytes,
Used: report.Metrics.Memory.UsedBytes,
Free: report.Metrics.Memory.FreeBytes,
Cache: report.Metrics.Memory.CacheBytes,
Usage: safeFloat(report.Metrics.Memory.Usage),
SwapTotal: report.Metrics.Memory.SwapTotal,
SwapUsed: report.Metrics.Memory.SwapUsed,
}
// Older agents don't report cache; clamp so used + cache never exceeds total.
if memory.Cache < 0 {
memory.Cache = 0
}
if memory.Total > 0 && memory.Used+memory.Cache > memory.Total {
memory.Cache = max(0, memory.Total-memory.Used)
}
// Fallback for LXC environments: gopsutil may read Total and Free correctly
// from cgroup limits but return 0 for Used. Calculate Used from Total - Free.
if memory.Used <= 0 && memory.Total > 0 && memory.Free > 0 {
memory.Used = memory.Total - memory.Free
if memory.Used < 0 {
memory.Used = 0
}
}
if memory.Usage <= 0 && memory.Total > 0 {
memory.Usage = safePercentage(float64(memory.Used), float64(memory.Total))
}
memory := normalizeAgentMemory(
report.Metrics.Memory.TotalBytes,
report.Metrics.Memory.UsedBytes,
report.Metrics.Memory.FreeBytes,
report.Metrics.Memory.CacheBytes,
report.Metrics.Memory.Usage,
report.Metrics.Memory.SwapTotal,
report.Metrics.Memory.SwapUsed,
)
disks := make([]models.Disk, 0, len(report.Disks))
for _, disk := range report.Disks {
@@ -2766,7 +2790,9 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config.
if !shouldSkipNativeMockStateMetricWrites() {
if m.metricsHistory != nil {
m.metricsHistory.AddGuestMetric(hostMetricKey, "cpu", host.CPUUsage, now)
m.metricsHistory.AddGuestMetric(hostMetricKey, "memory", host.Memory.Usage, now)
if host.Memory.HasKnownUsage() {
m.metricsHistory.AddGuestMetric(hostMetricKey, "memory", host.Memory.Usage, now)
}
m.metricsHistory.AddGuestMetric(hostMetricKey, "disk", hostDiskPercent, now)
if hostTemperature != nil {
m.metricsHistory.AddGuestMetric(hostMetricKey, "temperature", *hostTemperature, now)
@@ -2790,7 +2816,9 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config.
if m.metricsStore != nil {
m.metricsStore.Write("agent", host.ID, "cpu", host.CPUUsage, now)
m.metricsStore.Write("agent", host.ID, "memory", host.Memory.Usage, now)
if host.Memory.HasKnownUsage() {
m.metricsStore.Write("agent", host.ID, "memory", host.Memory.Usage, now)
}
m.metricsStore.Write("agent", host.ID, "disk", hostDiskPercent, now)
if hostTemperature != nil {
m.metricsStore.Write("agent", host.ID, "temperature", *hostTemperature, now)
@@ -47,6 +47,96 @@ func TestMonitoringBroadcastCarriesEveryAvailabilityProjection(t *testing.T) {
}
}
func TestNormalizeAgentMemoryCacheAwareFallbacks(t *testing.T) {
const gib = int64(1024 * 1024 * 1024)
tests := []struct {
name string
total int64
used int64
free int64
cache int64
usage float64
swapTotal int64
swapUsed int64
wantUsed int64
wantUsage float64
wantUnavailable bool
}{
{
name: "explicit agent usage remains authoritative",
total: 8 * gib,
used: 6 * gib,
free: gib / 2,
cache: 2 * gib,
wantUsed: 6 * gib,
wantUsage: 75,
},
{
name: "explicit bytes correct a conflicting reported percentage",
total: 8 * gib,
used: 4 * gib,
free: gib,
usage: 94,
wantUsed: 4 * gib,
wantUsage: 50,
},
{
name: "complete Linux cache evidence derives used without swap",
total: 8 * gib,
free: gib,
cache: 2 * gib,
wantUsed: 5 * gib,
wantUsage: 62.5,
},
{
name: "total and free alone stay unknown",
total: 8 * gib,
free: gib,
wantUnavailable: true,
},
{
name: "reported percentage can establish usage",
total: 8 * gib,
free: gib,
usage: 76,
wantUsed: 8 * gib * 76 / 100,
wantUsage: 76,
},
{
name: "zero swap does not invalidate cache aware memory",
total: 8 * gib,
free: 2 * gib,
cache: gib,
swapTotal: 0,
swapUsed: 0,
wantUsed: 5 * gib,
wantUsage: 62.5,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := normalizeAgentMemory(tt.total, tt.used, tt.free, tt.cache, tt.usage, tt.swapTotal, tt.swapUsed)
if got.UsageUnavailable != tt.wantUnavailable {
t.Fatalf("UsageUnavailable = %t, want %t", got.UsageUnavailable, tt.wantUnavailable)
}
if tt.wantUnavailable {
if got.HasKnownUsage() {
t.Fatalf("HasKnownUsage() = true for unavailable memory: %+v", got)
}
return
}
if got.Used != tt.wantUsed {
t.Fatalf("Used = %d, want %d", got.Used, tt.wantUsed)
}
if got.Usage != tt.wantUsage {
t.Fatalf("Usage = %.2f, want %.2f", got.Usage, tt.wantUsage)
}
})
}
}
func TestApplyHostReportOperationReceiptProtocolReplacesCapabilityAuthority(t *testing.T) {
now := time.Now().UTC()
baseReport := func(version int) agentshost.Report {
@@ -250,6 +250,31 @@ func TestPollPVEInstanceUsesRRDMemUsedFallback(t *testing.T) {
}
}
func TestNodeRRDMemoryCacheIsScopedByInstance(t *testing.T) {
mon := &Monitor{nodeRRDMemCache: make(map[string]rrdMemCacheEntry)}
const gib = uint64(1024 * 1024 * 1024)
first, err := mon.getNodeRRDMetrics(context.Background(), &stubPVEClient{
rrdPoints: []proxmox.NodeRRDPoint{{MemUsed: floatPtr(float64(2 * gib))}},
}, "pve-a", "node1")
if err != nil {
t.Fatalf("first getNodeRRDMetrics() error = %v", err)
}
second, err := mon.getNodeRRDMetrics(context.Background(), &stubPVEClient{
rrdPoints: []proxmox.NodeRRDPoint{{MemUsed: floatPtr(float64(6 * gib))}},
}, "pve-b", "node1")
if err != nil {
t.Fatalf("second getNodeRRDMetrics() error = %v", err)
}
if first.used != 2*gib || second.used != 6*gib {
t.Fatalf("cross-instance node RRD values leaked: first=%d second=%d", first.used, second.used)
}
if len(mon.nodeRRDMemCache) != 2 {
t.Fatalf("node RRD cache entries = %d, want two instance-scoped entries", len(mon.nodeRRDMemCache))
}
}
func TestPollPVEInstancePreservesRecentNodesWhenGetNodesReturnsEmpty(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
@@ -141,6 +141,15 @@ func (m *Monitor) pollContainersWithNodes(ctx context.Context, instanceName stri
MaxMem: container.MaxMem,
Mem: container.Mem,
}, client)
memUsed, memorySource, _ = stabilizeGuestLowTrustMemory(
m.previousGuestSnapshot(instanceName, "lxc", n.Node, int(container.VMID)),
container.Status,
memorySource,
memTotal,
memUsed,
sampleTime,
false,
)
memTotalBytes := clampToInt64(memTotal)
memUsedBytes := clampToInt64(memUsed)
@@ -152,6 +161,15 @@ func (m *Monitor) pollContainersWithNodes(ctx context.Context, instanceName stri
memFreeBytes = 0
}
memUsagePercent := safePercentage(float64(memUsedBytes), float64(memTotalBytes))
memory := models.UnavailableMemory(memTotalBytes)
if CanonicalMemorySource(memorySource) != "unavailable" {
memory = models.Memory{
Total: memTotalBytes,
Used: memUsedBytes,
Free: memFreeBytes,
Usage: memUsagePercent,
}
}
diskTotalBytes := clampToInt64(container.MaxDisk)
diskUsedBytes := clampToInt64(container.Disk)
@@ -176,12 +194,7 @@ func (m *Monitor) pollContainersWithNodes(ctx context.Context, instanceName stri
Type: "lxc",
CPU: cpuUsage,
CPUs: int(container.CPUs),
Memory: models.Memory{
Total: memTotalBytes,
Used: memUsedBytes,
Free: memFreeBytes,
Usage: memUsagePercent,
},
Memory: memory,
Disk: models.Disk{
Total: diskTotalBytes,
Used: diskUsedBytes,
@@ -337,7 +350,7 @@ func (m *Monitor) pollContainersWithNodes(ctx context.Context, instanceName stri
}
// IO/network series are not recorded on the traditional polling
// path (parity with the historical inline writes).
m.recordGuestMetric("container", ct.ID, unifiedresources.ProxmoxGuestCPUPercent(ct.CPU), ct.Memory.Usage, ct.Disk.Usage, -1, -1, -1, -1, now)
m.recordGuestMetric("container", ct.ID, unifiedresources.ProxmoxGuestCPUPercent(ct.CPU), historyMemoryUsage(ct.Memory), ct.Disk.Usage, -1, -1, -1, -1, now)
}
}
+52 -30
View File
@@ -11,6 +11,31 @@ import (
"github.com/rs/zerolog/log"
)
const nodeMemoryCarryForwardMaxAge = 2 * time.Minute
func (m *Monitor) canCarryForwardNodeMemory(instance, node string, now time.Time) bool {
if m == nil {
return false
}
m.diagMu.RLock()
snapshot, ok := m.nodeSnapshots[makeNodeSnapshotKey(instance, node)]
m.diagMu.RUnlock()
if !ok || !snapshot.Memory.HasKnownUsage() || snapshot.RetrievedAt.IsZero() {
return false
}
age := now.Sub(snapshot.RetrievedAt)
if age < 0 || age > nodeMemoryCarryForwardMaxAge {
return false
}
switch CanonicalMemorySource(snapshot.MemorySource) {
case "available-field", "derived-free-buffers-cached", "derived-total-minus-used",
"rrd-memavailable", "rrd-memused":
return true
default:
return false
}
}
func (m *Monitor) pollPVENode(
ctx context.Context,
instanceName string,
@@ -25,23 +50,22 @@ func (m *Monitor) pollPVENode(
displayName := getNodeDisplayName(instanceCfg, node.Node)
connectionHost, guestURL := resolveNodeConnectionInfo(instanceCfg, monitorDiscoveryConfig(m), node.Node)
nodeID, effectiveStatus := m.determineNodeIDAndStatus(instanceName, instanceCfg, node)
nodeFallbackFree := uint64(0)
if node.MaxMem >= node.Mem {
nodeFallbackFree = node.MaxMem - node.Mem
}
modelNode := models.Node{
ID: nodeID,
Name: node.Node,
DisplayName: displayName,
Instance: instanceName,
Host: connectionHost,
GuestURL: guestURL,
Status: effectiveStatus,
Type: "node",
CPU: safeFloat(node.CPU), // Proxmox returns 0-1 ratio (e.g., 0.15 = 15%)
Memory: models.Memory{
Total: int64(node.MaxMem),
Used: int64(node.Mem),
Free: int64(node.MaxMem - node.Mem),
Usage: safePercentage(float64(node.Mem), float64(node.MaxMem)),
},
ID: nodeID,
Name: node.Node,
DisplayName: displayName,
Instance: instanceName,
Host: connectionHost,
GuestURL: guestURL,
Status: effectiveStatus,
Type: "node",
CPU: safeFloat(node.CPU), // Proxmox returns 0-1 ratio (e.g., 0.15 = 15%)
Memory: models.UnavailableMemory(clampToInt64(node.MaxMem)),
Uptime: int64(node.Uptime),
LoadAverage: []float64{},
LastSeen: time.Now(),
@@ -56,15 +80,15 @@ func (m *Monitor) pollPVENode(
nodeSnapshotRaw := NodeMemoryRaw{
Total: node.MaxMem,
Used: node.Mem,
Free: node.MaxMem - node.Mem,
Free: nodeFallbackFree,
FallbackTotal: node.MaxMem,
FallbackUsed: node.Mem,
FallbackFree: node.MaxMem - node.Mem,
FallbackFree: nodeFallbackFree,
FallbackCalculated: true,
ProxmoxMemorySource: "nodes-endpoint",
}
nodeMemorySource := "nodes-endpoint"
var nodeFallbackReason string
nodeMemorySource := "unavailable"
nodeFallbackReason := "cache-aware-memory-unavailable"
// Debug logging for disk metrics - note that these values can fluctuate
// due to thin provisioning and dynamic allocation
@@ -93,7 +117,7 @@ func (m *Monitor) pollPVENode(
Err(nodeErr).
Uint64("usingDisk", node.Disk).
Uint64("usingMaxDisk", node.MaxDisk).
Msg("Could not get node status - using fallback metrics (memory will include cache/buffers)")
Msg("Could not get node status - disk fallback retained; memory usage unavailable")
} else {
log.Warn().
Str("instance", instanceName).
@@ -101,7 +125,7 @@ func (m *Monitor) pollPVENode(
Err(nodeErr).
Uint64("disk", node.Disk).
Uint64("maxDisk", node.MaxDisk).
Msg("Could not get node status - no fallback metrics available (memory will include cache/buffers)")
Msg("Could not get node status - memory usage unavailable")
}
} else if nodeInfo != nil {
if nodeInfo.Memory != nil {
@@ -159,11 +183,9 @@ func (m *Monitor) pollPVENode(
if ok {
modelNode.Memory = resolvedMemory
nodeMemorySource = resolvedSource
if resolvedFallback != "" {
nodeFallbackReason = resolvedFallback
}
nodeFallbackReason = resolvedFallback
nodeSnapshotRaw = resolvedRaw
memoryUpdated = true
memoryUpdated = resolvedMemory.HasKnownUsage()
}
}
@@ -195,8 +217,10 @@ func (m *Monitor) pollPVENode(
}
// If we couldn't update memory metrics using detailed status, preserve previous accurate values if available
if !memoryUpdated && effectiveStatus == "online" {
if prevMem, exists := prevNodeMemory[modelNode.ID]; exists && prevMem.Total > 0 {
if !memoryUpdated &&
effectiveStatus == "online" &&
m.canCarryForwardNodeMemory(instanceName, node.Node, time.Now()) {
if prevMem, exists := prevNodeMemory[modelNode.ID]; exists && prevMem.HasKnownUsage() {
total := int64(node.MaxMem)
if total == 0 {
total = prevMem.Total
@@ -222,9 +246,7 @@ func (m *Monitor) pollPVENode(
Str("node", node.Node).
Msg("Preserving previous memory metrics - node status unavailable this cycle")
if nodeFallbackReason == "" {
nodeFallbackReason = "preserved-previous-snapshot"
}
nodeFallbackReason = "preserved-previous-snapshot"
nodeMemorySource = "previous-snapshot"
if nodeSnapshotRaw.ProxmoxMemorySource == "node-status" && nodeSnapshotRaw.Total == 0 {
nodeSnapshotRaw.ProxmoxMemorySource = "previous-snapshot"
+1 -1
View File
@@ -124,7 +124,7 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, clu
}
// IO/network series are not recorded on the traditional polling
// path (parity with the historical inline writes).
m.recordGuestMetric("vm", vm.ID, unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), vm.Memory.Usage, vm.Disk.Usage, -1, -1, -1, -1, now)
m.recordGuestMetric("vm", vm.ID, unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), historyMemoryUsage(vm.Memory), vm.Disk.Usage, -1, -1, -1, -1, now)
}
}
@@ -316,11 +316,6 @@ func (m *Monitor) buildVMFromClusterResource(
state.onBoot = m.fetchVMOnBoot(ctx, client, res.Node, res.VMID)
}
memFree := uint64(0)
if state.memTotal >= state.memUsed {
memFree = state.memTotal - state.memUsed
}
var snapshotNotes []string
state.memUsed, state.memorySource, snapshotNotes = stabilizeGuestLowTrustMemory(
prevSnapshot,
@@ -339,6 +334,10 @@ func (m *Monitor) buildVMFromClusterResource(
state.agentVersion,
),
)
memFree := uint64(0)
if state.memTotal >= state.memUsed {
memFree = state.memTotal - state.memUsed
}
currentMetrics := IOMetrics{
DiskRead: state.diskReadBytes,
DiskWrite: state.diskWriteBytes,
@@ -348,12 +347,14 @@ func (m *Monitor) buildVMFromClusterResource(
}
diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(guestID, currentMetrics)
memoryUsage := safePercentage(float64(state.memUsed), float64(state.memTotal))
memory := models.Memory{
Total: int64(state.memTotal),
Used: int64(state.memUsed),
Free: int64(memFree),
Usage: memoryUsage,
memory := models.UnavailableMemory(clampToInt64(state.memTotal))
if CanonicalMemorySource(state.memorySource) != "unavailable" {
memory = models.Memory{
Total: int64(state.memTotal),
Used: int64(state.memUsed),
Free: int64(memFree),
Usage: safePercentage(float64(state.memUsed), float64(state.memTotal)),
}
}
if memory.Free < 0 {
memory.Free = 0
@@ -363,7 +364,14 @@ func (m *Monitor) buildVMFromClusterResource(
}
// Free above is total-used, i.e. available. When the guest reported its
// truly-free pages (meminfo), split the reclaimable cache back out.
splitReclaimableMemory(&memory, state.guestRaw.MemInfoFree)
var trulyFree uint64
switch CanonicalMemorySource(state.memorySource) {
case "available-field", "derived-free-buffers-cached", "derived-total-minus-used":
trulyFree = state.guestRaw.MemInfoFree
case "guest-agent-meminfo", "guest-agent-meminfo-derived":
trulyFree = state.guestRaw.GuestAgentMemFree
}
splitReclaimableMemory(&memory, trulyFree)
if state.detailedStatus != nil && state.detailedStatus.Balloon > 0 {
memory.Balloon = int64(state.detailedStatus.Balloon)
}
@@ -110,16 +110,23 @@ func (m *Monitor) recordGuestMetrics(allVMs []models.VM, allContainers []models.
now := time.Now()
for _, vm := range allVMs {
if vm.Status == "running" {
m.recordGuestMetric("vm", vm.ID, unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), vm.Memory.Usage, vm.Disk.Usage, vm.DiskRead, vm.DiskWrite, vm.NetworkIn, vm.NetworkOut, now)
m.recordGuestMetric("vm", vm.ID, unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), historyMemoryUsage(vm.Memory), vm.Disk.Usage, vm.DiskRead, vm.DiskWrite, vm.NetworkIn, vm.NetworkOut, now)
}
}
for _, ct := range allContainers {
if ct.Status == "running" {
m.recordGuestMetric("container", ct.ID, unifiedresources.ProxmoxGuestCPUPercent(ct.CPU), ct.Memory.Usage, ct.Disk.Usage, ct.DiskRead, ct.DiskWrite, ct.NetworkIn, ct.NetworkOut, now)
m.recordGuestMetric("container", ct.ID, unifiedresources.ProxmoxGuestCPUPercent(ct.CPU), historyMemoryUsage(ct.Memory), ct.Disk.Usage, ct.DiskRead, ct.DiskWrite, ct.NetworkIn, ct.NetworkOut, now)
}
}
}
func historyMemoryUsage(memory models.Memory) float64 {
if !memory.HasKnownUsage() {
return -1
}
return memory.Usage
}
// recordGuestMetric records metrics for a single guest (VM or container) to both
// the in-memory metrics history and the persistent metrics store.
func (m *Monitor) recordGuestMetric(
@@ -134,7 +141,9 @@ func (m *Monitor) recordGuestMetric(
if m.metricsHistory != nil {
m.metricsHistory.AddGuestMetric(resourceID, "cpu", cpu, now)
m.metricsHistory.AddGuestMetric(resourceID, "memory", memory, now)
if memory >= 0 {
m.metricsHistory.AddGuestMetric(resourceID, "memory", memory, now)
}
if diskUsage >= 0 {
m.metricsHistory.AddGuestMetric(resourceID, "disk", diskUsage, now)
}
@@ -154,7 +163,9 @@ func (m *Monitor) recordGuestMetric(
if m.metricsStore != nil {
m.metricsStore.Write(resourceType, resourceID, "cpu", cpu, now)
m.metricsStore.Write(resourceType, resourceID, "memory", memory, now)
if memory >= 0 {
m.metricsStore.Write(resourceType, resourceID, "memory", memory, now)
}
if diskUsage >= 0 {
m.metricsStore.Write(resourceType, resourceID, "disk", diskUsage, now)
}
+51 -37
View File
@@ -2,6 +2,7 @@ package monitoring
import (
"context"
"math"
"strings"
"time"
@@ -20,8 +21,8 @@ func (m *Monitor) calculateLXCMemory(
// The cluster resources API returns mem from cgroup which includes cache/buffers (inflated).
// Try to get more accurate memory metrics from RRD data.
memTotal := res.MaxMem
memUsed := res.Mem
memorySource := "cluster-resources"
memUsed := uint64(0)
memorySource := "powered-off"
guestRaw := VMMemoryRaw{
ListingMem: res.Mem,
ListingMaxMem: res.MaxMem,
@@ -29,6 +30,7 @@ func (m *Monitor) calculateLXCMemory(
// For running containers, try to get RRD data for cache-aware memory calculation
if res.Status == "running" {
memorySource = "unavailable"
rrdCtx, rrdCancel := context.WithTimeout(ctx, 5*time.Second)
rrdPoints, err := client.GetLXCRRDData(rrdCtx, res.Node, res.VMID, "hour", "AVERAGE", []string{"memavailable", "memused", "maxmem"})
rrdCancel()
@@ -37,40 +39,39 @@ func (m *Monitor) calculateLXCMemory(
// Use the most recent RRD point
point := rrdPoints[len(rrdPoints)-1]
if point.MaxMem != nil && *point.MaxMem > 0 {
guestRaw.StatusMaxMem = uint64(*point.MaxMem)
if point.MaxMem != nil && !math.IsNaN(*point.MaxMem) && !math.IsInf(*point.MaxMem, 0) && *point.MaxMem > 0 && *point.MaxMem <= math.MaxUint64 {
guestRaw.RRDMaxMem = uint64(*point.MaxMem)
if memTotal == 0 {
memTotal = uint64(*point.MaxMem)
}
}
// Prefer memavailable-based calculation (excludes cache/buffers)
if point.MemAvailable != nil && *point.MemAvailable > 0 {
if point.MemAvailable != nil && !math.IsNaN(*point.MemAvailable) && !math.IsInf(*point.MemAvailable, 0) && *point.MemAvailable >= 0 && *point.MemAvailable <= math.MaxUint64 && memTotal > 0 && *point.MemAvailable <= float64(memTotal) {
memAvailable := uint64(*point.MemAvailable)
if memAvailable <= memTotal {
memUsed = memTotal - memAvailable
memorySource = "rrd-memavailable"
guestRaw.MemInfoAvailable = memAvailable
log.Debug().
Str("container", res.Name).
Str("node", res.Node).
Uint64("total", memTotal).
Uint64("available", memAvailable).
Uint64("used", memUsed).
Float64("usage", safePercentage(float64(memUsed), float64(memTotal))).
Msg("LXC memory: using RRD memavailable (excludes reclaimable cache)")
}
} else if point.MemUsed != nil && *point.MemUsed > 0 {
memUsed = memTotal - memAvailable
memorySource = "rrd-memavailable"
guestRaw.RRDMemAvailable = memAvailable
log.Debug().
Str("container", res.Name).
Str("node", res.Node).
Uint64("total", memTotal).
Uint64("available", memAvailable).
Uint64("used", memUsed).
Float64("usage", safePercentage(float64(memUsed), float64(memTotal))).
Msg("LXC memory: using RRD memavailable (excludes reclaimable cache)")
} else if point.MemUsed != nil && !math.IsNaN(*point.MemUsed) && !math.IsInf(*point.MemUsed, 0) && *point.MemUsed >= 0 && *point.MemUsed <= math.MaxUint64 && memTotal > 0 && *point.MemUsed <= float64(memTotal) {
// Fall back to memused from RRD if available
memUsed = uint64(*point.MemUsed)
if memUsed <= memTotal {
memorySource = "rrd-memused"
guestRaw.MemInfoUsed = memUsed
log.Debug().
Str("container", res.Name).
Str("node", res.Node).
Uint64("total", memTotal).
Uint64("used", memUsed).
Float64("usage", safePercentage(float64(memUsed), float64(memTotal))).
Msg("LXC memory: using RRD memused (excludes reclaimable cache)")
}
memorySource = "rrd-memused"
guestRaw.RRDMemUsed = memUsed
log.Debug().
Str("container", res.Name).
Str("node", res.Node).
Uint64("total", memTotal).
Uint64("used", memUsed).
Float64("usage", safePercentage(float64(memUsed), float64(memTotal))).
Msg("LXC memory: using RRD memused (excludes reclaimable cache)")
}
} else if err != nil {
log.Debug().
@@ -78,7 +79,7 @@ func (m *Monitor) calculateLXCMemory(
Str("instance", instanceName).
Str("container", res.Name).
Int("vmid", res.VMID).
Msg("RRD memory data unavailable for LXC, using cluster resources value")
Msg("RRD memory data unavailable for LXC; memory usage remains unavailable")
}
}
@@ -122,6 +123,15 @@ func (m *Monitor) buildContainerFromClusterResource(
diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(guestID, currentMetrics)
memTotal, memUsed, memorySource, guestRaw := m.calculateLXCMemory(ctx, instanceName, res, client)
memUsed, memorySource, _ = stabilizeGuestLowTrustMemory(
m.previousGuestSnapshot(instanceName, "lxc", res.Node, res.VMID),
res.Status,
memorySource,
memTotal,
memUsed,
sampleTime,
false,
)
// Clamp memory and disk values to prevent >100% usage
// (Proxmox can report used > total for LXC due to cgroup accounting,
@@ -134,6 +144,15 @@ func (m *Monitor) buildContainerFromClusterResource(
if memFree < 0 {
memFree = 0
}
memory := models.UnavailableMemory(clampToInt64(memTotal))
if CanonicalMemorySource(memorySource) != "unavailable" {
memory = models.Memory{
Total: int64(memTotal),
Used: int64(clampedMemUsed),
Free: memFree,
Usage: safePercentage(float64(clampedMemUsed), float64(memTotal)),
}
}
diskUsed := res.Disk
if diskUsed > res.MaxDisk && res.MaxDisk > 0 {
diskUsed = res.MaxDisk
@@ -155,12 +174,7 @@ func (m *Monitor) buildContainerFromClusterResource(
Type: "lxc",
CPU: safeFloat(res.CPU),
CPUs: res.MaxCPU,
Memory: models.Memory{
Total: int64(memTotal),
Used: int64(clampedMemUsed),
Free: memFree,
Usage: safePercentage(float64(clampedMemUsed), float64(memTotal)),
},
Memory: memory,
Disk: models.Disk{
Total: int64(res.MaxDisk),
Used: int64(diskUsed),
+7 -3
View File
@@ -196,13 +196,15 @@ func (m *Monitor) applyStorageFallbackAndRecordNodeMetrics(
now := time.Now()
var nodeNetMetrics *rrdMemCacheEntry
if client != nil {
if rrdMetrics, err := m.getNodeRRDMetrics(context.Background(), client, modelNodes[i].Name); err == nil {
if rrdMetrics, err := m.getNodeRRDMetrics(context.Background(), client, instanceName, modelNodes[i].Name); err == nil {
nodeNetMetrics = &rrdMetrics
}
}
m.metricsHistory.AddNodeMetric(modelNodes[i].ID, "cpu", modelNodes[i].CPU*100, now)
m.metricsHistory.AddNodeMetric(modelNodes[i].ID, "memory", modelNodes[i].Memory.Usage, now)
if modelNodes[i].Memory.HasKnownUsage() {
m.metricsHistory.AddNodeMetric(modelNodes[i].ID, "memory", modelNodes[i].Memory.Usage, now)
}
m.metricsHistory.AddNodeMetric(modelNodes[i].ID, "disk", modelNodes[i].Disk.Usage, now)
if nodeNetMetrics != nil {
if nodeNetMetrics.hasNetIn {
@@ -216,7 +218,9 @@ func (m *Monitor) applyStorageFallbackAndRecordNodeMetrics(
// Also write to persistent store
if m.metricsStore != nil {
m.metricsStore.Write("node", modelNodes[i].ID, "cpu", modelNodes[i].CPU*100, now)
m.metricsStore.Write("node", modelNodes[i].ID, "memory", modelNodes[i].Memory.Usage, now)
if modelNodes[i].Memory.HasKnownUsage() {
m.metricsStore.Write("node", modelNodes[i].ID, "memory", modelNodes[i].Memory.Usage, now)
}
m.metricsStore.Write("node", modelNodes[i].ID, "disk", modelNodes[i].Disk.Usage, now)
if nodeNetMetrics != nil {
if nodeNetMetrics.hasNetIn {
+90 -104
View File
@@ -29,11 +29,10 @@ func (m *Monitor) resolveNodeMemory(
raw.Buffers = memory.Buffers
raw.Cached = memory.Cached
raw.Shared = memory.Shared
raw.EffectiveAvailable = memory.EffectiveAvailable()
raw.EffectiveAvailable = 0
raw.ProxmoxMemorySource = "node-status"
raw.FallbackCalculated = false
effectiveAvailable := memory.EffectiveAvailable()
componentAvailable := memory.Free
if memory.Buffers > 0 {
if math.MaxUint64-componentAvailable < memory.Buffers {
@@ -49,9 +48,7 @@ func (m *Monitor) resolveNodeMemory(
componentAvailable += memory.Cached
}
}
if memory.Total > 0 && componentAvailable > memory.Total {
componentAvailable = memory.Total
}
hasCacheComponents := memory.Buffers > 0 || memory.Cached > 0
availableFromUsed := uint64(0)
if memory.Total > 0 && memory.Used > 0 && memory.Total >= memory.Used {
@@ -59,27 +56,12 @@ func (m *Monitor) resolveNodeMemory(
}
raw.TotalMinusUsed = availableFromUsed
missingCacheMetrics := memory.Available == 0 &&
memory.Avail == 0 &&
memory.Buffers == 0 &&
memory.Cached == 0
var rrdMetrics rrdMemCacheEntry
haveRRDMetrics := false
usedRRDAvailableFallback := false
rrdMemUsedFallback := false
if missingCacheMetrics {
if metrics, err := m.getNodeRRDMetrics(ctx, client, nodeName); err == nil {
if memory.Available == 0 && memory.Avail == 0 && !hasCacheComponents {
if metrics, err := m.getNodeRRDMetrics(ctx, client, instanceName, nodeName); err == nil {
haveRRDMetrics = true
rrdMetrics = metrics
if metrics.available > 0 {
effectiveAvailable = metrics.available
usedRRDAvailableFallback = true
}
if metrics.used > 0 {
rrdMemUsedFallback = true
}
} else if err != nil {
log.Debug().
Err(err).
@@ -90,98 +72,98 @@ func (m *Monitor) resolveNodeMemory(
}
const totalMinusUsedGapTolerance uint64 = 16 * 1024 * 1024
gapGreaterThanComponents := false
if availableFromUsed > componentAvailable {
gap := availableFromUsed - componentAvailable
if componentAvailable == 0 || gap >= totalMinusUsedGapTolerance {
gapGreaterThanComponents = true
}
}
derivedFromTotalMinusUsed := !usedRRDAvailableFallback &&
missingCacheMetrics &&
availableFromUsed > 0 &&
gapGreaterThanComponents &&
effectiveAvailable == availableFromUsed
var actualUsed uint64
var effectiveAvailable uint64
source := ""
fallbackReason := ""
switch {
case effectiveAvailable > 0 && effectiveAvailable <= memory.Total:
case memory.Available > 0 && memory.Available <= memory.Total:
effectiveAvailable = memory.Available
actualUsed = memory.Total - effectiveAvailable
if actualUsed > memory.Total {
actualUsed = memory.Total
}
logCtx := log.Debug().
source = "available-field"
log.Debug().
Str("node", nodeName).
Uint64("total", memory.Total).
Uint64("effectiveAvailable", effectiveAvailable).
Uint64("actualUsed", actualUsed).
Float64("usage", safePercentage(float64(actualUsed), float64(memory.Total)))
Uint64("available", effectiveAvailable).
Msg("Node memory: using available field (excludes reclaimable cache)")
case memory.Avail > 0 && memory.Avail <= memory.Total:
effectiveAvailable = memory.Avail
actualUsed = memory.Total - effectiveAvailable
source = "available-field"
log.Debug().
Str("node", nodeName).
Uint64("total", memory.Total).
Uint64("available", effectiveAvailable).
Msg("Node memory: using avail field (excludes reclaimable cache)")
case hasCacheComponents && componentAvailable <= memory.Total:
effectiveAvailable = componentAvailable
actualUsed = memory.Total - effectiveAvailable
source = "derived-free-buffers-cached"
log.Debug().
Str("node", nodeName).
Uint64("total", memory.Total).
Uint64("free", memory.Free).
Uint64("buffers", memory.Buffers).
Uint64("cached", memory.Cached).
Uint64("effectiveAvailable", effectiveAvailable).
Msg("Node memory: deriving availability from free+buffers+cached")
case haveRRDMetrics && rrdMetrics.hasAvail && rrdMetrics.available <= memory.Total:
effectiveAvailable = rrdMetrics.available
actualUsed = memory.Total - effectiveAvailable
source = "rrd-memavailable"
fallbackReason = "rrd-memavailable"
raw.FallbackCalculated = true
raw.ProxmoxMemorySource = "rrd-memavailable"
log.Debug().
Str("node", nodeName).
Uint64("total", memory.Total).
Uint64("rrdAvailable", rrdMetrics.available).
Msg("Node memory: using RRD memavailable fallback")
case haveRRDMetrics && rrdMetrics.hasUsed && rrdMetrics.used <= memory.Total:
actualUsed = rrdMetrics.used
effectiveAvailable = memory.Total - actualUsed
source = "rrd-memused"
fallbackReason = "rrd-memused"
raw.FallbackCalculated = true
raw.ProxmoxMemorySource = "rrd-memused"
log.Debug().
Str("node", nodeName).
Uint64("total", memory.Total).
Uint64("rrdUsed", rrdMetrics.used).
Msg("Node memory: using RRD memused fallback")
case availableFromUsed > memory.Free &&
availableFromUsed-memory.Free >= totalMinusUsedGapTolerance:
effectiveAvailable = availableFromUsed
actualUsed = memory.Used
source = "derived-total-minus-used"
fallbackReason = "node-status-total-minus-used"
raw.FallbackCalculated = true
raw.ProxmoxMemorySource = "node-status-total-minus-used"
log.Debug().
Str("node", nodeName).
Uint64("total", memory.Total).
Uint64("availableFromUsed", availableFromUsed).
Uint64("reportedFree", memory.Free).
Msg("Node memory: deriving availability from total-used cache gap")
if usedRRDAvailableFallback {
if haveRRDMetrics && rrdMetrics.available > 0 {
logCtx = logCtx.Uint64("rrdAvailable", rrdMetrics.available)
}
logCtx.Msg("node memory: using RRD memavailable fallback (excludes reclaimable cache)")
source = "rrd-memavailable"
fallbackReason = "rrd-memavailable"
raw.FallbackCalculated = true
raw.ProxmoxMemorySource = "rrd-memavailable"
} else if memory.Available > 0 {
logCtx.Msg("node memory: using available field (excludes reclaimable cache)")
source = "available-field"
} else if memory.Avail > 0 {
logCtx.Msg("node memory: using avail field (excludes reclaimable cache)")
source = "available-field"
} else if derivedFromTotalMinusUsed {
logCtx.
Uint64("availableFromUsed", availableFromUsed).
Uint64("reportedFree", memory.Free).
Msg("Node memory: derived available from total-used gap (cache fields missing)")
source = "derived-total-minus-used"
fallbackReason = "node-status-total-minus-used"
raw.FallbackCalculated = true
raw.ProxmoxMemorySource = "node-status-total-minus-used"
} else {
logCtx.
Uint64("free", memory.Free).
Uint64("buffers", memory.Buffers).
Uint64("cached", memory.Cached).
Msg("Node memory: derived available from free+buffers+cached (excludes reclaimable cache)")
source = "derived-free-buffers-cached"
}
default:
switch {
case rrdMemUsedFallback && haveRRDMetrics && rrdMetrics.used > 0:
actualUsed = rrdMetrics.used
if actualUsed > memory.Total {
actualUsed = memory.Total
}
log.Debug().
Str("node", nodeName).
Uint64("total", memory.Total).
Uint64("rrdUsed", rrdMetrics.used).
Msg("Node memory: using RRD memused fallback (excludes reclaimable cache)")
source = "rrd-memused"
fallbackReason = "rrd-memused"
raw.FallbackCalculated = true
raw.ProxmoxMemorySource = "rrd-memused"
default:
actualUsed = memory.Used
if actualUsed > memory.Total {
actualUsed = memory.Total
}
log.Debug().
Str("node", nodeName).
Uint64("total", memory.Total).
Uint64("used", actualUsed).
Msg("Node memory: no cache-aware metrics - using traditional calculation (includes cache)")
source = "node-status-used"
}
source = "unavailable"
fallbackReason = "cache-aware-memory-unavailable"
raw.FallbackCalculated = true
raw.ProxmoxMemorySource = "cache-aware-unavailable"
log.Warn().
Str("instance", instanceName).
Str("node", nodeName).
Uint64("total", memory.Total).
Uint64("used", memory.Used).
Uint64("free", memory.Free).
Msg("Node memory unavailable: no cache-aware source")
}
raw.EffectiveAvailable = effectiveAvailable
@@ -191,6 +173,10 @@ func (m *Monitor) resolveNodeMemory(
raw.RRDTotal = rrdMetrics.total
}
if source == "unavailable" {
return models.UnavailableMemory(clampToInt64(memory.Total)), source, fallbackReason, raw, true
}
free := int64(memory.Total - actualUsed)
if free < 0 {
free = 0
@@ -80,6 +80,22 @@ func TestResolveNodeMemoryCharacterization(t *testing.T) {
wantUsed: 4 * gib,
wantRawSource: "rrd-memavailable",
},
{
name: "explicit zero RRD memused is a valid idle sample",
memory: &proxmox.MemoryStatus{
Total: 8 * gib,
Used: 7 * gib,
Free: 1 * gib,
},
rrdPoints: []proxmox.NodeRRDPoint{{
MemTotal: floatPtr(float64(8 * gib)),
MemUsed: floatPtr(0),
}},
wantSource: "rrd-memused",
wantFallback: "rrd-memused",
wantUsed: 0,
wantRawSource: "rrd-memused",
},
}
for _, tt := range tests {
+38 -29
View File
@@ -66,6 +66,7 @@ func resourceFromProxmoxNode(node models.Node, linkedHost *models.Host) (Resourc
TemperatureMonitoringEnabled: cloneBoolPtr(node.TemperatureMonitoringEnabled),
PendingUpdatesCheckedAt: zeroTimeToPtr(node.PendingUpdatesCheckedAt),
MemoryCache: node.Memory.Cache,
Memory: &node.Memory,
SwapUsed: node.Memory.SwapUsed,
SwapTotal: node.Memory.SwapTotal,
LinkedAgentID: linkedAgentID,
@@ -129,35 +130,28 @@ func resourceFromHost(host models.Host) (Resource, ResourceIdentity) {
platform := agentRuntimePlatformForHost(host, hostProfile)
agent := &AgentData{
AgentID: host.ID,
AgentVersion: host.AgentVersion,
Hostname: host.Hostname,
MachineID: host.MachineID,
TokenID: host.TokenID,
TokenName: host.TokenName,
TokenHint: host.TokenHint,
TokenLastUsedAt: host.TokenLastUsedAt,
Platform: platform,
HostProfile: hostProfile,
OSName: host.OSName,
OSVersion: host.OSVersion,
KernelVersion: host.KernelVersion,
Architecture: host.Architecture,
CPUCount: host.CPUCount,
LoadAverage: append([]float64(nil), host.LoadAverage...),
UptimeSeconds: host.UptimeSeconds,
IntervalSeconds: host.IntervalSeconds,
Temperature: maxCPUTemp(host.Sensors),
NetworkInterfaces: convertInterfaces(host.NetworkInterfaces),
Disks: convertDisks(host.Disks),
Memory: &AgentMemoryMeta{
Total: host.Memory.Total,
Used: host.Memory.Used,
Free: host.Memory.Free,
Cache: host.Memory.Cache,
SwapUsed: host.Memory.SwapUsed,
SwapTotal: host.Memory.SwapTotal,
},
AgentID: host.ID,
AgentVersion: host.AgentVersion,
Hostname: host.Hostname,
MachineID: host.MachineID,
TokenID: host.TokenID,
TokenName: host.TokenName,
TokenHint: host.TokenHint,
TokenLastUsedAt: host.TokenLastUsedAt,
Platform: platform,
HostProfile: hostProfile,
OSName: host.OSName,
OSVersion: host.OSVersion,
KernelVersion: host.KernelVersion,
Architecture: host.Architecture,
CPUCount: host.CPUCount,
LoadAverage: append([]float64(nil), host.LoadAverage...),
UptimeSeconds: host.UptimeSeconds,
IntervalSeconds: host.IntervalSeconds,
Temperature: maxCPUTemp(host.Sensors),
NetworkInterfaces: convertInterfaces(host.NetworkInterfaces),
Disks: convertDisks(host.Disks),
Memory: agentMemoryMetaFromModel(host.Memory),
CommandsEnabled: host.CommandsEnabled,
OperationReceiptVersion: host.OperationReceiptVersion,
ReportIP: host.ReportIP,
@@ -1324,6 +1318,7 @@ func resourceFromDockerHost(host models.DockerHost) (Resource, ResourceIdentity)
AgentVersion: host.AgentVersion,
CPUs: host.CPUs,
TotalMemoryBytes: host.TotalMemoryBytes,
Memory: agentMemoryMetaFromModel(host.Memory),
UptimeSeconds: host.UptimeSeconds,
LoadAverage: append([]float64(nil), host.LoadAverage...),
IntervalSeconds: host.IntervalSeconds,
@@ -1389,6 +1384,18 @@ func resourceFromDockerHost(host models.DockerHost) (Resource, ResourceIdentity)
return resource, identity
}
func agentMemoryMetaFromModel(memory models.Memory) *AgentMemoryMeta {
return &AgentMemoryMeta{
Total: memory.Total,
Used: memory.Used,
Free: memory.Free,
Cache: memory.Cache,
UsageUnavailable: memory.UsageUnavailable,
SwapUsed: memory.SwapUsed,
SwapTotal: memory.SwapTotal,
}
}
func resourceFromPBSInstance(instance models.PBSInstance) (Resource, ResourceIdentity) {
name := instance.Name
if strings.TrimSpace(name) == "" {
@@ -1689,6 +1696,7 @@ func resourceFromVM(vm models.VM) (Resource, ResourceIdentity) {
SwapTotal: vm.Memory.SwapTotal,
Balloon: vm.Memory.Balloon,
MemoryCache: vm.Memory.Cache,
Memory: &vm.Memory,
Lock: vm.Lock,
}
resource := Resource{
@@ -1754,6 +1762,7 @@ func resourceFromContainer(ct models.Container) (Resource, ResourceIdentity) {
SwapTotal: ct.Memory.SwapTotal,
Balloon: ct.Memory.Balloon,
MemoryCache: ct.Memory.Cache,
Memory: &ct.Memory,
Lock: ct.Lock,
}
resource := Resource{
@@ -1788,3 +1788,46 @@ func TestPhysicalDiskWearoutFromSMARTAttributesClampsExhaustedMedia(t *testing.T
t.Fatalf("wearout remaining = %d, want 0 for percentage-used above 100", got)
}
}
func TestResourceAdaptersPreserveUnavailableMemoryAsRawEvidence(t *testing.T) {
unavailable := models.UnavailableMemory(8 << 30)
hostResource, _ := resourceFromHost(models.Host{
ID: "agent-1501",
Hostname: "linux-host",
Memory: unavailable,
})
if hostResource.Metrics == nil || hostResource.Metrics.Memory != nil {
t.Fatalf("agent metrics = %+v, want unavailable memory omitted", hostResource.Metrics)
}
if hostResource.Agent == nil || hostResource.Agent.Memory == nil ||
!hostResource.Agent.Memory.UsageUnavailable {
t.Fatalf("agent raw memory = %+v, want explicit unavailable evidence", hostResource.Agent)
}
vmResource, _ := resourceFromVM(models.VM{
VMID: 1501,
Name: "linux-guest",
Memory: unavailable,
})
if vmResource.Metrics == nil || vmResource.Metrics.Memory != nil {
t.Fatalf("Proxmox metrics = %+v, want unavailable memory omitted", vmResource.Metrics)
}
if vmResource.Proxmox == nil || vmResource.Proxmox.Memory == nil ||
!vmResource.Proxmox.Memory.UsageUnavailable {
t.Fatalf("Proxmox raw memory = %+v, want explicit unavailable evidence", vmResource.Proxmox)
}
dockerResource, _ := resourceFromDockerHost(models.DockerHost{
ID: "docker-1501",
Hostname: "docker-host",
Memory: unavailable,
})
if dockerResource.Metrics == nil || dockerResource.Metrics.Memory != nil {
t.Fatalf("Docker metrics = %+v, want unavailable memory omitted", dockerResource.Metrics)
}
if dockerResource.Docker == nil || dockerResource.Docker.Memory == nil ||
!dockerResource.Docker.Memory.UsageUnavailable {
t.Fatalf("Docker raw memory = %+v, want explicit unavailable evidence", dockerResource.Docker)
}
}
@@ -75,6 +75,60 @@ func TestRefreshCanonicalIdentityPrefersTargetsAndCanonicalHostData(t *testing.T
}
}
func TestUnavailableMemoryFacetsDoNotChangeCanonicalIdentity(t *testing.T) {
total := int64(8 << 30)
resource := Resource{
ID: "vm:proxmox-a:1501",
Type: ResourceTypeVM,
Name: "linux-guest",
Identity: ResourceIdentity{
Hostnames: []string{"linux-guest.example"},
},
Proxmox: &ProxmoxData{
Instance: "proxmox-a",
VMID: 1501,
},
Agent: &AgentData{
AgentID: "agent-1501",
},
Docker: &DockerData{
HostSourceID: "docker-1501",
},
}
RefreshCanonicalIdentity(&resource)
if resource.Canonical == nil {
t.Fatal("canonical identity is nil")
}
wantPrimaryID := resource.Canonical.PrimaryID
wantHostname := resource.Canonical.Hostname
resource.Proxmox.Memory = &models.Memory{
Total: total,
UsageUnavailable: true,
}
resource.Agent.Memory = &AgentMemoryMeta{
Total: total,
UsageUnavailable: true,
}
resource.Docker.Memory = &AgentMemoryMeta{
Total: total,
UsageUnavailable: true,
}
RefreshCanonicalIdentity(&resource)
if resource.Canonical == nil ||
resource.Canonical.PrimaryID != wantPrimaryID ||
resource.Canonical.Hostname != wantHostname {
t.Fatalf(
"canonical identity = %+v, want primary ID %q and hostname %q unchanged",
resource.Canonical,
wantPrimaryID,
wantHostname,
)
}
}
func TestVMGuestAgentIncidentUsesCanonicalSourceID(t *testing.T) {
vm := models.VM{
ID: "cluster-a:pve-a:101",
+12
View File
@@ -144,6 +144,10 @@ func cloneProxmoxData(in *ProxmoxData) *ProxmoxData {
return nil
}
out := *in
if in.Memory != nil {
memory := *in.Memory
out.Memory = &memory
}
out.Temperature = cloneFloat64Ptr(in.Temperature)
out.TemperatureDetails = cloneTemperature(in.TemperatureDetails)
out.CPUInfo = cloneCPUInfo(in.CPUInfo)
@@ -191,6 +195,10 @@ func cloneAgentData(in *AgentData) *AgentData {
return nil
}
out := *in
if in.Memory != nil {
memory := *in.Memory
out.Memory = &memory
}
out.Temperature = cloneFloat64Ptr(in.Temperature)
out.LoadAverage = cloneFloat64Slice(in.LoadAverage)
out.NetworkInterfaces = cloneNetworkInterfaces(in.NetworkInterfaces)
@@ -229,6 +237,10 @@ func cloneDockerData(in *DockerData) *DockerData {
return nil
}
out := *in
if in.Memory != nil {
memory := *in.Memory
out.Memory = &memory
}
out.OOMKilled = cloneBoolPtr(in.OOMKilled)
out.Temperature = cloneFloat64Ptr(in.Temperature)
out.LoadAverage = cloneFloat64Slice(in.LoadAverage)
+28
View File
@@ -377,6 +377,34 @@ func TestCloneProxmoxData_NetworkInterfaceIsolation(t *testing.T) {
}
}
func TestCloneProxmoxDataMemoryIsolation(t *testing.T) {
original := &ProxmoxData{
Memory: &models.Memory{Total: 8 << 30, Used: 4 << 30, Usage: 50},
}
cloned := cloneProxmoxData(original)
cloned.Memory.Used = 7 << 30
if original.Memory.Used != 4<<30 {
t.Error("mutating cloned memory should not affect original")
}
}
func TestCloneAgentAndDockerMemoryIsolation(t *testing.T) {
agent := &AgentData{Memory: &AgentMemoryMeta{Total: 8 << 30}}
clonedAgent := cloneAgentData(agent)
clonedAgent.Memory.Total = 4 << 30
if agent.Memory.Total != 8<<30 {
t.Error("mutating cloned agent memory should not affect original")
}
docker := &DockerData{Memory: &AgentMemoryMeta{Total: 8 << 30}}
clonedDocker := cloneDockerData(docker)
clonedDocker.Memory.Total = 4 << 30
if docker.Memory.Total != 8<<30 {
t.Error("mutating cloned Docker memory should not affect original")
}
}
// --- cloneStorageMeta ---
func TestCloneStorageMeta_Nil(t *testing.T) {
+3 -3
View File
@@ -15,7 +15,7 @@ func metricsFromProxmoxNode(node models.Node) *ResourceMetrics {
cpuPercent := percentFromUsage(node.CPU)
metrics.CPU = &MetricValue{Value: cpuPercent, Percent: cpuPercent, Unit: "percent", Source: SourceProxmox}
if node.Memory.Total > 0 {
if node.Memory.Total > 0 && node.Memory.HasKnownUsage() {
percent := percentFromUsage(node.Memory.Usage)
metrics.Memory = &MetricValue{Used: &node.Memory.Used, Total: &node.Memory.Total, Percent: percent, Unit: "bytes", Source: SourceProxmox}
}
@@ -89,7 +89,7 @@ func buildHostMetricPayload(
metrics := &ResourceMetrics{}
cpuPercent := percentFromReportedPercent(cpuUsage)
metrics.CPU = &MetricValue{Value: cpuPercent, Percent: cpuPercent, Unit: "percent", Source: source}
if memory.Total > 0 {
if memory.Total > 0 && memory.HasKnownUsage() {
percent := percentFromReportedPercent(memory.Usage)
metrics.Memory = &MetricValue{Used: &memory.Used, Total: &memory.Total, Percent: percent, Unit: "bytes", Source: source}
}
@@ -186,7 +186,7 @@ func buildVMMetricPayload(
metrics := &ResourceMetrics{}
cpuPercent := ProxmoxGuestCPUPercent(cpu)
metrics.CPU = &MetricValue{Value: cpuPercent, Percent: cpuPercent, Unit: "percent", Source: source}
if memory.Total > 0 {
if memory.Total > 0 && memory.HasKnownUsage() {
percent := percentFromUsage(memory.Usage)
metrics.Memory = &MetricValue{Used: &memory.Used, Total: &memory.Total, Percent: percent, Unit: "bytes", Source: source}
}
+87
View File
@@ -2,6 +2,7 @@ package unifiedresources
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
@@ -32,6 +33,92 @@ func TestMetricsFromDockerHostIncludesIORates(t *testing.T) {
}
}
func TestUnavailableMemoryDoesNotProjectOrOverwriteTrustedCrossSourceMetric(t *testing.T) {
unavailable := models.UnavailableMemory(8 * 1024 * 1024 * 1024)
for name, metrics := range map[string]*ResourceMetrics{
"proxmox-node": metricsFromProxmoxNode(models.Node{Memory: unavailable}),
"proxmox-vm": metricsFromVM(models.VM{Memory: unavailable}),
"proxmox-lxc": metricsFromContainer(models.Container{Memory: unavailable}),
"agent-host": metricsFromHost(models.Host{Memory: unavailable}),
"docker-host": metricsFromDockerHost(models.DockerHost{Memory: unavailable}),
} {
t.Run(name, func(t *testing.T) {
if metrics.Memory != nil {
t.Fatalf("Memory = %+v, want nil for unavailable usage", metrics.Memory)
}
})
}
trustedTotal := int64(8 * 1024 * 1024 * 1024)
trustedUsed := int64(4 * 1024 * 1024 * 1024)
trusted := &ResourceMetrics{
Memory: &MetricValue{
Used: &trustedUsed,
Total: &trustedTotal,
Percent: 50,
Source: SourceAgent,
},
}
merged := mergeMetrics(
&Resource{Type: ResourceTypeVM, Proxmox: &ProxmoxData{VMID: 1501}},
trusted,
metricsFromVM(models.VM{Memory: unavailable}),
SourceProxmox,
time.Now().UTC(),
nil,
nil,
)
if merged.Memory == nil || merged.Memory.Percent != 50 || merged.Memory.Source != SourceAgent {
t.Fatalf("merged memory = %+v, want trusted agent metric preserved", merged.Memory)
}
resource, _ := resourceFromVM(models.VM{VMID: 1501, Memory: unavailable})
if resource.Metrics == nil || resource.Metrics.Memory != nil {
t.Fatalf("canonical metrics = %+v, want absent unavailable memory metric", resource.Metrics)
}
if resource.Proxmox == nil || resource.Proxmox.Memory == nil || !resource.Proxmox.Memory.UsageUnavailable {
t.Fatalf("Proxmox memory fallback = %+v, want explicit unavailable capacity", resource.Proxmox)
}
for name, tc := range map[string]struct {
source DataSource
resource *Resource
}{
"proxmox": {
source: SourceProxmox,
resource: &Resource{Proxmox: &ProxmoxData{Memory: &unavailable}},
},
"agent": {
source: SourceAgent,
resource: &Resource{Agent: &AgentData{Memory: &AgentMemoryMeta{
Total: unavailable.Total,
UsageUnavailable: true,
}}},
},
"docker": {
source: SourceDocker,
resource: &Resource{Docker: &DockerData{Memory: &AgentMemoryMeta{
Total: unavailable.Total,
UsageUnavailable: true,
}}},
},
} {
t.Run("same-source-"+name, func(t *testing.T) {
sameSource := &ResourceMetrics{Memory: &MetricValue{Percent: 50, Source: tc.source}}
cleared := clearUnavailableSourceMemoryMetric(sameSource, tc.resource, tc.source)
if cleared.Memory != nil {
t.Fatalf("Memory = %+v, want same-source stale value cleared", cleared.Memory)
}
})
}
dockerResource, _ := resourceFromDockerHost(models.DockerHost{Memory: unavailable})
if dockerResource.Docker == nil || dockerResource.Docker.Memory == nil || !dockerResource.Docker.Memory.UsageUnavailable {
t.Fatalf("Docker memory fallback = %+v, want explicit unavailable capacity", dockerResource.Docker)
}
}
func TestMetricsFromHostKeepsReportedPercentValues(t *testing.T) {
host := models.Host{
CPUUsage: 1,
+35
View File
@@ -3306,6 +3306,7 @@ func (rr *ResourceRegistry) mergeInto(existing *Resource, incoming Resource, sou
existing.Status = chooseStatus(existing.Status, incoming.Status, source)
existing.Metrics = mergeMetrics(existing, existing.Metrics, incoming.Metrics, source, now, existing.SourceStatus, nil)
existing.Metrics = clearUnavailableSourceMemoryMetric(existing.Metrics, &incoming, source)
// Prefer agent naming when available
if incoming.Name != "" {
@@ -3315,6 +3316,35 @@ func (rr *ResourceRegistry) mergeInto(existing *Resource, incoming Resource, sou
}
}
func clearUnavailableSourceMemoryMetric(metrics *ResourceMetrics, incoming *Resource, source DataSource) *ResourceMetrics {
if metrics == nil || metrics.Memory == nil || metrics.Memory.Source != source || incoming == nil {
return metrics
}
unavailable := false
switch source {
case SourceProxmox:
unavailable = incoming.Proxmox != nil &&
incoming.Proxmox.Memory != nil &&
incoming.Proxmox.Memory.UsageUnavailable
case SourceAgent:
unavailable = incoming.Agent != nil &&
incoming.Agent.Memory != nil &&
incoming.Agent.Memory.UsageUnavailable
case SourceDocker:
unavailable = incoming.Docker != nil &&
incoming.Docker.Memory != nil &&
incoming.Docker.Memory.UsageUnavailable
}
if !unavailable {
return metrics
}
cleared := *metrics
cleared.Memory = nil
return &cleared
}
func mergeProxmoxData(existing *ProxmoxData, incoming *ProxmoxData) *ProxmoxData {
if existing == nil {
return incoming
@@ -3389,6 +3419,11 @@ func mergeProxmoxData(existing *ProxmoxData, incoming *ProxmoxData) *ProxmoxData
if incoming.Balloon != 0 {
merged.Balloon = incoming.Balloon
}
if incoming.Memory != nil {
memory := *incoming.Memory
merged.Memory = &memory
merged.MemoryCache = incoming.MemoryCache
}
if incoming.Lock != "" {
merged.Lock = incoming.Lock
}
@@ -4,6 +4,8 @@ import (
"strings"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
func TestLinkedMergeAllowsOneSidedNodeHostLinkWhenHostnameCorroborates(t *testing.T) {
@@ -417,3 +419,72 @@ func containsDataSource(sources []DataSource, want DataSource) bool {
}
return false
}
func TestRegistryMemoryUnavailableClearsOnlySameSourceMetric(t *testing.T) {
total := int64(8 << 30)
used := total / 2
unavailable := &AgentMemoryMeta{
Total: total,
UsageUnavailable: true,
}
lastSeen := time.Now().UTC()
registry := NewRegistry(nil)
registry.IngestRecords(SourceAgent, []IngestRecord{{
SourceID: "agent-1501",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "linux-host",
Status: StatusOnline,
LastSeen: lastSeen,
Metrics: &ResourceMetrics{Memory: &MetricValue{
Used: &used,
Total: &total,
Percent: 50,
Source: SourceAgent,
}},
Agent: &AgentData{Memory: &AgentMemoryMeta{Total: total, Used: used}},
},
}})
registry.IngestRecords(SourceAgent, []IngestRecord{{
SourceID: "agent-1501",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "linux-host",
Status: StatusOnline,
LastSeen: lastSeen.Add(time.Second),
Metrics: &ResourceMetrics{},
Agent: &AgentData{Memory: unavailable},
},
}})
resources := registry.ListByType(ResourceTypeAgent)
if len(resources) != 1 || resources[0].Metrics == nil || resources[0].Metrics.Memory != nil {
t.Fatalf("same-source resources = %+v, want stale memory metric cleared", resources)
}
proxmoxUnavailable := &Resource{
Type: ResourceTypeVM,
Proxmox: &ProxmoxData{Memory: &models.Memory{Total: total, UsageUnavailable: true}},
Metrics: &ResourceMetrics{},
LastSeen: lastSeen.Add(2 * time.Second),
}
trusted := &ResourceMetrics{Memory: &MetricValue{
Used: &used,
Total: &total,
Percent: 50,
Source: SourceAgent,
}}
merged := mergeMetrics(
proxmoxUnavailable,
trusted,
proxmoxUnavailable.Metrics,
SourceProxmox,
proxmoxUnavailable.LastSeen,
nil,
nil,
)
if merged.Memory == nil || merged.Memory.Percent != 50 || merged.Memory.Source != SourceAgent {
t.Fatalf("cross-source memory = %+v, want trusted agent metric preserved", merged.Memory)
}
}
+33 -30
View File
@@ -371,8 +371,9 @@ type ProxmoxData struct {
SwapTotal int64 `json:"swapTotal,omitempty"`
Balloon int64 `json:"balloon,omitempty"`
// Reclaimable buff/cache split out of the memory metric's free bytes.
MemoryCache int64 `json:"memoryCache,omitempty"`
Lock string `json:"lock,omitempty"` // Proxmox lock state (e.g. "backup", "migrate", "snapshot")
MemoryCache int64 `json:"memoryCache,omitempty"`
Memory *models.Memory `json:"memory,omitempty"`
Lock string `json:"lock,omitempty"` // Proxmox lock state (e.g. "backup", "migrate", "snapshot")
// Internal link hint to a host agent resource.
LinkedAgentID string `json:"-"`
}
@@ -802,9 +803,10 @@ type AgentMemoryMeta struct {
Used int64 `json:"used,omitempty"`
Free int64 `json:"free,omitempty"`
// Cache is the reclaimable page cache; used + cache + free ≈ total.
Cache int64 `json:"cache,omitempty"`
SwapUsed int64 `json:"swapUsed,omitempty"`
SwapTotal int64 `json:"swapTotal,omitempty"`
Cache int64 `json:"cache,omitempty"`
UsageUnavailable bool `json:"usageUnavailable,omitempty"`
SwapUsed int64 `json:"swapUsed,omitempty"`
SwapTotal int64 `json:"swapTotal,omitempty"`
}
const HostPackageUpdateFreshness = 45 * time.Minute
@@ -989,31 +991,32 @@ type DockerPodmanContainerMeta struct {
// DockerData contains Docker host- and container-specific data.
type DockerData struct {
HostSourceID string `json:"hostSourceId,omitempty"` // raw model ID for the docker host
AgentID string `json:"agentId,omitempty"`
ContainerID string `json:"containerId,omitempty"`
Hostname string `json:"hostname,omitempty"`
DisplayName string `json:"displayName,omitempty"`
CustomDisplayName string `json:"customDisplayName,omitempty"`
MachineID string `json:"machineId,omitempty"`
Image string `json:"image,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
Runtime string `json:"runtime,omitempty"`
RuntimeVersion string `json:"runtimeVersion,omitempty"`
DockerVersion string `json:"dockerVersion,omitempty"`
OS string `json:"os,omitempty"`
KernelVersion string `json:"kernelVersion,omitempty"`
Architecture string `json:"architecture,omitempty"`
AgentVersion string `json:"agentVersion,omitempty"`
CPUs int `json:"cpus,omitempty"`
TotalMemoryBytes int64 `json:"totalMemoryBytes,omitempty"`
UptimeSeconds int64 `json:"uptimeSeconds,omitempty"`
LoadAverage []float64 `json:"loadAverage,omitempty"`
IntervalSeconds int `json:"intervalSeconds,omitempty"`
NetInRate float64 `json:"netInRate,omitempty"`
NetOutRate float64 `json:"netOutRate,omitempty"`
DiskReadRate float64 `json:"diskReadRate,omitempty"`
DiskWriteRate float64 `json:"diskWriteRate,omitempty"`
HostSourceID string `json:"hostSourceId,omitempty"` // raw model ID for the docker host
AgentID string `json:"agentId,omitempty"`
ContainerID string `json:"containerId,omitempty"`
Hostname string `json:"hostname,omitempty"`
DisplayName string `json:"displayName,omitempty"`
CustomDisplayName string `json:"customDisplayName,omitempty"`
MachineID string `json:"machineId,omitempty"`
Image string `json:"image,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
Runtime string `json:"runtime,omitempty"`
RuntimeVersion string `json:"runtimeVersion,omitempty"`
DockerVersion string `json:"dockerVersion,omitempty"`
OS string `json:"os,omitempty"`
KernelVersion string `json:"kernelVersion,omitempty"`
Architecture string `json:"architecture,omitempty"`
AgentVersion string `json:"agentVersion,omitempty"`
CPUs int `json:"cpus,omitempty"`
TotalMemoryBytes int64 `json:"totalMemoryBytes,omitempty"`
Memory *AgentMemoryMeta `json:"memory,omitempty"`
UptimeSeconds int64 `json:"uptimeSeconds,omitempty"`
LoadAverage []float64 `json:"loadAverage,omitempty"`
IntervalSeconds int `json:"intervalSeconds,omitempty"`
NetInRate float64 `json:"netInRate,omitempty"`
NetOutRate float64 `json:"netOutRate,omitempty"`
DiskReadRate float64 `json:"diskReadRate,omitempty"`
DiskWriteRate float64 `json:"diskWriteRate,omitempty"`
// Host-level summary fields (populated when Resource.Type == ResourceTypeAgent and Docker != nil)
ContainerCount int `json:"containerCount,omitempty"`
+150 -32
View File
@@ -734,34 +734,45 @@ func (m *MemoryStatus) UnmarshalJSON(data []byte) error {
// EffectiveAvailable returns the best-effort estimate of reclaimable memory.
// Prefer the dedicated "available"/"avail" fields when present, otherwise derive
// from free + buffers + cached which mirrors Linux's MemAvailable calculation.
// from free + buffers + cached when reclaimable-cache evidence is present.
// MemFree by itself is not an availability estimate.
func (m *MemoryStatus) EffectiveAvailable() uint64 {
if m == nil {
return 0
}
if m.Available > 0 {
if m.Total > 0 && m.Available > m.Total {
return 0
}
return m.Available
}
if m.Avail > 0 {
if m.Total > 0 && m.Avail > m.Total {
return 0
}
return m.Avail
}
derived := m.Free + m.Buffers + m.Cached
if m.Total > 0 && m.Used > 0 && m.Total >= m.Used {
availableFromUsed := m.Total - m.Used
if availableFromUsed > derived {
derived = availableFromUsed
}
if m.Buffers == 0 && m.Cached == 0 {
return 0
}
derived := m.Free
for _, value := range []uint64{m.Buffers, m.Cached} {
if math.MaxUint64-derived < value {
derived = math.MaxUint64
break
}
derived += value
}
if derived == 0 {
return 0
}
// Cap at total to guard against over-reporting when buffers/cached exceed total.
// Conflicting components are not a usable availability estimate.
if m.Total > 0 && derived > m.Total {
return m.Total
return 0
}
return derived
@@ -2112,13 +2123,124 @@ func vmNetworkInterfaceHasUsefulData(iface VMNetworkInterface) bool {
iface.HasIp6Gateway
}
// GetVMMemAvailableFromAgent reads /proc/meminfo via the QEMU guest agent's
// file-read endpoint and returns MemAvailable in bytes.
func (c *Client) GetVMMemAvailableFromAgent(ctx context.Context, node string, vmid int) (uint64, error) {
// LinuxMemoryAvailability records the cache-aware fields used to establish a
// Linux guest's available memory. Source is either meminfo-available or
// meminfo-derived.
type LinuxMemoryAvailability struct {
Total uint64
Free uint64
Available uint64
Buffers uint64
Cached uint64
SReclaimable uint64
Shmem uint64
SwapTotal uint64
SwapFree uint64
EffectiveAvailable uint64
Source string
}
// ParseLinuxMemoryAvailability parses /proc/meminfo. New kernels provide
// MemAvailable directly. Older kernels use the conservative cache-aware
// estimate Free + Buffers + Cached + SReclaimable - Shmem. A partial payload
// containing only MemTotal/MemFree is deliberately rejected.
func ParseLinuxMemoryAvailability(content string, truncated bool) (LinuxMemoryAvailability, error) {
var result LinuxMemoryAvailability
seen := make(map[string]struct{})
fieldsByName := map[string]*uint64{
"MemTotal": &result.Total,
"MemFree": &result.Free,
"MemAvailable": &result.Available,
"Buffers": &result.Buffers,
"Cached": &result.Cached,
"SReclaimable": &result.SReclaimable,
"Shmem": &result.Shmem,
"SwapTotal": &result.SwapTotal,
"SwapFree": &result.SwapFree,
}
for _, line := range strings.Split(content, "\n") {
nameValue := strings.SplitN(strings.TrimSpace(line), ":", 2)
if len(nameValue) != 2 {
continue
}
target, ok := fieldsByName[nameValue[0]]
if !ok {
continue
}
if _, duplicate := seen[nameValue[0]]; duplicate {
return LinuxMemoryAvailability{}, fmt.Errorf("duplicate %s field in /proc/meminfo", nameValue[0])
}
seen[nameValue[0]] = struct{}{}
fields := strings.Fields(nameValue[1])
if len(fields) < 2 || fields[1] != "kB" {
return LinuxMemoryAvailability{}, fmt.Errorf("invalid %s field in /proc/meminfo", nameValue[0])
}
kB, err := strconv.ParseUint(fields[0], 10, 64)
if err != nil {
return LinuxMemoryAvailability{}, fmt.Errorf("parse %s value %q: %w", nameValue[0], fields[0], err)
}
if kB > math.MaxUint64/1024 {
return LinuxMemoryAvailability{}, fmt.Errorf("%s value %d kB overflows uint64", nameValue[0], kB)
}
*target = kB * 1024
}
if _, present := seen["MemAvailable"]; present {
if _, ok := seen["MemTotal"]; !ok || result.Total == 0 {
return LinuxMemoryAvailability{}, fmt.Errorf("MemAvailable has no valid MemTotal")
}
if result.Available > result.Total {
return LinuxMemoryAvailability{}, fmt.Errorf("MemAvailable exceeds MemTotal")
}
result.EffectiveAvailable = result.Available
result.Source = "meminfo-available"
return result, nil
}
if truncated {
return LinuxMemoryAvailability{}, fmt.Errorf("truncated /proc/meminfo has no MemAvailable")
}
for _, required := range []string{"MemTotal", "MemFree", "Buffers", "Cached"} {
if _, ok := seen[required]; !ok {
return LinuxMemoryAvailability{}, fmt.Errorf("old-kernel /proc/meminfo missing %s", required)
}
}
if result.Total == 0 || result.Free == 0 && result.Buffers == 0 && result.Cached == 0 && result.SReclaimable == 0 {
return LinuxMemoryAvailability{}, fmt.Errorf("insufficient cache-aware fields in /proc/meminfo")
}
if result.Buffers == 0 && result.Cached == 0 && result.SReclaimable == 0 {
return LinuxMemoryAvailability{}, fmt.Errorf("MemAvailable and reclaimable cache fields not found in /proc/meminfo")
}
available := result.Free
for _, value := range []uint64{result.Buffers, result.Cached, result.SReclaimable} {
if math.MaxUint64-available < value {
available = math.MaxUint64
break
}
available += value
}
if result.Shmem >= available {
available = 0
} else {
available -= result.Shmem
}
if available > result.Total {
available = result.Total
}
result.EffectiveAvailable = available
result.Source = "meminfo-derived"
return result, nil
}
// GetVMMemoryAvailabilityFromAgent reads /proc/meminfo via the QEMU guest
// agent and returns the raw evidence plus the selected cache-aware value.
func (c *Client) GetVMMemoryAvailabilityFromAgent(ctx context.Context, node string, vmid int) (LinuxMemoryAvailability, error) {
fileParam := url.QueryEscape("/proc/meminfo")
resp, err := c.get(ctx, fmt.Sprintf("/nodes/%s/qemu/%d/agent/file-read?file=%s", node, vmid, fileParam))
if err != nil {
return 0, fmt.Errorf("guest agent file-read /proc/meminfo: %w", err)
return LinuxMemoryAvailability{}, fmt.Errorf("guest agent file-read /proc/meminfo: %w", err)
}
defer resp.Body.Close()
@@ -2129,29 +2251,25 @@ func (c *Client) GetVMMemAvailableFromAgent(ctx context.Context, node string, vm
} `json:"data"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return 0, fmt.Errorf("decode file-read response: %w", err)
return LinuxMemoryAvailability{}, fmt.Errorf("decode file-read response: %w", err)
}
for _, line := range strings.Split(result.Data.Content, "\n") {
line = strings.TrimSpace(line)
if !strings.HasPrefix(line, "MemAvailable:") {
continue
}
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
kB, err := strconv.ParseUint(fields[1], 10, 64)
if err != nil {
return 0, fmt.Errorf("parse MemAvailable value %q: %w", fields[1], err)
}
if kB > math.MaxUint64/1024 {
return 0, fmt.Errorf("MemAvailable value %d kB overflows uint64", kB)
}
return kB * 1024, nil
truncated := result.Data.Truncated != nil && *result.Data.Truncated
availability, err := ParseLinuxMemoryAvailability(result.Data.Content, truncated)
if err != nil {
return LinuxMemoryAvailability{}, err
}
return availability, nil
}
return 0, fmt.Errorf("MemAvailable not found in /proc/meminfo")
// GetVMMemAvailableFromAgent is retained for callers that only need the
// selected availability value.
func (c *Client) GetVMMemAvailableFromAgent(ctx context.Context, node string, vmid int) (uint64, error) {
availability, err := c.GetVMMemoryAvailabilityFromAgent(ctx, node, vmid)
if err != nil {
return 0, err
}
return availability.EffectiveAvailable, nil
}
// GetVMStatus returns detailed VM status including balloon info
+126 -9
View File
@@ -435,9 +435,14 @@ func TestMemoryStatusEffectiveAvailable(t *testing.T) {
want: 24 * 1024,
},
{
name: "caps derived value at total",
name: "rejects conflicting components above total",
status: MemoryStatus{Total: 8 * 1024, Free: 4 * 1024, Buffers: 4 * 1024, Cached: 4 * 1024},
want: 8 * 1024,
want: 0,
},
{
name: "rejects explicit available above total",
status: MemoryStatus{Total: 8 * 1024, Available: 9 * 1024},
want: 0,
},
{
name: "returns zero when no data",
@@ -455,6 +460,118 @@ func TestMemoryStatusEffectiveAvailable(t *testing.T) {
}
}
func TestParseLinuxMemoryAvailability(t *testing.T) {
tests := []struct {
name string
content string
truncated bool
wantSource string
wantAvailable uint64
wantErr bool
}{
{
name: "issue 1501 uses explicit MemAvailable without swap",
content: `MemTotal: 7796964 kB
MemFree: 444824 kB
MemAvailable: 1872820 kB
Buffers: 51360 kB
Cached: 1580464 kB
SReclaimable: 191812 kB
Shmem: 31200 kB`,
wantSource: "meminfo-available",
wantAvailable: 1872820 * 1024,
},
{
name: "explicit zero MemAvailable is valid full pressure",
content: `MemTotal: 8388608 kB
MemFree: 0 kB
MemAvailable: 0 kB`,
wantSource: "meminfo-available",
wantAvailable: 0,
},
{
name: "old kernel derives cache aware availability",
content: `MemTotal: 8388608 kB
MemFree: 524288 kB
Buffers: 262144 kB
Cached: 1572864 kB
SReclaimable: 262144 kB
Shmem: 131072 kB`,
wantSource: "meminfo-derived",
wantAvailable: (524288 + 262144 + 1572864 + 262144 - 131072) * 1024,
},
{
name: "partial total and free is unknown",
content: `MemTotal: 8388608 kB
MemFree: 524288 kB`,
wantErr: true,
},
{
name: "partial old kernel components are unknown even when not marked truncated",
content: `MemTotal: 8388608 kB
MemFree: 524288 kB
Cached: 1572864 kB`,
wantErr: true,
},
{
name: "truncated old kernel evidence is unknown",
content: `MemTotal: 8388608 kB
MemFree: 524288 kB
Cached: 1572864 kB`,
truncated: true,
wantErr: true,
},
{
name: "conflicting available above total is rejected",
content: `MemTotal: 1024 kB
MemFree: 512 kB
MemAvailable: 2048 kB`,
wantErr: true,
},
{
name: "malformed cache field is rejected",
content: `MemTotal: 1024 kB
MemFree: 512 kB
Cached: nope kB`,
wantErr: true,
},
{
name: "duplicate source evidence is rejected",
content: `MemTotal: 1024 kB
MemAvailable: 512 kB
MemAvailable: 256 kB`,
wantErr: true,
},
{
name: "non kB units are rejected",
content: `MemTotal: 1024 bytes
MemAvailable: 512 bytes`,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ParseLinuxMemoryAvailability(tt.content, tt.truncated)
if tt.wantErr {
if err == nil {
t.Fatal("ParseLinuxMemoryAvailability() error = nil, want error")
}
return
}
if err != nil {
t.Fatalf("ParseLinuxMemoryAvailability() error = %v", err)
}
if got.Source != tt.wantSource {
t.Fatalf("Source = %q, want %q", got.Source, tt.wantSource)
}
if got.EffectiveAvailable != tt.wantAvailable {
t.Fatalf("EffectiveAvailable = %d, want %d", got.EffectiveAvailable, tt.wantAvailable)
}
})
}
}
func TestMemoryStatusUnmarshalFlexibleValues(t *testing.T) {
tests := []struct {
name string
@@ -1248,7 +1365,7 @@ func TestMemoryStatusEffectiveAvailable_RegressionIssue435(t *testing.T) {
description: "When available/avail missing, derive from free+buffers+cached",
},
{
name: "proxmox 8.4 hides cache fields - derive from total-minus-used gap",
name: "proxmox 8.4 hides cache fields - availability remains unknown",
status: MemoryStatus{
Total: 134794743808, // ~125.6GB
Used: 107351023616, // ~100GB actual usage
@@ -1256,9 +1373,9 @@ func TestMemoryStatusEffectiveAvailable_RegressionIssue435(t *testing.T) {
Buffers: 0,
Cached: 0,
},
wantAvailable: 27443720192, // total - used => ~25.6GB reclaimable (free + cache)
wantUsedPct: 79.6, // Matches Proxmox node dashboard
description: "Proxmox 8.4 stops reporting buffers/cached; use total-used gap to recover cache-aware metric",
wantAvailable: 0,
wantUsedPct: 0,
description: "MemoryStatus alone does not treat total-used or MemFree as MemAvailable",
},
{
name: "issue #435 specific case - 86% vs 42% real usage",
@@ -1292,9 +1409,9 @@ func TestMemoryStatusEffectiveAvailable_RegressionIssue435(t *testing.T) {
Used: 6871947674,
Free: 0, // All fields missing
},
wantAvailable: 1717986918, // Derived from total - used
wantUsedPct: 80.0, // Still aligns with cache-inclusive calculation when nothing else reported
description: "When all cache fields missing, fall back to total-used gap instead of zero",
wantAvailable: 0,
wantUsedPct: 0,
description: "When all cache-aware fields are missing, preserve an honest unknown value",
},
}
+15
View File
@@ -1377,6 +1377,21 @@ func (cc *ClusterClient) GetVMMemAvailableFromAgent(ctx context.Context, node st
return result, err
}
// GetVMMemoryAvailabilityFromAgent reads cache-aware /proc/meminfo evidence
// through the first healthy cluster client.
func (cc *ClusterClient) GetVMMemoryAvailabilityFromAgent(ctx context.Context, node string, vmid int) (LinuxMemoryAvailability, error) {
var result LinuxMemoryAvailability
err := cc.executeWithFailover(ctx, func(client *Client) error {
availability, err := client.GetVMMemoryAvailabilityFromAgent(ctx, node, vmid)
if err != nil {
return err
}
result = availability
return nil
})
return result, err
}
// GetClusterResources returns all resources (VMs, containers) across the cluster in a single call
func (cc *ClusterClient) GetClusterResources(ctx context.Context, resourceType string) ([]ClusterResource, error) {
var result []ClusterResource
+32
View File
@@ -34,6 +34,38 @@ func TestClusterClient_GetCephStatus(t *testing.T) {
}
}
func TestClusterClient_GetVMMemoryAvailabilityFromAgent(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api2/json/nodes":
fmt.Fprint(w, `{"data":[{"node":"node1","status":"online"}]}`)
case "/api2/json/nodes/node1/qemu/1501/agent/file-read":
fmt.Fprint(w, `{"data":{"content":"MemTotal: 7796964 kB\nMemFree: 444824 kB\nMemAvailable: 1872820 kB\nCached: 1580464 kB\n","truncated":false}}`)
default:
http.NotFound(w, r)
}
}))
defer server.Close()
cc := NewClusterClient(
"test",
ClientConfig{Host: server.URL, TokenName: "u@p!t", TokenValue: "v"},
[]string{server.URL},
nil,
)
availability, err := cc.GetVMMemoryAvailabilityFromAgent(context.Background(), "node1", 1501)
if err != nil {
t.Fatalf("GetVMMemoryAvailabilityFromAgent() error = %v", err)
}
if availability.Source != "meminfo-available" {
t.Fatalf("source = %q, want meminfo-available", availability.Source)
}
if availability.EffectiveAvailable != 1872820*1024 {
t.Fatalf("available = %d, want %d", availability.EffectiveAvailable, 1872820*1024)
}
}
func TestClusterClient_GetVMSnapshots(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")