diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index c13fd6f26..d4a7a72d2 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -567,6 +567,21 @@ runtime failure. The client should preserve the usable base snapshot, record degraded enrichment issues on the snapshot, and let the poller publish those as `observed.degraded` plus summarized issue metadata instead of clearing the observed contribution or pretending the refresh was fully healthy. +That same VMware inventory floor also owns operator-visible uptime and guest +filesystem usage. `vmware.InventoryMetrics` carries `UptimeSeconds`, +`DiskUsedBytes`, `DiskTotalBytes`, and `DiskPercent` for hosts and VMs so the +canonical `Resource.Uptime` field and `ResourceMetrics.Disk` series populate +on vSphere-backed workloads — without these the workloads table renders +empty "0s" and blank disk cells for every vSphere row. Real collection uses +PerformanceManager `sys.uptime.latest` (host + VM) plus +`sys.osUptime.latest` for VMs (Tools-reported guest OS uptime; preferred +when present), and `GET /api/vcenter/vm/{vm}/guest/local-filesystem` +aggregated across mount points for disk usage. A 503 from that REST +endpoint (Tools not running) is recorded as a non-fatal `unavailable` +enrichment issue rather than failing the poll. Mock fixtures +(`internal/mock/platform_fixtures.go`) must synthesize the same fields per +powered-on VM and drop them for powered-off VMs so the demo estate +exercises the same workload-table contract as live vCenter would. That same poller-owned partial-success model must also keep runtime observability non-noisy. Repeated polls with the same degraded optional-read issue classes should not emit a fresh warning every interval; monitoring diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 47dc521cc..bce2f60c7 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -166,6 +166,13 @@ regression protection. `frontend-modern/src/hooks/useWorkloads.ts` must consume canonical `platformScopes`; page surfaces must not rebuild platform membership with ad hoc source or runtime scans. + The same `useWorkloads.ts` mapping owns canonical-field fallback for + per-row scalars the workload table reads. `WorkloadGuest.uptime` must + fall back to the canonical `Resource.Uptime` field after the + platform-specific carve-outs (`proxmox.uptime`, `agent.uptimeSeconds`, + `docker.uptimeSeconds`, `kubernetes.uptimeSeconds`); platforms whose + adapters only populate the canonical field — vSphere is the working + example — would otherwise render blank uptime cells for every row. 4. Keep shared auth gating in `internal/api/router.go` cheap and local: pre-auth quick-setup and recovery routing may short-circuit on loopback/session/token checks, but they must not trigger chart, metrics, or broad persistence fan-out on the protected request hot path. The same rule applies to public setup-script lifecycle routes: `/api/auto-register` and `/api/auto-unregister` may bypass the global auth wall so their handlers can diff --git a/frontend-modern/src/hooks/__tests__/useWorkloads.test.ts b/frontend-modern/src/hooks/__tests__/useWorkloads.test.ts index d793a4a83..7ac67e17e 100644 --- a/frontend-modern/src/hooks/__tests__/useWorkloads.test.ts +++ b/frontend-modern/src/hooks/__tests__/useWorkloads.test.ts @@ -319,6 +319,52 @@ describe('useWorkloads', () => { dispose(); }); + it('falls back to canonical resource.uptime when no platform-specific carve-out exists (vSphere)', async () => { + // vSphere VMs surface uptime only on the canonical Resource.Uptime + // field; there is no proxmox/agent/docker/kubernetes carve-out. The + // useWorkloads mapping must land on resource.uptime so the workloads + // table renders real "N days" cells rather than the blank "0s" + // placeholder. + apiFetchJSONMock.mockResolvedValueOnce({ + data: [ + { + ...sampleResource, + id: 'vmware-vm-uptime', + name: 'lab-app-01', + status: 'online', + uptime: 12345678, + sources: ['vmware'], + parentName: 'esxi-01', + proxmox: undefined, + vmware: { + managedObjectId: 'vm-901', + runtimeHostName: 'esxi-01', + clusterName: 'Compute-A', + connectionName: 'Production vCenter', + powerState: 'poweredOn', + }, + }, + ], + meta: { totalPages: 1 }, + }); + + let dispose = () => {}; + let result: ReturnType | undefined; + createRoot((d) => { + dispose = d; + const [enabled] = createSignal(true); + result = useWorkloads(enabled); + }); + + await flushAsync(); + await waitForWorkloadCount(() => result!.workloads().length, 1); + + const vm = result!.workloads().find((workload) => workload.name === 'lab-app-01'); + expect(vm?.uptime).toBe(12345678); + + dispose(); + }); + it('preserves canonical discovery targets for workloads instead of inferring them from platform type', async () => { apiFetchJSONMock.mockResolvedValueOnce({ data: [ diff --git a/frontend-modern/src/hooks/useWorkloads.ts b/frontend-modern/src/hooks/useWorkloads.ts index 13909f084..019a40cb8 100644 --- a/frontend-modern/src/hooks/useWorkloads.ts +++ b/frontend-modern/src/hooks/useWorkloads.ts @@ -55,6 +55,7 @@ type APIResource = { type?: string; name?: string; status?: string; + uptime?: number; lastSeen?: string; sources?: string[]; platformScopes?: string[]; @@ -474,6 +475,11 @@ const mapResourceToWorkload = (resource: APIResource): WorkloadGuest | null => { resource.agent?.uptimeSeconds ?? resource.docker?.uptimeSeconds ?? resource.kubernetes?.uptimeSeconds ?? + // Canonical Resource.Uptime is the universal fallback — vSphere's + // adapter populates only this field (no platform-specific + // vmware.uptime carve-out), so the chain has to land here for VMware + // VMs to surface uptime in the workloads table. + resource.uptime ?? 0, template: resource.proxmox?.template ?? false, lastBackup: (() => { diff --git a/internal/mock/platform_fixtures.go b/internal/mock/platform_fixtures.go index 30bb323c8..664b93218 100644 --- a/internal/mock/platform_fixtures.go +++ b/internal/mock/platform_fixtures.go @@ -409,6 +409,14 @@ func refreshVMwarePlatformFixture(snapshot vmware.InventorySnapshot, at time.Tim *out.VMs[i].Metrics.NetOutBytesPerSecond = 0 *out.VMs[i].Metrics.DiskReadBytesPerSecond = 0 *out.VMs[i].Metrics.DiskWriteBytesPerSecond = 0 + // A powered-off VM has no current uptime, and VMware Tools is + // obviously not reporting guest filesystem usage. Drop the + // pointers entirely so the frontend renders "—" rather than 0, + // matching how Pulse signals "no data" for an offline guest. + out.VMs[i].Metrics.UptimeSeconds = nil + out.VMs[i].Metrics.DiskUsedBytes = nil + out.VMs[i].Metrics.DiskTotalBytes = nil + out.VMs[i].Metrics.DiskPercent = nil } } for i := range out.Datastores { @@ -526,6 +534,74 @@ func refreshVMwareInventoryMetrics(metrics *vmware.InventoryMetrics, resourceCla if memoryTotal > 0 { *ensureInt64Ptr(&metrics.MemoryUsedBytes) = bytesFromPercent(memoryTotal, *metrics.MemoryPercent) } + + // Uptime: a per-resource stable base age (1-30 days) that climbs forward + // with `at`. Matches what a real vCenter would report from + // sys.uptime.latest (and sys.osUptime.latest for VMs with Tools running). + *ensureInt64Ptr(&metrics.UptimeSeconds) = mockUptimeSeconds(resourceClass, resourceID, at) + + // Guest filesystem usage: only meaningful for VMs (hosts don't have a + // `guest` shape in vSphere). For VMs we synthesize a stable total + // capacity and let SampleMetric oscillate usage naturally so the table + // renders realistic-looking bars instead of always-empty cells. + if resourceClass == "vm" { + total := mockGuestDiskTotalBytes(resourceID) + usage := clampFloat(SampleMetric(resourceClass, resourceID, "diskusage", at), 0, 100) + *ensureInt64Ptr(&metrics.DiskTotalBytes) = total + *ensureInt64Ptr(&metrics.DiskUsedBytes) = bytesFromPercent(total, usage) + *ensureFloat64Ptr(&metrics.DiskPercent) = usage + } +} + +// mockUptimeSeconds returns a stable, slowly-incrementing uptime for a +// vSphere mock resource. The base age is derived from the resource ID so +// each host / VM has its own multi-day uptime, then `at` shifts it forward +// so the column ticks naturally between snapshots. +func mockUptimeSeconds(resourceClass, resourceID string, at time.Time) int64 { + seed := fnv64a(resourceClass + "|" + resourceID + "|uptime") + // Base age between 1 hour and 30 days + const minSeconds = uint64(3600) + const maxSeconds = uint64(30 * 24 * 60 * 60) + span := maxSeconds - minSeconds + baseSeconds := int64(minSeconds + (seed % span)) + if at.IsZero() { + return baseSeconds + } + // Anchor the climb at a stable epoch so successive refreshes ramp up + // without resetting; use 2026-01-01 UTC. + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if at.Before(anchor) { + return baseSeconds + } + return baseSeconds + int64(at.Sub(anchor)/time.Second) +} + +// mockGuestDiskTotalBytes returns a per-VM stable guest filesystem total +// (between 32 GiB and 256 GiB) so the same VM always appears at the same +// size across refreshes. +func mockGuestDiskTotalBytes(resourceID string) int64 { + const ( + minGiB = int64(32) + maxGiB = int64(256) + ) + seed := fnv64a(resourceID + "|disktotal") + gib := minGiB + int64(seed%uint64(maxGiB-minGiB+1)) + return gib * 1024 * 1024 * 1024 +} + +// fnv64a is a tiny deterministic hash so mock helpers can derive stable +// per-resource seeds without pulling in a math/rand source. +func fnv64a(input string) uint64 { + const ( + offset uint64 = 14695981039346656037 + prime uint64 = 1099511628211 + ) + hash := offset + for i := 0; i < len(input); i++ { + hash ^= uint64(input[i]) + hash *= prime + } + return hash } func ensureFloat64Ptr(target **float64) *float64 { diff --git a/internal/mock/platform_fixtures_test.go b/internal/mock/platform_fixtures_test.go index c215156bf..ef794d3c4 100644 --- a/internal/mock/platform_fixtures_test.go +++ b/internal/mock/platform_fixtures_test.go @@ -287,6 +287,54 @@ func TestBuildFixtureGraphRefreshesPlatformFixtureMetricsFromCanonicalModel(t *t if got, want := *host.Metrics.CPUPercent, SampleMetric("agent", "vc-mock-1:host:host-101", "cpu", now); math.Abs(got-want) > 1e-9 { t.Fatalf("expected refreshed VMware host cpu %.6f, got %.6f", want, got) } + // Mock fixture must also synthesize uptime so the workloads table + // renders real "N days" cells for vSphere hosts and VMs instead of + // blank "0s" placeholders. Hosts get sys.uptime-style uptime only; + // guest disk usage is VM-only because ESXi exposes no `guest` shape. + if host.Metrics.UptimeSeconds == nil || *host.Metrics.UptimeSeconds <= 0 { + t.Fatalf("expected refreshed VMware host uptime seconds, got %+v", host.Metrics.UptimeSeconds) + } + if host.Metrics.DiskTotalBytes != nil || host.Metrics.DiskUsedBytes != nil || host.Metrics.DiskPercent != nil { + t.Fatalf("expected VMware host guest filesystem fields to stay nil, got total=%+v used=%+v percent=%+v", host.Metrics.DiskTotalBytes, host.Metrics.DiskUsedBytes, host.Metrics.DiskPercent) + } + + var poweredOnVM *struct{} + for _, vm := range graph.PlatformFixtures.VMware.VMs { + if vm.PowerState != "POWERED_ON" || vm.Metrics == nil { + continue + } + if vm.Metrics.UptimeSeconds == nil || *vm.Metrics.UptimeSeconds <= 0 { + t.Fatalf("expected powered-on VMware VM %q to surface uptime, got %+v", vm.Name, vm.Metrics.UptimeSeconds) + } + if vm.Metrics.DiskTotalBytes == nil || *vm.Metrics.DiskTotalBytes <= 0 { + t.Fatalf("expected powered-on VMware VM %q to surface guest disk total, got %+v", vm.Name, vm.Metrics.DiskTotalBytes) + } + if vm.Metrics.DiskUsedBytes == nil || *vm.Metrics.DiskUsedBytes < 0 { + t.Fatalf("expected powered-on VMware VM %q to surface guest disk used, got %+v", vm.Name, vm.Metrics.DiskUsedBytes) + } + if vm.Metrics.DiskPercent == nil { + t.Fatalf("expected powered-on VMware VM %q to surface guest disk percent", vm.Name) + } + marker := struct{}{} + poweredOnVM = &marker + break + } + if poweredOnVM == nil { + t.Fatal("expected at least one powered-on VMware VM fixture to assert uptime + guest disk projection") + } + + for _, vm := range graph.PlatformFixtures.VMware.VMs { + if vm.PowerState != "POWERED_OFF" || vm.Metrics == nil { + continue + } + if vm.Metrics.UptimeSeconds != nil { + t.Fatalf("expected powered-off VMware VM %q to drop uptime, got %+v", vm.Name, vm.Metrics.UptimeSeconds) + } + if vm.Metrics.DiskTotalBytes != nil || vm.Metrics.DiskUsedBytes != nil || vm.Metrics.DiskPercent != nil { + t.Fatalf("expected powered-off VMware VM %q to drop guest disk fields, got total=%+v used=%+v percent=%+v", vm.Name, vm.Metrics.DiskTotalBytes, vm.Metrics.DiskUsedBytes, vm.Metrics.DiskPercent) + } + break + } datastore := graph.PlatformFixtures.VMware.Datastores[0] wantFree := datastore.Capacity - bytesFromPercent(datastore.Capacity, SampleMetric("storage", "vc-mock-1:datastore:"+datastore.Datastore, "usage", now)) diff --git a/internal/vmware/client.go b/internal/vmware/client.go index fd6a6b9db..39a237df6 100644 --- a/internal/vmware/client.go +++ b/internal/vmware/client.go @@ -200,7 +200,7 @@ func (c *Client) CollectInventory(ctx context.Context) (*InventorySnapshot, erro if err != nil { return nil, err } - signalIssues, err := c.enrichInventorySnapshot(ctx, release, sessionID, refs.PerfManagerMoID, refs.EventManagerMoID, perfCounters, inventory) + signalIssues, err := c.enrichInventorySnapshot(ctx, automationSessionID, release, sessionID, refs.PerfManagerMoID, refs.EventManagerMoID, perfCounters, inventory) if err != nil { return nil, err } diff --git a/internal/vmware/client_metrics.go b/internal/vmware/client_metrics.go index 0aefbaf96..1d7218942 100644 --- a/internal/vmware/client_metrics.go +++ b/internal/vmware/client_metrics.go @@ -70,6 +70,15 @@ const ( vmwarePerfLogicalNetOut = "net_out" vmwarePerfLogicalDiskRead = "disk_read" vmwarePerfLogicalDiskWrite = "disk_write" + // sys.uptime.latest returns total seconds since last system startup + // (VMX-process clock for VMs, kernel uptime for ESXi hosts). Stats + // level 1, available on every interval. + vmwarePerfLogicalSysUptime = "sys_uptime" + // sys.osUptime.latest returns guest OS uptime via VMware Tools, in + // seconds. VM-only and stats level 4, so it may be absent from the + // historical rollups vCenter exposes by default; treat its presence + // as preferred over sys.uptime, otherwise fall back. + vmwarePerfLogicalGuestOSUptime = "guest_os_uptime" ) var vmwareHostPerfCounters = []perfCounterDefinition{ @@ -80,6 +89,7 @@ var vmwareHostPerfCounters = []perfCounterDefinition{ {logical: vmwarePerfLogicalNetOut, group: "net", name: "bytesTx", rollup: "average"}, {logical: vmwarePerfLogicalDiskRead, group: "disk", name: "read", rollup: "average"}, {logical: vmwarePerfLogicalDiskWrite, group: "disk", name: "write", rollup: "average"}, + {logical: vmwarePerfLogicalSysUptime, group: "sys", name: "uptime", rollup: "latest"}, } var vmwareVMPerfCounters = []perfCounterDefinition{ @@ -89,6 +99,8 @@ var vmwareVMPerfCounters = []perfCounterDefinition{ {logical: vmwarePerfLogicalNetOut, group: "net", name: "bytesTx", rollup: "average"}, {logical: vmwarePerfLogicalDiskRead, group: "disk", name: "read", rollup: "average"}, {logical: vmwarePerfLogicalDiskWrite, group: "disk", name: "write", rollup: "average"}, + {logical: vmwarePerfLogicalSysUptime, group: "sys", name: "uptime", rollup: "latest"}, + {logical: vmwarePerfLogicalGuestOSUptime, group: "sys", name: "osUptime", rollup: "latest"}, } func (c *Client) loadPerfCounterCatalog(ctx context.Context, release, sessionID, perfManagerMoID string) (perfCounterCatalog, error) { @@ -425,6 +437,15 @@ func inventoryMetricsFromPerf( if accumulator, ok := accumulators[vmwarePerfLogicalDiskWrite]; ok && accumulator.count > 0 { metrics.DiskWriteBytesPerSecond = float64Ptr(accumulator.sum * 1024.0) } + // Uptime: prefer guest OS uptime (VMware Tools) when present, otherwise + // fall back to VMX-process / host uptime. Both counters are in seconds. + // PerformanceManager sums across entity series, but uptime is a single + // value per entity, so the average is the value itself. + if accumulator, ok := accumulators[vmwarePerfLogicalGuestOSUptime]; ok && accumulator.count > 0 { + metrics.UptimeSeconds = int64Ptr(int64(math.Round(accumulator.sum / float64(accumulator.count)))) + } else if accumulator, ok := accumulators[vmwarePerfLogicalSysUptime]; ok && accumulator.count > 0 { + metrics.UptimeSeconds = int64Ptr(int64(math.Round(accumulator.sum / float64(accumulator.count)))) + } if !hasInventoryMetrics(metrics) { return nil @@ -474,7 +495,11 @@ func hasInventoryMetrics(metrics *InventoryMetrics) bool { metrics.NetInBytesPerSecond != nil || metrics.NetOutBytesPerSecond != nil || metrics.DiskReadBytesPerSecond != nil || - metrics.DiskWriteBytesPerSecond != nil + metrics.DiskWriteBytesPerSecond != nil || + metrics.UptimeSeconds != nil || + metrics.DiskTotalBytes != nil || + metrics.DiskUsedBytes != nil || + metrics.DiskPercent != nil } func float64Ptr(value float64) *float64 { diff --git a/internal/vmware/client_signals.go b/internal/vmware/client_signals.go index 91fb9e63b..3c8eb8626 100644 --- a/internal/vmware/client_signals.go +++ b/internal/vmware/client_signals.go @@ -154,6 +154,7 @@ func (c *Client) validateSignalFloor( func (c *Client) enrichInventorySnapshot( ctx context.Context, + automationSessionID string, release string, sessionID string, perfManagerMoID string, @@ -251,6 +252,28 @@ func (c *Client) enrichInventorySnapshot( } else if err != nil { return err } + // Aggregate guest filesystem usage from VMware Tools. 503 means + // Tools is not currently reporting (powered off, Tools missing, + // or guest agent not started); classify as a non-fatal issue + // and continue without disk usage. Hosts have no equivalent + // because ESXi exposes no `guest` shape. + diskTotal, diskUsed, diskPercent, diskOK, diskErr := c.collectVMGuestLocalFilesystem(ctx, automationSessionID, snapshot.VMs[i].VM) + if issue, ok := classifyInventoryEnrichmentIssue("signals", "vm", snapshot.VMs[i].VM, diskErr); ok { + recordIssue(issue) + } else if diskErr != nil && !isAutomationNotFound(diskErr) && !isAutomationUnavailable(diskErr) { + return diskErr + } + if diskOK { + if metrics == nil { + metrics = &InventoryMetrics{} + } + total := diskTotal + used := diskUsed + metrics.DiskTotalBytes = &total + metrics.DiskUsedBytes = &used + percent := diskPercent + metrics.DiskPercent = &percent + } snapshot.VMs[i].Metrics = metrics return nil }) diff --git a/internal/vmware/client_test.go b/internal/vmware/client_test.go index 9485a2d8d..0095cc739 100644 --- a/internal/vmware/client_test.go +++ b/internal/vmware/client_test.go @@ -232,17 +232,37 @@ func TestClientCollectInventoryPreservesBaseInventoryWhenOptionalEnrichmentDegra if len(snapshot.Hosts) != 1 || len(snapshot.VMs) != 1 || len(snapshot.Datastores) != 1 || len(snapshot.Networks) != 1 { t.Fatalf("unexpected inventory sizes: hosts=%d vms=%d datastores=%d networks=%d", len(snapshot.Hosts), len(snapshot.VMs), len(snapshot.Datastores), len(snapshot.Networks)) } - if len(snapshot.EnrichmentIssues) != 3 { - t.Fatalf("expected 3 enrichment issues, got %+v", snapshot.EnrichmentIssues) + // The unavailableVMGuestInfo knob now degrades two REST reads: the + // guest identity endpoint (topology stage) and the guest local + // filesystem endpoint (signals stage). That plus the per-stage + // permission denials gives four issues. + if len(snapshot.EnrichmentIssues) != 4 { + t.Fatalf("expected 4 enrichment issues, got %+v", snapshot.EnrichmentIssues) } - if snapshot.EnrichmentIssues[0].Category != "permission" || snapshot.EnrichmentIssues[0].Stage != "signals" { - t.Fatalf("unexpected first enrichment issue: %+v", snapshot.EnrichmentIssues[0]) + seen := make(map[string]InventoryEnrichmentIssue, len(snapshot.EnrichmentIssues)) + for _, issue := range snapshot.EnrichmentIssues { + key := issue.Stage + "/" + issue.EntityType + "/" + issue.Category + seen[key] = issue } - if snapshot.EnrichmentIssues[1].Category != "permission" || snapshot.EnrichmentIssues[1].Stage != "topology" || snapshot.EnrichmentIssues[1].EntityType != "cluster" { - t.Fatalf("unexpected second enrichment issue: %+v", snapshot.EnrichmentIssues[1]) + if issue, ok := seen["signals/host/permission"]; !ok { + t.Fatalf("expected signals/host permission issue, got %+v", snapshot.EnrichmentIssues) + } else if !strings.Contains(issue.Message, "overall status") { + t.Fatalf("unexpected signals/host permission message: %+v", issue) } - if snapshot.EnrichmentIssues[2].Category != "unavailable" || snapshot.EnrichmentIssues[2].Stage != "topology" { - t.Fatalf("unexpected third enrichment issue: %+v", snapshot.EnrichmentIssues[2]) + if issue, ok := seen["signals/vm/unavailable"]; !ok { + t.Fatalf("expected signals/vm unavailable issue for guest local filesystem, got %+v", snapshot.EnrichmentIssues) + } else if !strings.Contains(issue.Message, "guest local filesystem") { + t.Fatalf("unexpected signals/vm unavailable message: %+v", issue) + } + if issue, ok := seen["topology/cluster/permission"]; !ok { + t.Fatalf("expected topology/cluster permission issue, got %+v", snapshot.EnrichmentIssues) + } else if !strings.Contains(issue.Message, "cluster inventory") { + t.Fatalf("unexpected topology/cluster message: %+v", issue) + } + if issue, ok := seen["topology/vm/unavailable"]; !ok { + t.Fatalf("expected topology/vm unavailable issue for guest identity, got %+v", snapshot.EnrichmentIssues) + } else if !strings.Contains(issue.Message, "guest identity") { + t.Fatalf("unexpected topology/vm unavailable message: %+v", issue) } host := snapshot.Hosts[0] @@ -515,6 +535,27 @@ func newVMwareTestServer(t *testing.T, cfg vmwareTestServerConfig) *httptest.Ser "ip_address": "10.0.0.21", }) }) + mux.HandleFunc("/api/vcenter/vm/vm-201/guest/local-filesystem", func(w http.ResponseWriter, r *http.Request) { + requireAutomationSession(t, r) + // VMware Tools reports guest filesystem usage; the unavailable knob + // stands in for "Tools not running" which vCenter signals via 503. + if cfg.unavailableVMGuestInfo { + http.Error(w, "service unavailable", http.StatusServiceUnavailable) + return + } + writeJSON(w, map[string]any{ + "/": map[string]any{ + "capacity": int64(50 * 1024 * 1024 * 1024), + "free_space": int64(20 * 1024 * 1024 * 1024), + "filesystem": "ext4", + }, + "/var": map[string]any{ + "capacity": int64(20 * 1024 * 1024 * 1024), + "free_space": int64(5 * 1024 * 1024 * 1024), + "filesystem": "ext4", + }, + }) + }) mux.HandleFunc("/api/vcenter/vm/vm-201/tools", func(w http.ResponseWriter, r *http.Request) { requireAutomationSession(t, r) writeJSON(w, map[string]any{ diff --git a/internal/vmware/client_topology.go b/internal/vmware/client_topology.go index 26ff68ac1..3806f6937 100644 --- a/internal/vmware/client_topology.go +++ b/internal/vmware/client_topology.go @@ -796,6 +796,69 @@ func (c *Client) collectVMGuestIdentity( return &payload, nil } +// vcenterVMGuestLocalFilesystemInfo mirrors vSphere Automation REST +// `VmGuestLocalFilesystemInfo`. The endpoint returns a JSON map keyed by +// the filesystem mount-point string. Documented at +// https://developer.vmware.com/apis/vsphere-automation/latest/vcenter/api/vcenter/vm/vm/guest/local-filesystem/get/ +// Fields: +// - capacity / free_space: integer bytes (required by the schema). +// - filesystem: optional string (e.g. "ntfs", "ext4"). +// - mappings: optional `VirtualDiskMapping` array tying the filesystem +// back to a virtual disk; not yet consumed here. +type vcenterVMGuestLocalFilesystemInfo struct { + Capacity int64 `json:"capacity"` + FreeSpace int64 `json:"free_space"` + Filesystem string `json:"filesystem,omitempty"` +} + +// collectVMGuestLocalFilesystem fetches `/api/vcenter/vm/{vm}/guest/local-filesystem` +// and aggregates capacity / used bytes across every reported guest +// filesystem. Returns ok=false when the VM has no guest filesystem data +// (Tools not running, VM powered off, filesystem list empty); the caller +// should not mutate metrics in that case. A 503 from vCenter ("guest +// agent not running") is surfaced as an "unavailable" ConnectionError so +// the enrichment classifier treats it as a soft issue rather than a hard +// collection failure. +func (c *Client) collectVMGuestLocalFilesystem( + ctx context.Context, + automationSessionID string, + vmID string, +) (totalBytes int64, usedBytes int64, percent float64, ok bool, err error) { + vmID = strings.TrimSpace(vmID) + if vmID == "" || strings.TrimSpace(automationSessionID) == "" { + return 0, 0, 0, false, nil + } + var payload map[string]vcenterVMGuestLocalFilesystemInfo + path := fmt.Sprintf("/api/vcenter/vm/%s/guest/local-filesystem", url.PathEscape(vmID)) + if err := c.getAutomationJSON(ctx, automationSessionID, path, "vm guest local filesystem", &payload); err != nil { + return 0, 0, 0, false, err + } + if len(payload) == 0 { + return 0, 0, 0, false, nil + } + for _, info := range payload { + if info.Capacity <= 0 { + continue + } + totalBytes += info.Capacity + used := info.Capacity - info.FreeSpace + if used < 0 { + used = 0 + } + usedBytes += used + } + if totalBytes <= 0 { + return 0, 0, 0, false, nil + } + percent = float64(usedBytes) / float64(totalBytes) * 100 + if percent < 0 { + percent = 0 + } else if percent > 100 { + percent = 100 + } + return totalBytes, usedBytes, percent, true, nil +} + func (c *Client) collectVMTools( ctx context.Context, automationSessionID string, diff --git a/internal/vmware/provider.go b/internal/vmware/provider.go index ca3873bf3..a7a4d28ec 100644 --- a/internal/vmware/provider.go +++ b/internal/vmware/provider.go @@ -150,7 +150,19 @@ type InventoryVMHardware struct { } // InventoryMetrics captures the current runtime metric floor projected onto -// canonical Pulse metrics for VMware-backed hosts and VMs. +// canonical Pulse metrics for VMware-backed hosts and VMs. Sources: +// - Throughput / utilisation (cpu, mem, net*, disk*BytesPerSecond) come from +// VI/JSON PerformanceManager rollups (see client_metrics.go). +// - UptimeSeconds comes from PerformanceManager counters: prefer +// sys.osUptime.latest for VMs (guest OS uptime, requires VMware Tools) +// with fall back to sys.uptime.latest (VMX-process uptime; also the only +// uptime source for ESXi hosts). +// - DiskUsedBytes / DiskTotalBytes / DiskPercent are aggregated from the +// vSphere Automation REST endpoint +// `GET /api/vcenter/vm/{vm}/guest/local-filesystem`, which returns a map +// of mount-point -> {capacity, free_space} when VMware Tools is running. +// Host-level guest disk usage does not exist in vSphere; these fields +// stay nil for hosts. type InventoryMetrics struct { CPUPercent *float64 `json:"cpu_percent,omitempty"` MemoryPercent *float64 `json:"memory_percent,omitempty"` @@ -160,6 +172,10 @@ type InventoryMetrics struct { NetOutBytesPerSecond *float64 `json:"net_out_bytes_per_second,omitempty"` DiskReadBytesPerSecond *float64 `json:"disk_read_bytes_per_second,omitempty"` DiskWriteBytesPerSecond *float64 `json:"disk_write_bytes_per_second,omitempty"` + UptimeSeconds *int64 `json:"uptime_seconds,omitempty"` + DiskUsedBytes *int64 `json:"disk_used_bytes,omitempty"` + DiskTotalBytes *int64 `json:"disk_total_bytes,omitempty"` + DiskPercent *float64 `json:"disk_percent,omitempty"` } // InventoryEnrichmentIssue captures one optional VMware read that degraded a @@ -519,6 +535,7 @@ func vmwareRecordsFromSnapshot(snapshot *InventorySnapshot, now func() time.Time Status: unifiedresources.IncidentsStatus(hostStatus(host), incidents), LastSeen: collectedAt, UpdatedAt: collectedAt, + Uptime: inventoryUptimeSeconds(host.Metrics), Incidents: incidents, Metrics: inventoryMetricsResourceMetrics(host.Metrics), Agent: vmwareHostAgentData(snapshot, host), @@ -584,6 +601,7 @@ func vmwareRecordsFromSnapshot(snapshot *InventorySnapshot, now func() time.Time Status: unifiedresources.IncidentsStatus(vmStatus(vm), incidents), LastSeen: collectedAt, UpdatedAt: collectedAt, + Uptime: inventoryUptimeSeconds(vm.Metrics), Incidents: incidents, Metrics: inventoryMetricsResourceMetrics(vm.Metrics), ParentName: strings.TrimSpace(vm.RuntimeHostName), @@ -1058,6 +1076,10 @@ func cloneInventoryMetrics(in *InventoryMetrics) *InventoryMetrics { out.NetOutBytesPerSecond = cloneFloat64Pointer(in.NetOutBytesPerSecond) out.DiskReadBytesPerSecond = cloneFloat64Pointer(in.DiskReadBytesPerSecond) out.DiskWriteBytesPerSecond = cloneFloat64Pointer(in.DiskWriteBytesPerSecond) + out.UptimeSeconds = cloneInt64Pointer(in.UptimeSeconds) + out.DiskUsedBytes = cloneInt64Pointer(in.DiskUsedBytes) + out.DiskTotalBytes = cloneInt64Pointer(in.DiskTotalBytes) + out.DiskPercent = cloneFloat64Pointer(in.DiskPercent) return &out } @@ -1584,13 +1606,41 @@ func inventoryMetricsResourceMetrics(in *InventoryMetrics) *unifiedresources.Res Source: unifiedresources.SourceVMware, } } + // Guest filesystem capacity / usage from /api/vcenter/vm/{vm}/guest/local-filesystem. + // Total/Used populate only when both are known; Percent populates from the + // adapter's computed value (or derived from used/total if absent). + if in.DiskTotalBytes != nil || in.DiskPercent != nil { + disk := &unifiedresources.MetricValue{ + Unit: "bytes", + Source: unifiedresources.SourceVMware, + } + if in.DiskUsedBytes != nil { + used := *in.DiskUsedBytes + disk.Used = &used + } + if in.DiskTotalBytes != nil { + total := *in.DiskTotalBytes + disk.Total = &total + } + switch { + case in.DiskPercent != nil: + disk.Percent = *in.DiskPercent + disk.Value = *in.DiskPercent + case in.DiskUsedBytes != nil && in.DiskTotalBytes != nil && *in.DiskTotalBytes > 0: + percent := float64(*in.DiskUsedBytes) / float64(*in.DiskTotalBytes) * 100 + disk.Percent = percent + disk.Value = percent + } + metrics.Disk = disk + } if metrics.CPU == nil && metrics.Memory == nil && metrics.NetIn == nil && metrics.NetOut == nil && metrics.DiskRead == nil && - metrics.DiskWrite == nil { + metrics.DiskWrite == nil && + metrics.Disk == nil { return nil } return metrics @@ -1645,6 +1695,22 @@ func inventoryMetricFloat64(metrics *InventoryMetrics, pick func(*InventoryMetri return *value } +// inventoryUptimeSeconds returns the uptime stamp on the InventoryMetrics +// payload, or 0 when unknown. Pulse's canonical Resource.Uptime is int64 +// seconds; the vSphere adapter prefers guest OS uptime (sys.osUptime.latest +// from VMware Tools) and falls back to VMX-process uptime +// (sys.uptime.latest) for VMs without Tools and for ESXi hosts. +func inventoryUptimeSeconds(metrics *InventoryMetrics) int64 { + if metrics == nil || metrics.UptimeSeconds == nil { + return 0 + } + value := *metrics.UptimeSeconds + if value < 0 { + return 0 + } + return value +} + func cloneFloat64Pointer(in *float64) *float64 { if in == nil { return nil