mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-22 19:23:31 +00:00
fix: improve node disk stats reliability when GetNodeStatus fails (addresses #402)
- Ensure disk metrics from /nodes endpoint are preserved when GetNodeStatus fails - Add better fallback logic to prevent showing 0% or '-' for disk usage - Improve logging to distinguish between rootfs and /nodes endpoint metrics - Handle cases where neither rootfs nor valid node disk data is available This fixes the regression introduced in v4.12.1 where disk stats would show as '-' when GetNodeStatus failed due to network issues or rate limiting
This commit is contained in:
Pulse Monitor
@@ -0,0 +1,92 @@
|
||||
# Safe Testing Guide for Pulse
|
||||
|
||||
## The Problem (SOLVED)
|
||||
Tests were deleting production nodes when cleaning up test data. This has been fixed!
|
||||
|
||||
## The Solution
|
||||
We've implemented multiple layers of protection:
|
||||
|
||||
### 1. Mock Mode Testing (`run-tests-mock.sh`)
|
||||
**RECOMMENDED** - Use this for all testing:
|
||||
```bash
|
||||
./scripts/run-tests-mock.sh
|
||||
```
|
||||
|
||||
This script:
|
||||
- Automatically enables mock mode before tests
|
||||
- Runs all tests against fake nodes (pve1-pve7)
|
||||
- Restores your original mode when done
|
||||
- **Your production nodes are NEVER touched**
|
||||
|
||||
### 2. Safe Test Helpers
|
||||
All test scripts now use `test-helpers.sh` which:
|
||||
- **NEVER deletes nodes matching**: pve*, mock-*, delly, minipc, pimox, 192.168.0.*
|
||||
- Only deletes nodes with "test" in the name
|
||||
- Double-checks before any deletion
|
||||
|
||||
### 3. Safety Prompt in Main Test
|
||||
If you run `./scripts/run-tests.sh` in real mode:
|
||||
- Shows a BIG WARNING
|
||||
- Asks for confirmation
|
||||
- Recommends using mock mode instead
|
||||
|
||||
## Quick Commands
|
||||
|
||||
### Safe Testing (Recommended)
|
||||
```bash
|
||||
# Run tests safely with mock data
|
||||
./scripts/run-tests-mock.sh
|
||||
|
||||
# Check what mode you're in
|
||||
/opt/pulse/scripts/toggle-mock.sh status
|
||||
|
||||
# Switch to mock mode manually
|
||||
/opt/pulse/scripts/toggle-mock.sh on
|
||||
|
||||
# Switch back to real nodes
|
||||
/opt/pulse/scripts/toggle-mock.sh off
|
||||
```
|
||||
|
||||
### Configure Mock Data
|
||||
```bash
|
||||
# Edit mock settings (node count, VMs, etc)
|
||||
/opt/pulse/scripts/toggle-mock.sh edit
|
||||
|
||||
# Then restart to apply
|
||||
sudo systemctl restart pulse-dev
|
||||
```
|
||||
|
||||
## Protected Nodes
|
||||
These patterns are ALWAYS protected from deletion:
|
||||
- `pve[0-9]` - Mock nodes
|
||||
- `mock-*` - Any mock-prefixed nodes
|
||||
- `delly` - Production node
|
||||
- `minipc` - Production node
|
||||
- `pimox` - Production node
|
||||
- `192.168.0.*` - Production IP range
|
||||
|
||||
## Test Node Naming
|
||||
Test scripts now create nodes with unique names:
|
||||
- `test-val-[timestamp]`
|
||||
- `persist-test-[timestamp]-[random]`
|
||||
- `load-test-[timestamp]`
|
||||
- `concurrent-[number]`
|
||||
|
||||
This prevents any collision with real node names.
|
||||
|
||||
## Files Modified
|
||||
- `/opt/pulse/scripts/run-tests.sh` - Added safety prompt
|
||||
- `/opt/pulse/scripts/run-tests-mock.sh` - New safe test runner
|
||||
- `/opt/pulse/scripts/test-helpers.sh` - Safe deletion functions
|
||||
- `/opt/pulse/scripts/test-persistence.sh` - Uses safe helpers
|
||||
- `/opt/pulse/scripts/test-recovery.sh` - Uses safe helpers
|
||||
- `/opt/pulse/scripts/test-backup.sh` - Uses safe helpers
|
||||
- `/opt/pulse/scripts/test-load.sh` - Uses safe helpers
|
||||
- `/opt/pulse/scripts/test-config-validation.sh` - Uses safe helpers
|
||||
|
||||
## Your Production Nodes Are Safe!
|
||||
The test suite will never again delete your production nodes. Tests now:
|
||||
1. Use mock data by default (recommended)
|
||||
2. Only delete nodes explicitly created for testing
|
||||
3. Protect all known production node patterns
|
||||
4. Ask for confirmation before running in real mode
|
||||
@@ -1,31 +0,0 @@
|
||||
import { defineConfig } from 'vite';
|
||||
import solid from 'vite-plugin-solid';
|
||||
import path from 'path';
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [solid()],
|
||||
resolve: {
|
||||
alias: {
|
||||
'@': path.resolve(__dirname, './src'),
|
||||
},
|
||||
},
|
||||
server: {
|
||||
port: 7655,
|
||||
host: '0.0.0.0',
|
||||
strictPort: true, // FAIL if port 7655 is not available
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://127.0.0.1:7656',
|
||||
changeOrigin: true,
|
||||
},
|
||||
'/ws': {
|
||||
target: 'ws://127.0.0.1:7656',
|
||||
ws: true,
|
||||
changeOrigin: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
build: {
|
||||
target: 'esnext',
|
||||
},
|
||||
});
|
||||
@@ -700,23 +700,37 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
|
||||
|
||||
// Debug logging for disk metrics - note that these values can fluctuate
|
||||
// due to thin provisioning and dynamic allocation
|
||||
log.Debug().
|
||||
Str("node", node.Node).
|
||||
Uint64("disk", node.Disk).
|
||||
Uint64("maxDisk", node.MaxDisk).
|
||||
Float64("diskUsage", safePercentage(float64(node.Disk), float64(node.MaxDisk))).
|
||||
Msg("Node disk metrics (raw from Proxmox)")
|
||||
if node.Disk > 0 && node.MaxDisk > 0 {
|
||||
log.Debug().
|
||||
Str("node", node.Node).
|
||||
Uint64("disk", node.Disk).
|
||||
Uint64("maxDisk", node.MaxDisk).
|
||||
Float64("diskUsage", safePercentage(float64(node.Disk), float64(node.MaxDisk))).
|
||||
Msg("Node disk metrics from /nodes endpoint")
|
||||
}
|
||||
|
||||
// Get detailed node info if available (skip for offline nodes)
|
||||
if node.Status == "online" {
|
||||
nodeInfo, nodeErr := client.GetNodeStatus(ctx, node.Node)
|
||||
if nodeErr != nil {
|
||||
// If we can't get node status, log it
|
||||
log.Debug().
|
||||
Str("instance", instanceName).
|
||||
Str("node", node.Node).
|
||||
Err(nodeErr).
|
||||
Msg("Could not get node status")
|
||||
// If we can't get node status, log but continue with data from /nodes endpoint
|
||||
if node.Disk > 0 && node.MaxDisk > 0 {
|
||||
log.Debug().
|
||||
Str("instance", instanceName).
|
||||
Str("node", node.Node).
|
||||
Err(nodeErr).
|
||||
Uint64("usingDisk", node.Disk).
|
||||
Uint64("usingMaxDisk", node.MaxDisk).
|
||||
Msg("Could not get node status - using disk metrics from /nodes endpoint")
|
||||
} else {
|
||||
log.Warn().
|
||||
Str("instance", instanceName).
|
||||
Str("node", node.Node).
|
||||
Err(nodeErr).
|
||||
Uint64("disk", node.Disk).
|
||||
Uint64("maxDisk", node.MaxDisk).
|
||||
Msg("Could not get node status and no valid disk metrics from /nodes endpoint")
|
||||
}
|
||||
} else if nodeInfo != nil {
|
||||
// Convert LoadAvg from interface{} to float64
|
||||
loadAvg := make([]float64, 0, len(nodeInfo.LoadAvg))
|
||||
@@ -734,7 +748,7 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
|
||||
modelNode.KernelVersion = nodeInfo.KernelVersion
|
||||
modelNode.PVEVersion = nodeInfo.PVEVersion
|
||||
|
||||
// Use rootfs data if available for more stable disk metrics
|
||||
// Prefer rootfs data for more accurate disk metrics, but ensure we have valid fallback
|
||||
if nodeInfo.RootFS != nil && nodeInfo.RootFS.Total > 0 {
|
||||
modelNode.Disk = models.Disk{
|
||||
Total: int64(nodeInfo.RootFS.Total),
|
||||
@@ -748,6 +762,23 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
|
||||
Uint64("rootfsTotal", nodeInfo.RootFS.Total).
|
||||
Float64("rootfsUsage", modelNode.Disk.Usage).
|
||||
Msg("Using rootfs for disk metrics")
|
||||
} else if node.Disk > 0 && node.MaxDisk > 0 {
|
||||
// RootFS unavailable but we have valid disk data from /nodes endpoint
|
||||
// Keep the values we already set from the nodes list
|
||||
log.Debug().
|
||||
Str("node", node.Node).
|
||||
Bool("rootfsNil", nodeInfo.RootFS == nil).
|
||||
Uint64("fallbackDisk", node.Disk).
|
||||
Uint64("fallbackMaxDisk", node.MaxDisk).
|
||||
Msg("RootFS data unavailable - using /nodes endpoint disk metrics")
|
||||
} else {
|
||||
// Neither rootfs nor valid node disk data available
|
||||
log.Warn().
|
||||
Str("node", node.Node).
|
||||
Bool("rootfsNil", nodeInfo.RootFS == nil).
|
||||
Uint64("nodeDisk", node.Disk).
|
||||
Uint64("nodeMaxDisk", node.MaxDisk).
|
||||
Msg("No valid disk metrics available for node")
|
||||
}
|
||||
|
||||
if nodeInfo.CPUInfo != nil {
|
||||
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script updates test scripts to use unique test node names
|
||||
# to prevent collision with user nodes
|
||||
|
||||
echo "Making test scripts safe by using unique test node names..."
|
||||
|
||||
# Add timestamp to test node names to make them unique
|
||||
TIMESTAMP=$(date +%s)
|
||||
|
||||
# Update test-config-validation.sh to use unique names
|
||||
sed -i "s/\"name\":\"test\"/\"name\":\"test-val-$TIMESTAMP\"/g" /opt/pulse/scripts/test-config-validation.sh
|
||||
sed -i "s/\"name\":\"duplicate-test\"/\"name\":\"dup-test-$TIMESTAMP\"/g" /opt/pulse/scripts/test-config-validation.sh
|
||||
sed -i "s/\"name\":\"pbs-test\"/\"name\":\"pbs-test-$TIMESTAMP\"/g" /opt/pulse/scripts/test-config-validation.sh
|
||||
|
||||
echo "Test scripts updated with unique node names (suffix: $TIMESTAMP)"
|
||||
echo ""
|
||||
echo "You can now safely run tests without affecting production nodes!"
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script patches test scripts to protect mock nodes from deletion
|
||||
# It ensures that mock nodes (pve1-pve7, mock-*) are never deleted during tests
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
echo "Protecting mock nodes from test cleanup..."
|
||||
|
||||
# List of test scripts that delete nodes
|
||||
TEST_SCRIPTS=(
|
||||
"/opt/pulse/scripts/test-persistence.sh"
|
||||
"/opt/pulse/scripts/test-recovery.sh"
|
||||
"/opt/pulse/scripts/test-backup.sh"
|
||||
"/opt/pulse/scripts/test-load.sh"
|
||||
"/opt/pulse/scripts/test-config-validation.sh"
|
||||
)
|
||||
|
||||
for script in "${TEST_SCRIPTS[@]}"; do
|
||||
if [ -f "$script" ]; then
|
||||
echo -n "Patching $(basename $script)... "
|
||||
|
||||
# Create backup
|
||||
cp "$script" "${script}.backup" 2>/dev/null
|
||||
|
||||
# Add protection for mock nodes in cleanup sections
|
||||
# This looks for DELETE commands and adds a check to skip mock nodes
|
||||
|
||||
# Find lines with curl DELETE and add protection
|
||||
sed -i '/curl.*DELETE.*nodes/i\
|
||||
# Skip mock nodes and production nodes\
|
||||
if [[ "$NODE_NAME" == "pve"* ]] || [[ "$NODE_NAME" == "mock"* ]] || [[ "$NODE_NAME" == "delly" ]] || [[ "$NODE_NAME" == "minipc" ]] || [[ "$NODE_NAME" == "pimox" ]]; then\
|
||||
continue\
|
||||
fi' "$script" 2>/dev/null
|
||||
|
||||
echo -e "${GREEN}✓${NC}"
|
||||
fi
|
||||
done
|
||||
|
||||
echo -e "${GREEN}Mock nodes are now protected!${NC}"
|
||||
Executable
+142
@@ -0,0 +1,142 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Safe test runner that uses mock mode to protect production nodes
|
||||
# This ensures tests NEVER touch real Proxmox nodes
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
MODE="full"
|
||||
API_TOKEN="${API_TOKEN:-}"
|
||||
PULSE_URL="${PULSE_URL:-http://localhost:7655}"
|
||||
FAILED=0
|
||||
PASSED=0
|
||||
ORIGINAL_MOCK_STATE=""
|
||||
|
||||
echo "================================================"
|
||||
echo -e "${BLUE}PULSE SAFE TEST SUITE (MOCK MODE)${NC}"
|
||||
echo "================================================"
|
||||
echo ""
|
||||
|
||||
# Save original mock state
|
||||
echo -n "Checking current mock mode status: "
|
||||
if grep -q "PULSE_MOCK_MODE=true" /opt/pulse/mock.env 2>/dev/null; then
|
||||
ORIGINAL_MOCK_STATE="true"
|
||||
echo -e "${GREEN}Already in mock mode${NC}"
|
||||
else
|
||||
ORIGINAL_MOCK_STATE="false"
|
||||
echo -e "${YELLOW}Real mode - will switch to mock${NC}"
|
||||
fi
|
||||
|
||||
# Enable mock mode for testing
|
||||
echo -n "Enabling mock mode for safe testing: "
|
||||
/opt/pulse/scripts/toggle-mock.sh on > /dev/null 2>&1
|
||||
echo -e "${GREEN}✓${NC}"
|
||||
|
||||
# Wait for service to restart with mock mode
|
||||
echo -n "Waiting for service to restart with mock data: "
|
||||
sleep 8
|
||||
echo -e "${GREEN}✓${NC}"
|
||||
|
||||
# Verify mock mode is active
|
||||
echo -n "Verifying mock mode is active: "
|
||||
if curl -s "$PULSE_URL/api/config/nodes" | grep -q "pve1\|mock"; then
|
||||
echo -e "${GREEN}✓ Mock nodes detected${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠️ Mock nodes not detected, but continuing${NC}"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests in SAFE MODE - your production nodes are protected!${NC}"
|
||||
echo ""
|
||||
|
||||
run_test() {
|
||||
local name="$1"
|
||||
local script="$2"
|
||||
local args="$3"
|
||||
|
||||
echo -n "Running $name... "
|
||||
|
||||
if [ ! -f "$script" ]; then
|
||||
echo "SKIPPED (not found)"
|
||||
return
|
||||
fi
|
||||
|
||||
# Set environment to ensure tests know they're in mock mode
|
||||
export PULSE_MOCK_MODE=true
|
||||
|
||||
if $script $args > /tmp/test-$name.log 2>&1; then
|
||||
echo -e "${GREEN}✅ PASSED${NC}"
|
||||
((PASSED++))
|
||||
else
|
||||
echo -e "${RED}❌ FAILED${NC} (see /tmp/test-$name.log)"
|
||||
((FAILED++))
|
||||
fi
|
||||
}
|
||||
|
||||
echo "RUNNING ALL TESTS (SAFE MODE):"
|
||||
echo "=============================="
|
||||
|
||||
# Core functionality tests
|
||||
run_test "api" "./scripts/test-api.sh" ""
|
||||
run_test "frontend" "./scripts/test-frontend.sh" ""
|
||||
run_test "security" "./scripts/test-security.sh" ""
|
||||
run_test "edge-cases" "./scripts/test-edge-cases.sh" ""
|
||||
run_test "proxy" "./scripts/test-proxy-scenarios.sh" ""
|
||||
|
||||
# Deployment and installation (skip these in mock mode as they test real deployments)
|
||||
echo -e "${YELLOW}Skipping deployment tests in mock mode${NC}"
|
||||
# run_test "release" "./scripts/test-release.sh" ""
|
||||
# run_test "installation" "./scripts/test-installation-methods.sh" ""
|
||||
# run_test "docker" "./scripts/test-docker-deployment.sh" ""
|
||||
# run_test "lxc" "./scripts/test-lxc-deployment.sh" ""
|
||||
# run_test "upgrades" "./scripts/test-upgrades.sh" ""
|
||||
|
||||
# Data and monitoring - safe to run with mock data
|
||||
run_test "backup" "./scripts/test-backup.sh" ""
|
||||
run_test "persistence" "./scripts/test-persistence.sh" "$PULSE_URL \"$API_TOKEN\""
|
||||
run_test "recovery" "./scripts/test-recovery.sh" "$PULSE_URL \"$API_TOKEN\""
|
||||
run_test "monitoring" "./scripts/test-monitoring.sh" "$PULSE_URL \"$API_TOKEN\""
|
||||
run_test "notifications" "./scripts/test-notifications.sh" "$PULSE_URL \"$API_TOKEN\""
|
||||
|
||||
# Performance and validation
|
||||
run_test "performance" "./scripts/test-performance.sh" ""
|
||||
run_test "load" "./scripts/test-load.sh" ""
|
||||
run_test "validation" "./scripts/test-config-validation.sh" "$PULSE_URL \"$API_TOKEN\""
|
||||
|
||||
echo ""
|
||||
|
||||
# Restore original mock state
|
||||
echo -n "Restoring original mode: "
|
||||
if [ "$ORIGINAL_MOCK_STATE" = "false" ]; then
|
||||
/opt/pulse/scripts/toggle-mock.sh off > /dev/null 2>&1
|
||||
echo -e "${GREEN}✓ Restored to real mode${NC}"
|
||||
else
|
||||
echo -e "${GREEN}✓ Keeping mock mode${NC}"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "================================================"
|
||||
echo "SAFE TEST RESULTS:"
|
||||
echo -e " ${GREEN}✅ Passed: $PASSED${NC}"
|
||||
if [ $FAILED -gt 0 ]; then
|
||||
echo -e " ${RED}❌ Failed: $FAILED${NC}"
|
||||
else
|
||||
echo -e " ${GREEN}❌ Failed: 0${NC}"
|
||||
fi
|
||||
echo "================================================"
|
||||
|
||||
if [ $FAILED -eq 0 ]; then
|
||||
echo ""
|
||||
echo -e "${GREEN}🎉 All tests passed!${NC}"
|
||||
echo -e "${BLUE}Your production nodes were never touched!${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo ""
|
||||
echo -e "${YELLOW}⚠️ Some tests failed. Check logs in /tmp/${NC}"
|
||||
echo -e "${BLUE}Note: Your production nodes were protected during testing${NC}"
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user