pulse/tests/backupDataValidator.js

/**
 * Backup Data Validator
 * 
 * This module provides utilities to validate backup data against known ground truths
 * and help identify discrepancies in the backup system.
 */

// Ground truth data based on research
const groundTruthData = {
  totalGuests: 18, // Actual cluster count
  pbsBackupsTotal: 135,
  vmSnapshots: 3, // Only 3 actual VM/CT snapshots
  
  // Backup job schedules
  primaryBackupJob: {
    id: 'backup-2759a200-3e11',
    schedule: '02:00 AM',
    excludes: [102, 200, 400],
    retention: { daily: 7, weekly: 4, monthly: 3 }
  },
  secondaryBackupJob: {
    id: 'backup-79ce96ee-6527',
    schedule: '04:00 AM',
    includes: [102, 200, 400],
    retention: { keepLast: 3 }
  },
  
  // Expected backup ages (as of June 2, 12:50 PM BST)
  expectedBackupAges: {
    primaryJobGuests: { minHours: 10, maxHours: 11 }, // 2:00-2:10 AM backups
    secondaryJobGuests: { minHours: 8, maxHours: 9 }, // 4:00 AM backups
    vm102: 'no_recent_backup' // Issue found in research
  },
  
  // Known issues from research
  knownIssues: {
    guestCountDiscrepancy: true, // Pulse shows 20, actual is 18
    vm102BackupMissing: true,
    multipleEndpoints: 2, // proxmox.lan and pimox.lan
    snapshotLoggingConfusion: true // Logs incorrectly label PBS backups as snapshots
  }
};

/**
 * Validates guest count against expected values
 * @param {Object} discoveryData - The discovery data from fetchDiscoveryData
 * @returns {Object} Validation result with details
 */
function validateGuestCount(discoveryData) {
  const actualVMs = discoveryData.vms?.length || 0;
  const actualContainers = discoveryData.containers?.length || 0;
  const actualTotal = actualVMs + actualContainers;
  
  const result = {
    valid: actualTotal === groundTruthData.totalGuests,
    expected: groundTruthData.totalGuests,
    actual: actualTotal,
    vms: actualVMs,
    containers: actualContainers,
    discrepancy: actualTotal - groundTruthData.totalGuests,
    details: []
  };
  
  if (!result.valid) {
    result.details.push(`Guest count mismatch: Expected ${result.expected}, got ${result.actual}`);
    
    // Check for known issue
    if (actualTotal === 20 && groundTruthData.totalGuests === 18) {
      result.details.push('Known issue: Pulse showing 20 guests instead of actual 18');
    }
  }
  
  // Group by endpoint for detailed analysis
  const guestsByEndpoint = {};
  [...(discoveryData.vms || []), ...(discoveryData.containers || [])].forEach(guest => {
    const endpoint = guest.endpointId || 'unknown';
    if (!guestsByEndpoint[endpoint]) {
      guestsByEndpoint[endpoint] = { vms: 0, containers: 0 };
    }
    if (guest.type === 'qemu') {
      guestsByEndpoint[endpoint].vms++;
    } else {
      guestsByEndpoint[endpoint].containers++;
    }
  });
  
  result.byEndpoint = guestsByEndpoint;
  
  return result;
}

/**
 * Validates PBS backup counts vs VM snapshots
 * @param {Object} pbsData - PBS data from fetchPbsData
 * @param {Object} pveBackups - PVE backup data
 * @returns {Object} Validation result
 */
function validateBackupCounts(pbsData, pveBackups) {
  let pbsBackupCount = 0;
  let pbsBackupsByGuest = {};
  
  // Count PBS backups
  if (pbsData && pbsData[0]?.datastores) {
    pbsData[0].datastores.forEach(ds => {
      (ds.snapshots || []).forEach(snap => {
        pbsBackupCount++;
        const guestKey = `${snap['backup-type']}/${snap['backup-id']}`;
        pbsBackupsByGuest[guestKey] = (pbsBackupsByGuest[guestKey] || 0) + 1;
      });
    });
  }
  
  const vmSnapshotCount = pveBackups?.guestSnapshots?.length || 0;
  
  const result = {
    valid: pbsBackupCount > 100 && vmSnapshotCount < 10, // Expected pattern
    pbsBackups: {
      total: pbsBackupCount,
      expected: groundTruthData.pbsBackupsTotal,
      byGuest: pbsBackupsByGuest
    },
    vmSnapshots: {
      total: vmSnapshotCount,
      expected: groundTruthData.vmSnapshots,
      list: pveBackups?.guestSnapshots || []
    },
    details: []
  };
  
  if (Math.abs(pbsBackupCount - groundTruthData.pbsBackupsTotal) > 10) {
    result.details.push(`PBS backup count differs from expected: ${pbsBackupCount} vs ${groundTruthData.pbsBackupsTotal}`);
  }
  
  if (vmSnapshotCount > groundTruthData.vmSnapshots) {
    result.details.push(`More VM snapshots than expected: ${vmSnapshotCount} vs ${groundTruthData.vmSnapshots}`);
  }
  
  return result;
}

/**
 * Validates backup ages for all guests
 * @param {Object} pbsData - PBS data
 * @param {Date} currentTime - Current time for age calculations
 * @returns {Object} Validation result with age analysis
 */
function validateBackupAges(pbsData, currentTime = new Date()) {
  const backupAges = new Map();
  const guestsWithoutBackups = new Set();
  const expectedGuests = new Set();
  
  // Build expected guest list
  for (let i = 100; i <= 106; i++) {
    expectedGuests.add(String(i));
  }
  for (let i = 200; i <= 400; i += 100) {
    expectedGuests.add(String(i));
  }
  
  // Analyze PBS backups
  if (pbsData && pbsData[0]?.datastores) {
    pbsData[0].datastores.forEach(ds => {
      (ds.snapshots || []).forEach(snap => {
        const backupTime = snap['backup-time'] * 1000; // Convert to milliseconds
        const ageHours = (currentTime.getTime() - backupTime) / (1000 * 60 * 60);
        const guestId = snap['backup-id'];
        
        if (!backupAges.has(guestId) || ageHours < backupAges.get(guestId)) {
          backupAges.set(guestId, ageHours);
        }
      });
    });
  }
  
  // Find guests without recent backups
  expectedGuests.forEach(guestId => {
    if (!backupAges.has(guestId) || backupAges.get(guestId) > 24) {
      guestsWithoutBackups.add(guestId);
    }
  });
  
  // Categorize by backup schedule
  const primaryJobGuests = [];
  const secondaryJobGuests = [];
  const issues = [];
  
  backupAges.forEach((age, guestId) => {
    const id = parseInt(guestId);
    
    if ([102, 200, 400].includes(id)) {
      secondaryJobGuests.push({ id: guestId, age });
      if (age < groundTruthData.expectedBackupAges.secondaryJobGuests.minHours ||
          age > groundTruthData.expectedBackupAges.secondaryJobGuests.maxHours + 1) {
        issues.push(`Guest ${guestId} backup age ${age.toFixed(1)}h outside expected range`);
      }
    } else {
      primaryJobGuests.push({ id: guestId, age });
      if (age < groundTruthData.expectedBackupAges.primaryJobGuests.minHours ||
          age > groundTruthData.expectedBackupAges.primaryJobGuests.maxHours + 1) {
        issues.push(`Guest ${guestId} backup age ${age.toFixed(1)}h outside expected range`);
      }
    }
  });
  
  // Check for VM 102 issue
  if (guestsWithoutBackups.has('102')) {
    issues.push('VM 102 has no recent backup (known issue)');
  }
  
  return {
    valid: issues.length === 0,
    backupAges: Object.fromEntries(backupAges),
    primaryJobGuests,
    secondaryJobGuests,
    guestsWithoutBackups: Array.from(guestsWithoutBackups),
    issues,
    summary: {
      totalGuests: expectedGuests.size,
      guestsWithBackups: backupAges.size,
      guestsWithRecentBackups: Array.from(backupAges.entries())
        .filter(([_, age]) => age < 24).length
    }
  };
}

/**
 * Validates PBS task categorization
 * @param {Array} pbsTasks - Raw PBS tasks
 * @param {Object} processedTasks - Processed tasks from processPbsTasks
 * @returns {Object} Validation result
 */
function validateTaskProcessing(pbsTasks, processedTasks) {
  const result = {
    valid: true,
    totalTasks: pbsTasks?.length || 0,
    categorized: {
      backup: processedTasks.backupTasks?.summary?.total || 0,
      verify: processedTasks.verificationTasks?.summary?.total || 0,
      sync: processedTasks.syncTasks?.summary?.total || 0,
      prune: processedTasks.pruneTasks?.summary?.total || 0
    },
    uncategorized: [],
    issues: []
  };
  
  // Check if all tasks were categorized
  const categorizedTotal = Object.values(result.categorized).reduce((a, b) => a + b, 0);
  
  if (categorizedTotal !== result.totalTasks) {
    result.valid = false;
    result.issues.push(`Task count mismatch: ${categorizedTotal} categorized out of ${result.totalTasks} total`);
    
    // Find uncategorized tasks
    const taskTypeMap = {
      backup: 'backup',
      verify: 'verify',
      sync: 'sync',
      prune: 'prune',
      garbage_collection: 'prune',
      gc: 'prune'
    };
    
    pbsTasks?.forEach(task => {
      const type = task.worker_type || task.type;
      if (!taskTypeMap[type]) {
        result.uncategorized.push(type);
      }
    });
  }
  
  // Check for backup task details
  const backupTasks = processedTasks.backupTasks?.recentTasks || [];
  const pbsBackupTasks = backupTasks.filter(t => t.pbsBackupRun);
  
  if (pbsBackupTasks.length === 0 && result.categorized.backup > 0) {
    result.issues.push('No PBS backup runs found in recent tasks');
  }
  
  return result;
}

/**
 * Performs comprehensive validation of all backup data
 * @param {Object} data - Object containing discoveryData, pbsData, etc.
 * @returns {Object} Complete validation report
 */
function validateAllBackupData(data) {
  const report = {
    timestamp: new Date().toISOString(),
    validations: {},
    overallValid: true,
    criticalIssues: [],
    warnings: []
  };
  
  // Guest count validation
  if (data.discoveryData) {
    report.validations.guestCount = validateGuestCount(data.discoveryData);
    if (!report.validations.guestCount.valid) {
      report.warnings.push('Guest count discrepancy detected');
    }
  }
  
  // Backup count validation
  if (data.pbsData && data.discoveryData?.pveBackups) {
    report.validations.backupCounts = validateBackupCounts(
      data.pbsData,
      data.discoveryData.pveBackups
    );
    if (!report.validations.backupCounts.valid) {
      report.criticalIssues.push('Backup count validation failed');
      report.overallValid = false;
    }
  }
  
  // Backup age validation
  if (data.pbsData) {
    report.validations.backupAges = validateBackupAges(data.pbsData);
    if (!report.validations.backupAges.valid) {
      report.validations.backupAges.issues.forEach(issue => {
        if (issue.includes('VM 102')) {
          report.warnings.push(issue);
        } else {
          report.criticalIssues.push(issue);
          report.overallValid = false;
        }
      });
    }
  }
  
  // Task processing validation
  if (data.pbsTasks && data.processedTasks) {
    report.validations.taskProcessing = validateTaskProcessing(
      data.pbsTasks,
      data.processedTasks
    );
    if (!report.validations.taskProcessing.valid) {
      report.warnings.push('Task processing issues detected');
    }
  }
  
  // Summary
  report.summary = {
    criticalIssues: report.criticalIssues.length,
    warnings: report.warnings.length,
    recommendation: report.overallValid 
      ? 'Backup data appears valid'
      : 'Critical issues found - investigate backup system'
  };
  
  return report;
}

/**
 * Generates a human-readable report from validation results
 * @param {Object} validationReport - Report from validateAllBackupData
 * @returns {String} Formatted report
 */
function generateValidationReport(validationReport) {
  let report = `Backup Data Validation Report
Generated: ${validationReport.timestamp}
========================================

`;

  // Overall Status
  report += `Overall Status: ${validationReport.overallValid ? '✓ PASS' : '✗ FAIL'}\n`;
  report += `Critical Issues: ${validationReport.criticalIssues.length}\n`;
  report += `Warnings: ${validationReport.warnings.length}\n\n`;
  
  // Guest Count
  if (validationReport.validations.guestCount) {
    const gc = validationReport.validations.guestCount;
    report += `Guest Count Validation:\n`;
    report += `  Expected: ${gc.expected} guests\n`;
    report += `  Actual: ${gc.actual} guests (${gc.vms} VMs, ${gc.containers} CTs)\n`;
    if (gc.byEndpoint) {
      report += `  By Endpoint:\n`;
      Object.entries(gc.byEndpoint).forEach(([endpoint, counts]) => {
        report += `    ${endpoint}: ${counts.vms} VMs, ${counts.containers} CTs\n`;
      });
    }
    report += '\n';
  }
  
  // Backup Counts
  if (validationReport.validations.backupCounts) {
    const bc = validationReport.validations.backupCounts;
    report += `Backup Count Validation:\n`;
    report += `  PBS Backups: ${bc.pbsBackups.total} (expected ~${bc.pbsBackups.expected})\n`;
    report += `  VM Snapshots: ${bc.vmSnapshots.total} (expected ${bc.vmSnapshots.expected})\n`;
    report += '\n';
  }
  
  // Backup Ages
  if (validationReport.validations.backupAges) {
    const ba = validationReport.validations.backupAges;
    report += `Backup Age Validation:\n`;
    report += `  Total Guests: ${ba.summary.totalGuests}\n`;
    report += `  Guests with backups: ${ba.summary.guestsWithBackups}\n`;
    report += `  Guests with recent backups (<24h): ${ba.summary.guestsWithRecentBackups}\n`;
    if (ba.guestsWithoutBackups.length > 0) {
      report += `  Guests without recent backups: ${ba.guestsWithoutBackups.join(', ')}\n`;
    }
    report += '\n';
  }
  
  // Issues
  if (validationReport.criticalIssues.length > 0) {
    report += `Critical Issues:\n`;
    validationReport.criticalIssues.forEach(issue => {
      report += `  - ${issue}\n`;
    });
    report += '\n';
  }
  
  if (validationReport.warnings.length > 0) {
    report += `Warnings:\n`;
    validationReport.warnings.forEach(warning => {
      report += `  - ${warning}\n`;
    });
    report += '\n';
  }
  
  report += `Recommendation: ${validationReport.summary.recommendation}\n`;
  
  return report;
}

module.exports = {
  validateGuestCount,
  validateBackupCounts,
  validateBackupAges,
  validateTaskProcessing,
  validateAllBackupData,
  generateValidationReport
};