gsadmin/PSProxmoxVE
1
0
Fork 0
mirror of https://github.com/GoodOlClint/PSProxmoxVE.git synced 2026-07-26 07:58:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: retry re-auth during cluster join and always reconnect to node A

Browse Source 
- Extract re-auth retry into ReauthenticateWithRetry helper (fixes
  cognitive complexity warning)
- Retry auth up to 10x with 3s delay — node B's auth services need
  time to restart after joining the cluster
- Wrap join in try/finally so the test always reconnects to node A,
  even if the join fails

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Clint Branham
2026-03-25 08:47:17 -05:00
parent 8c131a0609
commit 4277ad73b9
2 changed files with 78 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/PSProxmoxVE/Cmdlets/Cluster/AddPveClusterMemberCmdlet.cs
+55 -16
View File
@@ -3,6 +3,7 @@ using System.Management.Automation;
using System.Net;
using System.Runtime.InteropServices;
using System.Security;
using System.Threading;
using PSProxmoxVE.Core.Authentication;
using PSProxmoxVE.Core.Exceptions;
using PSProxmoxVE.Core.Models.Vms;
@@ -57,6 +58,9 @@ namespace PSProxmoxVE.Cmdlets.Cluster
[Parameter(Mandatory = false, HelpMessage = "Wait for the join task to complete before returning.")]
public SwitchParameter Wait { get; set; }
private const int ReauthMaxAttempts = 10;
private const int ReauthDelayMs = 3000;
protected override void ProcessRecord()
{
if (!ShouldProcess($"this node to cluster via '{Hostname}'", "Join cluster"))
@@ -81,22 +85,7 @@ namespace PSProxmoxVE.Cmdlets.Cluster
if (Wait.IsPresent && !string.IsNullOrEmpty(upid))
{
var nodeName = upid.Split(':').Length > 1 ? upid.Split(':')[1] : session.Hostname;
var taskService = new TaskService();
try
{
task = taskService.WaitForTask(session, nodeName, upid);
}
catch (PveApiException ex) when (ex.StatusCode == HttpStatusCode.Unauthorized)
{
// Cluster join restarts auth services on this node, invalidating
// the ticket mid-poll. Re-authenticate and retry.
WriteVerbose("Session expired during join — re-authenticating to poll task status...");
var newSession = PveAuthenticator.AuthenticateWithCredentials(
session.Hostname, session.Port, session.SkipCertificateCheck,
"root@pam", plainPassword);
task = taskService.WaitForTask(newSession, nodeName, upid);
}
task = WaitForJoinTask(session, upid, plainPassword);
}
WriteObject(task);
@@ -107,5 +96,55 @@ namespace PSProxmoxVE.Cmdlets.Cluster
Marshal.ZeroFreeGlobalAllocUnicode(ptr);
}
}
/// <summary>
/// Waits for the join task to complete. If the ticket is invalidated mid-poll
/// (cluster join restarts auth services), re-authenticates with retry and
/// continues waiting.
/// </summary>
private PveTask WaitForJoinTask(PveSession session, string upid, string plainPassword)
{
var nodeName = upid.Split(':').Length > 1 ? upid.Split(':')[1] : session.Hostname;
var taskService = new TaskService();
try
{
return taskService.WaitForTask(session, nodeName, upid);
}
catch (PveApiException ex) when (ex.StatusCode == HttpStatusCode.Unauthorized)
{
WriteVerbose("Session expired during join — waiting for auth services to restart...");
var newSession = ReauthenticateWithRetry(session, plainPassword);
return taskService.WaitForTask(newSession, nodeName, upid);
}
}
/// <summary>
/// Retries authentication against this node until auth services come back online
/// after the cluster join restarts them.
/// </summary>
private PveSession ReauthenticateWithRetry(PveSession session, string plainPassword)
{
for (int attempt = 0; attempt < ReauthMaxAttempts; attempt++)
{
Thread.Sleep(ReauthDelayMs);
try
{
return PveAuthenticator.AuthenticateWithCredentials(
session.Hostname, session.Port, session.SkipCertificateCheck,
"root@pam", plainPassword);
}
catch (PveApiException retryEx) when (
retryEx.StatusCode == HttpStatusCode.Unauthorized ||
retryEx.StatusCode == HttpStatusCode.ServiceUnavailable)
{
WriteVerbose($"Re-auth attempt {attempt + 1}/{ReauthMaxAttempts} failed, retrying...");
}
}
throw new InvalidOperationException(
"Unable to re-authenticate after cluster join. " +
"Auth services on this node may still be restarting.");
}
}
}
tests/PSProxmoxVE.Tests/Integration/ClusterConfig.Integration.Tests.ps1
+23 -19
View File
@@ -261,27 +261,31 @@ Describe 'Cluster Config & HA Lifecycle — Integration' -Tag 'Integration' {
$fingerprint = $script:Fingerprint
$result = Add-PveClusterMember `
-Hostname $script:Host_ `
-Fingerprint $fingerprint `
-Password $secPw `
-Wait `
-Confirm:$false `
-ErrorAction Stop
try {
$result = Add-PveClusterMember `
-Hostname $script:Host_ `
-Fingerprint $fingerprint `
-Password $secPw `
-Wait `
-Confirm:$false `
-ErrorAction Stop
$result | Should -Not -BeNullOrEmpty
$result | Should -Not -BeNullOrEmpty
# Brief pause for pmxcfs to sync after task completion
Start-Sleep -Seconds 5
# Reconnect to node A with root@pam for subsequent cluster operations
$secPwA = ConvertTo-SecureString $script:Password -AsPlainText -Force
$credA = New-Object System.Management.Automation.PSCredential('root@pam', $secPwA)
Connect-PveServer `
-Server $script:Host_ `
-Port $script:Port `
-Credential $credA `
-SkipCertificateCheck
# Brief pause for pmxcfs to sync after task completion
Start-Sleep -Seconds 5
}
finally {
# Always reconnect to node A — whether join succeeded or failed,
# the current session points at node B which may be invalid.
$secPwA = ConvertTo-SecureString $script:Password -AsPlainText -Force
$credA = New-Object System.Management.Automation.PSCredential('root@pam', $secPwA)
Connect-PveServer `
-Server $script:Host_ `
-Port $script:Port `
-Credential $credA `
-SkipCertificateCheck
}
}
It 'Get-PveClusterConfigNode shows both nodes' {