fix: retry the qemu-server flock instead of predicting it

WaitForStatusTransition refused to return while snapshot.Locked, and its
comment quoted the exact error it was meant to prevent. Locked reads the
guest config's lock: property; the failure is the flock on
/var/lock/qemu-server/lock-<vmid>.conf, which PVE exposes nowhere.

The flock cannot be observed, so it is retried. GuestLockRetry reissues an
operation for a bounded 45s while PVE reports failing to enter lock_config
for a guest, which it raises before doing any work.

Two seams, because the failure has two surfaces. PveHttpClient.SendAsync
retries the request for operations PVE serialises in the API handler; it
takes a request factory because an HttpRequestMessage cannot be resent.
PveCmdletBase.InvokeGuestTask reissues the call and re-waits its task for
operations serialised in the forked worker, where the POST returns 200 and
only the task fails. WaitForStatusTransition routes through the latter,
hence Func<PveTask>.

The predicate is path-specific and anchored at the start of what PVE said:
lock_file uses identical wording for storage, LVM and HA locks, and a
qmclone that fails after allocating disks must not be reissued into
"VM already exists". That requires the raw text, so it reads
PveTaskFailedException.ExitStatus and PveApiException.ApiMessage.

The Locked check stays — it is correct for the config lock — with a comment
that says so.

Closes #113
This commit is contained in:
goodolclint-claude[bot]
2026-09-01 21:19:05 -05:00
parent 23ad9840c6
commit db416a3f03
20 changed files with 653 additions and 109 deletions
+36 -18
View File
@@ -10,6 +10,7 @@ using System.Threading.Tasks;
using Newtonsoft.Json.Linq;
using PSProxmoxVE.Core.Authentication;
using PSProxmoxVE.Core.Exceptions;
using PSProxmoxVE.Core.Utilities;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
@@ -62,7 +63,7 @@ namespace PSProxmoxVE.Core.Client
/// <summary>
/// Creates a bare HTTP client for pre-session use (e.g. initial authentication).
/// No auth headers are added to requests made with this constructor.
/// Requests it builds carry no authentication headers.
/// </summary>
internal PveHttpClient(string hostname, int port, bool skipCertificateCheck, TimeSpan? timeout = null)
{
@@ -95,8 +96,8 @@ namespace PSProxmoxVE.Core.Client
/// <returns>Raw JSON response body</returns>
public async Task<string> GetAsync(string resource)
{
var request = BuildRequest(HttpMethod.Get, resource);
return await SendAsync(request, resource, "GET").ConfigureAwait(false);
return await SendAsync(() => BuildRequest(HttpMethod.Get, resource), resource, "GET")
.ConfigureAwait(false);
}
/// <summary>Performs a POST request against the specified API resource path.</summary>
@@ -105,10 +106,13 @@ namespace PSProxmoxVE.Core.Client
/// <returns>Raw JSON response body</returns>
public async Task<string> PostAsync(string resource, Dictionary<string, string>? data = null)
{
var request = BuildRequest(HttpMethod.Post, resource, mutating: true);
if (data != null)
request.Content = BuildFormContent(data);
return await SendAsync(request, resource, "POST").ConfigureAwait(false);
return await SendAsync(() =>
{
var request = BuildRequest(HttpMethod.Post, resource, mutating: true);
if (data != null)
request.Content = BuildFormContent(data);
return request;
}, resource, "POST").ConfigureAwait(false);
}
/// <summary>
@@ -121,9 +125,12 @@ namespace PSProxmoxVE.Core.Client
public async Task<string> PostAsync(string resource, IEnumerable<KeyValuePair<string, string>> data)
{
if (data == null) throw new ArgumentNullException(nameof(data));
var request = BuildRequest(HttpMethod.Post, resource, mutating: true);
request.Content = BuildFormContent(data);
return await SendAsync(request, resource, "POST").ConfigureAwait(false);
return await SendAsync(() =>
{
var request = BuildRequest(HttpMethod.Post, resource, mutating: true);
request.Content = BuildFormContent(data);
return request;
}, resource, "POST").ConfigureAwait(false);
}
/// <summary>Performs a PUT request against the specified API resource path.</summary>
@@ -132,10 +139,13 @@ namespace PSProxmoxVE.Core.Client
/// <returns>Raw JSON response body</returns>
public async Task<string> PutAsync(string resource, Dictionary<string, string>? data = null)
{
var request = BuildRequest(HttpMethod.Put, resource, mutating: true);
if (data != null)
request.Content = BuildFormContent(data);
return await SendAsync(request, resource, "PUT").ConfigureAwait(false);
return await SendAsync(() =>
{
var request = BuildRequest(HttpMethod.Put, resource, mutating: true);
if (data != null)
request.Content = BuildFormContent(data);
return request;
}, resource, "PUT").ConfigureAwait(false);
}
/// <summary>Performs a DELETE request against the specified API resource path.</summary>
@@ -143,8 +153,8 @@ namespace PSProxmoxVE.Core.Client
/// <returns>Raw JSON response body</returns>
public async Task<string> DeleteAsync(string resource)
{
var request = BuildRequest(HttpMethod.Delete, resource, mutating: true);
return await SendAsync(request, resource, "DELETE").ConfigureAwait(false);
return await SendAsync(() => BuildRequest(HttpMethod.Delete, resource, mutating: true), resource, "DELETE")
.ConfigureAwait(false);
}
// -------------------------------------------------------------------------
@@ -331,7 +341,7 @@ namespace PSProxmoxVE.Core.Client
var request = BuildRequest(HttpMethod.Post, resource, mutating: true);
request.Content = multipart;
return await SendAsync(request, resource, "POST").ConfigureAwait(false);
return await SendOnceAsync(request, resource, "POST").ConfigureAwait(false);
}
finally
{
@@ -366,7 +376,15 @@ namespace PSProxmoxVE.Core.Client
return request;
}
private async Task<string> SendAsync(HttpRequestMessage request, string resource, string httpMethod)
/// <summary>
/// Sends a request, rebuilding it from <paramref name="buildRequest"/> for each attempt
/// while PVE rejects it for a guest's config flock. An <see cref="HttpRequestMessage"/>
/// cannot be resent, which is why this takes a factory rather than a request.
/// </summary>
private Task<string> SendAsync(Func<HttpRequestMessage> buildRequest, string resource, string httpMethod) =>
GuestLockRetry.ExecuteAsync(() => SendOnceAsync(buildRequest(), resource, httpMethod));
private async Task<string> SendOnceAsync(HttpRequestMessage request, string resource, string httpMethod)
{
HttpResponseMessage response;
try
@@ -15,6 +15,13 @@ namespace PSProxmoxVE.Core.Exceptions
/// <summary>The HTTP method used for the request (GET, POST, PUT, DELETE).</summary>
public string HttpMethod { get; }
/// <summary>
/// The error text PVE returned, without the status/resource prefix that
/// <see cref="Exception.Message"/> carries. Callers that match on what PVE said —
/// rather than on how this exception renders it — must read this.
/// </summary>
public string ApiMessage { get; }
/// <summary>Initializes a new instance for a failed PVE API request.</summary>
/// <param name="statusCode">The HTTP status code returned.</param>
/// <param name="message">The error message from the API.</param>
@@ -26,6 +33,7 @@ namespace PSProxmoxVE.Core.Exceptions
StatusCode = statusCode;
Resource = resource;
HttpMethod = httpMethod;
ApiMessage = message;
}
/// <summary>Initializes a new instance for a failed PVE API request, with an inner exception.</summary>
@@ -40,6 +48,7 @@ namespace PSProxmoxVE.Core.Exceptions
StatusCode = statusCode;
Resource = resource;
HttpMethod = httpMethod;
ApiMessage = message;
}
}
}
@@ -0,0 +1,103 @@
using System;
using System.Diagnostics;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using PSProxmoxVE.Core.Exceptions;
namespace PSProxmoxVE.Core.Utilities
{
/// <summary>
/// Retries an operation PVE rejected because it could not acquire a guest's config
/// flock (<c>/var/lock/qemu-server/lock-&lt;vmid&gt;.conf</c> for VMs,
/// <c>/run/lock/lxc/pve-config-&lt;vmid&gt;.lock</c> for containers).
///
/// That flock is held by <c>qm cleanup</c> after a guest stops and is not exposed
/// through the API in any form, so a caller cannot wait for it — only retry past it.
/// </summary>
public static class GuestLockRetry
{
/// <summary>
/// How long <see cref="Execute{T}"/> and <see cref="ExecuteAsync{T}"/> keep retrying.
/// <c>qm cleanup</c> polls <c>vm_running_locally</c> for up to 30s while holding the
/// flock, and each rejected attempt first burns PVE's own 10s <c>lock_config</c> timeout.
/// </summary>
public static readonly TimeSpan DefaultWindow = TimeSpan.FromSeconds(45);
private static readonly TimeSpan RetryInterval = TimeSpan.FromSeconds(2);
// Anchored, and specific to the two guest lock paths. `PVE::Tools::lock_file` emits this
// same wording for storage, LVM, HA and firewall locks, none of which carry the
// reissue-safety guarantee below. The anchor is what separates a failure to *enter*
// lock_config from one PVE prefixed with its own context ("clone failed: ..."), which
// means the worker had already done work.
private static readonly Regex GuestLockTimeout = new Regex(
@"^can't lock file '(?:/var/lock/qemu-server/lock-\d+\.conf|/run/lock/lxc/pve-config-\d+\.lock)' - got timeout",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
/// <summary>
/// True when <paramref name="ex"/> reports PVE failing to enter <c>lock_config</c> for a
/// guest. That is raised before the operation performs any work, so a call failing this
/// way is known not to have run and is safe to reissue.
///
/// Matched against what PVE actually said — <see cref="PveTaskFailedException.ExitStatus"/>
/// and <see cref="PveApiException.ApiMessage"/> — never against the composed
/// <see cref="Exception.Message"/>, whose prefix would defeat the anchor.
/// </summary>
/// <param name="ex">The exception to classify.</param>
public static bool IsLockTimeout(Exception ex) => ex switch
{
PveTaskFailedException task => GuestLockTimeout.IsMatch((task.ExitStatus ?? string.Empty).Trim()),
PveApiException api => GuestLockTimeout.IsMatch((api.ApiMessage ?? string.Empty).Trim()),
_ => false,
};
/// <summary>
/// Runs <paramref name="operation"/>, reissuing it while it fails on the guest config
/// flock and <paramref name="window"/> has not elapsed. Any other exception propagates
/// on the first attempt.
/// </summary>
/// <param name="operation">The operation to run.</param>
/// <param name="window">Retry budget. Defaults to <see cref="DefaultWindow"/>.</param>
public static T Execute<T>(Func<T> operation, TimeSpan? window = null)
{
if (operation == null) throw new ArgumentNullException(nameof(operation));
var budget = window ?? DefaultWindow;
var elapsed = Stopwatch.StartNew();
while (true)
{
try
{
return operation();
}
catch (Exception ex) when (IsLockTimeout(ex) && elapsed.Elapsed < budget)
{
Thread.Sleep(RetryInterval);
}
}
}
/// <summary>Asynchronous counterpart of <see cref="Execute{T}"/>.</summary>
/// <param name="operation">The operation to run.</param>
/// <param name="window">Retry budget. Defaults to <see cref="DefaultWindow"/>.</param>
public static async Task<T> ExecuteAsync<T>(Func<Task<T>> operation, TimeSpan? window = null)
{
if (operation == null) throw new ArgumentNullException(nameof(operation));
var budget = window ?? DefaultWindow;
var elapsed = Stopwatch.StartNew();
while (true)
{
try
{
return await operation().ConfigureAwait(false);
}
catch (Exception ex) when (IsLockTimeout(ex) && elapsed.Elapsed < budget)
{
await Task.Delay(RetryInterval).ConfigureAwait(false);
}
}
}
}
}
@@ -18,8 +18,9 @@ namespace PSProxmoxVE.Core.Utilities
/// StatusMatched: the guest reports <paramref name="expectedStatus"/>. qmpstatus is
/// preferred over status when present, because PVE reports status=running with
/// qmpstatus=paused for a suspended VM.
/// Locked: the guest config still carries a lock, so the next API call against it
/// would fail to acquire the lock file.
/// Locked: the guest config carries a `lock:` property (backup, clone, migrate,
/// snapshot). This is not the /var/lock/qemu-server flock, which PVE does not expose
/// through status/current or any other endpoint — see DECISIONS.md D015 and D020.
/// </returns>
public static (bool StatusMatched, bool Locked) Evaluate(string json, string expectedStatus)
{
@@ -56,17 +56,17 @@ namespace PSProxmoxVE.Cmdlets.Containers
WriteVerbose($"Restarting container {VmId} on node '{Node}'...");
// Graceful shutdown
var shutdownTask = containerService.ShutdownContainer(session, Node, VmId, Timeout);
PveTask Shutdown() => containerService.ShutdownContainer(session, Node, VmId, Timeout);
PveTask Start() => containerService.StartContainer(session, Node, VmId);
if (Wait.IsPresent)
WaitForStatusTransition(session, Node, shutdownTask, VmId, "stopped", Timeout, isContainer: true);
WaitForStatusTransition(session, Node, Shutdown, VmId, "stopped", Timeout, isContainer: true);
else
Shutdown();
// Start
var startTask = containerService.StartContainer(session, Node, VmId);
if (Wait.IsPresent)
startTask = WaitForStatusTransition(session, Node, startTask, VmId, "running", Timeout, isContainer: true);
var startTask = Wait.IsPresent
? WaitForStatusTransition(session, Node, Start, VmId, "running", Timeout, isContainer: true)
: Start();
WriteObject(startTask);
}
@@ -50,12 +50,11 @@ namespace PSProxmoxVE.Cmdlets.Containers
var containerService = new ContainerService();
WriteVerbose($"Starting container {VmId} on node '{Node}'...");
var task = containerService.StartContainer(session, Node, VmId);
PveTask Issue() => containerService.StartContainer(session, Node, VmId);
if (Wait.IsPresent)
{
task = WaitForStatusTransition(session, Node, task, VmId, "running", Timeout, isContainer: true);
}
var task = Wait.IsPresent
? WaitForStatusTransition(session, Node, Issue, VmId, "running", Timeout, isContainer: true)
: Issue();
WriteObject(task);
}
@@ -52,12 +52,11 @@ namespace PSProxmoxVE.Cmdlets.Containers
var containerService = new ContainerService();
WriteVerbose($"Stopping container {VmId} on node '{Node}'...");
var task = containerService.StopContainer(session, Node, VmId);
PveTask Issue() => containerService.StopContainer(session, Node, VmId);
if (Wait.IsPresent)
{
task = WaitForStatusTransition(session, Node, task, VmId, "stopped", Timeout, isContainer: true);
}
var task = Wait.IsPresent
? WaitForStatusTransition(session, Node, Issue, VmId, "stopped", Timeout, isContainer: true)
: Issue();
WriteObject(task);
}
+38 -18
View File
@@ -99,7 +99,10 @@ namespace PSProxmoxVE.Cmdlets
/// </summary>
/// <param name="session">The authenticated PVE session.</param>
/// <param name="node">The cluster node name.</param>
/// <param name="task">The task returned by the lifecycle API call.</param>
/// <param name="issueOperation">
/// Issues the lifecycle API call. Invoked again on each retry, so it must be safe to
/// repeat — see <see cref="InvokeGuestTask"/>.
/// </param>
/// <param name="vmid">The VM or container ID to poll.</param>
/// <param name="expectedStatus">The expected status string (e.g. "running", "stopped", "paused").</param>
/// <param name="timeoutSeconds">Maximum seconds to wait for the status transition. Default 60.</param>
@@ -108,19 +111,13 @@ namespace PSProxmoxVE.Cmdlets
protected PveTask WaitForStatusTransition(
PveSession session,
string node,
PveTask task,
Func<PveTask> issueOperation,
int vmid,
string expectedStatus,
int timeoutSeconds = 60,
bool isContainer = false)
{
var taskService = new TaskService();
// First wait for the PVE task to complete
if (!string.IsNullOrEmpty(task.Upid))
{
task = taskService.WaitForTask(session, node, task.Upid, null, null, null);
}
var task = InvokeGuestTask(session, node, issueOperation);
// Then poll status/current until VM/container reaches the expected status.
// We query the status/current endpoint directly instead of the list endpoint
@@ -141,15 +138,11 @@ namespace PSProxmoxVE.Cmdlets
var snapshot = GuestStatusSnapshot.Evaluate(json, expectedStatus);
lastMatched = snapshot.StatusMatched;
if (snapshot.StatusMatched)
{
// PVE reports the target status before the operation releases the
// config lock. A caller that issues its next request inside that
// window gets "can't lock file '/var/lock/qemu-server/lock-<vmid>.conf'
// - got timeout" from its own API call.
if (!snapshot.Locked)
return task;
}
// snapshot.Locked is the config `lock:` property (backup, clone, migrate,
// snapshot) — not the /var/lock/qemu-server flock, which PVE does not
// expose. The flock race is handled by retrying, not by waiting.
if (snapshot.StatusMatched && !snapshot.Locked)
return task;
}
catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException)
{
@@ -169,6 +162,33 @@ namespace PSProxmoxVE.Cmdlets
TimeSpan.FromSeconds(timeoutSeconds));
}
/// <summary>
/// Issues a guest operation and waits for the task it returns, reissuing the pair while
/// PVE rejects it for the guest's config flock.
///
/// PVE takes that flock inside the worker for most guest operations, so the failure
/// surfaces as a failed task rather than a failed request and cannot be retried at the
/// HTTP layer. <c>lock_config</c> raises it before doing any work, so a reissue repeats
/// nothing.
/// </summary>
/// <param name="session">The authenticated PVE session.</param>
/// <param name="node">The node the task runs on.</param>
/// <param name="issueOperation">Issues the API call; invoked again on each retry.</param>
/// <returns>The completed task, or the issued task when the call returned no UPID.</returns>
protected PveTask InvokeGuestTask(PveSession session, string node, Func<PveTask> issueOperation)
{
if (issueOperation == null) throw new ArgumentNullException(nameof(issueOperation));
var taskService = new TaskService();
return GuestLockRetry.Execute(() =>
{
var task = issueOperation();
return string.IsNullOrEmpty(task.Upid)
? task
: taskService.WaitForTask(session, node, task.Upid, null, null, null);
});
}
/// <summary>
/// Extracts the node name from a UPID string (format: UPID:node:...).
/// Falls back to <paramref name="fallback"/> if the UPID is empty or cannot be parsed.
@@ -81,13 +81,11 @@ namespace PSProxmoxVE.Cmdlets.Vms
WriteVerbose($"Cloning VM {VmId}...");
var newid = NewVmId ?? 0;
var task = vmService.CloneVm(session, SourceNode, VmId, newid, NewName, TargetNode, Full.IsPresent);
PveTask Issue() => vmService.CloneVm(session, SourceNode, VmId, newid, NewName, TargetNode, Full.IsPresent);
if (Wait.IsPresent)
{
var taskService = new TaskService();
task = taskService.WaitForTask(session, SourceNode, task.Upid, null, null, null);
}
var task = Wait.IsPresent
? InvokeGuestTask(session, SourceNode, Issue)
: Issue();
WriteObject(task);
}
@@ -52,12 +52,11 @@ namespace PSProxmoxVE.Cmdlets.Vms
var vmService = new VmService();
WriteVerbose($"Resetting VM {VmId} on node '{Node}'...");
var task = vmService.ResetVm(session, Node, VmId);
PveTask Issue() => vmService.ResetVm(session, Node, VmId);
if (Wait.IsPresent)
{
task = WaitForStatusTransition(session, Node, task, VmId, "running", Timeout);
}
var task = Wait.IsPresent
? WaitForStatusTransition(session, Node, Issue, VmId, "running", Timeout)
: Issue();
WriteObject(task);
}
@@ -65,13 +65,11 @@ namespace PSProxmoxVE.Cmdlets.Vms
var vmService = new VmService();
WriteVerbose($"Resizing disk '{Disk}' on VM {VmId}...");
var task = vmService.ResizeDisk(session, Node, VmId, Disk, Size);
PveTask Issue() => vmService.ResizeDisk(session, Node, VmId, Disk, Size);
if (Wait.IsPresent)
{
var taskService = new TaskService();
task = taskService.WaitForTask(session, Node, task.Upid, null, null, null);
}
var task = Wait.IsPresent
? InvokeGuestTask(session, Node, Issue)
: Issue();
WriteObject(task);
}
@@ -57,10 +57,11 @@ namespace PSProxmoxVE.Cmdlets.Vms
WriteVerbose($"Restarting VM {VmId} on node '{Node}'...");
var task = vmService.RebootVm(session, Node, VmId, Timeout);
PveTask Issue() => vmService.RebootVm(session, Node, VmId, Timeout);
if (Wait.IsPresent)
task = WaitForStatusTransition(session, Node, task, VmId, "running", Timeout);
var task = Wait.IsPresent
? WaitForStatusTransition(session, Node, Issue, VmId, "running", Timeout)
: Issue();
WriteObject(task);
}
@@ -50,12 +50,11 @@ namespace PSProxmoxVE.Cmdlets.Vms
var vmService = new VmService();
WriteVerbose($"Resuming VM {VmId} on node '{Node}'...");
var task = vmService.ResumeVm(session, Node, VmId);
PveTask Issue() => vmService.ResumeVm(session, Node, VmId);
if (Wait.IsPresent)
{
task = WaitForStatusTransition(session, Node, task, VmId, "running", Timeout);
}
var task = Wait.IsPresent
? WaitForStatusTransition(session, Node, Issue, VmId, "running", Timeout)
: Issue();
WriteObject(task);
}
@@ -50,12 +50,11 @@ namespace PSProxmoxVE.Cmdlets.Vms
var vmService = new VmService();
WriteVerbose($"Starting VM {VmId} on node '{Node}'...");
var task = vmService.StartVm(session, Node, VmId);
PveTask Issue() => vmService.StartVm(session, Node, VmId);
if (Wait.IsPresent)
{
task = WaitForStatusTransition(session, Node, task, VmId, "running", Timeout);
}
var task = Wait.IsPresent
? WaitForStatusTransition(session, Node, Issue, VmId, "running", Timeout)
: Issue();
WriteObject(task);
}
@@ -52,12 +52,11 @@ namespace PSProxmoxVE.Cmdlets.Vms
var vmService = new VmService();
WriteVerbose($"Stopping VM {VmId} on node '{Node}'...");
var task = vmService.StopVm(session, Node, VmId);
PveTask Issue() => vmService.StopVm(session, Node, VmId);
if (Wait.IsPresent)
{
task = WaitForStatusTransition(session, Node, task, VmId, "stopped", Timeout);
}
var task = Wait.IsPresent
? WaitForStatusTransition(session, Node, Issue, VmId, "stopped", Timeout)
: Issue();
WriteObject(task);
}
@@ -51,12 +51,11 @@ namespace PSProxmoxVE.Cmdlets.Vms
var vmService = new VmService();
WriteVerbose($"Suspending VM {VmId} on node '{Node}'...");
var task = vmService.SuspendVm(session, Node, VmId);
PveTask Issue() => vmService.SuspendVm(session, Node, VmId);
if (Wait.IsPresent)
{
task = WaitForStatusTransition(session, Node, task, VmId, "paused", Timeout);
}
var task = Wait.IsPresent
? WaitForStatusTransition(session, Node, Issue, VmId, "paused", Timeout)
: Issue();
WriteObject(task);
}