mirror of
https://github.com/GoodOlClint/PSProxmoxVE.git
synced 2026-09-03 18:55:33 +00:00
fix: retry the qemu-server flock instead of predicting it
WaitForStatusTransition refused to return while snapshot.Locked, and its comment quoted the exact error it was meant to prevent. Locked reads the guest config's lock: property; the failure is the flock on /var/lock/qemu-server/lock-<vmid>.conf, which PVE exposes nowhere. The flock cannot be observed, so it is retried. GuestLockRetry reissues an operation for a bounded 45s while PVE reports failing to enter lock_config for a guest, which it raises before doing any work. Two seams, because the failure has two surfaces. PveHttpClient.SendAsync retries the request for operations PVE serialises in the API handler; it takes a request factory because an HttpRequestMessage cannot be resent. PveCmdletBase.InvokeGuestTask reissues the call and re-waits its task for operations serialised in the forked worker, where the POST returns 200 and only the task fails. WaitForStatusTransition routes through the latter, hence Func<PveTask>. The predicate is path-specific and anchored at the start of what PVE said: lock_file uses identical wording for storage, LVM and HA locks, and a qmclone that fails after allocating disks must not be reissued into "VM already exists". That requires the raw text, so it reads PveTaskFailedException.ExitStatus and PveApiException.ApiMessage. The Locked check stays — it is correct for the config lock — with a comment that says so. Closes #113
This commit is contained in:
goodolclint-claude[bot]
@@ -0,0 +1,143 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using PSProxmoxVE.Core.Authentication;
|
||||
using PSProxmoxVE.Core.Client;
|
||||
using PSProxmoxVE.Core.Exceptions;
|
||||
using Xunit;
|
||||
|
||||
namespace PSProxmoxVE.Core.Tests.Client
|
||||
{
|
||||
public class PveHttpClientLockRetryTests
|
||||
{
|
||||
private const string LockTimeoutBody =
|
||||
"{\"message\":\"can't lock file '/var/lock/qemu-server/lock-100.conf' - got timeout\"}";
|
||||
|
||||
private static void SetInnerHttpClient(PveHttpClient client, HttpClient newInner)
|
||||
{
|
||||
var field = typeof(PveHttpClient).GetField("_httpClient",
|
||||
BindingFlags.Instance | BindingFlags.NonPublic)!;
|
||||
((HttpClient)field.GetValue(client)!).Dispose();
|
||||
field.SetValue(client, newInner);
|
||||
}
|
||||
|
||||
private static (PveHttpClient client, ScriptedHandler handler) NewClient(
|
||||
params (HttpStatusCode status, string body)[] responses)
|
||||
{
|
||||
var session = new PveSession("pve.example.com", 8006, false,
|
||||
"root@pam!token=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee");
|
||||
var client = new PveHttpClient(session);
|
||||
var handler = new ScriptedHandler(responses);
|
||||
SetInnerHttpClient(client, new HttpClient(handler));
|
||||
return (client, handler);
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> ConfigBody() =>
|
||||
new Dictionary<string, string> { ["scsi0"] = "local-lvm:1" };
|
||||
|
||||
[Fact]
|
||||
public async Task PutAsync_ReissuesTheRequestWhilePveReportsTheGuestFlock()
|
||||
{
|
||||
var (client, handler) = NewClient(
|
||||
(HttpStatusCode.InternalServerError, LockTimeoutBody),
|
||||
(HttpStatusCode.InternalServerError, LockTimeoutBody),
|
||||
(HttpStatusCode.OK, "{\"data\":null}"));
|
||||
|
||||
using (client)
|
||||
{
|
||||
var result = await client.PutAsync("nodes/pve9a/qemu/100/config", ConfigBody());
|
||||
Assert.Equal("{\"data\":null}", result);
|
||||
}
|
||||
|
||||
Assert.Equal(3, handler.Bodies.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PutAsync_RebuildsTheRequestSoEveryAttemptCarriesTheSameBody()
|
||||
{
|
||||
var (client, handler) = NewClient(
|
||||
(HttpStatusCode.InternalServerError, LockTimeoutBody),
|
||||
(HttpStatusCode.OK, "{\"data\":null}"));
|
||||
|
||||
using (client)
|
||||
{
|
||||
await client.PutAsync("nodes/pve9a/qemu/100/config", ConfigBody());
|
||||
}
|
||||
|
||||
Assert.Equal(2, handler.Bodies.Count);
|
||||
Assert.Equal("scsi0=local-lvm:1", handler.Bodies[0]);
|
||||
Assert.Equal(handler.Bodies[0], handler.Bodies[1]);
|
||||
Assert.All(handler.Methods, m => Assert.Equal(HttpMethod.Put, m));
|
||||
Assert.Equal(handler.Uris[0], handler.Uris[1]);
|
||||
Assert.EndsWith("nodes/pve9a/qemu/100/config", handler.Uris[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PostAsync_DoesNotReissueApiErrorsThatAreNotTheFlock()
|
||||
{
|
||||
var (client, handler) = NewClient(
|
||||
(HttpStatusCode.InternalServerError, "{\"message\":\"VM 100 not running\"}"),
|
||||
(HttpStatusCode.OK, "{\"data\":null}"));
|
||||
|
||||
using (client)
|
||||
{
|
||||
var ex = await Assert.ThrowsAsync<PveApiException>(
|
||||
() => client.PostAsync("nodes/pve9a/qemu/100/status/reset"));
|
||||
Assert.Contains("VM 100 not running", ex.Message);
|
||||
}
|
||||
|
||||
Assert.Single(handler.Bodies);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAsync_ReissuesWithoutCarryingContent()
|
||||
{
|
||||
var (client, handler) = NewClient(
|
||||
(HttpStatusCode.InternalServerError, LockTimeoutBody),
|
||||
(HttpStatusCode.OK, "{\"data\":{}}"));
|
||||
|
||||
using (client)
|
||||
{
|
||||
await client.GetAsync("nodes/pve9a/qemu/100/status/current");
|
||||
}
|
||||
|
||||
Assert.Equal(2, handler.Bodies.Count);
|
||||
Assert.All(handler.Bodies, b => Assert.Equal(string.Empty, b));
|
||||
}
|
||||
|
||||
private sealed class ScriptedHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly (HttpStatusCode status, string body)[] _responses;
|
||||
private int _index;
|
||||
|
||||
public List<string> Bodies { get; } = new List<string>();
|
||||
public List<HttpMethod> Methods { get; } = new List<HttpMethod>();
|
||||
public List<string> Uris { get; } = new List<string>();
|
||||
|
||||
public ScriptedHandler((HttpStatusCode status, string body)[] responses)
|
||||
{
|
||||
_responses = responses;
|
||||
}
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
Bodies.Add(request.Content == null
|
||||
? string.Empty
|
||||
: await request.Content.ReadAsStringAsync().ConfigureAwait(false));
|
||||
Methods.Add(request.Method);
|
||||
Uris.Add(request.RequestUri!.ToString());
|
||||
|
||||
if (_index >= _responses.Length)
|
||||
throw new InvalidOperationException("ScriptedHandler ran out of responses.");
|
||||
|
||||
var (status, body) = _responses[_index++];
|
||||
return new HttpResponseMessage(status) { Content = new StringContent(body) };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using PSProxmoxVE.Core.Exceptions;
|
||||
using PSProxmoxVE.Core.Utilities;
|
||||
using Xunit;
|
||||
|
||||
namespace PSProxmoxVE.Core.Tests.Utilities
|
||||
{
|
||||
public class GuestLockRetryTests
|
||||
{
|
||||
private const string VmLockError =
|
||||
"can't lock file '/var/lock/qemu-server/lock-100.conf' - got timeout";
|
||||
|
||||
private const string LxcLockError =
|
||||
"can't lock file '/run/lock/lxc/pve-config-100.lock' - got timeout";
|
||||
|
||||
private static PveApiException ApiError(string message) =>
|
||||
new PveApiException(HttpStatusCode.InternalServerError, message, "nodes/pve9a/qemu/100/config", "PUT");
|
||||
|
||||
private static PveTaskFailedException TaskError(string exitStatus) =>
|
||||
new PveTaskFailedException("UPID:pve9a:00000001:qmreset:100:root@pam:", exitStatus);
|
||||
|
||||
[Fact]
|
||||
public void IsLockTimeout_MatchesTheVmFlockErrorFromBothSurfaces()
|
||||
{
|
||||
Assert.True(GuestLockRetry.IsLockTimeout(ApiError(VmLockError)));
|
||||
Assert.True(GuestLockRetry.IsLockTimeout(TaskError(VmLockError)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsLockTimeout_MatchesTheContainerFlockError()
|
||||
{
|
||||
Assert.True(GuestLockRetry.IsLockTimeout(TaskError(LxcLockError)));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("VM 100 not running")]
|
||||
[InlineData("can't lock file '/var/lock/qemu-server/lock-100.conf'")]
|
||||
[InlineData("got timeout")]
|
||||
// PVE::Tools::lock_file uses this same wording for locks that carry no
|
||||
// reissue-safety guarantee. Only the two guest config paths may retry.
|
||||
[InlineData("can't lock file '/run/lock/pve-manager/pve-storage-local' - got timeout")]
|
||||
[InlineData("can't lock file '/var/lock/pve-manager/pve-backup' - got timeout")]
|
||||
[InlineData("can't lock file '/run/lock/lvm/V_pve' - got timeout")]
|
||||
// A message PVE prefixed with its own context means the worker had already
|
||||
// started; reissuing it could repeat work that landed.
|
||||
[InlineData("clone failed: can't lock file '/var/lock/qemu-server/lock-100.conf' - got timeout")]
|
||||
[InlineData("unable to resize: can't lock file '/var/lock/qemu-server/lock-100.conf' - got timeout")]
|
||||
public void IsLockTimeout_RejectsEveryOtherFailure(string message)
|
||||
{
|
||||
Assert.False(GuestLockRetry.IsLockTimeout(ApiError(message)));
|
||||
Assert.False(GuestLockRetry.IsLockTimeout(TaskError(message)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsLockTimeout_ReadsWhatPveSaidRatherThanTheComposedMessage()
|
||||
{
|
||||
// Both exception types prefix Message with their own context, which would
|
||||
// defeat the anchor if the predicate matched on Message.
|
||||
var api = ApiError(VmLockError);
|
||||
var task = TaskError(VmLockError);
|
||||
|
||||
Assert.StartsWith("PVE API error", api.Message);
|
||||
Assert.StartsWith("Task UPID:", task.Message);
|
||||
Assert.True(GuestLockRetry.IsLockTimeout(api));
|
||||
Assert.True(GuestLockRetry.IsLockTimeout(task));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsLockTimeout_RejectsExceptionTypesThatAreNotApiFailures()
|
||||
{
|
||||
Assert.False(GuestLockRetry.IsLockTimeout(new InvalidOperationException(VmLockError)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Execute_ReturnsWithoutRetryingWhenTheOperationSucceeds()
|
||||
{
|
||||
var attempts = 0;
|
||||
|
||||
var result = GuestLockRetry.Execute(() => { attempts++; return 42; });
|
||||
|
||||
Assert.Equal(42, result);
|
||||
Assert.Equal(1, attempts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Execute_ReissuesUntilTheLockClears()
|
||||
{
|
||||
var attempts = 0;
|
||||
|
||||
var result = GuestLockRetry.Execute(() =>
|
||||
{
|
||||
attempts++;
|
||||
if (attempts < 2) throw TaskError(VmLockError);
|
||||
return "cloned";
|
||||
});
|
||||
|
||||
Assert.Equal("cloned", result);
|
||||
Assert.Equal(2, attempts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Execute_DoesNotRetryOtherFailures()
|
||||
{
|
||||
var attempts = 0;
|
||||
|
||||
Assert.Throws<PveApiException>(() => GuestLockRetry.Execute<int>(() =>
|
||||
{
|
||||
attempts++;
|
||||
throw ApiError("VM 100 not running");
|
||||
}));
|
||||
|
||||
Assert.Equal(1, attempts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_ReissuesUntilTheLockClears()
|
||||
{
|
||||
var attempts = 0;
|
||||
|
||||
var result = await GuestLockRetry.ExecuteAsync(() =>
|
||||
{
|
||||
attempts++;
|
||||
if (attempts < 2) throw ApiError(VmLockError);
|
||||
return Task.FromResult("written");
|
||||
});
|
||||
|
||||
Assert.Equal("written", result);
|
||||
Assert.Equal(2, attempts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_DoesNotRetryOtherFailures()
|
||||
{
|
||||
var attempts = 0;
|
||||
|
||||
await Assert.ThrowsAsync<PveApiException>(() => GuestLockRetry.ExecuteAsync<int>(() =>
|
||||
{
|
||||
attempts++;
|
||||
throw ApiError("can't lock file '/run/lock/lvm/V_pve' - got timeout");
|
||||
}));
|
||||
|
||||
Assert.Equal(1, attempts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Execute_GivesUpAndRethrowsOnceTheWindowElapses()
|
||||
{
|
||||
var attempts = 0;
|
||||
|
||||
var ex = Assert.Throws<PveTaskFailedException>(() => GuestLockRetry.Execute<int>(
|
||||
() => { attempts++; throw TaskError(VmLockError); },
|
||||
TimeSpan.Zero));
|
||||
|
||||
Assert.Contains("got timeout", ex.Message);
|
||||
Assert.Equal(1, attempts);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user