using System;
using System.Diagnostics;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using PSProxmoxVE.Core.Exceptions;
namespace PSProxmoxVE.Core.Utilities
{
///
/// Retries an operation PVE rejected because it could not acquire a guest's config
/// flock (/var/lock/qemu-server/lock-<vmid>.conf for VMs,
/// /run/lock/lxc/pve-config-<vmid>.lock for containers).
///
/// That flock is held by qm cleanup after a guest stops and is not exposed
/// through the API in any form, so a caller cannot wait for it — only retry past it.
///
public static class GuestLockRetry
{
///
/// How long and keep retrying.
/// qm cleanup polls vm_running_locally for up to 30s while holding the
/// flock, and each rejected attempt first burns PVE's own 10s lock_config timeout.
///
public static readonly TimeSpan DefaultWindow = TimeSpan.FromSeconds(45);
private static readonly TimeSpan MaxRetryInterval = TimeSpan.FromSeconds(2);
// The gap must never eat a meaningful share of a short budget: a caller passing a
// small window wants a fast answer, not one long sleep.
private static TimeSpan RetryInterval(TimeSpan budget)
{
var quarter = TimeSpan.FromMilliseconds(budget.TotalMilliseconds / 4);
return quarter < MaxRetryInterval ? quarter : MaxRetryInterval;
}
// Anchored, and specific to the two guest lock paths. `PVE::Tools::lock_file` emits this
// same wording for storage, LVM, HA and firewall locks, none of which carry the
// reissue-safety guarantee below. The anchor is what separates a failure to *enter*
// lock_config from one PVE prefixed with its own context ("clone failed: ..."), which
// means the worker had already done work.
private static readonly Regex GuestLockTimeout = new Regex(
@"^can't lock file '(?:/var/lock/qemu-server/lock-\d+\.conf|/run/lock/lxc/pve-config-\d+\.lock)' - got timeout",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
///
/// True when reports PVE failing to enter lock_config for a
/// guest. That is raised before the operation performs any work, so a call failing this
/// way is known not to have run and is safe to reissue.
///
/// Matched against what PVE actually said —
/// and — never against the composed
/// , whose prefix would defeat the anchor.
///
/// The exception to classify.
public static bool IsLockTimeout(Exception ex) => ex switch
{
PveTaskFailedException task => GuestLockTimeout.IsMatch((task.ExitStatus ?? string.Empty).Trim()),
PveApiException api => GuestLockTimeout.IsMatch((api.ApiMessage ?? string.Empty).Trim()),
_ => false,
};
///
/// Runs , reissuing it while it fails on the guest config
/// flock and has not elapsed. Any other exception propagates
/// on the first attempt.
///
/// The operation to run.
/// Retry budget. Defaults to .
///
/// Invoked with the rejection before each reissue. A caller with somewhere to report
/// progress should pass one — a wait this long is otherwise indistinguishable from a hang.
///
public static T Execute(Func operation, TimeSpan? window = null, Action? onRetry = null)
{
if (operation == null) throw new ArgumentNullException(nameof(operation));
var budget = window ?? DefaultWindow;
var elapsed = Stopwatch.StartNew();
while (true)
{
try
{
return operation();
}
catch (Exception ex) when (IsLockTimeout(ex) && elapsed.Elapsed < budget)
{
onRetry?.Invoke(ex);
Thread.Sleep(RetryInterval(budget));
}
}
}
/// Asynchronous counterpart of .
/// The operation to run.
/// Retry budget. Defaults to .
public static async Task ExecuteAsync(Func> operation, TimeSpan? window = null)
{
if (operation == null) throw new ArgumentNullException(nameof(operation));
var budget = window ?? DefaultWindow;
var elapsed = Stopwatch.StartNew();
while (true)
{
try
{
return await operation().ConfigureAwait(false);
}
catch (Exception ex) when (IsLockTimeout(ex) && elapsed.Elapsed < budget)
{
await Task.Delay(RetryInterval(budget)).ConfigureAwait(false);
}
}
}
}
}