using System; using System.Diagnostics; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using PSProxmoxVE.Core.Exceptions; namespace PSProxmoxVE.Core.Utilities { /// /// Retries an operation PVE rejected because it could not acquire a guest's config /// flock (/var/lock/qemu-server/lock-<vmid>.conf for VMs, /// /run/lock/lxc/pve-config-<vmid>.lock for containers). /// /// That flock is held by qm cleanup after a guest stops and is not exposed /// through the API in any form, so a caller cannot wait for it — only retry past it. /// public static class GuestLockRetry { /// /// How long and keep retrying. /// qm cleanup polls vm_running_locally for up to 30s while holding the /// flock, and each rejected attempt first burns PVE's own 10s lock_config timeout. /// public static readonly TimeSpan DefaultWindow = TimeSpan.FromSeconds(45); private static readonly TimeSpan MaxRetryInterval = TimeSpan.FromSeconds(2); // The gap must never eat a meaningful share of a short budget: a caller passing a // small window wants a fast answer, not one long sleep. private static TimeSpan RetryInterval(TimeSpan budget) { var quarter = TimeSpan.FromMilliseconds(budget.TotalMilliseconds / 4); return quarter < MaxRetryInterval ? quarter : MaxRetryInterval; } // Anchored, and specific to the two guest lock paths. `PVE::Tools::lock_file` emits this // same wording for storage, LVM, HA and firewall locks, none of which carry the // reissue-safety guarantee below. The anchor is what separates a failure to *enter* // lock_config from one PVE prefixed with its own context ("clone failed: ..."), which // means the worker had already done work. private static readonly Regex GuestLockTimeout = new Regex( @"^can't lock file '(?:/var/lock/qemu-server/lock-\d+\.conf|/run/lock/lxc/pve-config-\d+\.lock)' - got timeout", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); /// /// True when reports PVE failing to enter lock_config for a /// guest. That is raised before the operation performs any work, so a call failing this /// way is known not to have run and is safe to reissue. /// /// Matched against what PVE actually said — /// and — never against the composed /// , whose prefix would defeat the anchor. /// /// The exception to classify. public static bool IsLockTimeout(Exception ex) => ex switch { PveTaskFailedException task => GuestLockTimeout.IsMatch((task.ExitStatus ?? string.Empty).Trim()), PveApiException api => GuestLockTimeout.IsMatch((api.ApiMessage ?? string.Empty).Trim()), _ => false, }; /// /// Runs , reissuing it while it fails on the guest config /// flock and has not elapsed. Any other exception propagates /// on the first attempt. /// /// The operation to run. /// Retry budget. Defaults to . /// /// Invoked with the rejection before each reissue. A caller with somewhere to report /// progress should pass one — a wait this long is otherwise indistinguishable from a hang. /// public static T Execute(Func operation, TimeSpan? window = null, Action? onRetry = null) { if (operation == null) throw new ArgumentNullException(nameof(operation)); var budget = window ?? DefaultWindow; var elapsed = Stopwatch.StartNew(); while (true) { try { return operation(); } catch (Exception ex) when (IsLockTimeout(ex) && elapsed.Elapsed < budget) { onRetry?.Invoke(ex); Thread.Sleep(RetryInterval(budget)); } } } /// Asynchronous counterpart of . /// The operation to run. /// Retry budget. Defaults to . public static async Task ExecuteAsync(Func> operation, TimeSpan? window = null) { if (operation == null) throw new ArgumentNullException(nameof(operation)); var budget = window ?? DefaultWindow; var elapsed = Stopwatch.StartNew(); while (true) { try { return await operation().ConfigureAwait(false); } catch (Exception ex) when (IsLockTimeout(ex) && elapsed.Elapsed < budget) { await Task.Delay(RetryInterval(budget)).ConfigureAwait(false); } } } } }