refactor: share one transport per host and poll tasks with backoff (#151) (#193)

Every service method built and disposed its own PveHttpClient, and each
client owned a fresh HttpClientHandler, so every API call was a TCP connect
plus a TLS handshake and WaitForTask paid that 300 times over a ten-minute
wait.

PveServiceBase now owns the injected-or-fresh client lifetime behind one
Invoke helper; the 201 hand-written try/finally blocks across the 16
services collapse to calls on it, and the nested NodeService/VmService
instances receive the injected client. PveHttpClient takes its handler
from a process-wide PveHandlerCache keyed on (host, port,
skipCertificateCheck) and never disposes it, so the connection pool
outlives any one client. WaitForTask holds one client for the whole wait
and, when no pollInterval is supplied, backs off from 1 s toward a 10 s
cap, never sleeping past the deadline.

Co-authored-by: goodolclint-claude[bot] <323206664+goodolclint-claude[bot]@users.noreply.github.com>

tests/PSProxmoxVE.Core.Tests/Client/PveHandlerCacheTests.cs

@@ -0,0 +1,144 @@
+using System;
+using System.Collections.Generic;
+using System.Net;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using PSProxmoxVE.Core.Authentication;
+using PSProxmoxVE.Core.Client;
+using Xunit;
+
+namespace PSProxmoxVE.Core.Tests.Client
+{
+    public class PveHandlerCacheTests
+    {
+        private static PveSession NewSession(bool skipCertificateCheck = true, string host = "pve.example.com") =>
+            new PveSession(host, 8006, skipCertificateCheck,
+                "root@pam!token=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee");
+
+        private sealed class CannedHandler : HttpClientHandler
+        {
+            public int Sends;
+            public bool IsDisposed;
+
+            protected override void Dispose(bool disposing)
+            {
+                IsDisposed = true;
+                base.Dispose(disposing);
+            }
+
+            protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+            {
+                if (IsDisposed) throw new ObjectDisposedException(nameof(CannedHandler));
+                Interlocked.Increment(ref Sends);
+                return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
+                {
+                    Content = new StringContent("{\"data\":{\"version\":\"8.2\"}}")
+                });
+            }
+        }
+
+        private static (PveHandlerCache cache, List<CannedHandler> built) NewCache()
+        {
+            var built = new List<CannedHandler>();
+            var cache = new PveHandlerCache(_ =>
+            {
+                var h = new CannedHandler();
+                built.Add(h);
+                return h;
+            });
+            return (cache, built);
+        }
+
+        [Fact]
+        public void TwoClientsForTheSameEndpointShareOneHandler()
+        {
+            var (cache, built) = NewCache();
+
+            using var first = new PveHttpClient(NewSession(), cache);
+            using var second = new PveHttpClient(NewSession(), cache);
+
+            Assert.Single(built);
+            Assert.Same(first.Handler, second.Handler);
+            Assert.Equal(1, cache.Count);
+        }
+
+        [Fact]
+        public void DifferentSkipCertificateCheckGetsADifferentHandler()
+        {
+            var (cache, built) = NewCache();
+
+            using var insecure = new PveHttpClient(NewSession(skipCertificateCheck: true), cache);
+            using var verified = new PveHttpClient(NewSession(skipCertificateCheck: false), cache);
+
+            Assert.Equal(2, built.Count);
+            Assert.NotSame(insecure.Handler, verified.Handler);
+        }
+
+        [Fact]
+        public void DifferentHostGetsADifferentHandler()
+        {
+            var (cache, _) = NewCache();
+
+            using var a = new PveHttpClient(NewSession(host: "pve-a.example.com"), cache);
+            using var b = new PveHttpClient(NewSession(host: "pve-b.example.com"), cache);
+
+            Assert.NotSame(a.Handler, b.Handler);
+        }
+
+        [Fact]
+        public async Task DisposingOneClientLeavesTheSharedHandlerUsableByAnother()
+        {
+            var (cache, built) = NewCache();
+
+            var first = new PveHttpClient(NewSession(), cache);
+            using var second = new PveHttpClient(NewSession(), cache);
+            first.Dispose();
+
+            var body = await second.GetAsync("version");
+
+            Assert.Contains("8.2", body);
+            Assert.Equal(1, built[0].Sends);
+            Assert.Single(built);
+            Assert.False(built[0].IsDisposed);
+        }
+
+        [Fact]
+        public void AnExplicitHandlerBypassesTheCacheAndIsOwnedByTheClient()
+        {
+            var (cache, built) = NewCache();
+            var own = new CannedHandler();
+
+            var client = new PveHttpClient(NewSession(), timeoutOverride: null,
+                guestLockRetryWindow: null, handler: own, guestLockRetryDelay: null, handlerCache: cache);
+
+            Assert.Same(own, client.Handler);
+            Assert.Empty(built);
+
+            client.Dispose();
+
+            Assert.True(own.IsDisposed);
+        }
+
+        [Fact]
+        public void SharedHandlersDoNotCarryACookieContainer()
+        {
+            var handler = PveHandlerCache.Shared.Get("cookies.example.invalid", 8006, skipCertificateCheck: false);
+
+            Assert.False(handler.UseCookies);
+        }
+
+        [Fact]
+        public void GetBuildsEachKeyOnceUnderConcurrentCallers()
+        {
+            var (cache, built) = NewCache();
+            var handlers = new HttpClientHandler[32];
+
+            Parallel.For(0, handlers.Length, i =>
+                handlers[i] = cache.Get("pve.example.com", 8006, skipCertificateCheck: true));
+
+            Assert.Single(built);
+            Assert.All(handlers, h => Assert.Same(built[0], h));
+        }
+    }
+}

tests/PSProxmoxVE.Core.Tests/Services/ContainerServiceTests.cs

@@ -2,7 +2,6 @@ using System;
 using System.Collections.Generic;
 using System.Net;
 using System.Net.Http;
-using System.Reflection;
 using Moq;
 using PSProxmoxVE.Core.Authentication;
 using PSProxmoxVE.Core.Client;
@@ -123,20 +122,6 @@ namespace PSProxmoxVE.Core.Tests.Services
         // GetContainers multi-node aggregation: issue #142
         // ---------------------------------------------------------------------
 
-        /// <summary>
-        /// Points the private NodeService field a ContainerService constructs at a mock
-        /// client, so the "list all nodes" call the multi-node overload issues is reachable
-        /// without a real HTTP connection. ContainerService(client) only injects the client
-        /// used for the per-node lxc calls; NodeService is never constructor-injectable from
-        /// ContainerService, so this is the only offline path to the aggregation loop.
-        /// </summary>
-        private static void InjectNodeServiceClient(ContainerService service, IPveHttpClient client)
-        {
-            var field = typeof(ContainerService).GetField("_nodeService", BindingFlags.NonPublic | BindingFlags.Instance)
-                ?? throw new InvalidOperationException("ContainerService._nodeService field not found.");
-            field.SetValue(service, new NodeService(client));
-        }
-
         private static Mock<IPveHttpClient> SetupTwoNodeCluster()
         {
             var mockClient = new Mock<IPveHttpClient>();
@@ -158,7 +143,6 @@ namespace PSProxmoxVE.Core.Tests.Services
                 .ThrowsAsync(new PveApiException(HttpStatusCode.InternalServerError, "internal error", "nodes/pve2/lxc", "GET"));
 
             var service = new ContainerService(mockClient.Object);
-            InjectNodeServiceClient(service, mockClient.Object);
 
             var skipped = new List<string>();
             var containers = service.GetContainers(CreateSession(), onNodeSkipped: (node, ex) => skipped.Add(node));
@@ -180,7 +164,6 @@ namespace PSProxmoxVE.Core.Tests.Services
                 .ThrowsAsync(new PveApiException(HttpStatusCode.Forbidden, "permission denied", "nodes/pve2/lxc", "GET"));
 
             var service = new ContainerService(mockClient.Object);
-            InjectNodeServiceClient(service, mockClient.Object);
 
             var ex = Assert.Throws<PveApiException>(() => service.GetContainers(CreateSession()));
             Assert.Equal(HttpStatusCode.Forbidden, ex.StatusCode);
@@ -203,7 +186,6 @@ namespace PSProxmoxVE.Core.Tests.Services
                     "nodes/pve2/lxc", "GET", new HttpRequestException("connection refused")));
 
             var service = new ContainerService(mockClient.Object);
-            InjectNodeServiceClient(service, mockClient.Object);
 
             var skipped = new List<string>();
             var containers = service.GetContainers(CreateSession(), onNodeSkipped: (node, ex) => skipped.Add(node));
@@ -230,7 +212,6 @@ namespace PSProxmoxVE.Core.Tests.Services
                     "nodes/pve2/lxc", "GET"));
 
             var service = new ContainerService(mockClient.Object);
-            InjectNodeServiceClient(service, mockClient.Object);
 
             var skipped = new List<string>();
             var containers = service.GetContainers(CreateSession(), onNodeSkipped: (node, ex) => skipped.Add(node));

tests/PSProxmoxVE.Core.Tests/Services/PveServiceBaseTests.cs

@@ -0,0 +1,142 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using PSProxmoxVE.Core.Authentication;
+using PSProxmoxVE.Core.Client;
+using PSProxmoxVE.Core.Services;
+using Xunit;
+
+namespace PSProxmoxVE.Core.Tests.Services
+{
+    public class PveServiceBaseTests
+    {
+        private static PveSession CreateSession() =>
+            new PveSession("pve1.example.com", 8006, true, "PVE:root@pam:TEST_TOKEN");
+
+        private sealed class RecordingClient : IPveHttpClient
+        {
+            public int DisposeCalls;
+            public int Gets;
+
+            public Task<string> GetAsync(string resource) { Gets++; return Task.FromResult("{\"data\":null}"); }
+            public Task<string> PostAsync(string resource, Dictionary<string, string>? data = null) => throw new NotSupportedException();
+            public Task<string> PostAsync(string resource, IEnumerable<KeyValuePair<string, string>> data) => throw new NotSupportedException();
+            public Task<string> PutAsync(string resource, Dictionary<string, string>? data = null) => throw new NotSupportedException();
+            public Task<string> DeleteAsync(string resource) => throw new NotSupportedException();
+            public string Get(string resource) => GetAsync(resource).GetAwaiter().GetResult();
+            public string Post(string resource, Dictionary<string, string>? data = null) => throw new NotSupportedException();
+            public string Put(string resource, Dictionary<string, string>? data = null) => throw new NotSupportedException();
+            public string Delete(string resource) => throw new NotSupportedException();
+            public Task<string> UploadFileAsync(string resource, string filePath, Dictionary<string, string>? formFields = null,
+                string? checksum = null, string? checksumAlgorithm = null, Action<long, long>? progressCallback = null) =>
+                throw new NotSupportedException();
+            public void Dispose() => DisposeCalls++;
+        }
+
+        private sealed class ProbeService : PveServiceBase
+        {
+            private readonly RecordingClient? _built;
+            public TimeSpan? SeenTimeoutOverride;
+
+            private ProbeService(RecordingClient built) { _built = built; }
+            private ProbeService(IPveHttpClient injected) : base(injected) { }
+
+            public static ProbeService Building(RecordingClient built) => new ProbeService(built);
+            public static ProbeService Using(IPveHttpClient injected) => new ProbeService(injected);
+
+            internal override IPveHttpClient CreateClient(PveSession session, TimeSpan? timeoutOverride)
+            {
+                SeenTimeoutOverride = timeoutOverride;
+                return _built ?? throw new InvalidOperationException("CreateClient reached with an injected client.");
+            }
+
+            public string Fetch(PveSession session) => Invoke(session, c => c.Get("version"));
+            public string FetchWithTimeout(PveSession session, TimeSpan timeout) => Invoke(session, timeout, c => c.Get("version"));
+            public void Touch(PveSession session) => Invoke(session, c => { c.Get("version"); });
+            public string Throw(PveSession session) => Invoke<string>(session, c => throw new InvalidOperationException("boom"));
+        }
+
+        [Fact]
+        public void WithNoInjectedClient_TheClientBuiltForTheCallIsDisposed()
+        {
+            var built = new RecordingClient();
+            var service = ProbeService.Building(built);
+
+            service.Fetch(CreateSession());
+
+            Assert.Equal(1, built.Gets);
+            Assert.Equal(1, built.DisposeCalls);
+        }
+
+        [Fact]
+        public void WithNoInjectedClient_TheClientIsDisposedWhenTheActionThrows()
+        {
+            var built = new RecordingClient();
+            var service = ProbeService.Building(built);
+
+            Assert.Throws<InvalidOperationException>(() => service.Throw(CreateSession()));
+
+            Assert.Equal(1, built.DisposeCalls);
+        }
+
+        [Fact]
+        public void WithAnInjectedClient_ItIsUsedAndNeverDisposed()
+        {
+            var injected = new RecordingClient();
+            var service = ProbeService.Using(injected);
+
+            service.Fetch(CreateSession());
+            service.Touch(CreateSession());
+
+            Assert.Equal(2, injected.Gets);
+            Assert.Equal(0, injected.DisposeCalls);
+        }
+
+        [Fact]
+        public void TheVoidOverloadDisposesTheClientItBuilt()
+        {
+            var built = new RecordingClient();
+            var service = ProbeService.Building(built);
+
+            service.Touch(CreateSession());
+
+            Assert.Equal(1, built.DisposeCalls);
+        }
+
+        [Fact]
+        public void TheTimeoutOverloadPassesTheOverrideToTheClientFactory()
+        {
+            var built = new RecordingClient();
+            var service = ProbeService.Building(built);
+
+            service.FetchWithTimeout(CreateSession(), TimeSpan.FromMinutes(30));
+
+            Assert.Equal(TimeSpan.FromMinutes(30), service.SeenTimeoutOverride);
+        }
+
+        [Fact]
+        public void ANullSessionIsRejectedBeforeAnyClientIsBuilt()
+        {
+            var built = new RecordingClient();
+            var service = ProbeService.Building(built);
+
+            var ex = Assert.Throws<ArgumentNullException>(() => service.Fetch(null!));
+
+            Assert.Equal("session", ex.ParamName);
+            Assert.Equal(0, built.DisposeCalls);
+        }
+
+        [Fact]
+        public void NestedServicesShareTheInjectedClient()
+        {
+            var injected = new RecordingClient();
+
+            new VmService(injected).GetVms(CreateSession());
+            new ContainerService(injected).GetContainers(CreateSession());
+            new TemplateService(injected).GetTemplates(CreateSession());
+
+            Assert.Equal(3, injected.Gets);
+            Assert.Equal(0, injected.DisposeCalls);
+        }
+    }
+}

tests/PSProxmoxVE.Core.Tests/Services/TaskServiceTests.cs

@@ -336,6 +336,129 @@ namespace PSProxmoxVE.Core.Tests.Services
             Assert.Equal(new List<string?> { "running", "running", "stopped" }, seenStatuses);
         }
 
+        private const string RunningJson = @"{ ""data"": { ""status"": ""running"", ""user"": ""root@pam"" } }";
+        private const string StoppedJson = @"{ ""data"": { ""status"": ""stopped"", ""exitstatus"": ""OK"", ""user"": ""root@pam"" } }";
+
+        private static (TaskService service, Mock<IPveHttpClient> client, List<TimeSpan> delays) ServiceWithRecordedDelays(int runningPolls)
+        {
+            var mockClient = new Mock<IPveHttpClient>();
+            var sequence = mockClient.SetupSequence(c => c.GetAsync(It.IsAny<string>()));
+            for (var i = 0; i < runningPolls; i++)
+                sequence = sequence.ReturnsAsync(RunningJson);
+            sequence.ReturnsAsync(StoppedJson);
+
+            var delays = new List<TimeSpan>();
+            var service = new TaskService(mockClient.Object, d => { delays.Add(d); return Task.CompletedTask; });
+            return (service, mockClient, delays);
+        }
+
+        [Fact]
+        public void WaitForTask_WithNoPollInterval_BacksOffFromOneSecondToATenSecondCap()
+        {
+            var (service, mockClient, delays) = ServiceWithRecordedDelays(runningPolls: 12);
+
+            var task = service.WaitForTask(CreateSession(), TestNode, TestUpid, timeout: TimeSpan.FromMinutes(5));
+
+            Assert.True(task.IsSuccessful);
+            mockClient.Verify(c => c.GetAsync(It.IsAny<string>()), Times.Exactly(13));
+            Assert.Equal(12, delays.Count);
+            Assert.Equal(TimeSpan.FromSeconds(1), delays[0]);
+            Assert.Equal(TimeSpan.FromSeconds(2), delays[1]);
+            for (var i = 1; i < delays.Count; i++)
+                Assert.True(delays[i] >= delays[i - 1], $"delay {i} ({delays[i]}) shrank from {delays[i - 1]}");
+            Assert.All(delays, d => Assert.True(d <= TimeSpan.FromSeconds(10), $"delay {d} exceeds the cap"));
+            Assert.Equal(TimeSpan.FromSeconds(10), delays[delays.Count - 1]);
+            Assert.Contains(delays, d => d > TimeSpan.FromSeconds(1) && d < TimeSpan.FromSeconds(10));
+        }
+
+        [Fact]
+        public void WaitForTask_NeverSleepsPastTheDeadline()
+        {
+            var mockClient = new Mock<IPveHttpClient>();
+            mockClient.Setup(c => c.GetAsync(It.IsAny<string>())).ReturnsAsync(RunningJson);
+            var delays = new List<TimeSpan>();
+            var service = new TaskService(mockClient.Object, d => { delays.Add(d); return Task.CompletedTask; });
+            var timeout = TimeSpan.FromMilliseconds(300);
+
+            Assert.Throws<PveTaskTimeoutException>(() =>
+                service.WaitForTask(CreateSession(), TestNode, TestUpid, timeout: timeout));
+
+            Assert.NotEmpty(delays);
+            Assert.All(delays, d => Assert.True(d <= timeout, $"slept {d} against a {timeout} timeout"));
+        }
+
+        [Fact]
+        public void WaitForTask_WithAnExplicitPollInterval_EveryDelayEqualsIt()
+        {
+            var (service, mockClient, delays) = ServiceWithRecordedDelays(runningPolls: 5);
+
+            service.WaitForTask(CreateSession(), TestNode, TestUpid,
+                timeout: TimeSpan.FromMinutes(5), pollInterval: TimeSpan.FromSeconds(3));
+
+            mockClient.Verify(c => c.GetAsync(It.IsAny<string>()), Times.Exactly(6));
+            Assert.Equal(5, delays.Count);
+            Assert.All(delays, d => Assert.Equal(TimeSpan.FromSeconds(3), d));
+        }
+
+        [Fact]
+        public void WaitForTask_WithAnExplicitPollIntervalBelowTheMinimum_EveryDelayIsOneSecond()
+        {
+            var (service, _, delays) = ServiceWithRecordedDelays(runningPolls: 3);
+
+            service.WaitForTask(CreateSession(), TestNode, TestUpid,
+                timeout: TimeSpan.FromMinutes(5), pollInterval: TimeSpan.FromMilliseconds(50));
+
+            Assert.Equal(3, delays.Count);
+            Assert.All(delays, d => Assert.Equal(TimeSpan.FromSeconds(1), d));
+        }
+
+        [Fact]
+        public void WaitForTask_PollsTheStatusEndpointOncePerPollThroughTheInjectedClient()
+        {
+            var (service, mockClient, _) = ServiceWithRecordedDelays(runningPolls: 2);
+
+            service.WaitForTask(CreateSession(), TestNode, TestUpid, timeout: TimeSpan.FromMinutes(5));
+
+            var expected = $"nodes/{TestNode}/tasks/{Uri.EscapeDataString(TestUpid)}/status";
+            mockClient.Verify(c => c.GetAsync(expected), Times.Exactly(3));
+            mockClient.Verify(c => c.Dispose(), Times.Never);
+        }
+
+        private sealed class ClientCountingTaskService : TaskService
+        {
+            public readonly List<Mock<IPveHttpClient>> Built = new List<Mock<IPveHttpClient>>();
+            private readonly int _runningPolls;
+
+            public ClientCountingTaskService(int runningPolls) : base(_ => Task.CompletedTask)
+            {
+                _runningPolls = runningPolls;
+            }
+
+            internal override IPveHttpClient CreateClient(PveSession session, TimeSpan? timeoutOverride)
+            {
+                var mock = new Mock<IPveHttpClient>();
+                var sequence = mock.SetupSequence(c => c.GetAsync(It.IsAny<string>()));
+                for (var i = 0; i < _runningPolls; i++)
+                    sequence = sequence.ReturnsAsync(RunningJson);
+                sequence.ReturnsAsync(StoppedJson);
+                Built.Add(mock);
+                return mock.Object;
+            }
+        }
+
+        [Fact]
+        public void WaitForTask_WithNoInjectedClient_OpensOneClientForTheWholeWaitAndDisposesItAfter()
+        {
+            var service = new ClientCountingTaskService(runningPolls: 4);
+
+            var task = service.WaitForTask(CreateSession(), TestNode, TestUpid, timeout: TimeSpan.FromMinutes(5));
+
+            Assert.True(task.IsSuccessful);
+            var client = Assert.Single(service.Built);
+            client.Verify(c => c.GetAsync(It.IsAny<string>()), Times.Exactly(5));
+            client.Verify(c => c.Dispose(), Times.Once);
+        }
+
         [Fact]
         public void StopTask_CallsDeleteAsyncWithCorrectPath()
         {

tests/PSProxmoxVE.Core.Tests/Services/VmServiceTests.cs

@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
-using System.Reflection;
 using Moq;
 using PSProxmoxVE.Core.Authentication;
 using PSProxmoxVE.Core.Client;
@@ -296,20 +295,6 @@ namespace PSProxmoxVE.Core.Tests.Services
         // GetVms multi-node aggregation: issue #142
         // ---------------------------------------------------------------------
 
-        /// <summary>
-        /// Points the private NodeService field a VmService constructs at a mock client,
-        /// so the "list all nodes" call the multi-node overload issues is reachable
-        /// without a real HTTP connection. VmService(client) only injects the client used
-        /// for the per-node qemu calls; NodeService is never constructor-injectable from
-        /// VmService, so this is the only offline path to the aggregation loop.
-        /// </summary>
-        private static void InjectNodeServiceClient(VmService service, IPveHttpClient client)
-        {
-            var field = typeof(VmService).GetField("_nodeService", BindingFlags.NonPublic | BindingFlags.Instance)
-                ?? throw new InvalidOperationException("VmService._nodeService field not found.");
-            field.SetValue(service, new NodeService(client));
-        }
-
         private static Mock<IPveHttpClient> SetupTwoNodeCluster()
         {
             var mockClient = new Mock<IPveHttpClient>();
@@ -331,7 +316,6 @@ namespace PSProxmoxVE.Core.Tests.Services
                 .ThrowsAsync(new PveApiException(HttpStatusCode.InternalServerError, "internal error", "nodes/pve2/qemu", "GET"));
 
             var service = new VmService(mockClient.Object);
-            InjectNodeServiceClient(service, mockClient.Object);
 
             var skipped = new List<string>();
             var vms = service.GetVms(CreateSession(), onNodeSkipped: (node, ex) => skipped.Add(node));
@@ -353,7 +337,6 @@ namespace PSProxmoxVE.Core.Tests.Services
                 .ThrowsAsync(new PveApiException(HttpStatusCode.Forbidden, "permission denied", "nodes/pve2/qemu", "GET"));
 
             var service = new VmService(mockClient.Object);
-            InjectNodeServiceClient(service, mockClient.Object);
 
             var ex = Assert.Throws<PveApiException>(() => service.GetVms(CreateSession()));
             Assert.Equal(HttpStatusCode.Forbidden, ex.StatusCode);
@@ -376,7 +359,6 @@ namespace PSProxmoxVE.Core.Tests.Services
                     "nodes/pve2/qemu", "GET", new HttpRequestException("connection refused")));
 
             var service = new VmService(mockClient.Object);
-            InjectNodeServiceClient(service, mockClient.Object);
 
             var skipped = new List<string>();
             var vms = service.GetVms(CreateSession(), onNodeSkipped: (node, ex) => skipped.Add(node));
@@ -403,7 +385,6 @@ namespace PSProxmoxVE.Core.Tests.Services
                     "nodes/pve2/qemu", "GET"));
 
             var service = new VmService(mockClient.Object);
-            InjectNodeServiceClient(service, mockClient.Object);
 
             var skipped = new List<string>();
             var vms = service.GetVms(CreateSession(), onNodeSkipped: (node, ex) => skipped.Add(node));