diff --git a/.claude/commands b/.claude/commands
index acaf0040..95b417d3 160000
--- a/.claude/commands
+++ b/.claude/commands
@@ -1 +1 @@
-Subproject commit acaf00402e10e8f7e7d06020a736757434c14f10
+Subproject commit 95b417d3c70f7ec4cdf372e9432382f428ff71f7
diff --git a/internal b/internal
index a71f08b7..c5861a3e 160000
--- a/internal
+++ b/internal
@@ -1 +1 @@
-Subproject commit a71f08b7a1afd17a74146e0d1d4d3975879451e6
+Subproject commit c5861a3e1bf58aa128a2031d7c78a5a84b61f190
diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/AdminWireParityTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/AdminWireParityTests.cs
index b488b06e..d0576a86 100644
--- a/src/Backend/AutopilotMonitor.Functions.Tests/AdminWireParityTests.cs
+++ b/src/Backend/AutopilotMonitor.Functions.Tests/AdminWireParityTests.cs
@@ -447,41 +447,27 @@ public class AdminWireParityTests
// ---- TriggerMaintenance --------------------------------------------------------------
[Fact]
- public void TriggerMaintenanceResponse_matches_the_manual_trigger_shape()
+ public void TriggerMaintenanceResponse_matches_the_queued_trigger_shape()
{
- var result = new MaintenanceResult
- {
- Success = true,
- TriggeredBy = "admin@contoso.com",
- TriggeredAt = new DateTime(2026, 8, 30, 13, 10, 0, DateTimeKind.Utc),
- DurationMs = 1234,
- StalledSessionsChecked = true,
- MetricsAggregated = true,
- AggregatedDate = "2026-08-29",
- DataCleanupExecuted = false,
- PlatformStatsRecomputed = true,
- DevicesBlockedForExcessiveData = 0,
- ContactEmailsBackfilled = 2,
- };
var userEmail = "admin@contoso.com";
var triggeredAt = new DateTime(2026, 8, 30, 13, 10, 5, DateTimeKind.Utc);
AssertParity(
new
{
- success = true,
- message = "Maintenance tasks completed",
- result = result,
+ message = "Maintenance run queued",
triggeredBy = userEmail,
- triggeredAt
+ triggeredAt,
+ targetDate = "2026-08-29",
+ aggregateOnly = true
},
new TriggerMaintenanceResponse
{
- Success = true,
- Message = "Maintenance tasks completed",
- Result = result,
+ Message = "Maintenance run queued",
TriggeredBy = userEmail,
- TriggeredAt = triggeredAt
+ TriggeredAt = triggeredAt,
+ TargetDate = "2026-08-29",
+ AggregateOnly = true
});
}
diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceRunGateTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceRunGateTests.cs
new file mode 100644
index 00000000..34a16686
--- /dev/null
+++ b/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceRunGateTests.cs
@@ -0,0 +1,137 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using AutopilotMonitor.Functions.Services;
+using AutopilotMonitor.Functions.Services.Backup;
+using AutopilotMonitor.Functions.Services.Maintenance;
+using AutopilotMonitor.Shared.DataAccess;
+using AutopilotMonitor.Shared.Models;
+using Azure.Storage.Blobs.Specialized;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+using Xunit;
+
+namespace AutopilotMonitor.Functions.Tests;
+
+///
+/// Single-flight of the platform maintenance run: the 2h timer and the manual trigger enter
+/// through one gate, so a second run is skipped instead of aggregating, cleaning up and
+/// backfilling the same tables in parallel — and a skip never looks like an active run.
+///
+public class MaintenanceRunGateTests
+{
+ private sealed class Harness
+ {
+ public Mock LockStore { get; }
+ public Mock Lease { get; } = new();
+ public List OpsEvents { get; } = new();
+ public MaintenanceRunGate Sut { get; }
+
+ public Harness()
+ {
+ LockStore = new Mock(new SessionDeletionMaintenanceFunctionTests.MaintenanceBlobStub());
+ LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny(), It.IsAny()))
+ .ReturnsAsync(Lease.Object);
+
+ var opsRepo = new Mock();
+ opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny()))
+ .Returns(e => { lock (OpsEvents) OpsEvents.Add(e.EventType); return Task.CompletedTask; });
+ var adminConfig = new Mock(
+ Mock.Of(), NullLogger.Instance, new MemoryCache(new MemoryCacheOptions()));
+ var opsService = new OpsEventService(opsRepo.Object, NullLogger.Instance,
+ TestNotifications.InertOpsAlertDispatch(adminConfig.Object));
+
+ Sut = new MaintenanceRunGate(LockStore.Object, opsService, NullLogger.Instance);
+ }
+
+ public void HoldLease() =>
+ LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny(), It.IsAny()))
+ .ThrowsAsync(new LeaseHeldException("held", new Exception("409")));
+ }
+
+ [Fact]
+ public async Task A_held_lease_skips_the_run_and_never_reports_it_as_started()
+ {
+ var h = new Harness();
+ h.HoldLease();
+ var ran = false;
+
+ var result = await h.Sut.RunExclusiveAsync("Timer", () => { ran = true; return Task.CompletedTask; });
+
+ Assert.False(result);
+ Assert.False(ran);
+ Assert.Equal(new[] { OpsEventTypes.MaintenanceSkippedLocked }, h.OpsEvents);
+ }
+
+ [Fact]
+ public async Task Started_is_written_under_the_lease_before_the_body_and_the_lease_is_released_last()
+ {
+ var h = new Harness();
+ var order = new List();
+ h.Lease.Setup(l => l.ReleaseAsync(It.IsAny(), It.IsAny()))
+ .Callback(() => order.Add("release"))
+ .ReturnsAsync(Mock.Of>());
+
+ var result = await h.Sut.RunExclusiveAsync("admin@example.com", () =>
+ {
+ order.Add($"body after [{string.Join(",", h.OpsEvents)}]");
+ return Task.CompletedTask;
+ });
+
+ Assert.True(result);
+ Assert.Equal(new[] { $"body after [{OpsEventTypes.MaintenanceStarted}]", "release" }, order);
+ }
+
+ [Fact]
+ public async Task The_lease_is_released_when_the_body_throws()
+ {
+ var h = new Harness();
+ var released = false;
+ h.Lease.Setup(l => l.ReleaseAsync(It.IsAny(), It.IsAny()))
+ .Callback(() => released = true)
+ .ReturnsAsync(Mock.Of>());
+
+ await Assert.ThrowsAsync(() =>
+ h.Sut.RunExclusiveAsync("Timer", () => throw new InvalidOperationException("boom")));
+
+ Assert.True(released);
+ }
+
+ [Fact]
+ public async Task Probe_reports_an_active_run_while_the_lease_is_held()
+ {
+ var h = new Harness();
+ h.HoldLease();
+
+ Assert.True(await h.Sut.IsRunActiveAsync());
+ Assert.Empty(h.OpsEvents);
+ }
+
+ [Fact]
+ public async Task Probe_releases_its_lease_and_reports_no_active_run()
+ {
+ var h = new Harness();
+ var released = false;
+ h.Lease.Setup(l => l.ReleaseAsync(It.IsAny(), It.IsAny()))
+ .Callback(() => released = true)
+ .ReturnsAsync(Mock.Of>());
+
+ Assert.False(await h.Sut.IsRunActiveAsync());
+ Assert.True(released);
+ }
+
+ [Fact]
+ public async Task Probe_lets_a_failed_release_surface_so_nothing_is_queued_behind_a_stuck_lease()
+ {
+ // A queued run picked up seconds later would find the probe's own lease still held,
+ // skip as "locked" and delete the message — the operator's run would silently vanish.
+ var h = new Harness();
+ h.Lease.Setup(l => l.ReleaseAsync(It.IsAny(), It.IsAny()))
+ .ThrowsAsync(new Azure.RequestFailedException(500, "storage down"));
+
+ await Assert.ThrowsAsync(() => h.Sut.IsRunActiveAsync());
+ }
+}
diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TriggerMaintenanceFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TriggerMaintenanceFunction.cs
index f9003e9a..b64d81c1 100644
--- a/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TriggerMaintenanceFunction.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TriggerMaintenanceFunction.cs
@@ -2,7 +2,7 @@ using System;
using System.Net;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
-using AutopilotMonitor.Functions.Services;
+using AutopilotMonitor.Functions.Services.Maintenance;
using AutopilotMonitor.Shared.Models;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
@@ -11,24 +11,29 @@ using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Admin
{
///
- /// Allows Global Admins to manually trigger maintenance tasks
+ /// POST /api/maintenance/trigger — manual Global Admin trigger of the platform
+ /// maintenance run. The run takes minutes, so the request only queues it:
+ /// 202 Accepted when queued, 409 while a run (timer or manual) is active,
+ /// 500 when the enqueue failed — never a hollow success. Progress and the run report
+ /// surface as Maintenance* ops events.
///
public class TriggerMaintenanceFunction
{
private readonly ILogger _logger;
- private readonly MaintenanceService _maintenanceService;
+ private readonly IMaintenanceTriggerProducer _producer;
+ private readonly MaintenanceRunGate _runGate;
public TriggerMaintenanceFunction(
ILogger logger,
- MaintenanceService maintenanceService)
+ IMaintenanceTriggerProducer producer,
+ MaintenanceRunGate runGate)
{
_logger = logger;
- _maintenanceService = maintenanceService;
+ _producer = producer;
+ _runGate = runGate;
}
///
- /// POST /api/maintenance/trigger
- /// Manually trigger maintenance tasks (Global Admin only)
/// Query parameters:
/// - date: Optional date to aggregate (yyyy-MM-dd). If not provided, uses yesterday.
/// - aggregateOnly: If true, only runs aggregation (skips timeout and cleanup)
@@ -38,45 +43,42 @@ namespace AutopilotMonitor.Functions.Functions.Admin
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "maintenance/trigger")] HttpRequestData req,
FunctionContext context)
{
- _logger.LogInformation("Manual maintenance trigger requested");
-
// Authentication + GlobalAdminOnly authorization enforced by PolicyEnforcementMiddleware
- var userEmail = TenantHelper.GetUserIdentifier(req);
-
- _logger.LogInformation($"Maintenance trigger initiated by Global Admin: {userEmail}");
+ var userEmail = TenantHelper.GetUserIdentifier(req) ?? "GlobalAdmin";
+ var ct = context.CancellationToken;
try
{
- // Parse query parameters
- var dateParam = req.Query["date"];
- var aggregateOnlyParam = req.Query["aggregateOnly"];
-
- if (!QueryParams.TryUtcInstant(dateParam, "date", out var dateInstant, out _))
+ if (!QueryParams.TryUtcInstant(req.Query["date"], "date", out var dateInstant, out _))
{
return await req.BadRequestAsync("Invalid date format. Use yyyy-MM-dd");
}
DateTime? targetDate = dateInstant?.Date;
- if (targetDate.HasValue)
+ bool aggregateOnly = req.Query["aggregateOnly"]?.ToLower() == "true";
+
+ if (await _runGate.IsRunActiveAsync(ct))
{
- _logger.LogInformation($"Manual maintenance for date: {targetDate:yyyy-MM-dd}");
+ return await req.ConflictAsync("A maintenance run is already active.");
}
- bool aggregateOnly = aggregateOnlyParam?.ToLower() == "true";
-
- // Execute maintenance tasks
- var result = await _maintenanceService.RunManualAsync(targetDate, aggregateOnly, userEmail);
-
- var response = req.CreateResponse(HttpStatusCode.OK);
- await response.WriteAsJsonAsync(new TriggerMaintenanceResponse
+ await _producer.EnqueueAsync(new MaintenanceTriggerEnvelope
{
- Success = true,
- Message = "Maintenance tasks completed",
- Result = result,
TriggeredBy = userEmail,
- TriggeredAt = DateTime.UtcNow
- });
+ TargetDate = targetDate,
+ AggregateOnly = aggregateOnly,
+ }, ct);
- return response;
+ _logger.LogInformation("TriggerMaintenance: run queued (triggeredBy={TriggeredBy}, date={Date:yyyy-MM-dd}, aggregateOnly={AggregateOnly})",
+ userEmail, targetDate, aggregateOnly);
+
+ return await req.JsonAsync(HttpStatusCode.Accepted, new TriggerMaintenanceResponse
+ {
+ Message = "Maintenance run queued",
+ TriggeredBy = userEmail,
+ TriggeredAt = DateTime.UtcNow,
+ TargetDate = targetDate?.ToString("yyyy-MM-dd"),
+ AggregateOnly = aggregateOnly,
+ });
}
catch (Exception ex)
{
diff --git a/src/Backend/AutopilotMonitor.Functions/Program.cs b/src/Backend/AutopilotMonitor.Functions/Program.cs
index 25179a66..f6da9d08 100644
--- a/src/Backend/AutopilotMonitor.Functions/Program.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Program.cs
@@ -283,7 +283,17 @@ builder.Services.AddSingleton();
builder.Services.AddHttpClient()
.AddPolicyHandler((sp, _) => sp.GetRequiredService().ExternalDataApi);
+// Platform maintenance run: blob-lease gate (serializes the 2h timer, the manual trigger and a
+// second host instance), fail-hard queue producer for POST /api/maintenance/trigger, and the
+// BackgroundService worker that runs a queued manual run.
+builder.Services.AddSingleton();
+builder.Services.AddSingleton();
+builder.Services.AddSingleton<
+ AutopilotMonitor.Functions.Services.Maintenance.IMaintenanceTriggerProducer,
+ AutopilotMonitor.Functions.Services.Maintenance.AzureQueueMaintenanceTriggerProducer>();
builder.Services.AddSingleton();
+builder.Services.AddHostedService<
+ AutopilotMonitor.Functions.Services.Maintenance.MaintenanceTriggerQueueWorker>();
builder.Services.AddSingleton();
builder.Services.AddSingleton();
builder.Services.AddSingleton<
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionMaintenanceLockStore.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionMaintenanceLockStore.cs
index e370894f..4cd62ed7 100644
--- a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionMaintenanceLockStore.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionMaintenanceLockStore.cs
@@ -1,12 +1,6 @@
-using System;
-using System.IO;
-using System.Threading;
-using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Backup;
+using AutopilotMonitor.Functions.Services.Locking;
using AutopilotMonitor.Shared;
-using Azure;
-using Azure.Storage.Blobs.Models;
-using Azure.Storage.Blobs.Specialized;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Deletion
@@ -19,89 +13,25 @@ namespace AutopilotMonitor.Functions.Services.Deletion
/// Deliberately a DEDICATED sentinel ( in the
/// deletion-manifests container) rather than the backup sentinel in
/// — sharing that lease would serialize backups against the
- /// retention fanout, which have no reason to exclude each other. Lease mechanics
- /// (60s lease + 45s renewal,
- /// on contention) are identical to the backup path.
+ /// retention fanout, which have no reason to exclude each other.
///
///
/// The deletion-manifests container has a 30-day lifecycle policy that may delete an
- /// idle sentinel; re-ensures it on every acquire, so the
- /// lock self-heals. The manifest-TTL sweep skips the sentinel because _lock/… does
- /// not parse as a manifest blob name.
+ /// idle sentinel; the base class re-ensures it on every acquire, so the lock self-heals.
+ /// The manifest-TTL sweep skips the sentinel because _lock/… does not parse as a
+ /// manifest blob name.
///
///
- public class SessionDeletionMaintenanceLockStore
+ public class SessionDeletionMaintenanceLockStore : BlobLeaseLockStore
{
/// Sentinel blob path — leased by the timer / manual-trigger worker to serialize runs.
public const string LockBlobName = "_lock/session-deletion-maintenance.lock";
- /// Lease duration (Azure spec allows 15..60s for renewable leases).
- public static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(60);
-
- private readonly BlobStorageService _blobs;
- private readonly ILogger _logger;
- private int _containerEnsured;
-
public SessionDeletionMaintenanceLockStore(
BlobStorageService blobs,
ILogger logger)
+ : base(blobs, Constants.BlobContainers.DeletionManifests, LockBlobName, "session-deletion maintenance")
{
- _blobs = blobs;
- _logger = logger;
- }
-
- ///
- /// Attempts to acquire the session-deletion maintenance lease. Throws
- /// on 409 (another holder); other errors propagate.
- /// Caller is responsible for renewing (via ) and
- /// releasing. virtual so tests can stub it.
- ///
- public virtual async Task AcquireLeaseAsync(TimeSpan? leaseDuration = null, CancellationToken ct = default)
- {
- await EnsureSentinelAsync(ct).ConfigureAwait(false);
-
- var blob = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests).GetBlobClient(LockBlobName);
- var leaseClient = blob.GetBlobLeaseClient();
- try
- {
- await leaseClient.AcquireAsync(leaseDuration ?? LeaseDuration, conditions: null, ct).ConfigureAwait(false);
- return leaseClient;
- }
- catch (RequestFailedException ex) when (ex.Status == 409)
- {
- throw new LeaseHeldException("session-deletion maintenance lease is held by another run", ex);
- }
- }
-
- ///
- /// Idempotently ensures the sentinel blob exists — an AcquireLease on a missing blob
- /// throws 404. If-None-Match=* lets concurrent callers cooperate; both 409
- /// BlobAlreadyExists and 412 ConditionNotMet are the steady state.
- ///
- private async Task EnsureSentinelAsync(CancellationToken ct)
- {
- var container = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests);
- if (Interlocked.CompareExchange(ref _containerEnsured, 0, 0) == 0)
- {
- await container.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
- Interlocked.Exchange(ref _containerEnsured, 1);
- }
-
- var blob = container.GetBlobClient(LockBlobName);
- var options = new BlobUploadOptions
- {
- HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" },
- Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All },
- };
- try
- {
- using var empty = new MemoryStream(Array.Empty());
- await blob.UploadAsync(empty, options, ct).ConfigureAwait(false);
- }
- catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 412)
- {
- // Sentinel exists already — steady state.
- }
}
}
}
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Locking/BlobLeaseLockStore.cs b/src/Backend/AutopilotMonitor.Functions/Services/Locking/BlobLeaseLockStore.cs
new file mode 100644
index 00000000..60b6bbd2
--- /dev/null
+++ b/src/Backend/AutopilotMonitor.Functions/Services/Locking/BlobLeaseLockStore.cs
@@ -0,0 +1,95 @@
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using AutopilotMonitor.Functions.Services.Backup;
+using Azure;
+using Azure.Storage.Blobs.Models;
+using Azure.Storage.Blobs.Specialized;
+
+namespace AutopilotMonitor.Functions.Services.Locking
+{
+ ///
+ /// Blob-lease lock on a sentinel blob: serializes a run against every other entry point of
+ /// the same work (timer, manual trigger, a second host instance). One subclass per lock —
+ /// each names its own sentinel, because two kinds of work that have no reason to exclude
+ /// each other must not share a lease.
+ ///
+ /// Lease mechanics: 60s lease, renewed by every 45s,
+ /// on contention. The sentinel is re-ensured on every
+ /// acquire, so a lifecycle policy that deletes an idle sentinel cannot break the lock.
+ ///
+ ///
+ public abstract class BlobLeaseLockStore
+ {
+ /// Lease duration (Azure spec allows 15..60s for renewable leases).
+ public static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(60);
+
+ private readonly BlobStorageService _blobs;
+ private readonly string _containerName;
+ private readonly string _lockBlobName;
+ private readonly string _description;
+ private int _containerEnsured;
+
+ protected BlobLeaseLockStore(BlobStorageService blobs, string containerName, string lockBlobName, string description)
+ {
+ _blobs = blobs;
+ _containerName = containerName;
+ _lockBlobName = lockBlobName;
+ _description = description;
+ }
+
+ ///
+ /// Attempts to acquire the lease. Throws on 409 (another
+ /// holder); other errors propagate. Caller is responsible for renewing (via
+ /// ) and releasing. virtual so tests can stub it.
+ ///
+ public virtual async Task AcquireLeaseAsync(TimeSpan? leaseDuration = null, CancellationToken ct = default)
+ {
+ await EnsureSentinelAsync(ct).ConfigureAwait(false);
+
+ var blob = _blobs.GetContainerClient(_containerName).GetBlobClient(_lockBlobName);
+ var leaseClient = blob.GetBlobLeaseClient();
+ try
+ {
+ await leaseClient.AcquireAsync(leaseDuration ?? LeaseDuration, conditions: null, ct).ConfigureAwait(false);
+ return leaseClient;
+ }
+ catch (RequestFailedException ex) when (ex.Status == 409)
+ {
+ throw new LeaseHeldException($"{_description} lease is held by another run", ex);
+ }
+ }
+
+ ///
+ /// Idempotently ensures the sentinel blob exists — an AcquireLease on a missing blob
+ /// throws 404. If-None-Match=* lets concurrent callers cooperate; both 409
+ /// BlobAlreadyExists and 412 ConditionNotMet are the steady state.
+ ///
+ private async Task EnsureSentinelAsync(CancellationToken ct)
+ {
+ var container = _blobs.GetContainerClient(_containerName);
+ if (Interlocked.CompareExchange(ref _containerEnsured, 0, 0) == 0)
+ {
+ await container.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
+ Interlocked.Exchange(ref _containerEnsured, 1);
+ }
+
+ var blob = container.GetBlobClient(_lockBlobName);
+ var options = new BlobUploadOptions
+ {
+ HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" },
+ Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All },
+ };
+ try
+ {
+ using var empty = new MemoryStream(Array.Empty());
+ await blob.UploadAsync(empty, options, ct).ConfigureAwait(false);
+ }
+ catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 412)
+ {
+ // Sentinel exists already — steady state.
+ }
+ }
+ }
+}
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/AzureQueueMaintenanceTriggerProducer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/AzureQueueMaintenanceTriggerProducer.cs
new file mode 100644
index 00000000..343caab1
--- /dev/null
+++ b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/AzureQueueMaintenanceTriggerProducer.cs
@@ -0,0 +1,60 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using AutopilotMonitor.Functions.Services.Queueing;
+using AutopilotMonitor.Shared;
+using Azure.Storage.Queues;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+
+namespace AutopilotMonitor.Functions.Services.Maintenance
+{
+ ///
+ /// Azure-Storage-Queue producer for the manual maintenance trigger. Same shape as the
+ /// session-deletion and backup trigger producers:
+ /// (Managed Identity preferred), fail-hard.
+ ///
+ public sealed class AzureQueueMaintenanceTriggerProducer : IMaintenanceTriggerProducer
+ {
+ private readonly QueueClient _queueClient;
+ private readonly ILogger _logger;
+ private int _queueEnsured;
+
+ public AzureQueueMaintenanceTriggerProducer(
+ QueueClientFactory queueFactory,
+ ILogger logger)
+ {
+ _logger = logger;
+ _queueClient = queueFactory.Create(Constants.QueueNames.MaintenanceTrigger);
+ }
+
+ public async Task EnqueueAsync(MaintenanceTriggerEnvelope envelope, CancellationToken ct = default)
+ {
+ if (envelope == null) throw new ArgumentNullException(nameof(envelope));
+ if (string.IsNullOrEmpty(envelope.TriggeredBy)) throw new ArgumentException("TriggeredBy required", nameof(envelope));
+
+ await EnsureQueueExistsAsync(ct).ConfigureAwait(false);
+
+ // Fail-hard: do NOT swallow. Caller (HTTP trigger) catches and returns 500.
+ var body = JsonConvert.SerializeObject(envelope);
+ await _queueClient.SendMessageAsync(body, ct).ConfigureAwait(false);
+ _logger.LogInformation("Maintenance trigger enqueued (triggeredBy={TriggeredBy})", envelope.TriggeredBy);
+ }
+
+ private async Task EnsureQueueExistsAsync(CancellationToken ct)
+ {
+ if (_queueEnsured == 1) return;
+ try
+ {
+ await _queueClient.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
+ Interlocked.Exchange(ref _queueEnsured, 1);
+ }
+ catch (Exception ex)
+ {
+ // Ensure failure during enqueue is itself a fail-hard signal — surface to the caller.
+ _logger.LogError(ex, "Maintenance trigger queue CreateIfNotExists failed");
+ throw;
+ }
+ }
+ }
+}
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/IMaintenanceTriggerProducer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/IMaintenanceTriggerProducer.cs
new file mode 100644
index 00000000..79e766f4
--- /dev/null
+++ b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/IMaintenanceTriggerProducer.cs
@@ -0,0 +1,29 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace AutopilotMonitor.Functions.Services.Maintenance
+{
+ ///
+ /// Queue envelope of a manually triggered maintenance run. There is no per-run state row —
+ /// the Maintenance* ops events are the status surface.
+ ///
+ public sealed class MaintenanceTriggerEnvelope
+ {
+ public string TriggeredBy { get; set; } = string.Empty;
+
+ /// Date to aggregate (UTC date); null = yesterday.
+ public DateTime? TargetDate { get; set; }
+
+ public bool AggregateOnly { get; set; }
+ }
+
+ ///
+ /// Producer for the manual maintenance trigger. Fail-hard: send failures propagate so
+ /// the HTTP trigger answers 5xx instead of a hollow 202.
+ ///
+ public interface IMaintenanceTriggerProducer
+ {
+ Task EnqueueAsync(MaintenanceTriggerEnvelope envelope, CancellationToken ct = default);
+ }
+}
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceRunGate.cs b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceRunGate.cs
new file mode 100644
index 00000000..33357e6a
--- /dev/null
+++ b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceRunGate.cs
@@ -0,0 +1,92 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using AutopilotMonitor.Functions.Services.Backup;
+using Azure.Storage.Blobs.Specialized;
+using Microsoft.Extensions.Logging;
+
+namespace AutopilotMonitor.Functions.Services.Maintenance
+{
+ ///
+ /// Single-flight gate for the platform maintenance run: the 2h timer, the manual trigger
+ /// and a second host instance all enter through , so two
+ /// runs never aggregate, clean up and backfill the same tables at the same time.
+ ///
+ public class MaintenanceRunGate
+ {
+ private readonly MaintenanceRunLockStore _lockStore;
+ private readonly OpsEventService _opsEvents;
+ private readonly ILogger _logger;
+
+ public MaintenanceRunGate(MaintenanceRunLockStore lockStore, OpsEventService opsEvents, ILogger logger)
+ {
+ _lockStore = lockStore;
+ _opsEvents = opsEvents;
+ _logger = logger;
+ }
+
+ ///
+ /// Acquire+release probe for the manual trigger: a held lease means a run is active RIGHT
+ /// NOW, so the operator is told instead of queuing a message that would only produce a
+ /// SkippedLocked event. Racing the probe (a run starts between probe and worker pickup)
+ /// is benign: the worker's own acquire lands on the SkippedLocked path.
+ ///
+ public virtual async Task IsRunActiveAsync(CancellationToken ct = default)
+ {
+ BlobLeaseClient lease;
+ try
+ {
+ lease = await _lockStore.AcquireLeaseAsync(ct: ct).ConfigureAwait(false);
+ }
+ catch (LeaseHeldException)
+ {
+ return true;
+ }
+
+ // A probe lease that cannot be released would still be held when the worker picks the
+ // message up seconds later — the run would be skipped as "locked" and the message
+ // deleted. Let the failure surface: the trigger answers 5xx and nothing is queued.
+ await lease.ReleaseAsync(cancellationToken: ct).ConfigureAwait(false);
+ return false;
+ }
+
+ ///
+ /// Runs under the maintenance-run lease. Returns false without
+ /// running it when another run holds the lease (SkippedLocked ops event). The Started
+ /// event is written AFTER the lease is held, so a skip never looks like an active run,
+ /// and the lease is released LAST, so the body's terminal ops event is still exclusive.
+ ///
+ public virtual async Task RunExclusiveAsync(string triggeredBy, Func body)
+ {
+ BlobLeaseClient lease;
+ try
+ {
+ lease = await _lockStore.AcquireLeaseAsync().ConfigureAwait(false);
+ }
+ catch (LeaseHeldException)
+ {
+ _logger.LogWarning("Maintenance: lease held by another run — skipping (triggeredBy={TriggeredBy})", triggeredBy);
+ await _opsEvents.RecordMaintenanceSkippedLockedAsync(triggeredBy).ConfigureAwait(false);
+ return false;
+ }
+
+ // The maintenance body takes no cancellation token; a lost lease is reported, not
+ // enforced — the run is minutes long and the lease renews every 45s.
+ using var leaseLost = new CancellationTokenSource();
+ var leaseHolder = new MaintenanceLeaseHolder(lease, leaseLost, _logger);
+ try
+ {
+ await _opsEvents.RecordMaintenanceStartedAsync(triggeredBy).ConfigureAwait(false);
+ await body().ConfigureAwait(false);
+ return true;
+ }
+ finally
+ {
+ if (leaseHolder.RenewalFailureReason != null)
+ _logger.LogWarning("Maintenance: the run lease could not be renewed during the run ({Reason}) — another run may have overlapped",
+ leaseHolder.RenewalFailureReason);
+ await leaseHolder.DisposeAsync().ConfigureAwait(false);
+ }
+ }
+ }
+}
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceRunLockStore.cs b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceRunLockStore.cs
new file mode 100644
index 00000000..fce499af
--- /dev/null
+++ b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceRunLockStore.cs
@@ -0,0 +1,21 @@
+using AutopilotMonitor.Functions.Services.Locking;
+using AutopilotMonitor.Shared;
+
+namespace AutopilotMonitor.Functions.Services.Maintenance
+{
+ ///
+ /// Blob-lease lock for the platform maintenance run (2h timer and manual trigger). Its own
+ /// sentinel: the session-deletion maintenance and the critical-table backup have no reason
+ /// to exclude this run, so they keep their own leases.
+ ///
+ public class MaintenanceRunLockStore : BlobLeaseLockStore
+ {
+ /// Sentinel blob path. _lock/… does not parse as a manifest blob name, so the manifest-TTL sweep skips it.
+ public const string LockBlobName = "_lock/platform-maintenance.lock";
+
+ public MaintenanceRunLockStore(BlobStorageService blobs)
+ : base(blobs, Constants.BlobContainers.DeletionManifests, LockBlobName, "platform maintenance")
+ {
+ }
+ }
+}
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceTriggerQueueWorker.cs b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceTriggerQueueWorker.cs
new file mode 100644
index 00000000..3b923e85
--- /dev/null
+++ b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceTriggerQueueWorker.cs
@@ -0,0 +1,49 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using AutopilotMonitor.Functions.Services.Queueing;
+using AutopilotMonitor.Shared;
+using Microsoft.Extensions.Logging;
+
+namespace AutopilotMonitor.Functions.Services.Maintenance
+{
+ ///
+ /// Background worker for the maintenance-trigger queue — runs the manually triggered
+ /// maintenance off the HTTP request, which would otherwise have to stay open for minutes
+ /// and dies at the gateway limit.
+ ///
+ /// A self-polling worker, not a queue-triggered function, because the run holds a lease
+ /// for its whole lifetime. Concurrency against the 2h timer is handled inside
+ /// by the run gate: a lease-held run
+ /// returns normally (SkippedLocked ops event), so the message is deleted rather than
+ /// retried — the operator asked for "a run", and one is already active. A failing run
+ /// reports MaintenanceFailed and returns as well: maintenance is not retried blindly.
+ ///
+ ///
+ public sealed class MaintenanceTriggerQueueWorker : QueuePollingWorker
+ {
+ private readonly MaintenanceService _maintenance;
+
+ public MaintenanceTriggerQueueWorker(
+ QueueClientFactory queueFactory,
+ MaintenanceService maintenance,
+ ILogger logger)
+ : base(queueFactory, Constants.QueueNames.MaintenanceTrigger, logger, Constants.QueueNames.MaintenanceTriggerPoison)
+ {
+ _maintenance = maintenance ?? throw new ArgumentNullException(nameof(maintenance));
+ }
+
+ // The host functionTimeout (60min) is the upper bound of a run; the message must not
+ // reappear while its run is still going.
+ protected override TimeSpan VisibilityTimeout => TimeSpan.FromMinutes(60);
+
+ protected override bool TryValidate(MaintenanceTriggerEnvelope envelope)
+ => !string.IsNullOrEmpty(envelope.TriggeredBy);
+
+ protected override string DescribeForLog(MaintenanceTriggerEnvelope envelope)
+ => $"triggeredBy={envelope.TriggeredBy} targetDate={envelope.TargetDate:yyyy-MM-dd} aggregateOnly={envelope.AggregateOnly}";
+
+ protected override Task HandleAsync(MaintenanceTriggerEnvelope envelope, CancellationToken ct)
+ => _maintenance.RunManualAsync(envelope.TargetDate, envelope.AggregateOnly, envelope.TriggeredBy);
+ }
+}
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs
index 98f9d15d..729bbfb8 100644
--- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs
@@ -55,6 +55,7 @@ namespace AutopilotMonitor.Functions.Services
private readonly IHttpClientFactory _httpClientFactory;
private readonly PreviewWhitelistService _previewWhitelistService;
private readonly IStorageInitializer _storageInitializer;
+ private readonly Maintenance.MaintenanceRunGate _runGate;
private readonly ILogger _logger;
private const string PlatformStatsAliasFileName = "platform-stats.json";
@@ -90,6 +91,7 @@ namespace AutopilotMonitor.Functions.Services
IHttpClientFactory httpClientFactory,
PreviewWhitelistService previewWhitelistService,
IStorageInitializer storageInitializer,
+ Maintenance.MaintenanceRunGate runGate,
ILogger logger)
{
_maintenanceRepo = maintenanceRepo;
@@ -120,6 +122,7 @@ namespace AutopilotMonitor.Functions.Services
_httpClientFactory = httpClientFactory;
_previewWhitelistService = previewWhitelistService;
_storageInitializer = storageInitializer;
+ _runGate = runGate;
_logger = logger;
}
@@ -136,9 +139,12 @@ namespace AutopilotMonitor.Functions.Services
}
///
- /// Runs all maintenance tasks (used by the daily timer trigger)
+ /// Runs all maintenance tasks (used by the 2h timer trigger). Single-flight with the
+ /// manual trigger: skipped when another run holds the maintenance-run lease.
///
- public async Task RunAllAsync()
+ public Task RunAllAsync() => _runGate.RunExclusiveAsync("Timer", RunAllCoreAsync);
+
+ private async Task RunAllCoreAsync()
{
_logger.LogInformation($"Daily maintenance started at {DateTime.UtcNow}");
var maintenanceStart = Stopwatch.StartNew();
@@ -246,9 +252,27 @@ namespace AutopilotMonitor.Functions.Services
=> window.Where(s => s.StartedAt >= windowStart).ToList();
///
- /// Manually triggered maintenance with flexible date selection
+ /// Manually triggered maintenance with flexible date selection. Single-flight with the
+ /// timer: when another run holds the maintenance-run lease, nothing runs and the result
+ /// says so.
///
public async Task RunManualAsync(DateTime? targetDate = null, bool aggregateOnly = false, string triggeredBy = "Unknown")
+ {
+ MaintenanceResult? result = null;
+ var ran = await _runGate.RunExclusiveAsync(triggeredBy,
+ async () => result = await RunManualCoreAsync(targetDate, aggregateOnly, triggeredBy));
+ return ran && result != null
+ ? result
+ : new MaintenanceResult
+ {
+ TriggeredBy = triggeredBy,
+ TriggeredAt = DateTime.UtcNow,
+ Success = false,
+ Error = "Another maintenance run is active",
+ };
+ }
+
+ private async Task RunManualCoreAsync(DateTime? targetDate, bool aggregateOnly, string triggeredBy)
{
_logger.LogInformation($"Manual maintenance triggered by {triggeredBy} at {DateTime.UtcNow}");
var maintenanceStart = Stopwatch.StartNew();
@@ -329,7 +353,7 @@ namespace AutopilotMonitor.Functions.Services
result.Success = true;
_logger.LogInformation($"Manual maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms");
- await _opsEventService.RecordMaintenanceCompletedAsync(result.DurationMs, triggeredBy);
+ await _opsEventService.RecordMaintenanceCompletedAsync(result.DurationMs, triggeredBy, result);
return result;
}
catch (Exception ex)
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs
index ad1e84c4..5cb68acf 100644
--- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs
@@ -64,10 +64,42 @@ namespace AutopilotMonitor.Functions.Services
// ── Maintenance ────────────────────────────────────────────────────────
- public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy)
+ ///
+ /// Emitted after the maintenance-run lease was acquired, so a lease-skip never
+ /// masquerades as an active run — the portal treats "latest Started newer than latest
+ /// Completed/Failed" as run-active.
+ ///
+ public Task RecordMaintenanceStartedAsync(string triggeredBy)
+ => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceStarted, OpsEventSeverity.Info,
+ $"Maintenance run started (triggered by {triggeredBy})",
+ null, triggeredBy, new { triggeredBy });
+
+ /// Another run (timer, manual trigger or a second host instance) held the maintenance-run lease.
+ public Task RecordMaintenanceSkippedLockedAsync(string triggeredBy)
+ => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceSkippedLocked, OpsEventSeverity.Info,
+ $"Maintenance run skipped — another run holds the lease (triggered by {triggeredBy})",
+ null, triggeredBy, new { reason = "lease held by another run", triggeredBy });
+
+ ///
+ /// is the run report of a manual run: the trigger answers 202
+ /// before the run starts, so this event is where the operator reads what the run did.
+ ///
+ public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy, MaintenanceResult? report = null)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceCompleted, OpsEventSeverity.Info,
$"Maintenance completed in {durationMs}ms (triggered by {triggeredBy})",
- null, triggeredBy, new { durationMs });
+ null, triggeredBy,
+ report == null
+ ? (object)new { durationMs }
+ : new
+ {
+ durationMs,
+ aggregatedDate = report.AggregatedDate,
+ stalledSessionsChecked = report.StalledSessionsChecked,
+ dataCleanupExecuted = report.DataCleanupExecuted,
+ platformStatsRecomputed = report.PlatformStatsRecomputed,
+ devicesBlockedForExcessiveData = report.DevicesBlockedForExcessiveData,
+ contactEmailsBackfilled = report.ContactEmailsBackfilled,
+ });
public Task RecordMaintenanceFailedAsync(string error, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceFailed, OpsEventSeverity.Error,
diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts
index 760083d8..95d093a8 100644
--- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts
+++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts
@@ -2787,22 +2787,6 @@ export interface LogQueryRequest {
budgetSeconds?: number | null;
}
-/** Run report of a manual maintenance run (POST maintenance/trigger). */
-export interface MaintenanceResult {
- success: boolean;
- error?: string;
- triggeredBy: string;
- triggeredAt: string;
- durationMs: number;
- stalledSessionsChecked: boolean;
- metricsAggregated: boolean;
- aggregatedDate?: string;
- dataCleanupExecuted: boolean;
- platformStatsRecomputed: boolean;
- devicesBlockedForExcessiveData: number;
- contactEmailsBackfilled: number;
-}
-
/** One managed tenant as the managing tenant sees it. */
export interface ManagedTenantItem {
tenantId: string;
@@ -5158,13 +5142,14 @@ export interface TriggerEpssSyncResponse {
syncedAt: string;
}
-/** Response of POST maintenance/trigger. */
+/** Response of POST maintenance/trigger (202 Accepted): the run is queued, not done. Its progress and its report surface as Maintenance* ops events (Started, Completed, Failed, SkippedLocked). */
export interface TriggerMaintenanceResponse {
- success: boolean;
message: string;
- result: MaintenanceResult;
triggeredBy: string;
triggeredAt: string;
+ /** Date to aggregate (yyyy-MM-dd); null = yesterday. */
+ targetDate?: string;
+ aggregateOnly: boolean;
}
/** Response of POST vulnerability/sync-msrc (dual-purpose: success reflects whether the MSRC index refresh finished without error). */
diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts
index 9b05adfd..1f99b4ae 100644
--- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts
+++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts
@@ -64,6 +64,8 @@ export const OPS_EVENT_TYPES = [
"MaintenanceCompleted",
"MaintenanceFailed",
"MaintenanceLongRunning",
+ "MaintenanceStarted",
+ "MaintenanceSkippedLocked",
"SessionSweepCompleted",
"SessionSweepFailed",
"OpsEventCleanup",
diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs
index bbcd8dc9..9030ebeb 100644
--- a/src/Shared/AutopilotMonitor.Shared/Constants.cs
+++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs
@@ -1701,6 +1701,20 @@ namespace AutopilotMonitor.Shared
/// Poison sibling of .
public const string SessionDeletionMaintenancePoison = "session-deletion-maintenance-poison";
+ ///
+ /// Manual-trigger queue for the platform maintenance run. Producer = HTTP trigger
+ /// /api/maintenance/trigger (fail-hard, answers 202); consumer =
+ /// MaintenanceTriggerQueueWorker (self-managed BackgroundService,
+ /// VisibilityTimeout=60min = the host functionTimeout). Envelope
+ /// { triggeredBy, targetDate, aggregateOnly }; concurrency against the 2h timer
+ /// is serialized by the maintenance-run blob lease, not the queue.
+ /// Poison-suffix -poison, max-dequeue 5.
+ ///
+ public const string MaintenanceTrigger = "maintenance-trigger";
+
+ /// Poison sibling of .
+ public const string MaintenanceTriggerPoison = "maintenance-trigger-poison";
+
///
/// IME-MSI archive fan-out. Producer = EventIngestProcessor when
/// RecordImeVersionAsync reports a genuinely NEW IME version (first sighting
diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs
index 6ee5d8c8..5d89a9e8 100644
--- a/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs
+++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs
@@ -32,6 +32,8 @@ namespace AutopilotMonitor.Shared.DataAccess
public const string MaintenanceCompleted = "MaintenanceCompleted";
public const string MaintenanceFailed = "MaintenanceFailed";
public const string MaintenanceLongRunning = "MaintenanceLongRunning";
+ public const string MaintenanceStarted = "MaintenanceStarted";
+ public const string MaintenanceSkippedLocked = "MaintenanceSkippedLocked";
public const string SessionSweepCompleted = "SessionSweepCompleted";
public const string SessionSweepFailed = "SessionSweepFailed";
public const string OpsEventCleanup = "OpsEventCleanup";
@@ -135,7 +137,7 @@ namespace AutopilotMonitor.Shared.DataAccess
public static readonly IReadOnlyList All = new[]
{
ConsentFlowStarted, ConsentFlowSuccess, ConsentFlowFailed, ConsentRedirectUriMismatch, AppHomingFlipped, AppHomingFlippedWithEntraRoles,
- MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift,
+ MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift,
DeviceBlocked, VersionBlocked, SessionTenantConflict, SessionOwnerMismatch, KillSignalDelivered, EmbeddedCertExpiringSoon, EmbeddedCertExpiringUrgent, EmbeddedCertExpired, EmbeddedCertBundleEmpty, SignalRConnectionsHigh, SignalRConnectionsCritical, SignalRMessagesHigh, SignalRMessagesCritical, PoisonQueueBacklogHigh, PoisonQueueBacklogCritical, ExcessiveSessionEventsAutoActioned, PrivilegedRouteDenied, McpServicePrincipalFirstSeen,
OffboardingFeedbackReceived, TenantOffboarded, TenantOffboardingFailed, TenantAutoApproved, WelcomeEmailSent, WelcomeEmailSkipped, WelcomeEmailFailed, TenantTrialStarted, TenantTrialExpiring, TenantTrialExpired, TenantPlanDowngraded, TenantRetentionGraceExpiring, TenantRetentionGraceEnded, RuleFrequencyRegression, AppVersionDurationRegression, CollectLogsQuickConfigEnabled, DiagnosticsUploadEnabled, DiagnosticsUploadDisabled,
SessionActionQueued, SessionTimeouts, AgentEmergencyBreak, AgentBinaryIntegrityMismatch, CmTraceTimeSkewRegression, ExcessiveSessionEvents, NewImeVersionDetected, ImePatternDriftSuspected, BlobStorageMissing, BlobStorageUnreachable, TelemetryItemsRejected,
diff --git a/src/Shared/AutopilotMonitor.Shared/Models/AdminApiModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/AdminApiModels.cs
index ecf02cb4..cf88bf97 100644
--- a/src/Shared/AutopilotMonitor.Shared/Models/AdminApiModels.cs
+++ b/src/Shared/AutopilotMonitor.Shared/Models/AdminApiModels.cs
@@ -145,15 +145,20 @@ namespace AutopilotMonitor.Shared.Models
public DateTime TriggeredAt { get; set; }
}
- /// Response of POST maintenance/trigger.
+ ///
+ /// Response of POST maintenance/trigger (202 Accepted): the run is queued, not done. Its
+ /// progress and its report surface as Maintenance* ops events (Started, Completed, Failed,
+ /// SkippedLocked).
+ ///
// Declaration order == wire order.
public class TriggerMaintenanceResponse : IApiResponse
{
- public bool Success { get; set; }
public string Message { get; set; } = default!;
- public MaintenanceResult Result { get; set; } = default!;
public string TriggeredBy { get; set; } = default!;
public DateTime TriggeredAt { get; set; }
+ /// Date to aggregate (yyyy-MM-dd); null = yesterday.
+ public string? TargetDate { get; set; }
+ public bool AggregateOnly { get; set; }
}
///
@@ -218,7 +223,7 @@ namespace AutopilotMonitor.Shared.Models
public string Reason { get; set; } = string.Empty;
}
- /// Run report of a manual maintenance run (POST maintenance/trigger).
+ /// Run report of a manual maintenance run; its figures travel in the MaintenanceCompleted ops event.
// Declaration order == wire order.
public class MaintenanceResult
{
diff --git a/src/Web/autopilot-monitor-web/app/admin/components/MaintenanceTriggerSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/MaintenanceTriggerSection.tsx
index d45aee27..4b9b0b12 100644
--- a/src/Web/autopilot-monitor-web/app/admin/components/MaintenanceTriggerSection.tsx
+++ b/src/Web/autopilot-monitor-web/app/admin/components/MaintenanceTriggerSection.tsx
@@ -2,16 +2,32 @@
import { useState } from "react";
import { api } from "@/lib/api";
-import { apiErrorText, fetchJson, fetchOk } from "@/lib/apiClient";
+import { ApiError, apiErrorText, fetchJson, fetchOk } from "@/lib/apiClient";
+import { parseOpsDetails, useOpsRunStatus, type OpsRunLifecycle } from "@/hooks/useOpsRunStatus";
import { SectionCardHeader } from "@/components/SectionCardHeader";
import type { GetLatestVersionsResponse } from "@/utils/wire-types.generated";
+const LIFECYCLE: OpsRunLifecycle = {
+ category: "Maintenance",
+ started: "MaintenanceStarted",
+ completed: "MaintenanceCompleted",
+ failed: "MaintenanceFailed",
+};
+
+// A run takes a few minutes; while one is active the status line follows it on its own.
+const POLL_WHILE_ACTIVE_MS = 15_000;
+
interface MaintenanceTriggerSectionProps {
getAccessToken: () => Promise;
setError: (error: string | null) => void;
setSuccessMessage: (message: string | null) => void;
}
+function formatRunDuration(durationMs: number | undefined): string {
+ if (typeof durationMs !== "number") return "";
+ return durationMs < 60_000 ? ` in ${Math.round(durationMs / 1000)} s` : ` in ${Math.round(durationMs / 60_000)} min`;
+}
+
export function MaintenanceTriggerSection({
getAccessToken,
setError,
@@ -23,6 +39,8 @@ export function MaintenanceTriggerSection({
// (react-hooks/purity) and the cap doesn't need a live midnight rollover.
const [maxMaintenanceDate] = useState(() => new Date(Date.now() - 86400000).toISOString().split('T')[0]);
const [refreshingVersions, setRefreshingVersions] = useState(false);
+ const { status: runStatus, refresh: refreshRunStatus } = useOpsRunStatus(getAccessToken, setError, LIFECYCLE, POLL_WHILE_ACTIVE_MS);
+ const runActive = runStatus.kind === "active";
const handleRefreshLatestVersions = async () => {
try {
@@ -53,15 +71,20 @@ export function MaintenanceTriggerSection({
setError(null);
setSuccessMessage(null);
- // The run answers when it is done (minutes), far beyond the default fetch timeout;
- // the platform gateway cuts a request at 230 s, so waiting longer gains nothing.
- await fetchOk(api.maintenance.trigger(maintenanceDate || undefined), getAccessToken, {
- method: "POST",
- signal: AbortSignal.timeout(230_000),
- });
+ // 202 = queued (the run reports through Maintenance* ops events); 409 = a run is
+ // already active, which is not an error for the operator.
+ const queued = await fetchOk(api.maintenance.trigger(maintenanceDate || undefined), getAccessToken, { method: "POST" })
+ .then(() => true)
+ .catch((err: unknown) => {
+ if (err instanceof ApiError && err.status === 409) return false;
+ throw err;
+ });
const dateInfo = maintenanceDate ? ` for ${maintenanceDate}` : '';
- setSuccessMessage(`Maintenance job completed successfully${dateInfo}!`);
+ setSuccessMessage(queued
+ ? `Maintenance run queued${dateInfo} — its status appears below shortly.`
+ : "A maintenance run is already active.");
+ refreshRunStatus();
setTimeout(() => setSuccessMessage(null), 5000);
} catch (err) {
@@ -142,8 +165,8 @@ export function MaintenanceTriggerSection({
- Warning: This operation runs across all tenants and may take several minutes to complete.
- Use this only for testing or when immediate cleanup is needed.
+ Warning: This operation runs across all tenants. The run is queued and takes a few minutes in the background;
+ only one run is active at a time, the 2h timer included. Use this only for testing or when immediate cleanup is needed.
@@ -179,16 +202,35 @@ export function MaintenanceTriggerSection({
-
+
+
+ {runStatus.kind === "active" && (
+ <>
+ Maintenance run active — started{" "}
+ {new Date(runStatus.since).toLocaleString()} (triggered by {runStatus.triggeredBy}).
+
+ )}
+ {runStatus.kind === "failed" && (
+ <>
+ Last run failed ({new Date(runStatus.at).toLocaleString()}): {runStatus.message}
+
+ )}
+ {runStatus.kind === "completed" && (
+ <>
+ Last run completed {new Date(runStatus.at).toLocaleString()}
+ {formatRunDuration(parseOpsDetails<{ durationMs?: number }>(runStatus.details).durationMs)}.
+
+ )}
+