feat(backend,web): manual maintenance trigger answers 202, the run is single-flight (D-255)

The trigger ran the whole maintenance inside the HTTP request (about three
minutes against a 230 s gateway limit): the portal reported a timeout while the
run went on, a second click started a second run, and nothing serialized those
runs against each other or against the 2h timer.

Backend
- MaintenanceRunGate: the timer and the manual run enter through one blob
  lease. A held lease writes MaintenanceSkippedLocked and runs nothing;
  MaintenanceStarted is written after the lease is held, the lease is released
  last.
- POST maintenance/trigger validates the date, probes the lease (409 while a
  run is active), enqueues on maintenance-trigger (fail-hard) and answers 202
  with the typed TriggerMaintenanceResponse. MaintenanceTriggerQueueWorker
  (self-polling) runs it; a failing or skipped run is not retried.
- The run report travels in the details of MaintenanceCompleted.
- BlobLeaseLockStore is the shared base of the lease lock stores;
  SessionDeletionMaintenanceLockStore derives from it, behaviour unchanged.
- The probe lets a failed lease release surface as 5xx: a probe lease still
  held at worker pickup would turn the queued run into a silent skip.

Web
- useOpsRunStatus: run status from ops events (latest Started newer than the
  latest Completed/Failed = active), used by the session-cleanup banner
  (rendering unchanged) and the maintenance trigger section.
- Trigger section: queued / already-active messages, status line, polls every
  15 s while a run is active, button disabled meanwhile. The 230 s fetch
  timeout is gone.

Shared manifest, generated wire types and the MCP ops-event vocabulary
regenerated (two new ops event types, reshaped response).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-09-19 13:50:06 +02:00
parent d8c03d9d95
commit e164eac1eb
28 changed files with 865 additions and 395 deletions
@@ -447,41 +447,27 @@ public class AdminWireParityTests
// ---- TriggerMaintenance --------------------------------------------------------------
[Fact]
public void TriggerMaintenanceResponse_matches_the_manual_trigger_shape()
public void TriggerMaintenanceResponse_matches_the_queued_trigger_shape()
{
var result = new MaintenanceResult
{
Success = true,
TriggeredBy = "admin@contoso.com",
TriggeredAt = new DateTime(2026, 8, 30, 13, 10, 0, DateTimeKind.Utc),
DurationMs = 1234,
StalledSessionsChecked = true,
MetricsAggregated = true,
AggregatedDate = "2026-08-29",
DataCleanupExecuted = false,
PlatformStatsRecomputed = true,
DevicesBlockedForExcessiveData = 0,
ContactEmailsBackfilled = 2,
};
var userEmail = "admin@contoso.com";
var triggeredAt = new DateTime(2026, 8, 30, 13, 10, 5, DateTimeKind.Utc);
AssertParity(
new
{
success = true,
message = "Maintenance tasks completed",
result = result,
message = "Maintenance run queued",
triggeredBy = userEmail,
triggeredAt
triggeredAt,
targetDate = "2026-08-29",
aggregateOnly = true
},
new TriggerMaintenanceResponse
{
Success = true,
Message = "Maintenance tasks completed",
Result = result,
Message = "Maintenance run queued",
TriggeredBy = userEmail,
TriggeredAt = triggeredAt
TriggeredAt = triggeredAt,
TargetDate = "2026-08-29",
AggregateOnly = true
});
}
@@ -0,0 +1,137 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Backup;
using AutopilotMonitor.Functions.Services.Maintenance;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using Azure.Storage.Blobs.Specialized;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Single-flight of the platform maintenance run: the 2h timer and the manual trigger enter
/// through one gate, so a second run is skipped instead of aggregating, cleaning up and
/// backfilling the same tables in parallel — and a skip never looks like an active run.
/// </summary>
public class MaintenanceRunGateTests
{
private sealed class Harness
{
public Mock<MaintenanceRunLockStore> LockStore { get; }
public Mock<BlobLeaseClient> Lease { get; } = new();
public List<string> OpsEvents { get; } = new();
public MaintenanceRunGate Sut { get; }
public Harness()
{
LockStore = new Mock<MaintenanceRunLockStore>(new SessionDeletionMaintenanceFunctionTests.MaintenanceBlobStub());
LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(Lease.Object);
var opsRepo = new Mock<IOpsEventRepository>();
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
.Returns<OpsEventEntry>(e => { lock (OpsEvents) OpsEvents.Add(e.EventType); return Task.CompletedTask; });
var adminConfig = new Mock<AdminConfigurationService>(
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()));
var opsService = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance,
TestNotifications.InertOpsAlertDispatch(adminConfig.Object));
Sut = new MaintenanceRunGate(LockStore.Object, opsService, NullLogger<MaintenanceRunGate>.Instance);
}
public void HoldLease() =>
LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new LeaseHeldException("held", new Exception("409")));
}
[Fact]
public async Task A_held_lease_skips_the_run_and_never_reports_it_as_started()
{
var h = new Harness();
h.HoldLease();
var ran = false;
var result = await h.Sut.RunExclusiveAsync("Timer", () => { ran = true; return Task.CompletedTask; });
Assert.False(result);
Assert.False(ran);
Assert.Equal(new[] { OpsEventTypes.MaintenanceSkippedLocked }, h.OpsEvents);
}
[Fact]
public async Task Started_is_written_under_the_lease_before_the_body_and_the_lease_is_released_last()
{
var h = new Harness();
var order = new List<string>();
h.Lease.Setup(l => l.ReleaseAsync(It.IsAny<Azure.RequestConditions>(), It.IsAny<CancellationToken>()))
.Callback(() => order.Add("release"))
.ReturnsAsync(Mock.Of<Azure.Response<Azure.Storage.Blobs.Models.ReleasedObjectInfo>>());
var result = await h.Sut.RunExclusiveAsync("admin@example.com", () =>
{
order.Add($"body after [{string.Join(",", h.OpsEvents)}]");
return Task.CompletedTask;
});
Assert.True(result);
Assert.Equal(new[] { $"body after [{OpsEventTypes.MaintenanceStarted}]", "release" }, order);
}
[Fact]
public async Task The_lease_is_released_when_the_body_throws()
{
var h = new Harness();
var released = false;
h.Lease.Setup(l => l.ReleaseAsync(It.IsAny<Azure.RequestConditions>(), It.IsAny<CancellationToken>()))
.Callback(() => released = true)
.ReturnsAsync(Mock.Of<Azure.Response<Azure.Storage.Blobs.Models.ReleasedObjectInfo>>());
await Assert.ThrowsAsync<InvalidOperationException>(() =>
h.Sut.RunExclusiveAsync("Timer", () => throw new InvalidOperationException("boom")));
Assert.True(released);
}
[Fact]
public async Task Probe_reports_an_active_run_while_the_lease_is_held()
{
var h = new Harness();
h.HoldLease();
Assert.True(await h.Sut.IsRunActiveAsync());
Assert.Empty(h.OpsEvents);
}
[Fact]
public async Task Probe_releases_its_lease_and_reports_no_active_run()
{
var h = new Harness();
var released = false;
h.Lease.Setup(l => l.ReleaseAsync(It.IsAny<Azure.RequestConditions>(), It.IsAny<CancellationToken>()))
.Callback(() => released = true)
.ReturnsAsync(Mock.Of<Azure.Response<Azure.Storage.Blobs.Models.ReleasedObjectInfo>>());
Assert.False(await h.Sut.IsRunActiveAsync());
Assert.True(released);
}
[Fact]
public async Task Probe_lets_a_failed_release_surface_so_nothing_is_queued_behind_a_stuck_lease()
{
// A queued run picked up seconds later would find the probe's own lease still held,
// skip as "locked" and delete the message — the operator's run would silently vanish.
var h = new Harness();
h.Lease.Setup(l => l.ReleaseAsync(It.IsAny<Azure.RequestConditions>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new Azure.RequestFailedException(500, "storage down"));
await Assert.ThrowsAsync<Azure.RequestFailedException>(() => h.Sut.IsRunActiveAsync());
}
}
@@ -2,7 +2,7 @@ using System;
using System.Net;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Maintenance;
using AutopilotMonitor.Shared.Models;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
@@ -11,24 +11,29 @@ using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Admin
{
/// <summary>
/// Allows Global Admins to manually trigger maintenance tasks
/// <c>POST /api/maintenance/trigger</c> — manual Global Admin trigger of the platform
/// maintenance run. The run takes minutes, so the request only queues it:
/// <c>202 Accepted</c> when queued, <c>409</c> while a run (timer or manual) is active,
/// <c>500</c> when the enqueue failed — never a hollow success. Progress and the run report
/// surface as <c>Maintenance*</c> ops events.
/// </summary>
public class TriggerMaintenanceFunction
{
private readonly ILogger<TriggerMaintenanceFunction> _logger;
private readonly MaintenanceService _maintenanceService;
private readonly IMaintenanceTriggerProducer _producer;
private readonly MaintenanceRunGate _runGate;
public TriggerMaintenanceFunction(
ILogger<TriggerMaintenanceFunction> logger,
MaintenanceService maintenanceService)
IMaintenanceTriggerProducer producer,
MaintenanceRunGate runGate)
{
_logger = logger;
_maintenanceService = maintenanceService;
_producer = producer;
_runGate = runGate;
}
/// <summary>
/// POST /api/maintenance/trigger
/// Manually trigger maintenance tasks (Global Admin only)
/// Query parameters:
/// - date: Optional date to aggregate (yyyy-MM-dd). If not provided, uses yesterday.
/// - aggregateOnly: If true, only runs aggregation (skips timeout and cleanup)
@@ -38,45 +43,42 @@ namespace AutopilotMonitor.Functions.Functions.Admin
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "maintenance/trigger")] HttpRequestData req,
FunctionContext context)
{
_logger.LogInformation("Manual maintenance trigger requested");
// Authentication + GlobalAdminOnly authorization enforced by PolicyEnforcementMiddleware
var userEmail = TenantHelper.GetUserIdentifier(req);
_logger.LogInformation($"Maintenance trigger initiated by Global Admin: {userEmail}");
var userEmail = TenantHelper.GetUserIdentifier(req) ?? "GlobalAdmin";
var ct = context.CancellationToken;
try
{
// Parse query parameters
var dateParam = req.Query["date"];
var aggregateOnlyParam = req.Query["aggregateOnly"];
if (!QueryParams.TryUtcInstant(dateParam, "date", out var dateInstant, out _))
if (!QueryParams.TryUtcInstant(req.Query["date"], "date", out var dateInstant, out _))
{
return await req.BadRequestAsync("Invalid date format. Use yyyy-MM-dd");
}
DateTime? targetDate = dateInstant?.Date;
if (targetDate.HasValue)
bool aggregateOnly = req.Query["aggregateOnly"]?.ToLower() == "true";
if (await _runGate.IsRunActiveAsync(ct))
{
_logger.LogInformation($"Manual maintenance for date: {targetDate:yyyy-MM-dd}");
return await req.ConflictAsync("A maintenance run is already active.");
}
bool aggregateOnly = aggregateOnlyParam?.ToLower() == "true";
// Execute maintenance tasks
var result = await _maintenanceService.RunManualAsync(targetDate, aggregateOnly, userEmail);
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new TriggerMaintenanceResponse
await _producer.EnqueueAsync(new MaintenanceTriggerEnvelope
{
Success = true,
Message = "Maintenance tasks completed",
Result = result,
TriggeredBy = userEmail,
TriggeredAt = DateTime.UtcNow
});
TargetDate = targetDate,
AggregateOnly = aggregateOnly,
}, ct);
return response;
_logger.LogInformation("TriggerMaintenance: run queued (triggeredBy={TriggeredBy}, date={Date:yyyy-MM-dd}, aggregateOnly={AggregateOnly})",
userEmail, targetDate, aggregateOnly);
return await req.JsonAsync(HttpStatusCode.Accepted, new TriggerMaintenanceResponse
{
Message = "Maintenance run queued",
TriggeredBy = userEmail,
TriggeredAt = DateTime.UtcNow,
TargetDate = targetDate?.ToString("yyyy-MM-dd"),
AggregateOnly = aggregateOnly,
});
}
catch (Exception ex)
{
@@ -283,7 +283,17 @@ builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Analyze.Interi
builder.Services.AddSingleton<ImeLogPatternService>();
builder.Services.AddHttpClient<GitHubRuleRepository>()
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi);
// Platform maintenance run: blob-lease gate (serializes the 2h timer, the manual trigger and a
// second host instance), fail-hard queue producer for POST /api/maintenance/trigger, and the
// BackgroundService worker that runs a queued manual run.
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.MaintenanceRunLockStore>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.MaintenanceRunGate>();
builder.Services.AddSingleton<
AutopilotMonitor.Functions.Services.Maintenance.IMaintenanceTriggerProducer,
AutopilotMonitor.Functions.Services.Maintenance.AzureQueueMaintenanceTriggerProducer>();
builder.Services.AddSingleton<MaintenanceService>();
builder.Services.AddHostedService<
AutopilotMonitor.Functions.Services.Maintenance.MaintenanceTriggerQueueWorker>();
builder.Services.AddSingleton<LegacyReclassificationService>();
builder.Services.AddSingleton<OccurredUtcBackfillService>();
builder.Services.AddSingleton<
@@ -1,12 +1,6 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Backup;
using AutopilotMonitor.Functions.Services.Locking;
using AutopilotMonitor.Shared;
using Azure;
using Azure.Storage.Blobs.Models;
using Azure.Storage.Blobs.Specialized;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Deletion
@@ -19,89 +13,25 @@ namespace AutopilotMonitor.Functions.Services.Deletion
/// Deliberately a DEDICATED sentinel (<see cref="LockBlobName"/> in the
/// <c>deletion-manifests</c> container) rather than the backup sentinel in
/// <see cref="BlobBackupStore"/> — sharing that lease would serialize backups against the
/// retention fanout, which have no reason to exclude each other. Lease mechanics
/// (60s lease + <see cref="MaintenanceLeaseHolder"/> 45s renewal, <see cref="LeaseHeldException"/>
/// on contention) are identical to the backup path.
/// retention fanout, which have no reason to exclude each other.
/// </para>
/// <para>
/// The <c>deletion-manifests</c> container has a 30-day lifecycle policy that may delete an
/// idle sentinel; <see cref="AcquireLeaseAsync"/> re-ensures it on every acquire, so the
/// lock self-heals. The manifest-TTL sweep skips the sentinel because <c>_lock/…</c> does
/// not parse as a manifest blob name.
/// idle sentinel; the base class re-ensures it on every acquire, so the lock self-heals.
/// The manifest-TTL sweep skips the sentinel because <c>_lock/…</c> does not parse as a
/// manifest blob name.
/// </para>
/// </summary>
public class SessionDeletionMaintenanceLockStore
public class SessionDeletionMaintenanceLockStore : BlobLeaseLockStore
{
/// <summary>Sentinel blob path — leased by the timer / manual-trigger worker to serialize runs.</summary>
public const string LockBlobName = "_lock/session-deletion-maintenance.lock";
/// <summary>Lease duration (Azure spec allows 15..60s for renewable leases).</summary>
public static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(60);
private readonly BlobStorageService _blobs;
private readonly ILogger<SessionDeletionMaintenanceLockStore> _logger;
private int _containerEnsured;
public SessionDeletionMaintenanceLockStore(
BlobStorageService blobs,
ILogger<SessionDeletionMaintenanceLockStore> logger)
: base(blobs, Constants.BlobContainers.DeletionManifests, LockBlobName, "session-deletion maintenance")
{
_blobs = blobs;
_logger = logger;
}
/// <summary>
/// Attempts to acquire the session-deletion maintenance lease. Throws
/// <see cref="LeaseHeldException"/> on 409 (another holder); other errors propagate.
/// Caller is responsible for renewing (via <see cref="MaintenanceLeaseHolder"/>) and
/// releasing. <c>virtual</c> so tests can stub it.
/// </summary>
public virtual async Task<BlobLeaseClient> AcquireLeaseAsync(TimeSpan? leaseDuration = null, CancellationToken ct = default)
{
await EnsureSentinelAsync(ct).ConfigureAwait(false);
var blob = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests).GetBlobClient(LockBlobName);
var leaseClient = blob.GetBlobLeaseClient();
try
{
await leaseClient.AcquireAsync(leaseDuration ?? LeaseDuration, conditions: null, ct).ConfigureAwait(false);
return leaseClient;
}
catch (RequestFailedException ex) when (ex.Status == 409)
{
throw new LeaseHeldException("session-deletion maintenance lease is held by another run", ex);
}
}
/// <summary>
/// Idempotently ensures the sentinel blob exists — an AcquireLease on a missing blob
/// throws 404. If-None-Match=* lets concurrent callers cooperate; both 409
/// BlobAlreadyExists and 412 ConditionNotMet are the steady state.
/// </summary>
private async Task EnsureSentinelAsync(CancellationToken ct)
{
var container = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests);
if (Interlocked.CompareExchange(ref _containerEnsured, 0, 0) == 0)
{
await container.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
Interlocked.Exchange(ref _containerEnsured, 1);
}
var blob = container.GetBlobClient(LockBlobName);
var options = new BlobUploadOptions
{
HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" },
Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All },
};
try
{
using var empty = new MemoryStream(Array.Empty<byte>());
await blob.UploadAsync(empty, options, ct).ConfigureAwait(false);
}
catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 412)
{
// Sentinel exists already — steady state.
}
}
}
}
@@ -0,0 +1,95 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Backup;
using Azure;
using Azure.Storage.Blobs.Models;
using Azure.Storage.Blobs.Specialized;
namespace AutopilotMonitor.Functions.Services.Locking
{
/// <summary>
/// Blob-lease lock on a sentinel blob: serializes a run against every other entry point of
/// the same work (timer, manual trigger, a second host instance). One subclass per lock —
/// each names its own sentinel, because two kinds of work that have no reason to exclude
/// each other must not share a lease.
/// <para>
/// Lease mechanics: 60s lease, renewed by <see cref="MaintenanceLeaseHolder"/> every 45s,
/// <see cref="LeaseHeldException"/> on contention. The sentinel is re-ensured on every
/// acquire, so a lifecycle policy that deletes an idle sentinel cannot break the lock.
/// </para>
/// </summary>
public abstract class BlobLeaseLockStore
{
/// <summary>Lease duration (Azure spec allows 15..60s for renewable leases).</summary>
public static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(60);
private readonly BlobStorageService _blobs;
private readonly string _containerName;
private readonly string _lockBlobName;
private readonly string _description;
private int _containerEnsured;
protected BlobLeaseLockStore(BlobStorageService blobs, string containerName, string lockBlobName, string description)
{
_blobs = blobs;
_containerName = containerName;
_lockBlobName = lockBlobName;
_description = description;
}
/// <summary>
/// Attempts to acquire the lease. Throws <see cref="LeaseHeldException"/> on 409 (another
/// holder); other errors propagate. Caller is responsible for renewing (via
/// <see cref="MaintenanceLeaseHolder"/>) and releasing. <c>virtual</c> so tests can stub it.
/// </summary>
public virtual async Task<BlobLeaseClient> AcquireLeaseAsync(TimeSpan? leaseDuration = null, CancellationToken ct = default)
{
await EnsureSentinelAsync(ct).ConfigureAwait(false);
var blob = _blobs.GetContainerClient(_containerName).GetBlobClient(_lockBlobName);
var leaseClient = blob.GetBlobLeaseClient();
try
{
await leaseClient.AcquireAsync(leaseDuration ?? LeaseDuration, conditions: null, ct).ConfigureAwait(false);
return leaseClient;
}
catch (RequestFailedException ex) when (ex.Status == 409)
{
throw new LeaseHeldException($"{_description} lease is held by another run", ex);
}
}
/// <summary>
/// Idempotently ensures the sentinel blob exists — an AcquireLease on a missing blob
/// throws 404. If-None-Match=* lets concurrent callers cooperate; both 409
/// BlobAlreadyExists and 412 ConditionNotMet are the steady state.
/// </summary>
private async Task EnsureSentinelAsync(CancellationToken ct)
{
var container = _blobs.GetContainerClient(_containerName);
if (Interlocked.CompareExchange(ref _containerEnsured, 0, 0) == 0)
{
await container.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
Interlocked.Exchange(ref _containerEnsured, 1);
}
var blob = container.GetBlobClient(_lockBlobName);
var options = new BlobUploadOptions
{
HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" },
Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All },
};
try
{
using var empty = new MemoryStream(Array.Empty<byte>());
await blob.UploadAsync(empty, options, ct).ConfigureAwait(false);
}
catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 412)
{
// Sentinel exists already — steady state.
}
}
}
}
@@ -0,0 +1,60 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Queueing;
using AutopilotMonitor.Shared;
using Azure.Storage.Queues;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace AutopilotMonitor.Functions.Services.Maintenance
{
/// <summary>
/// Azure-Storage-Queue producer for the manual maintenance trigger. Same shape as the
/// session-deletion and backup trigger producers: <see cref="QueueClientFactory"/>
/// (Managed Identity preferred), fail-hard.
/// </summary>
public sealed class AzureQueueMaintenanceTriggerProducer : IMaintenanceTriggerProducer
{
private readonly QueueClient _queueClient;
private readonly ILogger<AzureQueueMaintenanceTriggerProducer> _logger;
private int _queueEnsured;
public AzureQueueMaintenanceTriggerProducer(
QueueClientFactory queueFactory,
ILogger<AzureQueueMaintenanceTriggerProducer> logger)
{
_logger = logger;
_queueClient = queueFactory.Create(Constants.QueueNames.MaintenanceTrigger);
}
public async Task EnqueueAsync(MaintenanceTriggerEnvelope envelope, CancellationToken ct = default)
{
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
if (string.IsNullOrEmpty(envelope.TriggeredBy)) throw new ArgumentException("TriggeredBy required", nameof(envelope));
await EnsureQueueExistsAsync(ct).ConfigureAwait(false);
// Fail-hard: do NOT swallow. Caller (HTTP trigger) catches and returns 500.
var body = JsonConvert.SerializeObject(envelope);
await _queueClient.SendMessageAsync(body, ct).ConfigureAwait(false);
_logger.LogInformation("Maintenance trigger enqueued (triggeredBy={TriggeredBy})", envelope.TriggeredBy);
}
private async Task EnsureQueueExistsAsync(CancellationToken ct)
{
if (_queueEnsured == 1) return;
try
{
await _queueClient.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
Interlocked.Exchange(ref _queueEnsured, 1);
}
catch (Exception ex)
{
// Ensure failure during enqueue is itself a fail-hard signal — surface to the caller.
_logger.LogError(ex, "Maintenance trigger queue CreateIfNotExists failed");
throw;
}
}
}
}
@@ -0,0 +1,29 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace AutopilotMonitor.Functions.Services.Maintenance
{
/// <summary>
/// Queue envelope of a manually triggered maintenance run. There is no per-run state row —
/// the Maintenance* ops events are the status surface.
/// </summary>
public sealed class MaintenanceTriggerEnvelope
{
public string TriggeredBy { get; set; } = string.Empty;
/// <summary>Date to aggregate (UTC date); null = yesterday.</summary>
public DateTime? TargetDate { get; set; }
public bool AggregateOnly { get; set; }
}
/// <summary>
/// Producer for the manual maintenance trigger. <b>Fail-hard</b>: send failures propagate so
/// the HTTP trigger answers 5xx instead of a hollow 202.
/// </summary>
public interface IMaintenanceTriggerProducer
{
Task EnqueueAsync(MaintenanceTriggerEnvelope envelope, CancellationToken ct = default);
}
}
@@ -0,0 +1,92 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Backup;
using Azure.Storage.Blobs.Specialized;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Maintenance
{
/// <summary>
/// Single-flight gate for the platform maintenance run: the 2h timer, the manual trigger
/// and a second host instance all enter through <see cref="RunExclusiveAsync"/>, so two
/// runs never aggregate, clean up and backfill the same tables at the same time.
/// </summary>
public class MaintenanceRunGate
{
private readonly MaintenanceRunLockStore _lockStore;
private readonly OpsEventService _opsEvents;
private readonly ILogger<MaintenanceRunGate> _logger;
public MaintenanceRunGate(MaintenanceRunLockStore lockStore, OpsEventService opsEvents, ILogger<MaintenanceRunGate> logger)
{
_lockStore = lockStore;
_opsEvents = opsEvents;
_logger = logger;
}
/// <summary>
/// Acquire+release probe for the manual trigger: a held lease means a run is active RIGHT
/// NOW, so the operator is told instead of queuing a message that would only produce a
/// SkippedLocked event. Racing the probe (a run starts between probe and worker pickup)
/// is benign: the worker's own acquire lands on the SkippedLocked path.
/// </summary>
public virtual async Task<bool> IsRunActiveAsync(CancellationToken ct = default)
{
BlobLeaseClient lease;
try
{
lease = await _lockStore.AcquireLeaseAsync(ct: ct).ConfigureAwait(false);
}
catch (LeaseHeldException)
{
return true;
}
// A probe lease that cannot be released would still be held when the worker picks the
// message up seconds later — the run would be skipped as "locked" and the message
// deleted. Let the failure surface: the trigger answers 5xx and nothing is queued.
await lease.ReleaseAsync(cancellationToken: ct).ConfigureAwait(false);
return false;
}
/// <summary>
/// Runs <paramref name="body"/> under the maintenance-run lease. Returns false without
/// running it when another run holds the lease (SkippedLocked ops event). The Started
/// event is written AFTER the lease is held, so a skip never looks like an active run,
/// and the lease is released LAST, so the body's terminal ops event is still exclusive.
/// </summary>
public virtual async Task<bool> RunExclusiveAsync(string triggeredBy, Func<Task> body)
{
BlobLeaseClient lease;
try
{
lease = await _lockStore.AcquireLeaseAsync().ConfigureAwait(false);
}
catch (LeaseHeldException)
{
_logger.LogWarning("Maintenance: lease held by another run — skipping (triggeredBy={TriggeredBy})", triggeredBy);
await _opsEvents.RecordMaintenanceSkippedLockedAsync(triggeredBy).ConfigureAwait(false);
return false;
}
// The maintenance body takes no cancellation token; a lost lease is reported, not
// enforced — the run is minutes long and the lease renews every 45s.
using var leaseLost = new CancellationTokenSource();
var leaseHolder = new MaintenanceLeaseHolder(lease, leaseLost, _logger);
try
{
await _opsEvents.RecordMaintenanceStartedAsync(triggeredBy).ConfigureAwait(false);
await body().ConfigureAwait(false);
return true;
}
finally
{
if (leaseHolder.RenewalFailureReason != null)
_logger.LogWarning("Maintenance: the run lease could not be renewed during the run ({Reason}) — another run may have overlapped",
leaseHolder.RenewalFailureReason);
await leaseHolder.DisposeAsync().ConfigureAwait(false);
}
}
}
}
@@ -0,0 +1,21 @@
using AutopilotMonitor.Functions.Services.Locking;
using AutopilotMonitor.Shared;
namespace AutopilotMonitor.Functions.Services.Maintenance
{
/// <summary>
/// Blob-lease lock for the platform maintenance run (2h timer and manual trigger). Its own
/// sentinel: the session-deletion maintenance and the critical-table backup have no reason
/// to exclude this run, so they keep their own leases.
/// </summary>
public class MaintenanceRunLockStore : BlobLeaseLockStore
{
/// <summary>Sentinel blob path. <c>_lock/…</c> does not parse as a manifest blob name, so the manifest-TTL sweep skips it.</summary>
public const string LockBlobName = "_lock/platform-maintenance.lock";
public MaintenanceRunLockStore(BlobStorageService blobs)
: base(blobs, Constants.BlobContainers.DeletionManifests, LockBlobName, "platform maintenance")
{
}
}
}
@@ -0,0 +1,49 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Queueing;
using AutopilotMonitor.Shared;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Maintenance
{
/// <summary>
/// Background worker for the <c>maintenance-trigger</c> queue — runs the manually triggered
/// maintenance off the HTTP request, which would otherwise have to stay open for minutes
/// and dies at the gateway limit.
/// <para>
/// A self-polling worker, not a queue-triggered function, because the run holds a lease
/// for its whole lifetime. Concurrency against the 2h timer is handled inside
/// <see cref="MaintenanceService.RunManualAsync"/> by the run gate: a lease-held run
/// returns normally (SkippedLocked ops event), so the message is deleted rather than
/// retried — the operator asked for "a run", and one is already active. A failing run
/// reports MaintenanceFailed and returns as well: maintenance is not retried blindly.
/// </para>
/// </summary>
public sealed class MaintenanceTriggerQueueWorker : QueuePollingWorker<MaintenanceTriggerEnvelope>
{
private readonly MaintenanceService _maintenance;
public MaintenanceTriggerQueueWorker(
QueueClientFactory queueFactory,
MaintenanceService maintenance,
ILogger<MaintenanceTriggerQueueWorker> logger)
: base(queueFactory, Constants.QueueNames.MaintenanceTrigger, logger, Constants.QueueNames.MaintenanceTriggerPoison)
{
_maintenance = maintenance ?? throw new ArgumentNullException(nameof(maintenance));
}
// The host functionTimeout (60min) is the upper bound of a run; the message must not
// reappear while its run is still going.
protected override TimeSpan VisibilityTimeout => TimeSpan.FromMinutes(60);
protected override bool TryValidate(MaintenanceTriggerEnvelope envelope)
=> !string.IsNullOrEmpty(envelope.TriggeredBy);
protected override string DescribeForLog(MaintenanceTriggerEnvelope envelope)
=> $"triggeredBy={envelope.TriggeredBy} targetDate={envelope.TargetDate:yyyy-MM-dd} aggregateOnly={envelope.AggregateOnly}";
protected override Task HandleAsync(MaintenanceTriggerEnvelope envelope, CancellationToken ct)
=> _maintenance.RunManualAsync(envelope.TargetDate, envelope.AggregateOnly, envelope.TriggeredBy);
}
}
@@ -55,6 +55,7 @@ namespace AutopilotMonitor.Functions.Services
private readonly IHttpClientFactory _httpClientFactory;
private readonly PreviewWhitelistService _previewWhitelistService;
private readonly IStorageInitializer _storageInitializer;
private readonly Maintenance.MaintenanceRunGate _runGate;
private readonly ILogger<MaintenanceService> _logger;
private const string PlatformStatsAliasFileName = "platform-stats.json";
@@ -90,6 +91,7 @@ namespace AutopilotMonitor.Functions.Services
IHttpClientFactory httpClientFactory,
PreviewWhitelistService previewWhitelistService,
IStorageInitializer storageInitializer,
Maintenance.MaintenanceRunGate runGate,
ILogger<MaintenanceService> logger)
{
_maintenanceRepo = maintenanceRepo;
@@ -120,6 +122,7 @@ namespace AutopilotMonitor.Functions.Services
_httpClientFactory = httpClientFactory;
_previewWhitelistService = previewWhitelistService;
_storageInitializer = storageInitializer;
_runGate = runGate;
_logger = logger;
}
@@ -136,9 +139,12 @@ namespace AutopilotMonitor.Functions.Services
}
/// <summary>
/// Runs all maintenance tasks (used by the daily timer trigger)
/// Runs all maintenance tasks (used by the 2h timer trigger). Single-flight with the
/// manual trigger: skipped when another run holds the maintenance-run lease.
/// </summary>
public async Task RunAllAsync()
public Task RunAllAsync() => _runGate.RunExclusiveAsync("Timer", RunAllCoreAsync);
private async Task RunAllCoreAsync()
{
_logger.LogInformation($"Daily maintenance started at {DateTime.UtcNow}");
var maintenanceStart = Stopwatch.StartNew();
@@ -246,9 +252,27 @@ namespace AutopilotMonitor.Functions.Services
=> window.Where(s => s.StartedAt >= windowStart).ToList();
/// <summary>
/// Manually triggered maintenance with flexible date selection
/// Manually triggered maintenance with flexible date selection. Single-flight with the
/// timer: when another run holds the maintenance-run lease, nothing runs and the result
/// says so.
/// </summary>
public async Task<MaintenanceResult> RunManualAsync(DateTime? targetDate = null, bool aggregateOnly = false, string triggeredBy = "Unknown")
{
MaintenanceResult? result = null;
var ran = await _runGate.RunExclusiveAsync(triggeredBy,
async () => result = await RunManualCoreAsync(targetDate, aggregateOnly, triggeredBy));
return ran && result != null
? result
: new MaintenanceResult
{
TriggeredBy = triggeredBy,
TriggeredAt = DateTime.UtcNow,
Success = false,
Error = "Another maintenance run is active",
};
}
private async Task<MaintenanceResult> RunManualCoreAsync(DateTime? targetDate, bool aggregateOnly, string triggeredBy)
{
_logger.LogInformation($"Manual maintenance triggered by {triggeredBy} at {DateTime.UtcNow}");
var maintenanceStart = Stopwatch.StartNew();
@@ -329,7 +353,7 @@ namespace AutopilotMonitor.Functions.Services
result.Success = true;
_logger.LogInformation($"Manual maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms");
await _opsEventService.RecordMaintenanceCompletedAsync(result.DurationMs, triggeredBy);
await _opsEventService.RecordMaintenanceCompletedAsync(result.DurationMs, triggeredBy, result);
return result;
}
catch (Exception ex)
@@ -64,10 +64,42 @@ namespace AutopilotMonitor.Functions.Services
// ── Maintenance ────────────────────────────────────────────────────────
public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy)
/// <summary>
/// Emitted after the maintenance-run lease was acquired, so a lease-skip never
/// masquerades as an active run — the portal treats "latest Started newer than latest
/// Completed/Failed" as run-active.
/// </summary>
public Task RecordMaintenanceStartedAsync(string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceStarted, OpsEventSeverity.Info,
$"Maintenance run started (triggered by {triggeredBy})",
null, triggeredBy, new { triggeredBy });
/// <summary>Another run (timer, manual trigger or a second host instance) held the maintenance-run lease.</summary>
public Task RecordMaintenanceSkippedLockedAsync(string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceSkippedLocked, OpsEventSeverity.Info,
$"Maintenance run skipped — another run holds the lease (triggered by {triggeredBy})",
null, triggeredBy, new { reason = "lease held by another run", triggeredBy });
/// <summary>
/// <paramref name="report"/> is the run report of a manual run: the trigger answers 202
/// before the run starts, so this event is where the operator reads what the run did.
/// </summary>
public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy, MaintenanceResult? report = null)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceCompleted, OpsEventSeverity.Info,
$"Maintenance completed in {durationMs}ms (triggered by {triggeredBy})",
null, triggeredBy, new { durationMs });
null, triggeredBy,
report == null
? (object)new { durationMs }
: new
{
durationMs,
aggregatedDate = report.AggregatedDate,
stalledSessionsChecked = report.StalledSessionsChecked,
dataCleanupExecuted = report.DataCleanupExecuted,
platformStatsRecomputed = report.PlatformStatsRecomputed,
devicesBlockedForExcessiveData = report.DevicesBlockedForExcessiveData,
contactEmailsBackfilled = report.ContactEmailsBackfilled,
});
public Task RecordMaintenanceFailedAsync(string error, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceFailed, OpsEventSeverity.Error,
@@ -2787,22 +2787,6 @@ export interface LogQueryRequest {
budgetSeconds?: number | null;
}
/** Run report of a manual maintenance run (POST maintenance/trigger). */
export interface MaintenanceResult {
success: boolean;
error?: string;
triggeredBy: string;
triggeredAt: string;
durationMs: number;
stalledSessionsChecked: boolean;
metricsAggregated: boolean;
aggregatedDate?: string;
dataCleanupExecuted: boolean;
platformStatsRecomputed: boolean;
devicesBlockedForExcessiveData: number;
contactEmailsBackfilled: number;
}
/** One managed tenant as the managing tenant sees it. */
export interface ManagedTenantItem {
tenantId: string;
@@ -5158,13 +5142,14 @@ export interface TriggerEpssSyncResponse {
syncedAt: string;
}
/** Response of POST maintenance/trigger. */
/** Response of POST maintenance/trigger (202 Accepted): the run is queued, not done. Its progress and its report surface as Maintenance* ops events (Started, Completed, Failed, SkippedLocked). */
export interface TriggerMaintenanceResponse {
success: boolean;
message: string;
result: MaintenanceResult;
triggeredBy: string;
triggeredAt: string;
/** Date to aggregate (yyyy-MM-dd); null = yesterday. */
targetDate?: string;
aggregateOnly: boolean;
}
/** Response of POST vulnerability/sync-msrc (dual-purpose: success reflects whether the MSRC index refresh finished without error). */
@@ -64,6 +64,8 @@ export const OPS_EVENT_TYPES = [
"MaintenanceCompleted",
"MaintenanceFailed",
"MaintenanceLongRunning",
"MaintenanceStarted",
"MaintenanceSkippedLocked",
"SessionSweepCompleted",
"SessionSweepFailed",
"OpsEventCleanup",
@@ -1701,6 +1701,20 @@ namespace AutopilotMonitor.Shared
/// <summary>Poison sibling of <see cref="SessionDeletionMaintenance"/>.</summary>
public const string SessionDeletionMaintenancePoison = "session-deletion-maintenance-poison";
/// <summary>
/// Manual-trigger queue for the platform maintenance run. Producer = HTTP trigger
/// <c>/api/maintenance/trigger</c> (fail-hard, answers 202); consumer =
/// <c>MaintenanceTriggerQueueWorker</c> (self-managed BackgroundService,
/// VisibilityTimeout=60min = the host functionTimeout). Envelope
/// <c>{ triggeredBy, targetDate, aggregateOnly }</c>; concurrency against the 2h timer
/// is serialized by the maintenance-run blob lease, not the queue.
/// Poison-suffix <c>-poison</c>, max-dequeue 5.
/// </summary>
public const string MaintenanceTrigger = "maintenance-trigger";
/// <summary>Poison sibling of <see cref="MaintenanceTrigger"/>.</summary>
public const string MaintenanceTriggerPoison = "maintenance-trigger-poison";
/// <summary>
/// IME-MSI archive fan-out. Producer = <c>EventIngestProcessor</c> when
/// <c>RecordImeVersionAsync</c> reports a genuinely NEW IME version (first sighting
@@ -32,6 +32,8 @@ namespace AutopilotMonitor.Shared.DataAccess
public const string MaintenanceCompleted = "MaintenanceCompleted";
public const string MaintenanceFailed = "MaintenanceFailed";
public const string MaintenanceLongRunning = "MaintenanceLongRunning";
public const string MaintenanceStarted = "MaintenanceStarted";
public const string MaintenanceSkippedLocked = "MaintenanceSkippedLocked";
public const string SessionSweepCompleted = "SessionSweepCompleted";
public const string SessionSweepFailed = "SessionSweepFailed";
public const string OpsEventCleanup = "OpsEventCleanup";
@@ -135,7 +137,7 @@ namespace AutopilotMonitor.Shared.DataAccess
public static readonly IReadOnlyList<string> All = new[]
{
ConsentFlowStarted, ConsentFlowSuccess, ConsentFlowFailed, ConsentRedirectUriMismatch, AppHomingFlipped, AppHomingFlippedWithEntraRoles,
MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift,
MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift,
DeviceBlocked, VersionBlocked, SessionTenantConflict, SessionOwnerMismatch, KillSignalDelivered, EmbeddedCertExpiringSoon, EmbeddedCertExpiringUrgent, EmbeddedCertExpired, EmbeddedCertBundleEmpty, SignalRConnectionsHigh, SignalRConnectionsCritical, SignalRMessagesHigh, SignalRMessagesCritical, PoisonQueueBacklogHigh, PoisonQueueBacklogCritical, ExcessiveSessionEventsAutoActioned, PrivilegedRouteDenied, McpServicePrincipalFirstSeen,
OffboardingFeedbackReceived, TenantOffboarded, TenantOffboardingFailed, TenantAutoApproved, WelcomeEmailSent, WelcomeEmailSkipped, WelcomeEmailFailed, TenantTrialStarted, TenantTrialExpiring, TenantTrialExpired, TenantPlanDowngraded, TenantRetentionGraceExpiring, TenantRetentionGraceEnded, RuleFrequencyRegression, AppVersionDurationRegression, CollectLogsQuickConfigEnabled, DiagnosticsUploadEnabled, DiagnosticsUploadDisabled,
SessionActionQueued, SessionTimeouts, AgentEmergencyBreak, AgentBinaryIntegrityMismatch, CmTraceTimeSkewRegression, ExcessiveSessionEvents, NewImeVersionDetected, ImePatternDriftSuspected, BlobStorageMissing, BlobStorageUnreachable, TelemetryItemsRejected,
@@ -145,15 +145,20 @@ namespace AutopilotMonitor.Shared.Models
public DateTime TriggeredAt { get; set; }
}
/// <summary>Response of POST maintenance/trigger.</summary>
/// <summary>
/// Response of POST maintenance/trigger (202 Accepted): the run is queued, not done. Its
/// progress and its report surface as Maintenance* ops events (Started, Completed, Failed,
/// SkippedLocked).
/// </summary>
// Declaration order == wire order.
public class TriggerMaintenanceResponse : IApiResponse
{
public bool Success { get; set; }
public string Message { get; set; } = default!;
public MaintenanceResult Result { get; set; } = default!;
public string TriggeredBy { get; set; } = default!;
public DateTime TriggeredAt { get; set; }
/// <summary>Date to aggregate (yyyy-MM-dd); null = yesterday.</summary>
public string? TargetDate { get; set; }
public bool AggregateOnly { get; set; }
}
/// <summary>
@@ -218,7 +223,7 @@ namespace AutopilotMonitor.Shared.Models
public string Reason { get; set; } = string.Empty;
}
/// <summary>Run report of a manual maintenance run (POST maintenance/trigger).</summary>
/// <summary>Run report of a manual maintenance run; its figures travel in the MaintenanceCompleted ops event.</summary>
// Declaration order == wire order.
public class MaintenanceResult
{
@@ -2,16 +2,32 @@
import { useState } from "react";
import { api } from "@/lib/api";
import { apiErrorText, fetchJson, fetchOk } from "@/lib/apiClient";
import { ApiError, apiErrorText, fetchJson, fetchOk } from "@/lib/apiClient";
import { parseOpsDetails, useOpsRunStatus, type OpsRunLifecycle } from "@/hooks/useOpsRunStatus";
import { SectionCardHeader } from "@/components/SectionCardHeader";
import type { GetLatestVersionsResponse } from "@/utils/wire-types.generated";
const LIFECYCLE: OpsRunLifecycle = {
category: "Maintenance",
started: "MaintenanceStarted",
completed: "MaintenanceCompleted",
failed: "MaintenanceFailed",
};
// A run takes a few minutes; while one is active the status line follows it on its own.
const POLL_WHILE_ACTIVE_MS = 15_000;
interface MaintenanceTriggerSectionProps {
getAccessToken: () => Promise<string | null>;
setError: (error: string | null) => void;
setSuccessMessage: (message: string | null) => void;
}
function formatRunDuration(durationMs: number | undefined): string {
if (typeof durationMs !== "number") return "";
return durationMs < 60_000 ? ` in ${Math.round(durationMs / 1000)} s` : ` in ${Math.round(durationMs / 60_000)} min`;
}
export function MaintenanceTriggerSection({
getAccessToken,
setError,
@@ -23,6 +39,8 @@ export function MaintenanceTriggerSection({
// (react-hooks/purity) and the cap doesn't need a live midnight rollover.
const [maxMaintenanceDate] = useState(() => new Date(Date.now() - 86400000).toISOString().split('T')[0]);
const [refreshingVersions, setRefreshingVersions] = useState(false);
const { status: runStatus, refresh: refreshRunStatus } = useOpsRunStatus(getAccessToken, setError, LIFECYCLE, POLL_WHILE_ACTIVE_MS);
const runActive = runStatus.kind === "active";
const handleRefreshLatestVersions = async () => {
try {
@@ -53,15 +71,20 @@ export function MaintenanceTriggerSection({
setError(null);
setSuccessMessage(null);
// The run answers when it is done (minutes), far beyond the default fetch timeout;
// the platform gateway cuts a request at 230 s, so waiting longer gains nothing.
await fetchOk(api.maintenance.trigger(maintenanceDate || undefined), getAccessToken, {
method: "POST",
signal: AbortSignal.timeout(230_000),
});
// 202 = queued (the run reports through Maintenance* ops events); 409 = a run is
// already active, which is not an error for the operator.
const queued = await fetchOk(api.maintenance.trigger(maintenanceDate || undefined), getAccessToken, { method: "POST" })
.then(() => true)
.catch((err: unknown) => {
if (err instanceof ApiError && err.status === 409) return false;
throw err;
});
const dateInfo = maintenanceDate ? ` for ${maintenanceDate}` : '';
setSuccessMessage(`Maintenance job completed successfully${dateInfo}!`);
setSuccessMessage(queued
? `Maintenance run queued${dateInfo} — its status appears below shortly.`
: "A maintenance run is already active.");
refreshRunStatus();
setTimeout(() => setSuccessMessage(null), 5000);
} catch (err) {
@@ -142,8 +165,8 @@ export function MaintenanceTriggerSection({
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
</svg>
<p className="text-sm text-gray-800 dark:text-gray-200">
<strong>Warning:</strong> This operation runs across all tenants and may take several minutes to complete.
Use this only for testing or when immediate cleanup is needed.
<strong>Warning:</strong> This operation runs across all tenants. The run is queued and takes a few minutes in the background;
only one run is active at a time, the 2h timer included. Use this only for testing or when immediate cleanup is needed.
</p>
</div>
</div>
@@ -179,16 +202,35 @@ export function MaintenanceTriggerSection({
</button>
</div>
</div>
<div className="flex justify-end pt-2">
<div className="flex items-center justify-between gap-4 pt-2">
<p className={`text-sm ${runStatus.kind === "failed" ? "text-red-700 dark:text-red-300" : "text-purple-900 dark:text-gray-200"}`}>
{runStatus.kind === "active" && (
<>
<span className="font-medium">Maintenance run active</span> — started{" "}
{new Date(runStatus.since).toLocaleString()} (triggered by {runStatus.triggeredBy}).
</>
)}
{runStatus.kind === "failed" && (
<>
<span className="font-medium">Last run failed</span> ({new Date(runStatus.at).toLocaleString()}): {runStatus.message}
</>
)}
{runStatus.kind === "completed" && (
<>
Last run completed {new Date(runStatus.at).toLocaleString()}
{formatRunDuration(parseOpsDetails<{ durationMs?: number }>(runStatus.details).durationMs)}.
</>
)}
</p>
<button
onClick={handleTriggerMaintenance}
disabled={triggeringMaintenance}
disabled={triggeringMaintenance || runActive}
className="px-6 py-3 bg-gradient-to-r from-purple-600 to-violet-600 text-white rounded-lg hover:from-purple-700 hover:to-violet-700 disabled:opacity-50 disabled:cursor-not-allowed transition-all shadow-md hover:shadow-lg flex items-center space-x-2"
>
{triggeringMaintenance ? (
<>
<div className="animate-spin rounded-full h-5 w-5 border-b-2 border-white"></div>
<span>Running...</span>
<span>Queuing...</span>
</>
) : (
<>
@@ -32,6 +32,10 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
// Soft watchdog (Warning): a maintenance run completed but exceeded the 10min threshold and is
// climbing toward the host's 60min functionTimeout. Backend helper RecordMaintenanceLongRunningAsync.
"MaintenanceLongRunning",
// Run lifecycle + lease serialization: Started marks run begin (timer or manual,
// details.triggeredBy), SkippedLocked means another run held the maintenance-run lease.
"MaintenanceStarted",
"MaintenanceSkippedLocked",
"OpsEventCleanup",
"SessionTimeouts",
// Hourly stalled-session sweep interleave (SessionSweepFunction, minute 30) — dual-register
@@ -1,22 +1,9 @@
"use client";
import { useCallback, useEffect, useState } from "react";
import { useCallback, useState } from "react";
import { api } from "@/lib/api";
import { TokenExpiredError } from "@/lib/authenticatedFetch";
import { ApiError, apiErrorText, fetchJson, fetchOk } from "@/lib/apiClient";
interface OpsEvent {
id: string;
eventType: string;
severity: string;
message: string;
details: string | null;
timestamp: string;
}
interface OpsEventsResponse {
events: OpsEvent[];
}
import { ApiError, apiErrorText, fetchOk } from "@/lib/apiClient";
import { parseOpsDetails, useOpsRunStatus, type OpsRunLifecycle } from "@/hooks/useOpsRunStatus";
interface CompletedDetails {
tenantsProcessed?: number;
@@ -34,14 +21,12 @@ type BannerState =
| { kind: "idle"; at: string; details: CompletedDetails }
| { kind: "none" }; // no lifecycle events in the window — render nothing
function parseDetails<T>(raw: string | null): T {
if (!raw) return {} as T;
try {
return JSON.parse(raw) as T;
} catch {
return {} as T;
}
}
const LIFECYCLE: OpsRunLifecycle = {
category: "Maintenance",
started: "SessionDeletionMaintenanceStarted",
completed: "SessionDeletionMaintenanceCompleted",
failed: "SessionDeletionMaintenanceFailed",
};
/**
* Status banner for the session-deletion maintenance run (12h retention fanout + GC sweeps).
@@ -59,71 +44,18 @@ export function MaintenanceStatusBanner({
setError: (error: string | null) => void;
setSuccessMessage: (message: string | null) => void;
}) {
const [state, setState] = useState<BannerState>({ kind: "loading" });
const { status, refresh } = useOpsRunStatus(getAccessToken, setError, LIFECYCLE);
const [triggering, setTriggering] = useState(false);
const [refreshKey, setRefreshKey] = useState(0);
const fetchStatus = useCallback(async () => {
try {
const data = await fetchJson<OpsEventsResponse>(
api.opsEvents.list("Maintenance", { pageSize: 100 }),
getAccessToken,
);
const lifecycle = (data.events ?? []).filter((e) =>
e.eventType.startsWith("SessionDeletionMaintenance"),
);
// Backend returns newest-first; keep it defensive anyway.
lifecycle.sort((a, b) => (a.timestamp < b.timestamp ? 1 : -1));
const started = lifecycle.find((e) => e.eventType === "SessionDeletionMaintenanceStarted");
const terminal = lifecycle.find(
(e) =>
e.eventType === "SessionDeletionMaintenanceCompleted" ||
e.eventType === "SessionDeletionMaintenanceFailed",
);
if (started && (!terminal || started.timestamp > terminal.timestamp)) {
const details = parseDetails<{ triggeredBy?: string }>(started.details);
setState({
kind: "active",
since: started.timestamp,
triggeredBy: details.triggeredBy ?? "unknown",
});
return;
}
if (!terminal) {
setState({ kind: "none" });
return;
}
if (terminal.eventType === "SessionDeletionMaintenanceFailed") {
setState({ kind: "failed", at: terminal.timestamp, message: terminal.message });
return;
}
const details = parseDetails<CompletedDetails>(terminal.details);
setState(
details.abortedByBudget
? { kind: "budget-exceeded", at: terminal.timestamp, details }
: { kind: "idle", at: terminal.timestamp, details },
);
} catch (err) {
if (err instanceof TokenExpiredError) {
setError("Session expired; reload the page and try again.");
}
// Non-auth failures degrade silently — the banner is auxiliary to the tabs.
setState({ kind: "none" });
}
}, [getAccessToken, setError]);
useEffect(() => {
const run = async () => {
await fetchStatus();
};
void run();
}, [fetchStatus, refreshKey]);
let state: BannerState;
if (status.kind === "completed") {
const details = parseOpsDetails<CompletedDetails>(status.details);
state = details.abortedByBudget
? { kind: "budget-exceeded", at: status.at, details }
: { kind: "idle", at: status.at, details };
} else {
state = status;
}
const triggerRun = useCallback(async () => {
setTriggering(true);
@@ -136,13 +68,13 @@ export function MaintenanceStatusBanner({
throw err;
});
setSuccessMessage(queued ? "Maintenance run queued — it will appear here as active shortly." : "A maintenance run is already active.");
setRefreshKey((k) => k + 1);
refresh();
} catch (err) {
setError(apiErrorText(err));
} finally {
setTriggering(false);
}
}, [getAccessToken, setError, setSuccessMessage]);
}, [getAccessToken, setError, setSuccessMessage, refresh]);
if (state.kind === "loading" || state.kind === "none") return null;
@@ -165,7 +97,7 @@ export function MaintenanceStatusBanner({
deletions enqueued by this run appear under In-Flight.
</p>
<button
onClick={() => setRefreshKey((k) => k + 1)}
onClick={refresh}
className="shrink-0 px-3 py-1.5 text-xs font-medium border border-purple-300 dark:border-purple-700 text-purple-700 dark:text-purple-200 rounded hover:bg-purple-100 dark:hover:bg-purple-900/40"
>
Refresh
@@ -0,0 +1,118 @@
"use client";
import { useCallback, useEffect, useState } from "react";
import { api } from "@/lib/api";
import { TokenExpiredError } from "@/lib/authenticatedFetch";
import { fetchJson } from "@/lib/apiClient";
interface OpsEvent {
id: string;
eventType: string;
severity: string;
message: string;
details: string | null;
timestamp: string;
}
interface OpsEventsResponse {
events: OpsEvent[];
}
/** The ops event types that make up one background run's lifecycle. */
export interface OpsRunLifecycle {
category: string;
started: string;
completed: string;
failed: string;
}
export type OpsRunStatus =
| { kind: "loading" }
| { kind: "active"; since: string; triggeredBy: string }
| { kind: "failed"; at: string; message: string }
| { kind: "completed"; at: string; details: string | null }
| { kind: "none" }; // no lifecycle events in the window
export function parseOpsDetails<T>(raw: string | null): T {
if (!raw) return {} as T;
try {
return JSON.parse(raw) as T;
} catch {
return {} as T;
}
}
/**
* Status of a background run that reports through ops events (the trigger answers 202, there
* is no job row): the latest Started newer than the latest terminal event (Completed/Failed)
* means a run is active; otherwise the latest terminal event is the last run's outcome.
* `lifecycle` must be a stable reference (module constant). With `pollWhileActiveMs` the
* status reloads on that cadence for as long as a run is active.
*/
export function useOpsRunStatus(
getAccessToken: (forceRefresh?: boolean) => Promise<string | null>,
setError: (error: string | null) => void,
lifecycle: OpsRunLifecycle,
pollWhileActiveMs?: number,
): { status: OpsRunStatus; refresh: () => void } {
const [status, setStatus] = useState<OpsRunStatus>({ kind: "loading" });
const [refreshKey, setRefreshKey] = useState(0);
const fetchStatus = useCallback(async () => {
try {
const data = await fetchJson<OpsEventsResponse>(
api.opsEvents.list(lifecycle.category, { pageSize: 100 }),
getAccessToken,
);
const events = [...(data.events ?? [])];
// Backend returns newest-first; keep it defensive anyway.
events.sort((a, b) => (a.timestamp < b.timestamp ? 1 : -1));
const started = events.find((e) => e.eventType === lifecycle.started);
const terminal = events.find(
(e) => e.eventType === lifecycle.completed || e.eventType === lifecycle.failed,
);
if (started && (!terminal || started.timestamp > terminal.timestamp)) {
const details = parseOpsDetails<{ triggeredBy?: string }>(started.details);
setStatus({ kind: "active", since: started.timestamp, triggeredBy: details.triggeredBy ?? "unknown" });
return;
}
if (!terminal) {
setStatus({ kind: "none" });
return;
}
setStatus(
terminal.eventType === lifecycle.failed
? { kind: "failed", at: terminal.timestamp, message: terminal.message }
: { kind: "completed", at: terminal.timestamp, details: terminal.details },
);
} catch (err) {
if (err instanceof TokenExpiredError) {
setError("Session expired; reload the page and try again.");
}
// Non-auth failures degrade silently — the run status is auxiliary to the page.
setStatus({ kind: "none" });
}
}, [getAccessToken, setError, lifecycle]);
useEffect(() => {
const run = async () => {
await fetchStatus();
};
void run();
}, [fetchStatus, refreshKey]);
const active = status.kind === "active";
useEffect(() => {
if (!active || !pollWhileActiveMs) return;
const timer = setInterval(() => setRefreshKey((k) => k + 1), pollWhileActiveMs);
return () => clearInterval(timer);
}, [active, pollWhileActiveMs]);
const refresh = useCallback(() => setRefreshKey((k) => k + 1), []);
return { status, refresh };
}
@@ -10,7 +10,7 @@ export const PERMANENT: Record<string, string> = {
"app/health-check/page.tsx": "/version.json of the SWA itself, not the API",
"app/dashboard/hooks/deleteSessionResponse.ts": "safeJson on the 202 Response of the delete-cascade classifier (refusals arrive as ApiError)",
"app/settings/TenantConfigContext.tsx": "consent return path routes on TokenExpiredError (router.replace); the access check rethrows it as a non-outcome",
"app/admin/ops/session-cleanup/components/MaintenanceStatusBanner.tsx": "auxiliary banner: only a token expiry is surfaced, backend refusals stay silent",
"hooks/useOpsRunStatus.ts": "auxiliary run status (session-cleanup banner, maintenance trigger): only a token expiry is surfaced, backend refusals stay silent",
"app/dashboard/hooks/useBlockDevice.ts": "bulk runner rethrows the expiry so it is toasted once, not per device",
"app/dashboard/hooks/useDeleteSession.ts": "bulk runner collects the expiry and toasts once after the batch",
"app/dashboard/hooks/useTenantSecurityConfig.ts": "fail-soft banner data; the expiry is the one failure the user must hear about",
@@ -50,7 +50,7 @@ export const BODY_STRINGIFY_BASELINE: Record<string, number> = {
};
export const TOKENEXPIRED_BASELINE: Record<string, number> = {
"app/admin/ops/session-cleanup/components/MaintenanceStatusBanner.tsx": 1,
"hooks/useOpsRunStatus.ts": 1,
"app/dashboard/hooks/useBlockDevice.ts": 1,
"app/dashboard/hooks/useDeleteSession.ts": 1,
"app/dashboard/hooks/useTenantSecurityConfig.ts": 1,
@@ -530,6 +530,8 @@ export const SHARED_MANIFEST = {
"MaintenanceCompleted",
"MaintenanceFailed",
"MaintenanceLongRunning",
"MaintenanceStarted",
"MaintenanceSkippedLocked",
"SessionSweepCompleted",
"SessionSweepFailed",
"OpsEventCleanup",
@@ -526,6 +526,8 @@
"MaintenanceCompleted",
"MaintenanceFailed",
"MaintenanceLongRunning",
"MaintenanceStarted",
"MaintenanceSkippedLocked",
"SessionSweepCompleted",
"SessionSweepFailed",
"OpsEventCleanup",
@@ -13149,98 +13151,6 @@
}
]
},
"MaintenanceResult": {
"kind": "object",
"doc": "Run report of a manual maintenance run (POST maintenance/trigger).",
"fields": [
{
"name": "success",
"type": {
"kind": "primitive",
"name": "boolean"
}
},
{
"name": "error",
"type": {
"kind": "primitive",
"name": "string"
},
"optional": true
},
{
"name": "triggeredBy",
"type": {
"kind": "primitive",
"name": "string"
}
},
{
"name": "triggeredAt",
"type": {
"kind": "primitive",
"name": "string"
}
},
{
"name": "durationMs",
"type": {
"kind": "primitive",
"name": "number"
}
},
{
"name": "stalledSessionsChecked",
"type": {
"kind": "primitive",
"name": "boolean"
}
},
{
"name": "metricsAggregated",
"type": {
"kind": "primitive",
"name": "boolean"
}
},
{
"name": "aggregatedDate",
"type": {
"kind": "primitive",
"name": "string"
},
"optional": true
},
{
"name": "dataCleanupExecuted",
"type": {
"kind": "primitive",
"name": "boolean"
}
},
{
"name": "platformStatsRecomputed",
"type": {
"kind": "primitive",
"name": "boolean"
}
},
{
"name": "devicesBlockedForExcessiveData",
"type": {
"kind": "primitive",
"name": "number"
}
},
{
"name": "contactEmailsBackfilled",
"type": {
"kind": "primitive",
"name": "number"
}
}
]
},
"ManagedTenantItem": {
"kind": "object",
"doc": "One managed tenant as the managing tenant sees it.",
@@ -23545,15 +23455,8 @@
},
"TriggerMaintenanceResponse": {
"kind": "object",
"doc": "Response of POST maintenance/trigger.",
"doc": "Response of POST maintenance/trigger (202 Accepted): the run is queued, not done. Its progress and its report surface as Maintenance* ops events (Started, Completed, Failed, SkippedLocked).",
"fields": [
{
"name": "success",
"type": {
"kind": "primitive",
"name": "boolean"
}
},
{
"name": "message",
"type": {
@@ -23561,13 +23464,6 @@
"name": "string"
}
},
{
"name": "result",
"type": {
"kind": "ref",
"name": "MaintenanceResult"
}
},
{
"name": "triggeredBy",
"type": {
@@ -23581,6 +23477,22 @@
"kind": "primitive",
"name": "string"
}
},
{
"name": "targetDate",
"type": {
"kind": "primitive",
"name": "string"
},
"optional": true,
"doc": "Date to aggregate (yyyy-MM-dd); null = yesterday."
},
{
"name": "aggregateOnly",
"type": {
"kind": "primitive",
"name": "boolean"
}
}
]
},
@@ -2787,22 +2787,6 @@ export interface LogQueryRequest {
budgetSeconds?: number | null;
}
/** Run report of a manual maintenance run (POST maintenance/trigger). */
export interface MaintenanceResult {
success: boolean;
error?: string;
triggeredBy: string;
triggeredAt: string;
durationMs: number;
stalledSessionsChecked: boolean;
metricsAggregated: boolean;
aggregatedDate?: string;
dataCleanupExecuted: boolean;
platformStatsRecomputed: boolean;
devicesBlockedForExcessiveData: number;
contactEmailsBackfilled: number;
}
/** One managed tenant as the managing tenant sees it. */
export interface ManagedTenantItem {
tenantId: string;
@@ -5158,13 +5142,14 @@ export interface TriggerEpssSyncResponse {
syncedAt: string;
}
/** Response of POST maintenance/trigger. */
/** Response of POST maintenance/trigger (202 Accepted): the run is queued, not done. Its progress and its report surface as Maintenance* ops events (Started, Completed, Failed, SkippedLocked). */
export interface TriggerMaintenanceResponse {
success: boolean;
message: string;
result: MaintenanceResult;
triggeredBy: string;
triggeredAt: string;
/** Date to aggregate (yyyy-MM-dd); null = yesterday. */
targetDate?: string;
aggregateOnly: boolean;
}
/** Response of POST vulnerability/sync-msrc (dual-purpose: success reflects whether the MSRC index refresh finished without error). */