mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
feat(backend,web): manual maintenance trigger answers 202, the run is single-flight (D-255)
The trigger ran the whole maintenance inside the HTTP request (about three minutes against a 230 s gateway limit): the portal reported a timeout while the run went on, a second click started a second run, and nothing serialized those runs against each other or against the 2h timer. Backend - MaintenanceRunGate: the timer and the manual run enter through one blob lease. A held lease writes MaintenanceSkippedLocked and runs nothing; MaintenanceStarted is written after the lease is held, the lease is released last. - POST maintenance/trigger validates the date, probes the lease (409 while a run is active), enqueues on maintenance-trigger (fail-hard) and answers 202 with the typed TriggerMaintenanceResponse. MaintenanceTriggerQueueWorker (self-polling) runs it; a failing or skipped run is not retried. - The run report travels in the details of MaintenanceCompleted. - BlobLeaseLockStore is the shared base of the lease lock stores; SessionDeletionMaintenanceLockStore derives from it, behaviour unchanged. - The probe lets a failed lease release surface as 5xx: a probe lease still held at worker pickup would turn the queued run into a silent skip. Web - useOpsRunStatus: run status from ops events (latest Started newer than the latest Completed/Failed = active), used by the session-cleanup banner (rendering unchanged) and the maintenance trigger section. - Trigger section: queued / already-active messages, status line, polls every 15 s while a run is active, button disabled meanwhile. The 230 s fetch timeout is gone. Shared manifest, generated wire types and the MCP ops-event vocabulary regenerated (two new ops event types, reshaped response). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
+1
-1
Submodule .claude/commands updated: acaf00402e...95b417d3c7
+1
-1
Submodule internal updated: a71f08b7a1...c5861a3e1b
@@ -447,41 +447,27 @@ public class AdminWireParityTests
|
||||
// ---- TriggerMaintenance --------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void TriggerMaintenanceResponse_matches_the_manual_trigger_shape()
|
||||
public void TriggerMaintenanceResponse_matches_the_queued_trigger_shape()
|
||||
{
|
||||
var result = new MaintenanceResult
|
||||
{
|
||||
Success = true,
|
||||
TriggeredBy = "admin@contoso.com",
|
||||
TriggeredAt = new DateTime(2026, 8, 30, 13, 10, 0, DateTimeKind.Utc),
|
||||
DurationMs = 1234,
|
||||
StalledSessionsChecked = true,
|
||||
MetricsAggregated = true,
|
||||
AggregatedDate = "2026-08-29",
|
||||
DataCleanupExecuted = false,
|
||||
PlatformStatsRecomputed = true,
|
||||
DevicesBlockedForExcessiveData = 0,
|
||||
ContactEmailsBackfilled = 2,
|
||||
};
|
||||
var userEmail = "admin@contoso.com";
|
||||
var triggeredAt = new DateTime(2026, 8, 30, 13, 10, 5, DateTimeKind.Utc);
|
||||
|
||||
AssertParity(
|
||||
new
|
||||
{
|
||||
success = true,
|
||||
message = "Maintenance tasks completed",
|
||||
result = result,
|
||||
message = "Maintenance run queued",
|
||||
triggeredBy = userEmail,
|
||||
triggeredAt
|
||||
triggeredAt,
|
||||
targetDate = "2026-08-29",
|
||||
aggregateOnly = true
|
||||
},
|
||||
new TriggerMaintenanceResponse
|
||||
{
|
||||
Success = true,
|
||||
Message = "Maintenance tasks completed",
|
||||
Result = result,
|
||||
Message = "Maintenance run queued",
|
||||
TriggeredBy = userEmail,
|
||||
TriggeredAt = triggeredAt
|
||||
TriggeredAt = triggeredAt,
|
||||
TargetDate = "2026-08-29",
|
||||
AggregateOnly = true
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Azure.Storage.Blobs.Specialized;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Single-flight of the platform maintenance run: the 2h timer and the manual trigger enter
|
||||
/// through one gate, so a second run is skipped instead of aggregating, cleaning up and
|
||||
/// backfilling the same tables in parallel — and a skip never looks like an active run.
|
||||
/// </summary>
|
||||
public class MaintenanceRunGateTests
|
||||
{
|
||||
private sealed class Harness
|
||||
{
|
||||
public Mock<MaintenanceRunLockStore> LockStore { get; }
|
||||
public Mock<BlobLeaseClient> Lease { get; } = new();
|
||||
public List<string> OpsEvents { get; } = new();
|
||||
public MaintenanceRunGate Sut { get; }
|
||||
|
||||
public Harness()
|
||||
{
|
||||
LockStore = new Mock<MaintenanceRunLockStore>(new SessionDeletionMaintenanceFunctionTests.MaintenanceBlobStub());
|
||||
LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Lease.Object);
|
||||
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Returns<OpsEventEntry>(e => { lock (OpsEvents) OpsEvents.Add(e.EventType); return Task.CompletedTask; });
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()));
|
||||
var opsService = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance,
|
||||
TestNotifications.InertOpsAlertDispatch(adminConfig.Object));
|
||||
|
||||
Sut = new MaintenanceRunGate(LockStore.Object, opsService, NullLogger<MaintenanceRunGate>.Instance);
|
||||
}
|
||||
|
||||
public void HoldLease() =>
|
||||
LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new LeaseHeldException("held", new Exception("409")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_held_lease_skips_the_run_and_never_reports_it_as_started()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.HoldLease();
|
||||
var ran = false;
|
||||
|
||||
var result = await h.Sut.RunExclusiveAsync("Timer", () => { ran = true; return Task.CompletedTask; });
|
||||
|
||||
Assert.False(result);
|
||||
Assert.False(ran);
|
||||
Assert.Equal(new[] { OpsEventTypes.MaintenanceSkippedLocked }, h.OpsEvents);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Started_is_written_under_the_lease_before_the_body_and_the_lease_is_released_last()
|
||||
{
|
||||
var h = new Harness();
|
||||
var order = new List<string>();
|
||||
h.Lease.Setup(l => l.ReleaseAsync(It.IsAny<Azure.RequestConditions>(), It.IsAny<CancellationToken>()))
|
||||
.Callback(() => order.Add("release"))
|
||||
.ReturnsAsync(Mock.Of<Azure.Response<Azure.Storage.Blobs.Models.ReleasedObjectInfo>>());
|
||||
|
||||
var result = await h.Sut.RunExclusiveAsync("admin@example.com", () =>
|
||||
{
|
||||
order.Add($"body after [{string.Join(",", h.OpsEvents)}]");
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
Assert.True(result);
|
||||
Assert.Equal(new[] { $"body after [{OpsEventTypes.MaintenanceStarted}]", "release" }, order);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_lease_is_released_when_the_body_throws()
|
||||
{
|
||||
var h = new Harness();
|
||||
var released = false;
|
||||
h.Lease.Setup(l => l.ReleaseAsync(It.IsAny<Azure.RequestConditions>(), It.IsAny<CancellationToken>()))
|
||||
.Callback(() => released = true)
|
||||
.ReturnsAsync(Mock.Of<Azure.Response<Azure.Storage.Blobs.Models.ReleasedObjectInfo>>());
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
h.Sut.RunExclusiveAsync("Timer", () => throw new InvalidOperationException("boom")));
|
||||
|
||||
Assert.True(released);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_reports_an_active_run_while_the_lease_is_held()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.HoldLease();
|
||||
|
||||
Assert.True(await h.Sut.IsRunActiveAsync());
|
||||
Assert.Empty(h.OpsEvents);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_releases_its_lease_and_reports_no_active_run()
|
||||
{
|
||||
var h = new Harness();
|
||||
var released = false;
|
||||
h.Lease.Setup(l => l.ReleaseAsync(It.IsAny<Azure.RequestConditions>(), It.IsAny<CancellationToken>()))
|
||||
.Callback(() => released = true)
|
||||
.ReturnsAsync(Mock.Of<Azure.Response<Azure.Storage.Blobs.Models.ReleasedObjectInfo>>());
|
||||
|
||||
Assert.False(await h.Sut.IsRunActiveAsync());
|
||||
Assert.True(released);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_lets_a_failed_release_surface_so_nothing_is_queued_behind_a_stuck_lease()
|
||||
{
|
||||
// A queued run picked up seconds later would find the probe's own lease still held,
|
||||
// skip as "locked" and delete the message — the operator's run would silently vanish.
|
||||
var h = new Harness();
|
||||
h.Lease.Setup(l => l.ReleaseAsync(It.IsAny<Azure.RequestConditions>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new Azure.RequestFailedException(500, "storage down"));
|
||||
|
||||
await Assert.ThrowsAsync<Azure.RequestFailedException>(() => h.Sut.IsRunActiveAsync());
|
||||
}
|
||||
}
|
||||
+34
-32
@@ -2,7 +2,7 @@ using System;
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
@@ -11,24 +11,29 @@ using Microsoft.Extensions.Logging;
|
||||
namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
{
|
||||
/// <summary>
|
||||
/// Allows Global Admins to manually trigger maintenance tasks
|
||||
/// <c>POST /api/maintenance/trigger</c> — manual Global Admin trigger of the platform
|
||||
/// maintenance run. The run takes minutes, so the request only queues it:
|
||||
/// <c>202 Accepted</c> when queued, <c>409</c> while a run (timer or manual) is active,
|
||||
/// <c>500</c> when the enqueue failed — never a hollow success. Progress and the run report
|
||||
/// surface as <c>Maintenance*</c> ops events.
|
||||
/// </summary>
|
||||
public class TriggerMaintenanceFunction
|
||||
{
|
||||
private readonly ILogger<TriggerMaintenanceFunction> _logger;
|
||||
private readonly MaintenanceService _maintenanceService;
|
||||
private readonly IMaintenanceTriggerProducer _producer;
|
||||
private readonly MaintenanceRunGate _runGate;
|
||||
|
||||
public TriggerMaintenanceFunction(
|
||||
ILogger<TriggerMaintenanceFunction> logger,
|
||||
MaintenanceService maintenanceService)
|
||||
IMaintenanceTriggerProducer producer,
|
||||
MaintenanceRunGate runGate)
|
||||
{
|
||||
_logger = logger;
|
||||
_maintenanceService = maintenanceService;
|
||||
_producer = producer;
|
||||
_runGate = runGate;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// POST /api/maintenance/trigger
|
||||
/// Manually trigger maintenance tasks (Global Admin only)
|
||||
/// Query parameters:
|
||||
/// - date: Optional date to aggregate (yyyy-MM-dd). If not provided, uses yesterday.
|
||||
/// - aggregateOnly: If true, only runs aggregation (skips timeout and cleanup)
|
||||
@@ -38,45 +43,42 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "maintenance/trigger")] HttpRequestData req,
|
||||
FunctionContext context)
|
||||
{
|
||||
_logger.LogInformation("Manual maintenance trigger requested");
|
||||
|
||||
// Authentication + GlobalAdminOnly authorization enforced by PolicyEnforcementMiddleware
|
||||
var userEmail = TenantHelper.GetUserIdentifier(req);
|
||||
|
||||
_logger.LogInformation($"Maintenance trigger initiated by Global Admin: {userEmail}");
|
||||
var userEmail = TenantHelper.GetUserIdentifier(req) ?? "GlobalAdmin";
|
||||
var ct = context.CancellationToken;
|
||||
|
||||
try
|
||||
{
|
||||
// Parse query parameters
|
||||
var dateParam = req.Query["date"];
|
||||
var aggregateOnlyParam = req.Query["aggregateOnly"];
|
||||
|
||||
if (!QueryParams.TryUtcInstant(dateParam, "date", out var dateInstant, out _))
|
||||
if (!QueryParams.TryUtcInstant(req.Query["date"], "date", out var dateInstant, out _))
|
||||
{
|
||||
return await req.BadRequestAsync("Invalid date format. Use yyyy-MM-dd");
|
||||
}
|
||||
DateTime? targetDate = dateInstant?.Date;
|
||||
if (targetDate.HasValue)
|
||||
bool aggregateOnly = req.Query["aggregateOnly"]?.ToLower() == "true";
|
||||
|
||||
if (await _runGate.IsRunActiveAsync(ct))
|
||||
{
|
||||
_logger.LogInformation($"Manual maintenance for date: {targetDate:yyyy-MM-dd}");
|
||||
return await req.ConflictAsync("A maintenance run is already active.");
|
||||
}
|
||||
|
||||
bool aggregateOnly = aggregateOnlyParam?.ToLower() == "true";
|
||||
|
||||
// Execute maintenance tasks
|
||||
var result = await _maintenanceService.RunManualAsync(targetDate, aggregateOnly, userEmail);
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new TriggerMaintenanceResponse
|
||||
await _producer.EnqueueAsync(new MaintenanceTriggerEnvelope
|
||||
{
|
||||
Success = true,
|
||||
Message = "Maintenance tasks completed",
|
||||
Result = result,
|
||||
TriggeredBy = userEmail,
|
||||
TriggeredAt = DateTime.UtcNow
|
||||
});
|
||||
TargetDate = targetDate,
|
||||
AggregateOnly = aggregateOnly,
|
||||
}, ct);
|
||||
|
||||
return response;
|
||||
_logger.LogInformation("TriggerMaintenance: run queued (triggeredBy={TriggeredBy}, date={Date:yyyy-MM-dd}, aggregateOnly={AggregateOnly})",
|
||||
userEmail, targetDate, aggregateOnly);
|
||||
|
||||
return await req.JsonAsync(HttpStatusCode.Accepted, new TriggerMaintenanceResponse
|
||||
{
|
||||
Message = "Maintenance run queued",
|
||||
TriggeredBy = userEmail,
|
||||
TriggeredAt = DateTime.UtcNow,
|
||||
TargetDate = targetDate?.ToString("yyyy-MM-dd"),
|
||||
AggregateOnly = aggregateOnly,
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -283,7 +283,17 @@ builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Analyze.Interi
|
||||
builder.Services.AddSingleton<ImeLogPatternService>();
|
||||
builder.Services.AddHttpClient<GitHubRuleRepository>()
|
||||
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi);
|
||||
// Platform maintenance run: blob-lease gate (serializes the 2h timer, the manual trigger and a
|
||||
// second host instance), fail-hard queue producer for POST /api/maintenance/trigger, and the
|
||||
// BackgroundService worker that runs a queued manual run.
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.MaintenanceRunLockStore>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.MaintenanceRunGate>();
|
||||
builder.Services.AddSingleton<
|
||||
AutopilotMonitor.Functions.Services.Maintenance.IMaintenanceTriggerProducer,
|
||||
AutopilotMonitor.Functions.Services.Maintenance.AzureQueueMaintenanceTriggerProducer>();
|
||||
builder.Services.AddSingleton<MaintenanceService>();
|
||||
builder.Services.AddHostedService<
|
||||
AutopilotMonitor.Functions.Services.Maintenance.MaintenanceTriggerQueueWorker>();
|
||||
builder.Services.AddSingleton<LegacyReclassificationService>();
|
||||
builder.Services.AddSingleton<OccurredUtcBackfillService>();
|
||||
builder.Services.AddSingleton<
|
||||
|
||||
+7
-77
@@ -1,12 +1,6 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Functions.Services.Locking;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Azure;
|
||||
using Azure.Storage.Blobs.Models;
|
||||
using Azure.Storage.Blobs.Specialized;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
@@ -19,89 +13,25 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
/// Deliberately a DEDICATED sentinel (<see cref="LockBlobName"/> in the
|
||||
/// <c>deletion-manifests</c> container) rather than the backup sentinel in
|
||||
/// <see cref="BlobBackupStore"/> — sharing that lease would serialize backups against the
|
||||
/// retention fanout, which have no reason to exclude each other. Lease mechanics
|
||||
/// (60s lease + <see cref="MaintenanceLeaseHolder"/> 45s renewal, <see cref="LeaseHeldException"/>
|
||||
/// on contention) are identical to the backup path.
|
||||
/// retention fanout, which have no reason to exclude each other.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The <c>deletion-manifests</c> container has a 30-day lifecycle policy that may delete an
|
||||
/// idle sentinel; <see cref="AcquireLeaseAsync"/> re-ensures it on every acquire, so the
|
||||
/// lock self-heals. The manifest-TTL sweep skips the sentinel because <c>_lock/…</c> does
|
||||
/// not parse as a manifest blob name.
|
||||
/// idle sentinel; the base class re-ensures it on every acquire, so the lock self-heals.
|
||||
/// The manifest-TTL sweep skips the sentinel because <c>_lock/…</c> does not parse as a
|
||||
/// manifest blob name.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class SessionDeletionMaintenanceLockStore
|
||||
public class SessionDeletionMaintenanceLockStore : BlobLeaseLockStore
|
||||
{
|
||||
/// <summary>Sentinel blob path — leased by the timer / manual-trigger worker to serialize runs.</summary>
|
||||
public const string LockBlobName = "_lock/session-deletion-maintenance.lock";
|
||||
|
||||
/// <summary>Lease duration (Azure spec allows 15..60s for renewable leases).</summary>
|
||||
public static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(60);
|
||||
|
||||
private readonly BlobStorageService _blobs;
|
||||
private readonly ILogger<SessionDeletionMaintenanceLockStore> _logger;
|
||||
private int _containerEnsured;
|
||||
|
||||
public SessionDeletionMaintenanceLockStore(
|
||||
BlobStorageService blobs,
|
||||
ILogger<SessionDeletionMaintenanceLockStore> logger)
|
||||
: base(blobs, Constants.BlobContainers.DeletionManifests, LockBlobName, "session-deletion maintenance")
|
||||
{
|
||||
_blobs = blobs;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to acquire the session-deletion maintenance lease. Throws
|
||||
/// <see cref="LeaseHeldException"/> on 409 (another holder); other errors propagate.
|
||||
/// Caller is responsible for renewing (via <see cref="MaintenanceLeaseHolder"/>) and
|
||||
/// releasing. <c>virtual</c> so tests can stub it.
|
||||
/// </summary>
|
||||
public virtual async Task<BlobLeaseClient> AcquireLeaseAsync(TimeSpan? leaseDuration = null, CancellationToken ct = default)
|
||||
{
|
||||
await EnsureSentinelAsync(ct).ConfigureAwait(false);
|
||||
|
||||
var blob = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests).GetBlobClient(LockBlobName);
|
||||
var leaseClient = blob.GetBlobLeaseClient();
|
||||
try
|
||||
{
|
||||
await leaseClient.AcquireAsync(leaseDuration ?? LeaseDuration, conditions: null, ct).ConfigureAwait(false);
|
||||
return leaseClient;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409)
|
||||
{
|
||||
throw new LeaseHeldException("session-deletion maintenance lease is held by another run", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Idempotently ensures the sentinel blob exists — an AcquireLease on a missing blob
|
||||
/// throws 404. If-None-Match=* lets concurrent callers cooperate; both 409
|
||||
/// BlobAlreadyExists and 412 ConditionNotMet are the steady state.
|
||||
/// </summary>
|
||||
private async Task EnsureSentinelAsync(CancellationToken ct)
|
||||
{
|
||||
var container = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests);
|
||||
if (Interlocked.CompareExchange(ref _containerEnsured, 0, 0) == 0)
|
||||
{
|
||||
await container.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
Interlocked.Exchange(ref _containerEnsured, 1);
|
||||
}
|
||||
|
||||
var blob = container.GetBlobClient(LockBlobName);
|
||||
var options = new BlobUploadOptions
|
||||
{
|
||||
HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" },
|
||||
Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All },
|
||||
};
|
||||
try
|
||||
{
|
||||
using var empty = new MemoryStream(Array.Empty<byte>());
|
||||
await blob.UploadAsync(empty, options, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 412)
|
||||
{
|
||||
// Sentinel exists already — steady state.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using Azure;
|
||||
using Azure.Storage.Blobs.Models;
|
||||
using Azure.Storage.Blobs.Specialized;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Locking
|
||||
{
|
||||
/// <summary>
|
||||
/// Blob-lease lock on a sentinel blob: serializes a run against every other entry point of
|
||||
/// the same work (timer, manual trigger, a second host instance). One subclass per lock —
|
||||
/// each names its own sentinel, because two kinds of work that have no reason to exclude
|
||||
/// each other must not share a lease.
|
||||
/// <para>
|
||||
/// Lease mechanics: 60s lease, renewed by <see cref="MaintenanceLeaseHolder"/> every 45s,
|
||||
/// <see cref="LeaseHeldException"/> on contention. The sentinel is re-ensured on every
|
||||
/// acquire, so a lifecycle policy that deletes an idle sentinel cannot break the lock.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public abstract class BlobLeaseLockStore
|
||||
{
|
||||
/// <summary>Lease duration (Azure spec allows 15..60s for renewable leases).</summary>
|
||||
public static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(60);
|
||||
|
||||
private readonly BlobStorageService _blobs;
|
||||
private readonly string _containerName;
|
||||
private readonly string _lockBlobName;
|
||||
private readonly string _description;
|
||||
private int _containerEnsured;
|
||||
|
||||
protected BlobLeaseLockStore(BlobStorageService blobs, string containerName, string lockBlobName, string description)
|
||||
{
|
||||
_blobs = blobs;
|
||||
_containerName = containerName;
|
||||
_lockBlobName = lockBlobName;
|
||||
_description = description;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to acquire the lease. Throws <see cref="LeaseHeldException"/> on 409 (another
|
||||
/// holder); other errors propagate. Caller is responsible for renewing (via
|
||||
/// <see cref="MaintenanceLeaseHolder"/>) and releasing. <c>virtual</c> so tests can stub it.
|
||||
/// </summary>
|
||||
public virtual async Task<BlobLeaseClient> AcquireLeaseAsync(TimeSpan? leaseDuration = null, CancellationToken ct = default)
|
||||
{
|
||||
await EnsureSentinelAsync(ct).ConfigureAwait(false);
|
||||
|
||||
var blob = _blobs.GetContainerClient(_containerName).GetBlobClient(_lockBlobName);
|
||||
var leaseClient = blob.GetBlobLeaseClient();
|
||||
try
|
||||
{
|
||||
await leaseClient.AcquireAsync(leaseDuration ?? LeaseDuration, conditions: null, ct).ConfigureAwait(false);
|
||||
return leaseClient;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409)
|
||||
{
|
||||
throw new LeaseHeldException($"{_description} lease is held by another run", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Idempotently ensures the sentinel blob exists — an AcquireLease on a missing blob
|
||||
/// throws 404. If-None-Match=* lets concurrent callers cooperate; both 409
|
||||
/// BlobAlreadyExists and 412 ConditionNotMet are the steady state.
|
||||
/// </summary>
|
||||
private async Task EnsureSentinelAsync(CancellationToken ct)
|
||||
{
|
||||
var container = _blobs.GetContainerClient(_containerName);
|
||||
if (Interlocked.CompareExchange(ref _containerEnsured, 0, 0) == 0)
|
||||
{
|
||||
await container.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
Interlocked.Exchange(ref _containerEnsured, 1);
|
||||
}
|
||||
|
||||
var blob = container.GetBlobClient(_lockBlobName);
|
||||
var options = new BlobUploadOptions
|
||||
{
|
||||
HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" },
|
||||
Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All },
|
||||
};
|
||||
try
|
||||
{
|
||||
using var empty = new MemoryStream(Array.Empty<byte>());
|
||||
await blob.UploadAsync(empty, options, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 412)
|
||||
{
|
||||
// Sentinel exists already — steady state.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Queueing;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Azure.Storage.Queues;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Azure-Storage-Queue producer for the manual maintenance trigger. Same shape as the
|
||||
/// session-deletion and backup trigger producers: <see cref="QueueClientFactory"/>
|
||||
/// (Managed Identity preferred), fail-hard.
|
||||
/// </summary>
|
||||
public sealed class AzureQueueMaintenanceTriggerProducer : IMaintenanceTriggerProducer
|
||||
{
|
||||
private readonly QueueClient _queueClient;
|
||||
private readonly ILogger<AzureQueueMaintenanceTriggerProducer> _logger;
|
||||
private int _queueEnsured;
|
||||
|
||||
public AzureQueueMaintenanceTriggerProducer(
|
||||
QueueClientFactory queueFactory,
|
||||
ILogger<AzureQueueMaintenanceTriggerProducer> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
_queueClient = queueFactory.Create(Constants.QueueNames.MaintenanceTrigger);
|
||||
}
|
||||
|
||||
public async Task EnqueueAsync(MaintenanceTriggerEnvelope envelope, CancellationToken ct = default)
|
||||
{
|
||||
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
|
||||
if (string.IsNullOrEmpty(envelope.TriggeredBy)) throw new ArgumentException("TriggeredBy required", nameof(envelope));
|
||||
|
||||
await EnsureQueueExistsAsync(ct).ConfigureAwait(false);
|
||||
|
||||
// Fail-hard: do NOT swallow. Caller (HTTP trigger) catches and returns 500.
|
||||
var body = JsonConvert.SerializeObject(envelope);
|
||||
await _queueClient.SendMessageAsync(body, ct).ConfigureAwait(false);
|
||||
_logger.LogInformation("Maintenance trigger enqueued (triggeredBy={TriggeredBy})", envelope.TriggeredBy);
|
||||
}
|
||||
|
||||
private async Task EnsureQueueExistsAsync(CancellationToken ct)
|
||||
{
|
||||
if (_queueEnsured == 1) return;
|
||||
try
|
||||
{
|
||||
await _queueClient.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
Interlocked.Exchange(ref _queueEnsured, 1);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Ensure failure during enqueue is itself a fail-hard signal — surface to the caller.
|
||||
_logger.LogError(ex, "Maintenance trigger queue CreateIfNotExists failed");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Queue envelope of a manually triggered maintenance run. There is no per-run state row —
|
||||
/// the Maintenance* ops events are the status surface.
|
||||
/// </summary>
|
||||
public sealed class MaintenanceTriggerEnvelope
|
||||
{
|
||||
public string TriggeredBy { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Date to aggregate (UTC date); null = yesterday.</summary>
|
||||
public DateTime? TargetDate { get; set; }
|
||||
|
||||
public bool AggregateOnly { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Producer for the manual maintenance trigger. <b>Fail-hard</b>: send failures propagate so
|
||||
/// the HTTP trigger answers 5xx instead of a hollow 202.
|
||||
/// </summary>
|
||||
public interface IMaintenanceTriggerProducer
|
||||
{
|
||||
Task EnqueueAsync(MaintenanceTriggerEnvelope envelope, CancellationToken ct = default);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using Azure.Storage.Blobs.Specialized;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Single-flight gate for the platform maintenance run: the 2h timer, the manual trigger
|
||||
/// and a second host instance all enter through <see cref="RunExclusiveAsync"/>, so two
|
||||
/// runs never aggregate, clean up and backfill the same tables at the same time.
|
||||
/// </summary>
|
||||
public class MaintenanceRunGate
|
||||
{
|
||||
private readonly MaintenanceRunLockStore _lockStore;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
private readonly ILogger<MaintenanceRunGate> _logger;
|
||||
|
||||
public MaintenanceRunGate(MaintenanceRunLockStore lockStore, OpsEventService opsEvents, ILogger<MaintenanceRunGate> logger)
|
||||
{
|
||||
_lockStore = lockStore;
|
||||
_opsEvents = opsEvents;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Acquire+release probe for the manual trigger: a held lease means a run is active RIGHT
|
||||
/// NOW, so the operator is told instead of queuing a message that would only produce a
|
||||
/// SkippedLocked event. Racing the probe (a run starts between probe and worker pickup)
|
||||
/// is benign: the worker's own acquire lands on the SkippedLocked path.
|
||||
/// </summary>
|
||||
public virtual async Task<bool> IsRunActiveAsync(CancellationToken ct = default)
|
||||
{
|
||||
BlobLeaseClient lease;
|
||||
try
|
||||
{
|
||||
lease = await _lockStore.AcquireLeaseAsync(ct: ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (LeaseHeldException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// A probe lease that cannot be released would still be held when the worker picks the
|
||||
// message up seconds later — the run would be skipped as "locked" and the message
|
||||
// deleted. Let the failure surface: the trigger answers 5xx and nothing is queued.
|
||||
await lease.ReleaseAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs <paramref name="body"/> under the maintenance-run lease. Returns false without
|
||||
/// running it when another run holds the lease (SkippedLocked ops event). The Started
|
||||
/// event is written AFTER the lease is held, so a skip never looks like an active run,
|
||||
/// and the lease is released LAST, so the body's terminal ops event is still exclusive.
|
||||
/// </summary>
|
||||
public virtual async Task<bool> RunExclusiveAsync(string triggeredBy, Func<Task> body)
|
||||
{
|
||||
BlobLeaseClient lease;
|
||||
try
|
||||
{
|
||||
lease = await _lockStore.AcquireLeaseAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (LeaseHeldException)
|
||||
{
|
||||
_logger.LogWarning("Maintenance: lease held by another run — skipping (triggeredBy={TriggeredBy})", triggeredBy);
|
||||
await _opsEvents.RecordMaintenanceSkippedLockedAsync(triggeredBy).ConfigureAwait(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
// The maintenance body takes no cancellation token; a lost lease is reported, not
|
||||
// enforced — the run is minutes long and the lease renews every 45s.
|
||||
using var leaseLost = new CancellationTokenSource();
|
||||
var leaseHolder = new MaintenanceLeaseHolder(lease, leaseLost, _logger);
|
||||
try
|
||||
{
|
||||
await _opsEvents.RecordMaintenanceStartedAsync(triggeredBy).ConfigureAwait(false);
|
||||
await body().ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (leaseHolder.RenewalFailureReason != null)
|
||||
_logger.LogWarning("Maintenance: the run lease could not be renewed during the run ({Reason}) — another run may have overlapped",
|
||||
leaseHolder.RenewalFailureReason);
|
||||
await leaseHolder.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
using AutopilotMonitor.Functions.Services.Locking;
|
||||
using AutopilotMonitor.Shared;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Blob-lease lock for the platform maintenance run (2h timer and manual trigger). Its own
|
||||
/// sentinel: the session-deletion maintenance and the critical-table backup have no reason
|
||||
/// to exclude this run, so they keep their own leases.
|
||||
/// </summary>
|
||||
public class MaintenanceRunLockStore : BlobLeaseLockStore
|
||||
{
|
||||
/// <summary>Sentinel blob path. <c>_lock/…</c> does not parse as a manifest blob name, so the manifest-TTL sweep skips it.</summary>
|
||||
public const string LockBlobName = "_lock/platform-maintenance.lock";
|
||||
|
||||
public MaintenanceRunLockStore(BlobStorageService blobs)
|
||||
: base(blobs, Constants.BlobContainers.DeletionManifests, LockBlobName, "platform maintenance")
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Queueing;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Background worker for the <c>maintenance-trigger</c> queue — runs the manually triggered
|
||||
/// maintenance off the HTTP request, which would otherwise have to stay open for minutes
|
||||
/// and dies at the gateway limit.
|
||||
/// <para>
|
||||
/// A self-polling worker, not a queue-triggered function, because the run holds a lease
|
||||
/// for its whole lifetime. Concurrency against the 2h timer is handled inside
|
||||
/// <see cref="MaintenanceService.RunManualAsync"/> by the run gate: a lease-held run
|
||||
/// returns normally (SkippedLocked ops event), so the message is deleted rather than
|
||||
/// retried — the operator asked for "a run", and one is already active. A failing run
|
||||
/// reports MaintenanceFailed and returns as well: maintenance is not retried blindly.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class MaintenanceTriggerQueueWorker : QueuePollingWorker<MaintenanceTriggerEnvelope>
|
||||
{
|
||||
private readonly MaintenanceService _maintenance;
|
||||
|
||||
public MaintenanceTriggerQueueWorker(
|
||||
QueueClientFactory queueFactory,
|
||||
MaintenanceService maintenance,
|
||||
ILogger<MaintenanceTriggerQueueWorker> logger)
|
||||
: base(queueFactory, Constants.QueueNames.MaintenanceTrigger, logger, Constants.QueueNames.MaintenanceTriggerPoison)
|
||||
{
|
||||
_maintenance = maintenance ?? throw new ArgumentNullException(nameof(maintenance));
|
||||
}
|
||||
|
||||
// The host functionTimeout (60min) is the upper bound of a run; the message must not
|
||||
// reappear while its run is still going.
|
||||
protected override TimeSpan VisibilityTimeout => TimeSpan.FromMinutes(60);
|
||||
|
||||
protected override bool TryValidate(MaintenanceTriggerEnvelope envelope)
|
||||
=> !string.IsNullOrEmpty(envelope.TriggeredBy);
|
||||
|
||||
protected override string DescribeForLog(MaintenanceTriggerEnvelope envelope)
|
||||
=> $"triggeredBy={envelope.TriggeredBy} targetDate={envelope.TargetDate:yyyy-MM-dd} aggregateOnly={envelope.AggregateOnly}";
|
||||
|
||||
protected override Task HandleAsync(MaintenanceTriggerEnvelope envelope, CancellationToken ct)
|
||||
=> _maintenance.RunManualAsync(envelope.TargetDate, envelope.AggregateOnly, envelope.TriggeredBy);
|
||||
}
|
||||
}
|
||||
@@ -55,6 +55,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly PreviewWhitelistService _previewWhitelistService;
|
||||
private readonly IStorageInitializer _storageInitializer;
|
||||
private readonly Maintenance.MaintenanceRunGate _runGate;
|
||||
private readonly ILogger<MaintenanceService> _logger;
|
||||
|
||||
private const string PlatformStatsAliasFileName = "platform-stats.json";
|
||||
@@ -90,6 +91,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
IHttpClientFactory httpClientFactory,
|
||||
PreviewWhitelistService previewWhitelistService,
|
||||
IStorageInitializer storageInitializer,
|
||||
Maintenance.MaintenanceRunGate runGate,
|
||||
ILogger<MaintenanceService> logger)
|
||||
{
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
@@ -120,6 +122,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_previewWhitelistService = previewWhitelistService;
|
||||
_storageInitializer = storageInitializer;
|
||||
_runGate = runGate;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -136,9 +139,12 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs all maintenance tasks (used by the daily timer trigger)
|
||||
/// Runs all maintenance tasks (used by the 2h timer trigger). Single-flight with the
|
||||
/// manual trigger: skipped when another run holds the maintenance-run lease.
|
||||
/// </summary>
|
||||
public async Task RunAllAsync()
|
||||
public Task RunAllAsync() => _runGate.RunExclusiveAsync("Timer", RunAllCoreAsync);
|
||||
|
||||
private async Task RunAllCoreAsync()
|
||||
{
|
||||
_logger.LogInformation($"Daily maintenance started at {DateTime.UtcNow}");
|
||||
var maintenanceStart = Stopwatch.StartNew();
|
||||
@@ -246,9 +252,27 @@ namespace AutopilotMonitor.Functions.Services
|
||||
=> window.Where(s => s.StartedAt >= windowStart).ToList();
|
||||
|
||||
/// <summary>
|
||||
/// Manually triggered maintenance with flexible date selection
|
||||
/// Manually triggered maintenance with flexible date selection. Single-flight with the
|
||||
/// timer: when another run holds the maintenance-run lease, nothing runs and the result
|
||||
/// says so.
|
||||
/// </summary>
|
||||
public async Task<MaintenanceResult> RunManualAsync(DateTime? targetDate = null, bool aggregateOnly = false, string triggeredBy = "Unknown")
|
||||
{
|
||||
MaintenanceResult? result = null;
|
||||
var ran = await _runGate.RunExclusiveAsync(triggeredBy,
|
||||
async () => result = await RunManualCoreAsync(targetDate, aggregateOnly, triggeredBy));
|
||||
return ran && result != null
|
||||
? result
|
||||
: new MaintenanceResult
|
||||
{
|
||||
TriggeredBy = triggeredBy,
|
||||
TriggeredAt = DateTime.UtcNow,
|
||||
Success = false,
|
||||
Error = "Another maintenance run is active",
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<MaintenanceResult> RunManualCoreAsync(DateTime? targetDate, bool aggregateOnly, string triggeredBy)
|
||||
{
|
||||
_logger.LogInformation($"Manual maintenance triggered by {triggeredBy} at {DateTime.UtcNow}");
|
||||
var maintenanceStart = Stopwatch.StartNew();
|
||||
@@ -329,7 +353,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
result.Success = true;
|
||||
|
||||
_logger.LogInformation($"Manual maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms");
|
||||
await _opsEventService.RecordMaintenanceCompletedAsync(result.DurationMs, triggeredBy);
|
||||
await _opsEventService.RecordMaintenanceCompletedAsync(result.DurationMs, triggeredBy, result);
|
||||
return result;
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
@@ -64,10 +64,42 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
// ── Maintenance ────────────────────────────────────────────────────────
|
||||
|
||||
public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy)
|
||||
/// <summary>
|
||||
/// Emitted after the maintenance-run lease was acquired, so a lease-skip never
|
||||
/// masquerades as an active run — the portal treats "latest Started newer than latest
|
||||
/// Completed/Failed" as run-active.
|
||||
/// </summary>
|
||||
public Task RecordMaintenanceStartedAsync(string triggeredBy)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceStarted, OpsEventSeverity.Info,
|
||||
$"Maintenance run started (triggered by {triggeredBy})",
|
||||
null, triggeredBy, new { triggeredBy });
|
||||
|
||||
/// <summary>Another run (timer, manual trigger or a second host instance) held the maintenance-run lease.</summary>
|
||||
public Task RecordMaintenanceSkippedLockedAsync(string triggeredBy)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceSkippedLocked, OpsEventSeverity.Info,
|
||||
$"Maintenance run skipped — another run holds the lease (triggered by {triggeredBy})",
|
||||
null, triggeredBy, new { reason = "lease held by another run", triggeredBy });
|
||||
|
||||
/// <summary>
|
||||
/// <paramref name="report"/> is the run report of a manual run: the trigger answers 202
|
||||
/// before the run starts, so this event is where the operator reads what the run did.
|
||||
/// </summary>
|
||||
public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy, MaintenanceResult? report = null)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceCompleted, OpsEventSeverity.Info,
|
||||
$"Maintenance completed in {durationMs}ms (triggered by {triggeredBy})",
|
||||
null, triggeredBy, new { durationMs });
|
||||
null, triggeredBy,
|
||||
report == null
|
||||
? (object)new { durationMs }
|
||||
: new
|
||||
{
|
||||
durationMs,
|
||||
aggregatedDate = report.AggregatedDate,
|
||||
stalledSessionsChecked = report.StalledSessionsChecked,
|
||||
dataCleanupExecuted = report.DataCleanupExecuted,
|
||||
platformStatsRecomputed = report.PlatformStatsRecomputed,
|
||||
devicesBlockedForExcessiveData = report.DevicesBlockedForExcessiveData,
|
||||
contactEmailsBackfilled = report.ContactEmailsBackfilled,
|
||||
});
|
||||
|
||||
public Task RecordMaintenanceFailedAsync(string error, string triggeredBy)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceFailed, OpsEventSeverity.Error,
|
||||
|
||||
@@ -2787,22 +2787,6 @@ export interface LogQueryRequest {
|
||||
budgetSeconds?: number | null;
|
||||
}
|
||||
|
||||
/** Run report of a manual maintenance run (POST maintenance/trigger). */
|
||||
export interface MaintenanceResult {
|
||||
success: boolean;
|
||||
error?: string;
|
||||
triggeredBy: string;
|
||||
triggeredAt: string;
|
||||
durationMs: number;
|
||||
stalledSessionsChecked: boolean;
|
||||
metricsAggregated: boolean;
|
||||
aggregatedDate?: string;
|
||||
dataCleanupExecuted: boolean;
|
||||
platformStatsRecomputed: boolean;
|
||||
devicesBlockedForExcessiveData: number;
|
||||
contactEmailsBackfilled: number;
|
||||
}
|
||||
|
||||
/** One managed tenant as the managing tenant sees it. */
|
||||
export interface ManagedTenantItem {
|
||||
tenantId: string;
|
||||
@@ -5158,13 +5142,14 @@ export interface TriggerEpssSyncResponse {
|
||||
syncedAt: string;
|
||||
}
|
||||
|
||||
/** Response of POST maintenance/trigger. */
|
||||
/** Response of POST maintenance/trigger (202 Accepted): the run is queued, not done. Its progress and its report surface as Maintenance* ops events (Started, Completed, Failed, SkippedLocked). */
|
||||
export interface TriggerMaintenanceResponse {
|
||||
success: boolean;
|
||||
message: string;
|
||||
result: MaintenanceResult;
|
||||
triggeredBy: string;
|
||||
triggeredAt: string;
|
||||
/** Date to aggregate (yyyy-MM-dd); null = yesterday. */
|
||||
targetDate?: string;
|
||||
aggregateOnly: boolean;
|
||||
}
|
||||
|
||||
/** Response of POST vulnerability/sync-msrc (dual-purpose: success reflects whether the MSRC index refresh finished without error). */
|
||||
|
||||
@@ -64,6 +64,8 @@ export const OPS_EVENT_TYPES = [
|
||||
"MaintenanceCompleted",
|
||||
"MaintenanceFailed",
|
||||
"MaintenanceLongRunning",
|
||||
"MaintenanceStarted",
|
||||
"MaintenanceSkippedLocked",
|
||||
"SessionSweepCompleted",
|
||||
"SessionSweepFailed",
|
||||
"OpsEventCleanup",
|
||||
|
||||
@@ -1701,6 +1701,20 @@ namespace AutopilotMonitor.Shared
|
||||
/// <summary>Poison sibling of <see cref="SessionDeletionMaintenance"/>.</summary>
|
||||
public const string SessionDeletionMaintenancePoison = "session-deletion-maintenance-poison";
|
||||
|
||||
/// <summary>
|
||||
/// Manual-trigger queue for the platform maintenance run. Producer = HTTP trigger
|
||||
/// <c>/api/maintenance/trigger</c> (fail-hard, answers 202); consumer =
|
||||
/// <c>MaintenanceTriggerQueueWorker</c> (self-managed BackgroundService,
|
||||
/// VisibilityTimeout=60min = the host functionTimeout). Envelope
|
||||
/// <c>{ triggeredBy, targetDate, aggregateOnly }</c>; concurrency against the 2h timer
|
||||
/// is serialized by the maintenance-run blob lease, not the queue.
|
||||
/// Poison-suffix <c>-poison</c>, max-dequeue 5.
|
||||
/// </summary>
|
||||
public const string MaintenanceTrigger = "maintenance-trigger";
|
||||
|
||||
/// <summary>Poison sibling of <see cref="MaintenanceTrigger"/>.</summary>
|
||||
public const string MaintenanceTriggerPoison = "maintenance-trigger-poison";
|
||||
|
||||
/// <summary>
|
||||
/// IME-MSI archive fan-out. Producer = <c>EventIngestProcessor</c> when
|
||||
/// <c>RecordImeVersionAsync</c> reports a genuinely NEW IME version (first sighting
|
||||
|
||||
@@ -32,6 +32,8 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
public const string MaintenanceCompleted = "MaintenanceCompleted";
|
||||
public const string MaintenanceFailed = "MaintenanceFailed";
|
||||
public const string MaintenanceLongRunning = "MaintenanceLongRunning";
|
||||
public const string MaintenanceStarted = "MaintenanceStarted";
|
||||
public const string MaintenanceSkippedLocked = "MaintenanceSkippedLocked";
|
||||
public const string SessionSweepCompleted = "SessionSweepCompleted";
|
||||
public const string SessionSweepFailed = "SessionSweepFailed";
|
||||
public const string OpsEventCleanup = "OpsEventCleanup";
|
||||
@@ -135,7 +137,7 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
public static readonly IReadOnlyList<string> All = new[]
|
||||
{
|
||||
ConsentFlowStarted, ConsentFlowSuccess, ConsentFlowFailed, ConsentRedirectUriMismatch, AppHomingFlipped, AppHomingFlippedWithEntraRoles,
|
||||
MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift,
|
||||
MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift,
|
||||
DeviceBlocked, VersionBlocked, SessionTenantConflict, SessionOwnerMismatch, KillSignalDelivered, EmbeddedCertExpiringSoon, EmbeddedCertExpiringUrgent, EmbeddedCertExpired, EmbeddedCertBundleEmpty, SignalRConnectionsHigh, SignalRConnectionsCritical, SignalRMessagesHigh, SignalRMessagesCritical, PoisonQueueBacklogHigh, PoisonQueueBacklogCritical, ExcessiveSessionEventsAutoActioned, PrivilegedRouteDenied, McpServicePrincipalFirstSeen,
|
||||
OffboardingFeedbackReceived, TenantOffboarded, TenantOffboardingFailed, TenantAutoApproved, WelcomeEmailSent, WelcomeEmailSkipped, WelcomeEmailFailed, TenantTrialStarted, TenantTrialExpiring, TenantTrialExpired, TenantPlanDowngraded, TenantRetentionGraceExpiring, TenantRetentionGraceEnded, RuleFrequencyRegression, AppVersionDurationRegression, CollectLogsQuickConfigEnabled, DiagnosticsUploadEnabled, DiagnosticsUploadDisabled,
|
||||
SessionActionQueued, SessionTimeouts, AgentEmergencyBreak, AgentBinaryIntegrityMismatch, CmTraceTimeSkewRegression, ExcessiveSessionEvents, NewImeVersionDetected, ImePatternDriftSuspected, BlobStorageMissing, BlobStorageUnreachable, TelemetryItemsRejected,
|
||||
|
||||
@@ -145,15 +145,20 @@ namespace AutopilotMonitor.Shared.Models
|
||||
public DateTime TriggeredAt { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Response of POST maintenance/trigger.</summary>
|
||||
/// <summary>
|
||||
/// Response of POST maintenance/trigger (202 Accepted): the run is queued, not done. Its
|
||||
/// progress and its report surface as Maintenance* ops events (Started, Completed, Failed,
|
||||
/// SkippedLocked).
|
||||
/// </summary>
|
||||
// Declaration order == wire order.
|
||||
public class TriggerMaintenanceResponse : IApiResponse
|
||||
{
|
||||
public bool Success { get; set; }
|
||||
public string Message { get; set; } = default!;
|
||||
public MaintenanceResult Result { get; set; } = default!;
|
||||
public string TriggeredBy { get; set; } = default!;
|
||||
public DateTime TriggeredAt { get; set; }
|
||||
/// <summary>Date to aggregate (yyyy-MM-dd); null = yesterday.</summary>
|
||||
public string? TargetDate { get; set; }
|
||||
public bool AggregateOnly { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -218,7 +223,7 @@ namespace AutopilotMonitor.Shared.Models
|
||||
public string Reason { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>Run report of a manual maintenance run (POST maintenance/trigger).</summary>
|
||||
/// <summary>Run report of a manual maintenance run; its figures travel in the MaintenanceCompleted ops event.</summary>
|
||||
// Declaration order == wire order.
|
||||
public class MaintenanceResult
|
||||
{
|
||||
|
||||
@@ -2,16 +2,32 @@
|
||||
|
||||
import { useState } from "react";
|
||||
import { api } from "@/lib/api";
|
||||
import { apiErrorText, fetchJson, fetchOk } from "@/lib/apiClient";
|
||||
import { ApiError, apiErrorText, fetchJson, fetchOk } from "@/lib/apiClient";
|
||||
import { parseOpsDetails, useOpsRunStatus, type OpsRunLifecycle } from "@/hooks/useOpsRunStatus";
|
||||
import { SectionCardHeader } from "@/components/SectionCardHeader";
|
||||
import type { GetLatestVersionsResponse } from "@/utils/wire-types.generated";
|
||||
|
||||
const LIFECYCLE: OpsRunLifecycle = {
|
||||
category: "Maintenance",
|
||||
started: "MaintenanceStarted",
|
||||
completed: "MaintenanceCompleted",
|
||||
failed: "MaintenanceFailed",
|
||||
};
|
||||
|
||||
// A run takes a few minutes; while one is active the status line follows it on its own.
|
||||
const POLL_WHILE_ACTIVE_MS = 15_000;
|
||||
|
||||
interface MaintenanceTriggerSectionProps {
|
||||
getAccessToken: () => Promise<string | null>;
|
||||
setError: (error: string | null) => void;
|
||||
setSuccessMessage: (message: string | null) => void;
|
||||
}
|
||||
|
||||
function formatRunDuration(durationMs: number | undefined): string {
|
||||
if (typeof durationMs !== "number") return "";
|
||||
return durationMs < 60_000 ? ` in ${Math.round(durationMs / 1000)} s` : ` in ${Math.round(durationMs / 60_000)} min`;
|
||||
}
|
||||
|
||||
export function MaintenanceTriggerSection({
|
||||
getAccessToken,
|
||||
setError,
|
||||
@@ -23,6 +39,8 @@ export function MaintenanceTriggerSection({
|
||||
// (react-hooks/purity) and the cap doesn't need a live midnight rollover.
|
||||
const [maxMaintenanceDate] = useState(() => new Date(Date.now() - 86400000).toISOString().split('T')[0]);
|
||||
const [refreshingVersions, setRefreshingVersions] = useState(false);
|
||||
const { status: runStatus, refresh: refreshRunStatus } = useOpsRunStatus(getAccessToken, setError, LIFECYCLE, POLL_WHILE_ACTIVE_MS);
|
||||
const runActive = runStatus.kind === "active";
|
||||
|
||||
const handleRefreshLatestVersions = async () => {
|
||||
try {
|
||||
@@ -53,15 +71,20 @@ export function MaintenanceTriggerSection({
|
||||
setError(null);
|
||||
setSuccessMessage(null);
|
||||
|
||||
// The run answers when it is done (minutes), far beyond the default fetch timeout;
|
||||
// the platform gateway cuts a request at 230 s, so waiting longer gains nothing.
|
||||
await fetchOk(api.maintenance.trigger(maintenanceDate || undefined), getAccessToken, {
|
||||
method: "POST",
|
||||
signal: AbortSignal.timeout(230_000),
|
||||
});
|
||||
// 202 = queued (the run reports through Maintenance* ops events); 409 = a run is
|
||||
// already active, which is not an error for the operator.
|
||||
const queued = await fetchOk(api.maintenance.trigger(maintenanceDate || undefined), getAccessToken, { method: "POST" })
|
||||
.then(() => true)
|
||||
.catch((err: unknown) => {
|
||||
if (err instanceof ApiError && err.status === 409) return false;
|
||||
throw err;
|
||||
});
|
||||
|
||||
const dateInfo = maintenanceDate ? ` for ${maintenanceDate}` : '';
|
||||
setSuccessMessage(`Maintenance job completed successfully${dateInfo}!`);
|
||||
setSuccessMessage(queued
|
||||
? `Maintenance run queued${dateInfo} — its status appears below shortly.`
|
||||
: "A maintenance run is already active.");
|
||||
refreshRunStatus();
|
||||
|
||||
setTimeout(() => setSuccessMessage(null), 5000);
|
||||
} catch (err) {
|
||||
@@ -142,8 +165,8 @@ export function MaintenanceTriggerSection({
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
|
||||
</svg>
|
||||
<p className="text-sm text-gray-800 dark:text-gray-200">
|
||||
<strong>Warning:</strong> This operation runs across all tenants and may take several minutes to complete.
|
||||
Use this only for testing or when immediate cleanup is needed.
|
||||
<strong>Warning:</strong> This operation runs across all tenants. The run is queued and takes a few minutes in the background;
|
||||
only one run is active at a time, the 2h timer included. Use this only for testing or when immediate cleanup is needed.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
@@ -179,16 +202,35 @@ export function MaintenanceTriggerSection({
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex justify-end pt-2">
|
||||
<div className="flex items-center justify-between gap-4 pt-2">
|
||||
<p className={`text-sm ${runStatus.kind === "failed" ? "text-red-700 dark:text-red-300" : "text-purple-900 dark:text-gray-200"}`}>
|
||||
{runStatus.kind === "active" && (
|
||||
<>
|
||||
<span className="font-medium">Maintenance run active</span> — started{" "}
|
||||
{new Date(runStatus.since).toLocaleString()} (triggered by {runStatus.triggeredBy}).
|
||||
</>
|
||||
)}
|
||||
{runStatus.kind === "failed" && (
|
||||
<>
|
||||
<span className="font-medium">Last run failed</span> ({new Date(runStatus.at).toLocaleString()}): {runStatus.message}
|
||||
</>
|
||||
)}
|
||||
{runStatus.kind === "completed" && (
|
||||
<>
|
||||
Last run completed {new Date(runStatus.at).toLocaleString()}
|
||||
{formatRunDuration(parseOpsDetails<{ durationMs?: number }>(runStatus.details).durationMs)}.
|
||||
</>
|
||||
)}
|
||||
</p>
|
||||
<button
|
||||
onClick={handleTriggerMaintenance}
|
||||
disabled={triggeringMaintenance}
|
||||
disabled={triggeringMaintenance || runActive}
|
||||
className="px-6 py-3 bg-gradient-to-r from-purple-600 to-violet-600 text-white rounded-lg hover:from-purple-700 hover:to-violet-700 disabled:opacity-50 disabled:cursor-not-allowed transition-all shadow-md hover:shadow-lg flex items-center space-x-2"
|
||||
>
|
||||
{triggeringMaintenance ? (
|
||||
<>
|
||||
<div className="animate-spin rounded-full h-5 w-5 border-b-2 border-white"></div>
|
||||
<span>Running...</span>
|
||||
<span>Queuing...</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
|
||||
@@ -32,6 +32,10 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
|
||||
// Soft watchdog (Warning): a maintenance run completed but exceeded the 10min threshold and is
|
||||
// climbing toward the host's 60min functionTimeout. Backend helper RecordMaintenanceLongRunningAsync.
|
||||
"MaintenanceLongRunning",
|
||||
// Run lifecycle + lease serialization: Started marks run begin (timer or manual,
|
||||
// details.triggeredBy), SkippedLocked means another run held the maintenance-run lease.
|
||||
"MaintenanceStarted",
|
||||
"MaintenanceSkippedLocked",
|
||||
"OpsEventCleanup",
|
||||
"SessionTimeouts",
|
||||
// Hourly stalled-session sweep interleave (SessionSweepFunction, minute 30) — dual-register
|
||||
|
||||
+22
-90
@@ -1,22 +1,9 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useCallback, useState } from "react";
|
||||
import { api } from "@/lib/api";
|
||||
import { TokenExpiredError } from "@/lib/authenticatedFetch";
|
||||
import { ApiError, apiErrorText, fetchJson, fetchOk } from "@/lib/apiClient";
|
||||
|
||||
interface OpsEvent {
|
||||
id: string;
|
||||
eventType: string;
|
||||
severity: string;
|
||||
message: string;
|
||||
details: string | null;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
interface OpsEventsResponse {
|
||||
events: OpsEvent[];
|
||||
}
|
||||
import { ApiError, apiErrorText, fetchOk } from "@/lib/apiClient";
|
||||
import { parseOpsDetails, useOpsRunStatus, type OpsRunLifecycle } from "@/hooks/useOpsRunStatus";
|
||||
|
||||
interface CompletedDetails {
|
||||
tenantsProcessed?: number;
|
||||
@@ -34,14 +21,12 @@ type BannerState =
|
||||
| { kind: "idle"; at: string; details: CompletedDetails }
|
||||
| { kind: "none" }; // no lifecycle events in the window — render nothing
|
||||
|
||||
function parseDetails<T>(raw: string | null): T {
|
||||
if (!raw) return {} as T;
|
||||
try {
|
||||
return JSON.parse(raw) as T;
|
||||
} catch {
|
||||
return {} as T;
|
||||
}
|
||||
}
|
||||
const LIFECYCLE: OpsRunLifecycle = {
|
||||
category: "Maintenance",
|
||||
started: "SessionDeletionMaintenanceStarted",
|
||||
completed: "SessionDeletionMaintenanceCompleted",
|
||||
failed: "SessionDeletionMaintenanceFailed",
|
||||
};
|
||||
|
||||
/**
|
||||
* Status banner for the session-deletion maintenance run (12h retention fanout + GC sweeps).
|
||||
@@ -59,71 +44,18 @@ export function MaintenanceStatusBanner({
|
||||
setError: (error: string | null) => void;
|
||||
setSuccessMessage: (message: string | null) => void;
|
||||
}) {
|
||||
const [state, setState] = useState<BannerState>({ kind: "loading" });
|
||||
const { status, refresh } = useOpsRunStatus(getAccessToken, setError, LIFECYCLE);
|
||||
const [triggering, setTriggering] = useState(false);
|
||||
const [refreshKey, setRefreshKey] = useState(0);
|
||||
|
||||
const fetchStatus = useCallback(async () => {
|
||||
try {
|
||||
const data = await fetchJson<OpsEventsResponse>(
|
||||
api.opsEvents.list("Maintenance", { pageSize: 100 }),
|
||||
getAccessToken,
|
||||
);
|
||||
|
||||
const lifecycle = (data.events ?? []).filter((e) =>
|
||||
e.eventType.startsWith("SessionDeletionMaintenance"),
|
||||
);
|
||||
// Backend returns newest-first; keep it defensive anyway.
|
||||
lifecycle.sort((a, b) => (a.timestamp < b.timestamp ? 1 : -1));
|
||||
|
||||
const started = lifecycle.find((e) => e.eventType === "SessionDeletionMaintenanceStarted");
|
||||
const terminal = lifecycle.find(
|
||||
(e) =>
|
||||
e.eventType === "SessionDeletionMaintenanceCompleted" ||
|
||||
e.eventType === "SessionDeletionMaintenanceFailed",
|
||||
);
|
||||
|
||||
if (started && (!terminal || started.timestamp > terminal.timestamp)) {
|
||||
const details = parseDetails<{ triggeredBy?: string }>(started.details);
|
||||
setState({
|
||||
kind: "active",
|
||||
since: started.timestamp,
|
||||
triggeredBy: details.triggeredBy ?? "unknown",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!terminal) {
|
||||
setState({ kind: "none" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (terminal.eventType === "SessionDeletionMaintenanceFailed") {
|
||||
setState({ kind: "failed", at: terminal.timestamp, message: terminal.message });
|
||||
return;
|
||||
}
|
||||
|
||||
const details = parseDetails<CompletedDetails>(terminal.details);
|
||||
setState(
|
||||
details.abortedByBudget
|
||||
? { kind: "budget-exceeded", at: terminal.timestamp, details }
|
||||
: { kind: "idle", at: terminal.timestamp, details },
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) {
|
||||
setError("Session expired; reload the page and try again.");
|
||||
}
|
||||
// Non-auth failures degrade silently — the banner is auxiliary to the tabs.
|
||||
setState({ kind: "none" });
|
||||
}
|
||||
}, [getAccessToken, setError]);
|
||||
|
||||
useEffect(() => {
|
||||
const run = async () => {
|
||||
await fetchStatus();
|
||||
};
|
||||
void run();
|
||||
}, [fetchStatus, refreshKey]);
|
||||
let state: BannerState;
|
||||
if (status.kind === "completed") {
|
||||
const details = parseOpsDetails<CompletedDetails>(status.details);
|
||||
state = details.abortedByBudget
|
||||
? { kind: "budget-exceeded", at: status.at, details }
|
||||
: { kind: "idle", at: status.at, details };
|
||||
} else {
|
||||
state = status;
|
||||
}
|
||||
|
||||
const triggerRun = useCallback(async () => {
|
||||
setTriggering(true);
|
||||
@@ -136,13 +68,13 @@ export function MaintenanceStatusBanner({
|
||||
throw err;
|
||||
});
|
||||
setSuccessMessage(queued ? "Maintenance run queued — it will appear here as active shortly." : "A maintenance run is already active.");
|
||||
setRefreshKey((k) => k + 1);
|
||||
refresh();
|
||||
} catch (err) {
|
||||
setError(apiErrorText(err));
|
||||
} finally {
|
||||
setTriggering(false);
|
||||
}
|
||||
}, [getAccessToken, setError, setSuccessMessage]);
|
||||
}, [getAccessToken, setError, setSuccessMessage, refresh]);
|
||||
|
||||
if (state.kind === "loading" || state.kind === "none") return null;
|
||||
|
||||
@@ -165,7 +97,7 @@ export function MaintenanceStatusBanner({
|
||||
deletions enqueued by this run appear under In-Flight.
|
||||
</p>
|
||||
<button
|
||||
onClick={() => setRefreshKey((k) => k + 1)}
|
||||
onClick={refresh}
|
||||
className="shrink-0 px-3 py-1.5 text-xs font-medium border border-purple-300 dark:border-purple-700 text-purple-700 dark:text-purple-200 rounded hover:bg-purple-100 dark:hover:bg-purple-900/40"
|
||||
>
|
||||
Refresh
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { api } from "@/lib/api";
|
||||
import { TokenExpiredError } from "@/lib/authenticatedFetch";
|
||||
import { fetchJson } from "@/lib/apiClient";
|
||||
|
||||
interface OpsEvent {
|
||||
id: string;
|
||||
eventType: string;
|
||||
severity: string;
|
||||
message: string;
|
||||
details: string | null;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
interface OpsEventsResponse {
|
||||
events: OpsEvent[];
|
||||
}
|
||||
|
||||
/** The ops event types that make up one background run's lifecycle. */
|
||||
export interface OpsRunLifecycle {
|
||||
category: string;
|
||||
started: string;
|
||||
completed: string;
|
||||
failed: string;
|
||||
}
|
||||
|
||||
export type OpsRunStatus =
|
||||
| { kind: "loading" }
|
||||
| { kind: "active"; since: string; triggeredBy: string }
|
||||
| { kind: "failed"; at: string; message: string }
|
||||
| { kind: "completed"; at: string; details: string | null }
|
||||
| { kind: "none" }; // no lifecycle events in the window
|
||||
|
||||
export function parseOpsDetails<T>(raw: string | null): T {
|
||||
if (!raw) return {} as T;
|
||||
try {
|
||||
return JSON.parse(raw) as T;
|
||||
} catch {
|
||||
return {} as T;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Status of a background run that reports through ops events (the trigger answers 202, there
|
||||
* is no job row): the latest Started newer than the latest terminal event (Completed/Failed)
|
||||
* means a run is active; otherwise the latest terminal event is the last run's outcome.
|
||||
* `lifecycle` must be a stable reference (module constant). With `pollWhileActiveMs` the
|
||||
* status reloads on that cadence for as long as a run is active.
|
||||
*/
|
||||
export function useOpsRunStatus(
|
||||
getAccessToken: (forceRefresh?: boolean) => Promise<string | null>,
|
||||
setError: (error: string | null) => void,
|
||||
lifecycle: OpsRunLifecycle,
|
||||
pollWhileActiveMs?: number,
|
||||
): { status: OpsRunStatus; refresh: () => void } {
|
||||
const [status, setStatus] = useState<OpsRunStatus>({ kind: "loading" });
|
||||
const [refreshKey, setRefreshKey] = useState(0);
|
||||
|
||||
const fetchStatus = useCallback(async () => {
|
||||
try {
|
||||
const data = await fetchJson<OpsEventsResponse>(
|
||||
api.opsEvents.list(lifecycle.category, { pageSize: 100 }),
|
||||
getAccessToken,
|
||||
);
|
||||
|
||||
const events = [...(data.events ?? [])];
|
||||
// Backend returns newest-first; keep it defensive anyway.
|
||||
events.sort((a, b) => (a.timestamp < b.timestamp ? 1 : -1));
|
||||
|
||||
const started = events.find((e) => e.eventType === lifecycle.started);
|
||||
const terminal = events.find(
|
||||
(e) => e.eventType === lifecycle.completed || e.eventType === lifecycle.failed,
|
||||
);
|
||||
|
||||
if (started && (!terminal || started.timestamp > terminal.timestamp)) {
|
||||
const details = parseOpsDetails<{ triggeredBy?: string }>(started.details);
|
||||
setStatus({ kind: "active", since: started.timestamp, triggeredBy: details.triggeredBy ?? "unknown" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!terminal) {
|
||||
setStatus({ kind: "none" });
|
||||
return;
|
||||
}
|
||||
|
||||
setStatus(
|
||||
terminal.eventType === lifecycle.failed
|
||||
? { kind: "failed", at: terminal.timestamp, message: terminal.message }
|
||||
: { kind: "completed", at: terminal.timestamp, details: terminal.details },
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) {
|
||||
setError("Session expired; reload the page and try again.");
|
||||
}
|
||||
// Non-auth failures degrade silently — the run status is auxiliary to the page.
|
||||
setStatus({ kind: "none" });
|
||||
}
|
||||
}, [getAccessToken, setError, lifecycle]);
|
||||
|
||||
useEffect(() => {
|
||||
const run = async () => {
|
||||
await fetchStatus();
|
||||
};
|
||||
void run();
|
||||
}, [fetchStatus, refreshKey]);
|
||||
|
||||
const active = status.kind === "active";
|
||||
useEffect(() => {
|
||||
if (!active || !pollWhileActiveMs) return;
|
||||
const timer = setInterval(() => setRefreshKey((k) => k + 1), pollWhileActiveMs);
|
||||
return () => clearInterval(timer);
|
||||
}, [active, pollWhileActiveMs]);
|
||||
|
||||
const refresh = useCallback(() => setRefreshKey((k) => k + 1), []);
|
||||
return { status, refresh };
|
||||
}
|
||||
@@ -10,7 +10,7 @@ export const PERMANENT: Record<string, string> = {
|
||||
"app/health-check/page.tsx": "/version.json of the SWA itself, not the API",
|
||||
"app/dashboard/hooks/deleteSessionResponse.ts": "safeJson on the 202 Response of the delete-cascade classifier (refusals arrive as ApiError)",
|
||||
"app/settings/TenantConfigContext.tsx": "consent return path routes on TokenExpiredError (router.replace); the access check rethrows it as a non-outcome",
|
||||
"app/admin/ops/session-cleanup/components/MaintenanceStatusBanner.tsx": "auxiliary banner: only a token expiry is surfaced, backend refusals stay silent",
|
||||
"hooks/useOpsRunStatus.ts": "auxiliary run status (session-cleanup banner, maintenance trigger): only a token expiry is surfaced, backend refusals stay silent",
|
||||
"app/dashboard/hooks/useBlockDevice.ts": "bulk runner rethrows the expiry so it is toasted once, not per device",
|
||||
"app/dashboard/hooks/useDeleteSession.ts": "bulk runner collects the expiry and toasts once after the batch",
|
||||
"app/dashboard/hooks/useTenantSecurityConfig.ts": "fail-soft banner data; the expiry is the one failure the user must hear about",
|
||||
@@ -50,7 +50,7 @@ export const BODY_STRINGIFY_BASELINE: Record<string, number> = {
|
||||
};
|
||||
|
||||
export const TOKENEXPIRED_BASELINE: Record<string, number> = {
|
||||
"app/admin/ops/session-cleanup/components/MaintenanceStatusBanner.tsx": 1,
|
||||
"hooks/useOpsRunStatus.ts": 1,
|
||||
"app/dashboard/hooks/useBlockDevice.ts": 1,
|
||||
"app/dashboard/hooks/useDeleteSession.ts": 1,
|
||||
"app/dashboard/hooks/useTenantSecurityConfig.ts": 1,
|
||||
|
||||
@@ -530,6 +530,8 @@ export const SHARED_MANIFEST = {
|
||||
"MaintenanceCompleted",
|
||||
"MaintenanceFailed",
|
||||
"MaintenanceLongRunning",
|
||||
"MaintenanceStarted",
|
||||
"MaintenanceSkippedLocked",
|
||||
"SessionSweepCompleted",
|
||||
"SessionSweepFailed",
|
||||
"OpsEventCleanup",
|
||||
|
||||
@@ -526,6 +526,8 @@
|
||||
"MaintenanceCompleted",
|
||||
"MaintenanceFailed",
|
||||
"MaintenanceLongRunning",
|
||||
"MaintenanceStarted",
|
||||
"MaintenanceSkippedLocked",
|
||||
"SessionSweepCompleted",
|
||||
"SessionSweepFailed",
|
||||
"OpsEventCleanup",
|
||||
@@ -13149,98 +13151,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"MaintenanceResult": {
|
||||
"kind": "object",
|
||||
"doc": "Run report of a manual maintenance run (POST maintenance/trigger).",
|
||||
"fields": [
|
||||
{
|
||||
"name": "success",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "boolean"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "error",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
},
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "triggeredBy",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "triggeredAt",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "durationMs",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "number"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "stalledSessionsChecked",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "boolean"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "metricsAggregated",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "boolean"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "aggregatedDate",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
},
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "dataCleanupExecuted",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "boolean"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "platformStatsRecomputed",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "boolean"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "devicesBlockedForExcessiveData",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "number"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "contactEmailsBackfilled",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "number"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"ManagedTenantItem": {
|
||||
"kind": "object",
|
||||
"doc": "One managed tenant as the managing tenant sees it.",
|
||||
@@ -23545,15 +23455,8 @@
|
||||
},
|
||||
"TriggerMaintenanceResponse": {
|
||||
"kind": "object",
|
||||
"doc": "Response of POST maintenance/trigger.",
|
||||
"doc": "Response of POST maintenance/trigger (202 Accepted): the run is queued, not done. Its progress and its report surface as Maintenance* ops events (Started, Completed, Failed, SkippedLocked).",
|
||||
"fields": [
|
||||
{
|
||||
"name": "success",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "boolean"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "message",
|
||||
"type": {
|
||||
@@ -23561,13 +23464,6 @@
|
||||
"name": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "result",
|
||||
"type": {
|
||||
"kind": "ref",
|
||||
"name": "MaintenanceResult"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "triggeredBy",
|
||||
"type": {
|
||||
@@ -23581,6 +23477,22 @@
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "targetDate",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
},
|
||||
"optional": true,
|
||||
"doc": "Date to aggregate (yyyy-MM-dd); null = yesterday."
|
||||
},
|
||||
{
|
||||
"name": "aggregateOnly",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "boolean"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -2787,22 +2787,6 @@ export interface LogQueryRequest {
|
||||
budgetSeconds?: number | null;
|
||||
}
|
||||
|
||||
/** Run report of a manual maintenance run (POST maintenance/trigger). */
|
||||
export interface MaintenanceResult {
|
||||
success: boolean;
|
||||
error?: string;
|
||||
triggeredBy: string;
|
||||
triggeredAt: string;
|
||||
durationMs: number;
|
||||
stalledSessionsChecked: boolean;
|
||||
metricsAggregated: boolean;
|
||||
aggregatedDate?: string;
|
||||
dataCleanupExecuted: boolean;
|
||||
platformStatsRecomputed: boolean;
|
||||
devicesBlockedForExcessiveData: number;
|
||||
contactEmailsBackfilled: number;
|
||||
}
|
||||
|
||||
/** One managed tenant as the managing tenant sees it. */
|
||||
export interface ManagedTenantItem {
|
||||
tenantId: string;
|
||||
@@ -5158,13 +5142,14 @@ export interface TriggerEpssSyncResponse {
|
||||
syncedAt: string;
|
||||
}
|
||||
|
||||
/** Response of POST maintenance/trigger. */
|
||||
/** Response of POST maintenance/trigger (202 Accepted): the run is queued, not done. Its progress and its report surface as Maintenance* ops events (Started, Completed, Failed, SkippedLocked). */
|
||||
export interface TriggerMaintenanceResponse {
|
||||
success: boolean;
|
||||
message: string;
|
||||
result: MaintenanceResult;
|
||||
triggeredBy: string;
|
||||
triggeredAt: string;
|
||||
/** Date to aggregate (yyyy-MM-dd); null = yesterday. */
|
||||
targetDate?: string;
|
||||
aggregateOnly: boolean;
|
||||
}
|
||||
|
||||
/** Response of POST vulnerability/sync-msrc (dual-purpose: success reflects whether the MSRC index refresh finished without error). */
|
||||
|
||||
Reference in New Issue
Block a user