AutopilotMonitor/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TenantOffboardFunction.cs

using System;
using System.Collections.Generic;
using System.Net;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure;
using Microsoft.AspNetCore.Authorization;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging;

namespace AutopilotMonitor.Functions.Functions.Admin;

/// <summary>
/// Tenant offboarding endpoint (Phase 1 of the tenant-offboarding cascade plan).
/// Replaces the previous synchronous <c>DeleteAllTenantDataAsync</c> teardown with a
/// queued cascade. Phase 1 commits the History/Pointer/Marker rows, flips
/// <c>TenantConfiguration.Disabled=true</c> (the actual auth gate), audits, and enqueues
/// a <see cref="TenantOffboardingEnvelope"/> with a <see cref="DrainBarrier"/> visibility
/// delay. The worker picks the message up after the cache-drain window, by which point
/// every function-host has expired its 5-min <c>TenantConfigurationService</c> cache and
/// the existing Disabled-gate carries the 403. Returns 202.
/// </summary>
public class TenantOffboardFunction
{
    private const int PointerCasMaxAttempts = 5;

    /// <summary>
    /// Cache-Drain-Barrier (plan v2 §2.2). Worker MUST NOT begin Phase 2 before this elapsed.
    /// 5 min = <see cref="TenantConfigurationService.CacheDuration"/> (absolute expiration).
    /// 1 min = clock-skew + GC + Storage-replication safety buffer.
    /// Hardcoded by design: this is a structural invariant tied to CacheDuration, not an
    /// operational knob. If CacheDuration is ever raised, this constant must move with it.
    /// </summary>
    internal static readonly TimeSpan DrainBarrier = TimeSpan.FromMinutes(6);

    private readonly ILogger<TenantOffboardFunction> _logger;
    private readonly IConfigRepository _configRepo;
    private readonly TenantConfigurationService _tenantConfigService;
    private readonly IMaintenanceRepository _maintenanceRepo;
    private readonly IOffboardingAuditRepository _offboardingRepo;
    private readonly ITenantOffboardingEnqueuer _offboardingEnqueuer;
    private readonly PreviewWhitelistService _previewWhitelistService;

    public TenantOffboardFunction(
        ILogger<TenantOffboardFunction> logger,
        IConfigRepository configRepo,
        TenantConfigurationService tenantConfigService,
        IMaintenanceRepository maintenanceRepo,
        IOffboardingAuditRepository offboardingRepo,
        ITenantOffboardingEnqueuer offboardingEnqueuer,
        PreviewWhitelistService previewWhitelistService)
    {
        _logger = logger;
        _configRepo = configRepo;
        _tenantConfigService = tenantConfigService;
        _maintenanceRepo = maintenanceRepo;
        _offboardingRepo = offboardingRepo;
        _offboardingEnqueuer = offboardingEnqueuer;
        _previewWhitelistService = previewWhitelistService;
    }

    /// <summary>
    /// DELETE /api/tenants/{tenantId}/offboard
    /// Phase 1 — synchronously commits the offboarding state (History/Pointer/Marker),
    /// flips <c>TenantConfiguration.Disabled=true</c> (the SOLE active auth-gate in the
    /// PR3-revised architecture — the agent + web hotpaths read <c>config.Disabled</c>
    /// only, never the marker), and enqueues a single tenant-offboarding envelope with
    /// a <see cref="DrainBarrier"/> visibility-delay so warm function-host caches drain
    /// before Phase 2 begins.
    /// Accessible by: Tenant Admins of the same tenant OR Global Admins (enforced by
    /// PolicyEnforcementMiddleware).
    /// </summary>
    [Function("OffboardTenant")]
    [Authorize]
    public async Task<HttpResponseData> OffboardTenant(
        [HttpTrigger(AuthorizationLevel.Anonymous, "delete", Route = "tenants/{tenantId}/offboard")] HttpRequestData req,
        string tenantId,
        FunctionContext context)
    {
        var requestCtx = context.GetRequestContext();
        var upn = requestCtx.UserPrincipalName;
        var targetTenantId = requestCtx.TargetTenantId;

        if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId))
        {
            return await req.BadRequestAsync("tenantId must be a valid GUID");
        }

        var normalizedTenantId = targetTenantId.ToLowerInvariant();

        _logger.LogWarning(
            "TENANT OFFBOARD initiated for tenant {TenantId} by {Upn}",
            normalizedTenantId, upn);

        // 2. Capture what must outlive the TenantConfiguration row the cascade deletes: DomainName
        //    for the worker's Completed/Failed OpsEvents ("{domain} ({tenantId})"), and TrialConsumed
        //    for the pointer, which carries it to the returning tenant's first login (D-288).
        //    Fail-loud: without the read the one-trial-per-tenant rule would silently reset.
        string? domainName;
        bool trialConsumed;
        try
        {
            var existingConfig = await _configRepo.GetTenantConfigurationAsync(targetTenantId);
            domainName = existingConfig?.DomainName;
            trialConsumed = existingConfig?.TrialConsumed ?? false;
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to read tenant configuration for {TenantId} before offboard", normalizedTenantId);
            return await Build500Async(req, "Failed to read tenant configuration");
        }

        // 2.b — Capture the Preview-Notification-Email before Phase 2.D wipes the
        // PreviewWhitelist table. Stored on the OffboardingHistory row (survives the wipe)
        // and consumed by the post-completion farewell-email side-effect in
        // TenantOffboardingHandler. Fail-soft: null on lookup failure — the
        // farewell email is a courtesy, not a correctness contract.
        var notificationEmail = await CaptureNotificationEmailAsync(normalizedTenantId);

        // 3. Idempotency: if an active marker exists, replay the queue-enqueue defensively
        //    when the offboarding is still in-flight. The previous implementation returned
        //    200 unconditionally, which left the tenant stuck whenever the first enqueue
        //    failed between marker-insert and queue-send: the marker would block all future
        //    re-clicks via the idempotency-200 path while the worker queue was empty.
        //    Resume-paths per marker status:
        //      - Initiated/InProgress → defensive re-enqueue (TenantOffboardingHandler is
        //        idempotent via History.DrainCompletedAt Rev-9-F1 skip-gate; worst case the
        //        queue carries two envelopes that both resolve cleanly).
        //      - Completed            → no re-enqueue (worker already finished; marker is
        //        the 15-min Rev-5-F2 grace blocker).
        //      - Failed               → no re-enqueue (R1 fail-closed by design; operator
        //        must inspect History.FailedPhase + clear the marker manually before retry).
        try
        {
            var existingMarker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
            if (existingMarker != null)
            {
                return await HandleExistingMarkerAsync(
                    req, existingMarker, upn, normalizedTenantId, isRace: false);
            }
        }
        catch (Exception ex)
        {
            _logger.LogError(ex,
                "Failed to read offboarding marker for {TenantId} during idempotency check",
                normalizedTenantId);
            return await Build500Async(req, "Failed to read offboarding state");
        }

        var now = DateTime.UtcNow;
        var historyRowKey = $"{now:yyyyMMddHHmmssfff}_{normalizedTenantId}";
        var earliestProcessingAt = now + DrainBarrier;

        // 4. Insert History row FIRST (audit trail must exist before any side-effect).
        var history = new OffboardingHistoryEntry
        {
            PartitionKey = Constants.OffboardingPartitionKeys.History,
            RowKey = historyRowKey,
            TenantId = normalizedTenantId,
            DomainName = domainName ?? string.Empty,
            InitiatedBy = upn,
            OffboardedAt = now,
            EarliestProcessingAt = earliestProcessingAt,
            Status = "Initiated",
            RetryCount = 0,
            NotificationEmail = notificationEmail,
        };

        try
        {
            await _offboardingRepo.InsertHistoryAsync(history);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to insert OffboardingHistory row for {TenantId}", normalizedTenantId);
            return await Build500Async(req, "Failed to record offboarding history");
        }

        // 5. Upsert ByTenant pointer with ETag-CAS so OffboardCount increments race-safely.
        try
        {
            await UpsertPointerWithCasAsync(normalizedTenantId, historyRowKey, now, trialConsumed);
        }
        catch (Exception ex)
        {
            // History row is already committed — operator can find it under
            // PartitionKey="OffboardingHistory". Nothing destructive has started: without the
            // pointer the returning tenant would get its trial back, so refuse and let the
            // admin re-click.
            _logger.LogError(ex,
                "Failed to upsert OffboardingByTenant pointer for {TenantId} after history row inserted",
                normalizedTenantId);
            return await Build500Async(req, "Failed to update offboarding pointer");
        }

        // 6. Insert Marker LAST among the three rows. The marker is the idempotency anchor
        //    for admin re-clicks (the next click resolves through TryGetMarkerAsync and
        //    hits the resume path) and is later read by OffboardingMarkerCleanupFunction
        //    to drive post-completion cleanup sweeps. The Marker is NOT an auth-gate any
        //    more — TenantConfiguration.Disabled=true (committed in step 7 below) is the
        //    sole active gate.
        var marker = new OffboardingMarkerEntry
        {
            PartitionKey = Constants.OffboardingPartitionKeys.Marker,
            RowKey = normalizedTenantId,
            TenantId = normalizedTenantId,
            OffboardingHistoryRowKey = historyRowKey,
            InitiatedAt = now,
            InitiatedBy = upn,
            Status = "Initiated",
        };

        try
        {
            await _offboardingRepo.InsertMarkerAsync(marker);
        }
        catch (RequestFailedException ex) when (ex.Status == 409)
        {
            // Race: another caller inserted a marker between our TryGet at step 3 and now.
            // Resolve to the existing marker and run the same defensive re-enqueue path as
            // the idempotency branch — the winner may still have failed its own enqueue and
            // we cannot tell from here, so re-enqueue is the safe default. The worker is
            // idempotent via the DrainCompletedAt skip-gate (Rev-9-F1).
            _logger.LogInformation(
                "OffboardingMarker insert raced for {TenantId}; another initiator won. Resolving to existing marker.",
                normalizedTenantId);

            var raced = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
            if (raced == null)
            {
                _logger.LogError(
                    "OffboardingMarker 409 race for {TenantId} but follow-up read returned null; cannot resolve race",
                    normalizedTenantId);
                return await Build500Async(req, "Failed to resolve offboarding race");
            }
            return await HandleExistingMarkerAsync(
                req, raced, upn, normalizedTenantId, isRace: true);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to insert OffboardingMarker for {TenantId}", normalizedTenantId);
            return await Build500Async(req, "Failed to commit offboarding marker");
        }

        // 7. Flip TenantConfiguration.Disabled=true. This is the SOLE active auth-gate now
        //    that the hotpath does not read the OffboardingMarker any more (PR3-revised).
        //    Fail-loud — without Disabled=true the cache-drain barrier offers no protection
        //    and Phase 2 would wipe data while warm function-hosts still accept agent
        //    traffic. Marker stays Initiated → next admin re-click hits the resume path
        //    which also calls EnsureTenantDisabledAsync before re-enqueuing.
        try
        {
            await EnsureTenantDisabledAsync(targetTenantId, upn);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex,
                "Failed to set TenantConfiguration.Disabled=true for {TenantId}; refusing to enqueue (auth-gate not committed)",
                normalizedTenantId);
            return await Build500Async(req, "Failed to commit Disabled-gate; tenant offboarding NOT enqueued");
        }

        // 8. Audit row under AuditGlobalTenantId so it survives the eventual TenantConfiguration wipe.
        await _maintenanceRepo.LogAuditEntryAsync(
            Constants.AuditGlobalTenantId,
            "DELETE",
            "Tenant",
            normalizedTenantId,
            upn,
            new Dictionary<string, string>
            {
                { "Action", "Offboard" },
                { "Phase", "Initiated" },
                { "HistoryRowKey", historyRowKey },
                { "DomainName", domainName ?? string.Empty },
            });

        // 9. Enqueue with full DrainBarrier as visibility delay. The worker won't see this
        //    message until ~6min from now, by which time every function-host has refreshed
        //    its TenantConfiguration cache and the existing Disabled-gate is the active
        //    block. Fail-loud — marker stays Initiated on failure; the next admin re-click
        //    hits the idempotency branch which defensively re-enqueues with the remaining
        //    barrier so the tenant cannot get stuck.
        try
        {
            await EnqueueEnvelopeAsync(normalizedTenantId, historyRowKey, upn, now, DrainBarrier);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex,
                "Failed to enqueue tenant-offboarding envelope for {TenantId}; marker remains Initiated for re-click resume",
                normalizedTenantId);
            return await Build500Async(req, "Failed to enqueue tenant offboarding");
        }

        _logger.LogWarning(
            "TENANT OFFBOARD queued for {TenantId} by {Upn}; historyRowKey={History}; earliestProcessingAt={ProcessAt}",
            normalizedTenantId, upn, historyRowKey, earliestProcessingAt);

        var response = req.CreateResponse(HttpStatusCode.Accepted);
        await response.WriteAsJsonAsync(new OffboardResponse
        {
            TenantId = normalizedTenantId,
            Status = "Queued",
            HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
            HistoryRowKey = historyRowKey,
            EarliestProcessingAt = earliestProcessingAt,
            Message = "Tenant offboarding queued for asynchronous processing"
        });
        return response;
    }

    // Resume-path for re-clicks and 409-race winners. Thin HTTP shell on top of
    // ResumeExistingMarkerAsync — the actual decision (re-enqueue or not, message text,
    // remaining drain-barrier) lives in the pure helper below so it can be unit-tested
    // without faking HttpRequestData. Plan Review Finding 1 (stuck-tenant fix).
    internal async Task<HttpResponseData> HandleExistingMarkerAsync(
        HttpRequestData req,
        OffboardingMarkerEntry existing,
        string upn,
        string normalizedTenantId,
        bool isRace)
    {
        var resume = await ResumeExistingMarkerAsync(existing, upn, normalizedTenantId, isRace);

        if (resume.Kind == ResumeOutcomeKind.ReEnqueueFailed)
        {
            return await Build500Async(req, "Failed to re-enqueue tenant offboarding");
        }

        var response = req.CreateResponse(resume.StatusCode);
        await response.WriteAsJsonAsync(new OffboardResponse
        {
            TenantId = normalizedTenantId,
            Status = existing.Status,
            HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
            HistoryRowKey = existing.OffboardingHistoryRowKey,
            EarliestProcessingAt = resume.EarliestProcessingAt,
            Message = resume.Message,
        });
        return response;
    }

    /// <summary>
    /// Pure resume decision: classifies the existing marker, optionally re-enqueues, and
    /// returns the (StatusCode, Message, ReEnqueued?) tuple the HTTP shell wraps. Exposed
    /// internal for unit tests — Finding-1 fix requires asserting "Initiated/InProgress
    /// triggers a queue write, Completed/Failed does not".
    /// </summary>
    internal async Task<ResumeOutcome> ResumeExistingMarkerAsync(
        OffboardingMarkerEntry existing,
        string upn,
        string normalizedTenantId,
        bool isRace)
    {
        var status = existing.Status ?? "Initiated";

        if (string.Equals(status, "Completed", StringComparison.OrdinalIgnoreCase))
        {
            _logger.LogInformation(
                "TenantOffboard re-click for {TenantId}: marker is Completed; no re-enqueue (race={Race})",
                normalizedTenantId, isRace);
            return new ResumeOutcome(
                ResumeOutcomeKind.IdempotentCompleted,
                HttpStatusCode.OK,
                "Tenant offboarding already completed (marker in 15-min grace window)",
                ReEnqueued: false);
        }

        if (string.Equals(status, "Failed", StringComparison.OrdinalIgnoreCase))
        {
            // Failed is fail-closed by design (R1). Re-enqueuing a Failed offboarding would
            // just retrace the same failure path. Surface the FailedPhase to the operator
            // so they can clear the marker manually before retrying.
            _logger.LogWarning(
                "TenantOffboard re-click for {TenantId}: marker is Failed (phase={Phase}); operator action required, no re-enqueue",
                normalizedTenantId, existing.FailedPhase ?? "unknown");
            return new ResumeOutcome(
                ResumeOutcomeKind.IdempotentFailed,
                HttpStatusCode.OK,
                $"Tenant offboarding previously failed at phase '{existing.FailedPhase ?? "unknown"}' — operator action required before retry",
                ReEnqueued: false);
        }

        // Initiated / InProgress (or any unrecognised in-flight value): defensively
        // re-enqueue the envelope. The handler tolerates double-pickup via the
        // DrainCompletedAt skip-gate, so a duplicate envelope on the queue is safe.
        //
        // The visibility-delay computation below uses
        //   max(history.EarliestProcessingAt, tombstone.LastUpdated + DrainBarrier)
        // as the effective drain-deadline. History is best-effort here: if the read
        // fails or the field is missing, the tombstone anchor still gives a safe lower
        // bound (when the tombstone was just written, that's now + DrainBarrier; when
        // it was written long ago, the barrier is already elapsed).
        OffboardingHistoryEntry? history = null;
        try
        {
            history = await _offboardingRepo.TryGetHistoryAsync(existing.OffboardingHistoryRowKey);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex,
                "Failed to read History row {HistoryRowKey} for {TenantId} during resume; falling through to tombstone.LastUpdated-based deadline",
                existing.OffboardingHistoryRowKey, normalizedTenantId);
        }
        // Re-affirm the Disabled-gate FIRST so we know whether we just wrote it for the
        // first time (= cache-drain barrier restarts now) or it was already there from a
        // prior attempt (= the History row's deadline is still valid). The prior run may
        // have crashed between marker-insert and Disabled-flip, or an admin may have
        // cleared the flag manually. Either way, we MUST NOT re-enqueue without the
        // auth-gate committed — otherwise the cache-drain barrier guards nothing.
        EnsureDisabledResult ensureResult;
        try
        {
            ensureResult = await EnsureTenantDisabledAsync(normalizedTenantId, upn);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex,
                "Resume: failed to re-affirm TenantConfiguration.Disabled=true for {TenantId}; refusing to re-enqueue",
                normalizedTenantId);
            return new ResumeOutcome(
                ResumeOutcomeKind.ReEnqueueFailed,
                HttpStatusCode.InternalServerError,
                "Failed to re-affirm Disabled-gate; tenant offboarding NOT re-enqueued",
                ReEnqueued: false);
        }

        // Resume-path visibility-delay decision (Codex finding "Resume can bypass cache-drain"):
        // We compute the effective drain-deadline as the MAX of two sources, then derive
        // the queue-message visibility-delay from that:
        //
        //   (a) history.EarliestProcessingAt — set on Phase 1 commit; patched here on
        //       fresh writes when we can.
        //   (b) ensureResult.TombstoneLastUpdated + DrainBarrier — the authoritative
        //       "cache-drain started here" timestamp, written atomically with the
        //       Disabled-flip, ALWAYS present even if the History-patch later fails.
        //
        // Using max() of both makes the History-patch non-load-bearing for safety: even if
        // a fast follow-up re-click reads stale EarliestProcessingAt, the TombstoneLastUpdated
        // anchor gives us a correct lower bound. The History-patch is now best-effort
        // documentation, not a critical write.
        var now = DateTime.UtcNow;
        var tombstoneDeadline = ensureResult.TombstoneLastUpdated + DrainBarrier;
        var historyDeadline = history?.EarliestProcessingAt ?? tombstoneDeadline;
        var effectiveDeadline = tombstoneDeadline > historyDeadline ? tombstoneDeadline : historyDeadline;

        var visibilityDelay = effectiveDeadline > now ? effectiveDeadline - now : TimeSpan.Zero;
        var earliestProcessingAtForResponse = effectiveDeadline;

        // Patch History.EarliestProcessingAt for new tombstone writes so the UI countdown
        // + any later admin tools see the new deadline. Failure here is NOT a safety issue
        // anymore (tombstone.LastUpdated covers us above) — purely cosmetic / observability.
        if (ensureResult.Outcome == EnsureDisabledOutcome.WroteNewTombstone && history != null)
        {
            history.EarliestProcessingAt = effectiveDeadline;
            try
            {
                await _offboardingRepo.UpsertHistoryAsync(history);
            }
            catch (Exception ex)
            {
                _logger.LogWarning(ex,
                    "Resume: failed to patch History.EarliestProcessingAt for {TenantId} {History} — UI countdown may show stale value, but the queue visibility-delay + tombstone.LastUpdated guarantee the drain barrier on the worker side",
                    normalizedTenantId, existing.OffboardingHistoryRowKey);
            }

            _logger.LogWarning(
                "Resume: Disabled-gate was freshly written; restarted full cache-drain barrier ({Barrier}) for tenant={TenantId}",
                DrainBarrier, normalizedTenantId);
        }

        try
        {
            await EnqueueEnvelopeAsync(
                normalizedTenantId, existing.OffboardingHistoryRowKey, upn, existing.InitiatedAt, visibilityDelay);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex,
                "Defensive re-enqueue failed for {TenantId} (existing marker status={Status}, race={Race})",
                normalizedTenantId, status, isRace);
            return new ResumeOutcome(
                ResumeOutcomeKind.ReEnqueueFailed,
                HttpStatusCode.InternalServerError,
                "Failed to re-enqueue tenant offboarding",
                ReEnqueued: false);
        }

        _logger.LogInformation(
            "TenantOffboard re-click for {TenantId}: defensively re-enqueued with visibilityDelay={Delay} outcome={Outcome} tombstoneLastUpdated={TombstoneLastUpdated} (existing status={Status}, race={Race})",
            normalizedTenantId, visibilityDelay, ensureResult.Outcome, ensureResult.TombstoneLastUpdated, status, isRace);

        return new ResumeOutcome(
            ResumeOutcomeKind.ReEnqueuedInFlight,
            HttpStatusCode.Accepted,
            isRace
                ? "Tenant offboarding already in progress (race-winner resolved, defensively re-enqueued)"
                : $"Tenant offboarding resumed (existing status={status}, defensively re-enqueued)",
            ReEnqueued: true,
            EarliestProcessingAt: earliestProcessingAtForResponse);
    }

    // ── Offboarding record + operator retry (Global Admin tooling) ────────────────────

    /// <summary>
    /// GET /api/global/tenants/{tenantId}/offboarding — the tenant's current offboarding record
    /// (marker + history) for the admin tenant editor. 404 without a marker: the tenant was
    /// never offboarded, or the Completed marker is already cleaned up
    /// (<see cref="OffboardingMarkerCleanupFunction"/>, 15-min grace) — the normal end state,
    /// nothing for an operator to do. Global Admin / Global Reader only (middleware-enforced).
    /// </summary>
    [Function("GetTenantOffboardingStatus")]
    [Authorize]
    public async Task<HttpResponseData> GetTenantOffboardingStatus(
        [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/tenants/{tenantId}/offboarding")] HttpRequestData req,
        string tenantId,
        FunctionContext context)
    {
        var targetTenantId = context.GetRequestContext().TargetTenantId;
        if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId))
        {
            return await req.BadRequestAsync("tenantId must be a valid GUID");
        }

        var normalizedTenantId = targetTenantId.ToLowerInvariant();
        var marker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
        if (marker == null)
        {
            return await req.NotFoundAsync("No offboarding on record for this tenant");
        }

        var history = await _offboardingRepo.TryGetHistoryAsync(marker.OffboardingHistoryRowKey);
        return await req.OkAsync(BuildStatusResponse(normalizedTenantId, marker, history));
    }

    internal static TenantOffboardingStatusResponse BuildStatusResponse(
        string normalizedTenantId, OffboardingMarkerEntry marker, OffboardingHistoryEntry? history)
        => new()
        {
            TenantId = normalizedTenantId,
            Status = marker.Status,
            HistoryRowKey = marker.OffboardingHistoryRowKey,
            InitiatedAt = marker.InitiatedAt,
            InitiatedBy = marker.InitiatedBy,
            RetryCount = history?.RetryCount ?? 0,
            EarliestProcessingAt = history?.EarliestProcessingAt,
            DrainCompletedAt = history?.DrainCompletedAt,
            CompletedAt = marker.CompletedAt,
            FailedAt = marker.FailedAt,
            FailedPhase = marker.FailedPhase,
            ErrorMessage = history?.ErrorMessage,
        };

    /// <summary>
    /// POST /api/global/tenants/{tenantId}/offboarding/retry — re-drives an offboarding whose
    /// marker is Failed. Failed is fail-closed by design (the worker returns, the offboard
    /// re-click refuses to re-enqueue), so this is the explicit operator action that design
    /// asks for; nothing retries on its own. History and marker go back to Initiated — the
    /// History row keeps its RowKey and <c>DrainCompletedAt</c> (the worker's resume anchor)
    /// and counts the retry — and the ordinary resume path re-affirms the Disabled-gate,
    /// computes the remaining drain barrier and enqueues the envelope. Global Admin only.
    /// </summary>
    [Function("RetryTenantOffboarding")]
    [Authorize]
    public async Task<HttpResponseData> RetryTenantOffboarding(
        [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "global/tenants/{tenantId}/offboarding/retry")] HttpRequestData req,
        string tenantId,
        FunctionContext context)
    {
        var requestCtx = context.GetRequestContext();
        var upn = requestCtx.UserPrincipalName;
        var targetTenantId = requestCtx.TargetTenantId;

        if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId))
        {
            return await req.BadRequestAsync("tenantId must be a valid GUID");
        }

        var normalizedTenantId = targetTenantId.ToLowerInvariant();
        var retry = await RetryFailedOffboardingAsync(normalizedTenantId, upn);

        switch (retry.Kind)
        {
            case RetryOutcomeKind.Retried:
            {
                var response = req.CreateResponse(HttpStatusCode.Accepted);
                await response.WriteAsJsonAsync(new OffboardResponse
                {
                    TenantId = normalizedTenantId,
                    Status = "Queued",
                    HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
                    HistoryRowKey = retry.HistoryRowKey ?? string.Empty,
                    EarliestProcessingAt = retry.EarliestProcessingAt,
                    Message = retry.Message,
                });
                return response;
            }
            case RetryOutcomeKind.NotFound:
                return await req.NotFoundAsync(retry.Message);
            case RetryOutcomeKind.NotFailed:
                return await req.ConflictAsync(retry.Message);
            default:
                return await Build500Async(req, retry.Message);
        }
    }

    /// <summary>
    /// Pure retry decision behind <see cref="RetryTenantOffboarding"/>; internal for unit tests.
    /// Write order matters: History first (the worker's authority — a Failed history makes
    /// it return early), Marker second (the HTTP idempotency anchor). A crash in between
    /// leaves History=Initiated + Marker=Failed, and the next retry click resolves it the
    /// same way. The worker's first pickup then flips History/Pointer/Marker to InProgress.
    /// </summary>
    internal async Task<RetryOutcome> RetryFailedOffboardingAsync(string normalizedTenantId, string upn)
    {
        var marker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
        if (marker == null)
        {
            return new RetryOutcome(RetryOutcomeKind.NotFound, "No offboarding on record for this tenant");
        }

        if (!string.Equals(marker.Status, "Failed", StringComparison.OrdinalIgnoreCase))
        {
            var message = string.Equals(marker.Status, "Completed", StringComparison.OrdinalIgnoreCase)
                ? "Tenant offboarding already completed — nothing to retry"
                : $"Tenant offboarding is {marker.Status} — not Failed; re-run the offboard action to resume it";
            _logger.LogInformation(
                "TenantOffboard retry for {TenantId} refused: marker status={Status}", normalizedTenantId, marker.Status);
            return new RetryOutcome(RetryOutcomeKind.NotFailed, message, marker.OffboardingHistoryRowKey);
        }

        var history = await _offboardingRepo.TryGetHistoryAsync(marker.OffboardingHistoryRowKey);
        if (history == null)
        {
            _logger.LogError(
                "TenantOffboard retry for {TenantId}: History row {History} is missing — operator inspection required",
                normalizedTenantId, marker.OffboardingHistoryRowKey);
            return new RetryOutcome(
                RetryOutcomeKind.HistoryMissing,
                "Offboarding history row is missing — operator inspection required",
                marker.OffboardingHistoryRowKey);
        }

        var failedPhase = marker.FailedPhase ?? "unknown";

        history.Status = "Initiated";
        history.RetryCount += 1;
        history.ErrorMessage = null;
        await _offboardingRepo.UpsertHistoryAsync(history);

        marker.Status = "Initiated";
        marker.FailedAt = null;
        marker.FailedPhase = null;
        await _offboardingRepo.UpsertMarkerAsync(marker);

        _logger.LogWarning(
            "TENANT OFFBOARD RETRY for {TenantId} by {Upn}: previous failure phase={Phase}; historyRowKey={History} retryCount={RetryCount}",
            normalizedTenantId, upn, failedPhase, history.RowKey, history.RetryCount);

        // Global audit row (survives the TenantConfiguration wipe like the initiation's DELETE row).
        await _maintenanceRepo.LogAuditEntryAsync(
            Constants.AuditGlobalTenantId,
            "RETRY",
            "TenantOffboarding",
            normalizedTenantId,
            upn,
            new Dictionary<string, string>
            {
                ["failedPhase"] = failedPhase,
                ["historyRowKey"] = history.RowKey,
                ["retryCount"] = history.RetryCount.ToString(System.Globalization.CultureInfo.InvariantCulture),
            });

        var resume = await ResumeExistingMarkerAsync(marker, upn, normalizedTenantId, isRace: false);
        if (resume.Kind == ResumeOutcomeKind.ReEnqueueFailed)
        {
            // Marker is Initiated now, so the ordinary offboard re-click resumes it.
            return new RetryOutcome(
                RetryOutcomeKind.ReEnqueueFailed,
                "Failed to re-enqueue tenant offboarding — re-run the offboard action to resume it",
                history.RowKey);
        }

        return new RetryOutcome(
            RetryOutcomeKind.Retried,
            $"Tenant offboarding retried after the failure in phase '{failedPhase}'",
            history.RowKey,
            resume.EarliestProcessingAt);
    }

    /// <summary>
    /// Magic-string DisabledReason that identifies a TenantConfiguration row as the
    /// offboarding tombstone. Used by <c>EnsureTenantDisabledAsync</c> for the idempotent
    /// no-write check AND by <see cref="OffboardingMarkerCleanupFunction"/> to gate the
    /// defense-in-depth TenantConfiguration sweep so a self-service-re-onboarded fresh
    /// config does not get wiped by mistake.
    /// </summary>
    internal const string OffboardingDisabledReason = "Offboarding in progress";

    /// <summary>
    /// True when the row is the offboarding tombstone written by
    /// <c>EnsureTenantDisabledAsync</c>. Shared predicate for every config writer and
    /// wiper that must distinguish the mid-offboarding freeze from an ordinary
    /// suspended (or re-onboarded) tenant: while this is true, the tenant's
    /// configuration is frozen and the offboarding cascade owns the row.
    /// </summary>
    internal static bool IsOffboardingTombstone(TenantConfiguration? config) =>
        config is { Disabled: true }
        && string.Equals(config.DisabledReason, OffboardingDisabledReason, StringComparison.Ordinal);

    /// <summary>
    /// Result of <see cref="EnsureTenantDisabledAsync"/>. Carries both whether a fresh
    /// write happened AND the tombstone's <c>LastUpdated</c> timestamp so the resume-path
    /// can compute a safe drain-deadline without depending on the (optionally-failing)
    /// <c>History.EarliestProcessingAt</c> patch.
    /// <para>
    /// Drain-deadline math the caller applies:
    /// <c>effectiveDeadline = max(history.EarliestProcessingAt, TombstoneLastUpdated + DrainBarrier)</c>.
    /// Because the tombstone's <c>LastUpdated</c> is part of the same atomic Save as the
    /// Disabled-flip, it is always present once the auth-gate is committed — even if
    /// the History-row patch later fails on a fast re-click.
    /// </para>
    /// </summary>
    internal readonly record struct EnsureDisabledResult(
        EnsureDisabledOutcome Outcome,
        DateTime TombstoneLastUpdated);

    internal enum EnsureDisabledOutcome
    {
        /// <summary>Row was already a Disabled=true tombstone with the offboarding reason —
        /// no Save was issued. Existing <c>LastUpdated</c> is returned.</summary>
        AlreadyTombstone,
        /// <summary>We actively wrote the Disabled=true gate (either flipped an existing
        /// row or created a fresh one). <c>LastUpdated</c> is the timestamp of this write.</summary>
        WroteNewTombstone,
    }

    /// <summary>
    /// Commits the SOLE active auth-gate (<c>TenantConfiguration.Disabled=true</c>) and
    /// invalidates the local <see cref="TenantConfigurationService"/> cache. Fail-loud:
    /// every caller (initial enqueue + resume re-enqueue) MUST treat an exception as
    /// "do not enqueue" — without this gate committed, the cache-drain barrier protects
    /// nothing and Phase 2 would wipe data while warm function-hosts still accept agent
    /// traffic.
    /// <para>
    /// Return value distinguishes the two outcomes so the resume-path can decide whether
    /// to honour the old <c>History.EarliestProcessingAt</c> deadline
    /// (<see cref="EnsureDisabledResult.AlreadyTombstone"/>) or restart the cache-drain
    /// barrier from now (<see cref="EnsureDisabledResult.WroteNewTombstone"/>) — see
    /// resume-path Codex finding "Resume can bypass cache-drain when first attempt failed
    /// before Disabled-write".
    /// </para>
    /// <para>
    /// Behaviour:
    /// <list type="bullet">
    ///   <item>Existing TenantConfiguration row → read-modify-write so signup data + custom
    ///   rate limits etc. are preserved. Idempotent: if already
    ///   <c>Disabled=true</c> with the offboarding reason, no Save is issued and
    ///   <see cref="EnsureDisabledResult.AlreadyTombstone"/> is returned.</item>
    ///   <item>Missing TenantConfiguration row (edge case: very old tenant / manual delete)
    ///   → create a fresh <c>CreateDefault</c> row with <c>Disabled=true</c>. Necessary
    ///   because <c>AuthFunction</c> would otherwise auto-create a <c>Disabled=false</c>
    ///   default on the next <c>/api/auth/me</c> and slip the offboarded tenant back through.</item>
    ///   <item><c>SaveTenantConfigurationAsync</c> returns <c>false</c> on transient storage
    ///   failure (the production repo does NOT throw; see Codex-finding TableConfigRepository).
    ///   We turn that into <see cref="InvalidOperationException"/> so the caller treats it
    ///   as a hard fail.</item>
    /// </list>
    /// </para>
    /// </summary>
    internal async Task<EnsureDisabledResult> EnsureTenantDisabledAsync(string targetTenantId, string upn)
    {
        var tenantConfig = await _configRepo.GetTenantConfigurationAsync(targetTenantId);

        if (tenantConfig != null)
        {
            // Idempotent: if Disabled is already true with our reason, no-op the write so
            // resume-clicks do not thrash the LastUpdated timestamp on the row.
            var alreadyDisabled = tenantConfig.Disabled
                && string.Equals(tenantConfig.DisabledReason, OffboardingDisabledReason, StringComparison.Ordinal);
            if (alreadyDisabled)
            {
                _tenantConfigService.InvalidateCache(targetTenantId);
                return new EnsureDisabledResult(EnsureDisabledOutcome.AlreadyTombstone, tenantConfig.LastUpdated);
            }
        }
        else
        {
            // Missing-row edge case: create a fresh Disabled=true row so AuthFunction's
            // auto-create-default cannot bring the tenant back enabled.
            tenantConfig = TenantConfiguration.CreateDefault(targetTenantId);
            _logger.LogWarning(
                "TenantOffboard: no TenantConfiguration row found for {TenantId} — creating fresh Disabled=true tombstone",
                targetTenantId);
        }

        tenantConfig.Disabled = true;
        tenantConfig.DisabledReason = OffboardingDisabledReason;
        tenantConfig.DisabledUntil = null;
        tenantConfig.UpdatedBy = upn;
        // Explicit LastUpdated stamp — we bypass TenantConfigurationService.SaveConfigurationAsync
        // (which would set this) and write to the repo directly, so we own the bookkeeping.
        // Critical for the resume-path drain-deadline math: this timestamp is the
        // authoritative "when did the cache-drain barrier start" anchor.
        var writtenAt = DateTime.UtcNow;
        tenantConfig.LastUpdated = writtenAt;

        var saved = await _configRepo.SaveTenantConfigurationAsync(tenantConfig, "offboard", "Disabled-gate before cascade");
        if (!saved)
        {
            // TableConfigRepository.SaveTenantConfigurationAsync returns false on transient
            // storage failure instead of throwing. Without throwing here, the caller would
            // proceed to enqueue without the auth-gate committed — defeating the whole
            // cache-drain barrier. Surface as a hard failure.
            throw new InvalidOperationException(
                $"SaveTenantConfigurationAsync returned false for tenant {targetTenantId}; Disabled-gate NOT committed");
        }
        _tenantConfigService.InvalidateCache(targetTenantId);
        return new EnsureDisabledResult(EnsureDisabledOutcome.WroteNewTombstone, writtenAt);
    }

    /// <summary>
    /// Enqueues the offboarding envelope with the given visibility delay (queue-side
    /// invisibility window). First-click: full <see cref="DrainBarrier"/>. Resume-click:
    /// remaining delay computed from <c>EarliestProcessingAt</c> so a re-enqueue does not
    /// reset the drain barrier (the message would otherwise become visible too early).
    /// </summary>
    internal Task EnqueueEnvelopeAsync(
        string normalizedTenantId, string historyRowKey, string upn, DateTime initiatedAt,
        TimeSpan? visibilityDelay)
    {
        var envelope = new TenantOffboardingEnvelope
        {
            EnvelopeVersion = "1",
            TenantId = normalizedTenantId,
            HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
            HistoryRowKey = historyRowKey,
            InitiatedBy = upn,
            InitiatedAt = initiatedAt,
            EnqueuedAt = DateTime.UtcNow,
            DrainPollCount = 0,
        };
        return _offboardingEnqueuer.EnqueueAsync(envelope, visibilityDelay);
    }

    // Reads the tenant's Preview-Notification-Email at Phase 1, BEFORE Phase 2.D wipes
    // the PreviewWhitelist table. Fail-soft: any lookup exception returns null + warn log
    // (the farewell email is a courtesy side-effect, not part of the correctness contract).
    // Internal so unit tests can drive the capture path without going through HttpRequestData.
    internal async Task<string?> CaptureNotificationEmailAsync(string normalizedTenantId)
    {
        try
        {
            return await _previewWhitelistService.GetNotificationEmailAsync(normalizedTenantId);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex,
                "Failed to read preview notification email for {TenantId} before offboard; farewell email will be skipped",
                normalizedTenantId);
            return null;
        }
    }

    // Plan §4.4 read-modify-write CAS-loop. Bounded retry; throws on exhaustion so the
    // caller turns it into a 500. The pointer is the O(1) re-onboarding index; TrialConsumed
    // is sticky across rounds (OR), never reset by an offboarding without a trial.
    internal async Task UpsertPointerWithCasAsync(
        string normalizedTenantId, string historyRowKey, DateTime now, bool trialConsumed)
    {
        for (var attempt = 1; attempt <= PointerCasMaxAttempts; attempt++)
        {
            var (existing, etag) = await _offboardingRepo.TryGetByTenantPointerAsync(normalizedTenantId);

            if (existing == null)
            {
                var fresh = new OffboardingByTenantPointer
                {
                    PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
                    RowKey = normalizedTenantId,
                    TenantId = normalizedTenantId,
                    LatestHistoryRowKey = historyRowKey,
                    LatestStatus = "Initiated",
                    LatestUpdatedAt = now,
                    OffboardCount = 1,
                    TrialConsumed = trialConsumed,
                };

                try
                {
                    await _offboardingRepo.InsertByTenantPointerAsync(fresh);
                    return;
                }
                catch (RequestFailedException ex) when (ex.Status == 409)
                {
                    _logger.LogInformation(
                        "OffboardingByTenant pointer raced on insert for {TenantId} (attempt {Attempt}); falling back to ETag update",
                        normalizedTenantId, attempt);
                    continue;
                }
            }

            existing.LatestHistoryRowKey = historyRowKey;
            existing.LatestStatus = "Initiated";
            existing.LatestUpdatedAt = now;
            existing.OffboardCount = existing.OffboardCount + 1;
            existing.TrialConsumed |= trialConsumed;

            try
            {
                await _offboardingRepo.UpdateByTenantPointerWithEtagAsync(existing, etag!);
                return;
            }
            catch (RequestFailedException ex) when (ex.Status == 412)
            {
                _logger.LogInformation(
                    "OffboardingByTenant pointer ETag mismatch for {TenantId} (attempt {Attempt}); retrying",
                    normalizedTenantId, attempt);
            }
        }

        throw new InvalidOperationException(
            $"OffboardingByTenant pointer CAS exhausted for tenant {normalizedTenantId} after {PointerCasMaxAttempts} attempts");
    }

    private static Task<HttpResponseData> Build500Async(HttpRequestData req, string message)
        => req.ErrorAsync(HttpStatusCode.InternalServerError, Constants.ApiErrorCodes.InternalError, message);
}

// OffboardResponse (the 202/200 offboarding body) moved to AutopilotMonitor.Shared.Models
// (Models/OffboardApiModels.cs) so the shared manifest exports it as a wire type.

/// <summary>
/// Decision the resume-path produced; consumed by <see cref="TenantOffboardFunction.HandleExistingMarkerAsync"/>
/// to shape the HTTP response. Test-visible so unit tests can assert the re-enqueue decision
/// without faking <c>HttpRequestData</c>.
/// <para>
/// <see cref="EarliestProcessingAt"/> is the History row's drain-barrier deadline for the
/// in-flight path (so the UI can keep counting down through a re-click); null on the
/// terminal idempotent branches.
/// </para>
/// </summary>
internal sealed record ResumeOutcome(
    ResumeOutcomeKind Kind,
    System.Net.HttpStatusCode StatusCode,
    string Message,
    bool ReEnqueued,
    DateTime? EarliestProcessingAt = null);

internal enum ResumeOutcomeKind
{
    /// <summary>Marker is Completed; 15-min grace blocker is held, no re-enqueue.</summary>
    IdempotentCompleted,
    /// <summary>Marker is Failed; operator action required, no re-enqueue.</summary>
    IdempotentFailed,
    /// <summary>Marker is Initiated/InProgress; defensive re-enqueue was sent.</summary>
    ReEnqueuedInFlight,
    /// <summary>Marker classified as in-flight but re-enqueue threw; surface 500.</summary>
    ReEnqueueFailed,
}

/// <summary>Result of <see cref="TenantOffboardFunction.RetryFailedOffboardingAsync"/>.</summary>
internal sealed record RetryOutcome(
    RetryOutcomeKind Kind,
    string Message,
    string? HistoryRowKey = null,
    DateTime? EarliestProcessingAt = null);

internal enum RetryOutcomeKind
{
    /// <summary>Marker was Failed; history + marker reset to Initiated and the envelope re-enqueued (202).</summary>
    Retried,
    /// <summary>No marker on record (404).</summary>
    NotFound,
    /// <summary>Marker is Completed or in flight — nothing to retry (409).</summary>
    NotFailed,
    /// <summary>Marker points at a History row that no longer exists (500).</summary>
    HistoryMissing,
    /// <summary>Rows were reset but the re-enqueue threw (500); the offboard re-click resumes.</summary>
    ReEnqueueFailed,
}