feat(recovery): make rollback-recovery image lifecycle visible and controllable (#1753)

* feat(recovery): make rollback-recovery image lifecycle visible and controllable GitHub discussion #1751 asked why Sencho creates sencho-rb/<id>/<service>:hold images during automatic updates and how to clean them up. That surfaced a real safety bug alongside the missing visibility: the manual single-image delete route did not consult the held-image predicate every other deletion path already honors, so a user could delete a rollback-protected image straight through the Images tab and silently break automatic recovery for that update. A short/truncated id also bypassed the predicate's full-id lookup. Fixes: - POST /images/delete now resolves the submitted id to its canonical form and checks the unified held-image predicate before deleting, returning 409 IMAGE_HELD_FOR_ROLLBACK for a protected image. - The Images tab no longer mislabels a protected image as plain "Unused"; a fully-synthetic hold image is kept out of the generic inventory entirely and surfaced instead in a new Resources -> Rollback tab, with an additive "Rollback protected" badge for images that still carry a normal tag too. New capability: - Two settings (Deploy Guardrails): superseded-generation retention (days, replaces a hardcoded 7) and a cap on retained generations per stack. - A new Resources -> Rollback tab lists every generation (stack, short id, state, retention) with an admin-gated manual release action, including releasing the current generation with an explicit warning that automatic rollback becomes unavailable until the next successful update. Release is a single atomic, server-revalidated transition so a stale UI read can never release a row that has since become ineligible. Also consolidated three near-duplicate implementations of the held-image predicate (two of which relied on a require() of a sibling .ts file that silently failed to resolve under the test runner and was never actually exercised by a real test before this change) into one shared module. Known follow-up, not fixed here: an orphaned sencho-rb tag whose recovery row no longer exists (DB restore, node re-add) is invisible in both the Images and Rollback tabs with no UI path to reclaim it. * fix(audit): add summary mapping for rollback generation release * fix(security): sanitize prune target in log sinks and cover release RBAC Closes two open js/log-injection findings on the system prune route by applying the same inline sanitizeForLog barrier the rest of the file already uses. The prune target is validated against an enum by parsePruneTargets before reaching these sinks, so the findings were false positives, but the barrier is cheap and removes the standing alerts on a file this change already touches. Also wraps the generation id in the release log line for consistency with the stack name beside it. Adds coverage for gaps a QA pass identified: - Release endpoint refuses a viewer and a deployer (Admin-only), leaving the generation and its artifacts untouched. - Viewer can still read the generations list, matching the sibling Resources routes. - The predicate the prune routes build reports full-stack rollback holds, not just service-scoped ones, and re-reads per call so a hold taken between plan and delete still gates the delete. - After releasing the current generation, no rollback point is claimed for the stack through any consumer of the current-generation lookup.
2026-08-12 19:57:37 +00:00 · 2026-08-02 21:55:22 -04:00
parent 97be019696
commit 41bf075eb0
31 changed files with 1734 additions and 93 deletions
@@ -30,6 +30,7 @@ import {
  MissingExternalNetworksError,
  type DeployInvocationContext,
 } from './network/missingExternalNetworksError';
+import { buildUnifiedHeldImagePredicate } from './recoveryHeldImages';
 import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
 import type { NotificationCategory } from './NotificationService';

@@ -1019,7 +1020,7 @@ export class ComposeService {
      try {
        const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1';
        if (pruneOnUpdate) {
-          const isImageHeld = recoverySvc.buildUnifiedHeldImagePredicate(this.nodeId);
+          const isImageHeld = buildUnifiedHeldImagePredicate(this.nodeId);
          const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld);
          const reclaimed = result.reclaimedBytes > 0
            ? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB`
@@ -71,6 +71,8 @@ export interface StackUpdateDetail {
 }

 const SERVICES_JSON_VERSION = 1;
+const DEFAULT_RECOVERY_RETENTION_DAYS = 7;
+const DEFAULT_RECOVERY_MAX_GENERATIONS = 0;

 function isStackServiceStatus(value: unknown): value is StackServiceStatus {
    if (!value || typeof value !== 'object') return false;
@@ -253,6 +255,9 @@ export interface StackUpdateRecoveryGenerationRow {
    updated_at: number;
    created_by: string | null;
    artifacts_retired: number;
+    /** Set when an operator manually released rollback protection early (see releaseStackUpdateRecoveryGeneration). */
+    released_at: number | null;
+    released_by: string | null;
 }

 /** Durable cleanup tombstone for stack/node deletion artifact sweep. */
@@ -1898,6 +1903,12 @@ export class DatabaseService {
        `);

        maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0');
+        // Manual release (operator gave up rollback protection early). Additive
+        // columns rather than a new `status` enum value, since `status` carries a
+        // CHECK constraint that would need the heavier table-rebuild migration
+        // pattern used for health_gate_runs below.
+        maybeAddCol('stack_update_recovery_generations', 'released_at', 'INTEGER');
+        maybeAddCol('stack_update_recovery_generations', 'released_by', 'TEXT');
        maybeAddCol('stack_update_cleanup_pending', 'required_blueprint_id', 'INTEGER');

        // Distributed API model columns
@@ -2040,6 +2051,11 @@ export class DatabaseService {
        stmt.run('reclaim_hero', '0');
        stmt.run('health_gate_enabled', '1');
        stmt.run('health_gate_window_seconds', '90');
+        // Superseded-generation retention (days) and a per-stack cap on total
+        // retained generations (0 = unlimited). Never applies to the current
+        // generation, which stays protected until superseded or released.
+        stmt.run('recovery_retention_days', '7');
+        stmt.run('recovery_max_generations', '0');
        stmt.run('image_update_check_interval_minutes', '120');
        stmt.run('image_update_check_mode', 'interval');
        stmt.run('image_update_check_cron', '');
@@ -4213,16 +4229,39 @@ export class DatabaseService {
        return result.changes === 1;
    }

+    /** Days a superseded generation's Docker/FS artifacts are retained before automatic cleanup. Never applies to the current generation. */
+    public getRecoveryRetentionDays(): number {
+        try {
+            const raw = parseInt(this.getGlobalSettings()['recovery_retention_days'] ?? '', 10);
+            return Number.isFinite(raw) && raw > 0 ? Math.min(raw, 90) : DEFAULT_RECOVERY_RETENTION_DAYS;
+        } catch (e) {
+            console.warn('[DatabaseService] recovery_retention_days read failed; using default:', (e as Error).message);
+            return DEFAULT_RECOVERY_RETENTION_DAYS;
+        }
+    }
+
+    /** Total generations retained per stack, current included (0 = unlimited). */
+    public getRecoveryMaxGenerations(): number {
+        try {
+            const raw = parseInt(this.getGlobalSettings()['recovery_max_generations'] ?? '', 10);
+            return Number.isFinite(raw) && raw >= 0 ? Math.min(raw, 50) : DEFAULT_RECOVERY_MAX_GENERATIONS;
+        } catch (e) {
+            console.warn('[DatabaseService] recovery_max_generations read failed; using default:', (e as Error).message);
+            return DEFAULT_RECOVERY_MAX_GENERATIONS;
+        }
+    }
+
    public casHandoffGeneration(candidateId: string, nodeId: number, stackName: string): boolean {
        const handoff = this.db.transaction(() => {
            const candidate = this.getStackUpdateRecoveryGeneration(candidateId);
            if (!candidate || candidate.node_id !== nodeId || candidate.stack_name !== stackName) return false;
            if (candidate.status !== 'candidate' || candidate.phase !== 'acquired') return false;
+            const retentionMs = this.getRecoveryRetentionDays() * 24 * 60 * 60 * 1000;
            this.db.prepare(
                `UPDATE stack_update_recovery_generations
                 SET status = 'superseded', is_current = 0, artifact_expires_at = ?, updated_at = ?
                 WHERE node_id = ? AND stack_name = ? AND is_current = 1 AND id != ?`
-            ).run(Date.now() + 7 * 24 * 60 * 60 * 1000, Date.now(), nodeId, stackName, candidateId);
+            ).run(Date.now() + retentionMs, Date.now(), nodeId, stackName, candidateId);
            const result = this.db.prepare(
                `UPDATE stack_update_recovery_generations
                 SET status = 'active', is_current = 1, phase = 'handoff_committed', updated_at = ?
@@ -4234,18 +4273,41 @@ export class DatabaseService {
    }


-    /** Generations whose Docker/FS artifacts can be retired (not actively held). */
+    /**
+     * Generations whose Docker/FS artifacts can be retired (not actively held).
+     * A manually released row (released_at set) is swept immediately regardless
+     * of its expiry timers; a naturally abandoned/superseded row still waits out
+     * artifact_expires_at / gate_retain_until.
+     */
    public listStackUpdateRecoveryGenerationsForArtifactRetirement(now: number): StackUpdateRecoveryGenerationRow[] {
        return this.db.prepare(
            `SELECT * FROM stack_update_recovery_generations
             WHERE artifacts_retired = 0
               AND is_current = 0
-               AND status IN ('abandoned', 'superseded')
-               AND (artifact_expires_at IS NULL OR artifact_expires_at <= ?)
-               AND (gate_retain_until IS NULL OR gate_retain_until <= ?)`
+               AND (
+                 released_at IS NOT NULL
+                 OR (
+                   status IN ('abandoned', 'superseded')
+                   AND (artifact_expires_at IS NULL OR artifact_expires_at <= ?)
+                   AND (gate_retain_until IS NULL OR gate_retain_until <= ?)
+                 )
+               )`
        ).all(now, now) as StackUpdateRecoveryGenerationRow[];
    }

+    /**
+     * Superseded, not-yet-retired, not-released generations across every node
+     * for cap enforcement (mirrors the other reconcile-sweep list methods,
+     * which are also unscoped by node), newest first per (node_id, stack_name).
+     */
+    public listActiveSupersededGenerations(): StackUpdateRecoveryGenerationRow[] {
+        return this.db.prepare(
+            `SELECT * FROM stack_update_recovery_generations
+             WHERE status = 'superseded' AND artifacts_retired = 0 AND released_at IS NULL
+             ORDER BY node_id, stack_name, created_at DESC, id DESC`
+        ).all() as StackUpdateRecoveryGenerationRow[];
+    }
+
    public markStackUpdateRecoveryArtifactsRetired(id: string): boolean {
        const result = this.db.prepare(
            `UPDATE stack_update_recovery_generations
@@ -4266,6 +4328,33 @@ export class DatabaseService {
        return result.changes === 1;
    }

+    /**
+     * Operator-initiated release of rollback protection. A single conditional
+     * UPDATE both revalidates eligibility and performs the transition
+     * atomically, so a stale caller can never release a row that has since
+     * become ineligible (e.g. it started a health gate observation, or moved
+     * to recovery_required). Only clears is_current/timestamps; Docker tag and
+     * override-file cleanup is the caller's job via retireGenerationArtifacts,
+     * matching how abandon() already separates the DB transition from cleanup.
+     */
+    public releaseStackUpdateRecoveryGeneration(id: string, releasedBy: string | null): boolean {
+        const now = Date.now();
+        const result = this.db.prepare(
+            `UPDATE stack_update_recovery_generations
+             SET released_at = ?, released_by = ?, is_current = 0, updated_at = ?
+             WHERE id = ?
+               AND released_at IS NULL
+               AND artifacts_retired = 0
+               AND phase = 'immediate_verified'
+               AND status IN ('active', 'restored_current', 'superseded')
+               AND (health_gate_id IS NULL OR NOT EXISTS (
+                 SELECT 1 FROM health_gate_runs g
+                 WHERE g.id = stack_update_recovery_generations.health_gate_id AND g.status = 'observing'
+               ))`
+        ).run(now, releasedBy, now, id);
+        return result.changes === 1;
+    }
+
    /** Pre-handoff candidates whose operation lease has expired. */
    public listStaleStackUpdateRecoveryCandidates(now: number): StackUpdateRecoveryGenerationRow[] {
        return this.db.prepare(
@@ -4303,6 +4392,7 @@ export class DatabaseService {
        const rows = this.db.prepare(
            `SELECT services_json FROM stack_update_recovery_generations
             WHERE node_id = ?
+               AND released_at IS NULL
               AND status IN ('candidate','active','restored_current','recovery_required')
               AND (artifact_expires_at IS NULL OR artifact_expires_at > ? OR gate_retain_until > ? OR is_current = 1)`
        ).all(nodeId, now, now) as Array<{ services_json: string }>;
@@ -4438,7 +4528,7 @@ export class DatabaseService {
        const categories = [
            'deploy_success', 'deploy_failure', 'stack_started', 'stack_stopped', 'stack_restarted',
            'image_update_applied', 'update_started', 'health_gate_passed', 'health_gate_failed',
-            'network_auto_created',
+            'network_auto_created', 'rollback_generation_released',
        ];
        const placeholders = categories.map(() => '?').join(', ');
        const sql = `
@@ -143,6 +143,9 @@ export interface ClassifiedImage {
  managedBy: string | null;
  managedStatus: 'managed' | 'unmanaged' | 'unused';
  isSencho: boolean;
+  /** True when a StackUpdateRecoveryService/ServiceUpdateRecoveryService hold protects this image from pruning. Additive: does not change managedStatus semantics. */
+  rollbackProtected: boolean;
+  rollbackProtectionKind?: 'stack' | 'service';
 }

 export interface PortInUseInfo {
@@ -561,23 +564,52 @@ class DockerController {

    const selfIdentity = SelfIdentityService.getInstance();

-    const images: ClassifiedImage[] = this.validateApiData<any[]>(rawImages).map((img: any) => {
-      const usedByStacks = [...(imageToStacks.get(img.Id) ?? [])].sort((a, b) => a.localeCompare(b));
-      const managedBy = usedByStacks[0] ?? null;
-      const managedStatus: ClassifiedImage['managedStatus'] =
-        img.Containers === 0 ? 'unused' :
-        managedBy ? 'managed' : 'unmanaged';
-      return {
-        Id: img.Id,
-        RepoTags: img.RepoTags ?? [],
-        Size: img.Size ?? 0,
-        Containers: img.Containers ?? 0,
-        usedByStacks,
-        managedBy,
-        managedStatus,
-        isSencho: selfIdentity.isOwnImage(img.Id),
-      };
-    });
+    // Dynamic (async) imports avoid a static cycle: StackUpdateRecoveryService
+    // imports DockerController directly, and ServiceUpdateRecoveryService
+    // reaches it transitively through ComposeService. It must be `await import`
+    // rather than require(), which does not resolve under Vitest's loader.
+    const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
+    const { ServiceUpdateRecoveryService } = await import('./ServiceUpdateRecoveryService');
+    const stackHeld = StackUpdateRecoveryService.getInstance().getHeldImageIds(this.nodeId);
+    const serviceHeld = ServiceUpdateRecoveryService.getInstance().getHeldImageIds(this.nodeId);
+    // A null lookup means "held state unknown" (the DB read failed); treat it
+    // as held so the badge never disagrees with the delete guard, which fails
+    // the same way (recoveryHeldImages.ts's buildUnifiedHeldImagePredicate).
+    const rollbackKind = (imageId: string): ClassifiedImage['rollbackProtectionKind'] => {
+      if (stackHeld === null || stackHeld.has(imageId)) return 'stack';
+      if (serviceHeld === null || serviceHeld.has(imageId)) return 'service';
+      return undefined;
+    };
+
+    // Only hide an image from the generic inventory when every visible tag is
+    // a synthetic sencho-rb hold tag; an image that also carries a normal
+    // registry tag stays visible here (with the badge below) so the generic
+    // inventory stays complete. Its generation still surfaces in the Rollback tab.
+    const isFullySyntheticHoldImage = (repoTags: string[]): boolean =>
+      repoTags.length > 0 && repoTags.every((tag) => tag.startsWith('sencho-rb/'));
+
+    const images: ClassifiedImage[] = this.validateApiData<any[]>(rawImages)
+      .map((img: any) => {
+        const usedByStacks = [...(imageToStacks.get(img.Id) ?? [])].sort((a, b) => a.localeCompare(b));
+        const managedBy = usedByStacks[0] ?? null;
+        const managedStatus: ClassifiedImage['managedStatus'] =
+          img.Containers === 0 ? 'unused' :
+          managedBy ? 'managed' : 'unmanaged';
+        const rollbackProtectionKind = rollbackKind(img.Id);
+        return {
+          Id: img.Id,
+          RepoTags: img.RepoTags ?? [],
+          Size: img.Size ?? 0,
+          Containers: img.Containers ?? 0,
+          usedByStacks,
+          managedBy,
+          managedStatus,
+          isSencho: selfIdentity.isOwnImage(img.Id),
+          rollbackProtected: rollbackProtectionKind !== undefined,
+          rollbackProtectionKind,
+        };
+      })
+      .filter((img) => !isFullySyntheticHoldImage(img.RepoTags));

    const volumes: ClassifiedVolume[] = rawVolumes.map((vol: any) => {
      const stack = DockerController.resolveProjectLabel(vol.Labels?.['com.docker.compose.project'], knownSet, projectToStack);
@@ -1504,6 +1536,23 @@ class DockerController {
    return { inspect, history };
  }

+  /**
+   * Resolve any valid Docker image reference (full ID, short ID, digest, or
+   * tag) to its canonical full sha256 ID. isValidDockerResourceId accepts
+   * short IDs down to 12 hex chars, which a held-image-id set lookup (always
+   * keyed on the full 64-char form) would miss without this resolve step.
+   * Returns null when the image does not exist.
+   */
+  public async resolveImageId(id: string): Promise<string | null> {
+    try {
+      const info = await this.docker.getImage(id).inspect();
+      return info.Id;
+    } catch (error) {
+      if ((error as { statusCode?: number })?.statusCode === 404) return null;
+      throw error;
+    }
+  }
+
  public async removeVolume(name: string) {
    const volume = this.docker.getVolume(name);
    await volume.remove({ force: true });
@@ -48,6 +48,9 @@ export type NotificationCategory =
    | 'update_started'
    | 'health_gate_passed'
    | 'health_gate_failed'
+    // Manual rollback-generation release (Resources → Rollback). History-only
+    // for the same reason as the drift pair above.
+    | 'rollback_generation_released'
    // Automatic external-network creation during deploy. History-only.
    | 'network_auto_created'
    | 'node_update_available'
@@ -67,7 +70,7 @@ export const ALL_SUPPRESSIBLE_CATEGORIES: readonly NotificationCategory[] = [
    ...ALL_NOTIFICATION_CATEGORIES,
    'drift_detected', 'drift_resolved',
    'update_started', 'health_gate_passed', 'health_gate_failed',
-    'network_auto_created',
+    'network_auto_created', 'rollback_generation_released',
 ];

 /** Webhook timeout: 10 seconds per external dispatch call. */
@@ -1,6 +1,7 @@
 import { randomUUID } from 'crypto';
 import { DatabaseService, type ServiceUpdateRecoveryRow } from './DatabaseService';
 import { getComposeCommandTimeoutMs } from './ComposeService';
+import { buildUnifiedHeldImagePredicate } from './recoveryHeldImages';
 import { getErrorMessage } from '../utils/errors';

 const SWEEP_INTERVAL_MS = 5 * 60_000;
@@ -229,26 +230,13 @@ export class ServiceUpdateRecoveryService {
  /**
   * A predicate a pruner can call immediately before deleting each candidate
   * image. Re-reads the held set on every call (rather than snapshotting it
-   * once) so a snapshot that becomes eligible between plan and delete is
-   * still honored. When the held set cannot be read, returns true for every
-   * id so prune skips deletes (fail closed).
+   * once, unlike recoveryHeldImages.buildUnifiedHeldImagePredicate) so a
+   * generation that becomes eligible between plan and delete is still
+   * honored. When the held set cannot be read, returns true for every id so
+   * prune skips deletes (fail closed).
   */
  public buildHeldImagePredicate(nodeId: number): (imageId: string) => boolean {
-    return (imageId: string) => {
-      const held = this.getHeldImageIds(nodeId);
-      if (held === null) return true;
-      if (held.has(imageId)) return true;
-      try {
-        // Dynamic import avoids a static cycle with StackUpdateRecoveryService.
-        // eslint-disable-next-line @typescript-eslint/no-require-imports
-        const { StackUpdateRecoveryService } = require('./StackUpdateRecoveryService') as typeof import('./StackUpdateRecoveryService');
-        const stackHeld = StackUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
-        if (stackHeld === null) return true;
-        return stackHeld.has(imageId);
-      } catch {
-        return true;
-      }
-    };
+    return (imageId: string) => buildUnifiedHeldImagePredicate(nodeId)(imageId);
  }

  private nextClaimExpiry(now: number): number {
@@ -70,9 +70,13 @@ function sanitizeServiceSlug(name: string): string {
  return name.replace(/[^a-zA-Z0-9._-]/g, '-').toLowerCase() || 'svc';
 }

+/** Same short form used in the opaque rollback tag, so the UI's "Generation" label matches the Docker tag. */
+export function shortGenerationId(generationId: string): string {
+  return generationId.replace(/-/g, '').slice(0, 12);
+}
+
 function opaqueRollbackTag(generationId: string, serviceName: string): string {
-  const short = generationId.replace(/-/g, '').slice(0, 12);
-  return `sencho-rb/${short}/${sanitizeServiceSlug(serviceName)}:hold`;
+  return `sencho-rb/${shortGenerationId(generationId)}/${sanitizeServiceSlug(serviceName)}:hold`;
 }

 function parseServicesJson(raw: string): StackRecoveryServiceCapture[] {
@@ -284,6 +288,8 @@ export class StackUpdateRecoveryService {
        updated_at: now,
        created_by: createdBy,
        artifacts_retired: 0,
+        released_at: null,
+        released_by: null,
      };
      DatabaseService.getInstance().insertStackUpdateRecoveryGeneration(row);
      return row;
@@ -329,7 +335,7 @@ export class StackUpdateRecoveryService {
      throw new Error('Stack directory escapes compose base');
    }

-    const short = generationId.replace(/-/g, '').slice(0, 12);
+    const short = shortGenerationId(generationId);
    if (!/^[a-f0-9]{12}$/i.test(short)) {
      throw new Error('Invalid recovery generation id');
    }
@@ -398,6 +404,74 @@ export class StackUpdateRecoveryService {
    return ok;
  }

+  /**
+   * Informational mirror of releaseStackUpdateRecoveryGeneration's WHERE
+   * clause, for the list endpoint to grey out a row it already knows is
+   * ineligible. Not authoritative: releaseGeneration revalidates for real.
+   */
+  public isReleaseEligible(row: StackUpdateRecoveryGenerationRow): boolean {
+    if (row.released_at !== null || row.artifacts_retired !== 0) return false;
+    if (row.phase !== 'immediate_verified') return false;
+    if (!['active', 'restored_current', 'superseded'].includes(row.status)) return false;
+    if (row.health_gate_id) {
+      const gate = DatabaseService.getInstance().getHealthGateRun(row.node_id, row.stack_name, row.health_gate_id);
+      if (gate?.status === 'observing') return false;
+    }
+    return true;
+  }
+
+  /**
+   * Operator-initiated release of rollback protection, current generation
+   * included. The DB transition (releaseStackUpdateRecoveryGeneration)
+   * atomically revalidates eligibility and clears is_current, which is what
+   * stops getCurrent()/isRestoredCurrentPinActive() from reporting a released
+   * row as the live rollback point. Docker tag + override cleanup reuses the
+   * same idempotent retireGenerationArtifacts() that abandon() already relies
+   * on, so a mid-cleanup Docker failure leaves artifacts_retired at 0 and is
+   * retried by the next reconcileIncomplete() sweep rather than silently
+   * "succeeding" in the UI.
+   */
+  public async releaseGeneration(
+    id: string,
+    releasedBy: string | null,
+  ): Promise<
+    | { ok: true; row: StackUpdateRecoveryGenerationRow; artifactsCleaned: boolean }
+    | { ok: false; reason: 'not_found' | 'already_released' | 'not_eligible' }
+  > {
+    const before = this.get(id);
+    if (!before) return { ok: false, reason: 'not_found' };
+    if (before.released_at !== null) return { ok: false, reason: 'already_released' };
+
+    const released = DatabaseService.getInstance().releaseStackUpdateRecoveryGeneration(id, releasedBy);
+    if (!released) return { ok: false, reason: 'not_eligible' };
+
+    const row = this.get(id);
+    if (!row) return { ok: false, reason: 'not_found' };
+    const artifactsCleaned = await this.retireGenerationArtifacts(row);
+
+    const wasCurrent = before.is_current === 1;
+    try {
+      DatabaseService.getInstance().addNotificationHistory(row.node_id, {
+        level: wasCurrent ? 'warning' : 'info',
+        category: 'rollback_generation_released',
+        message: wasCurrent
+          ? `${row.stack_name}: current rollback protection released. Automatic rollback is unavailable until the next successful full-stack update.`
+          : `${row.stack_name}: rollback protection released for generation ${shortGenerationId(row.id)}.`,
+        timestamp: Date.now(),
+        stack_name: row.stack_name,
+        actor_username: releasedBy,
+      });
+    } catch (error) {
+      console.warn(
+        '[StackUpdateRecovery] Failed to record release activity for %s:',
+        sanitizeForLog(id),
+        sanitizeForLog(getErrorMessage(error, 'unknown')),
+      );
+    }
+
+    return { ok: true, row, artifactsCleaned };
+  }
+
  public linkHealthGate(id: string, healthGateId: string): void {
    DatabaseService.getInstance().linkStackUpdateRecoveryHealthGate(id, healthGateId);
  }
@@ -460,22 +534,6 @@ export class StackUpdateRecoveryService {
    }
  }

-  /**
-   * Unified held-image predicate: service-scoped + full-stack holds.
-   * Fail closed (skip prune) when either lookup fails.
-   */
-  public buildUnifiedHeldImagePredicate(nodeId: number): (imageId: string) => boolean {
-    // Dynamic require avoids a static cycle with ServiceUpdateRecoveryService.
-    // eslint-disable-next-line @typescript-eslint/no-require-imports
-    const { ServiceUpdateRecoveryService } = require('./ServiceUpdateRecoveryService') as typeof import('./ServiceUpdateRecoveryService');
-    const serviceHeld = ServiceUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
-    const stackHeld = this.getHeldImageIds(nodeId);
-    if (serviceHeld === null || stackHeld === null) {
-      return () => true;
-    }
-    return (imageId: string) => serviceHeld.has(imageId) || stackHeld.has(imageId);
-  }
-
  /**
   * Post-handoff compensation: restore files + pinned up, then probe before
   * reporting restored_current / immediate_verified.
@@ -655,7 +713,20 @@ export class StackUpdateRecoveryService {
      }
    }
    if (!tagsOk || !overrideOk) return false;
-    DatabaseService.getInstance().markStackUpdateRecoveryArtifactsRetired(row.id);
+    try {
+      DatabaseService.getInstance().markStackUpdateRecoveryArtifactsRetired(row.id);
+    } catch (error) {
+      // Tags/override are already gone at this point; a DB write failure here
+      // must not surface as "release/abandon failed" to the caller (the
+      // mutation it asked for already happened). Leave artifacts_retired at 0
+      // so the next reconcileIncomplete() sweep retries the DB write alone.
+      console.warn(
+        '[StackUpdateRecovery] Failed to mark artifacts retired for %s: %s',
+        sanitizeForLog(row.id),
+        sanitizeForLog(getErrorMessage(error, 'unknown')),
+      );
+      return false;
+    }
    return true;
  }

@@ -680,16 +751,38 @@ export class StackUpdateRecoveryService {
        });
        flagged += 1;
      }
+      let capped = 0;
+      const maxGenerations = db.getRecoveryMaxGenerations();
+      if (maxGenerations > 0) {
+        // The current generation always counts as one of the cap, so the
+        // superseded budget is one less; it can never itself be evicted here.
+        const supersededBudget = Math.max(0, maxGenerations - 1);
+        const byStack = new Map<string, StackUpdateRecoveryGenerationRow[]>();
+        for (const row of db.listActiveSupersededGenerations()) {
+          const key = `${row.node_id}:${row.stack_name}`;
+          const list = byStack.get(key) ?? [];
+          list.push(row);
+          byStack.set(key, list);
+        }
+        for (const rows of byStack.values()) {
+          for (const row of rows.slice(supersededBudget)) {
+            if (row.artifact_expires_at === null || row.artifact_expires_at > now) {
+              db.updateStackUpdateRecoveryGeneration(row.id, { artifact_expires_at: now });
+              capped += 1;
+            }
+          }
+        }
+      }
      let retired = 0;
      for (const row of db.listStackUpdateRecoveryGenerationsForArtifactRetirement(now)) {
        // Never retire an active/current or recovery_required hold target.
        if (row.is_current === 1 || row.status === 'recovery_required') continue;
        if (await this.retireGenerationArtifacts(row)) retired += 1;
      }
-      if (abandoned > 0 || flagged > 0 || retired > 0) {
+      if (abandoned > 0 || flagged > 0 || capped > 0 || retired > 0) {
        console.log(
          `[StackUpdateRecovery] Reconciled ${abandoned} stale candidate(s), `
-          + `${flagged} stuck generation(s), retired ${retired} artifact set(s)`,
+          + `${flagged} stuck generation(s), ${capped} generation(s) over cap, retired ${retired} artifact set(s)`,
        );
      }
    } catch (error) {
@@ -0,0 +1,18 @@
+import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService';
+import { StackUpdateRecoveryService } from './StackUpdateRecoveryService';
+
+/**
+ * Unified held-image predicate: service-scoped + full-stack rollback holds.
+ * Lives in its own module (rather than on either service) so both can be
+ * imported here statically without a cycle -- ServiceUpdateRecoveryService
+ * and StackUpdateRecoveryService intentionally do not import each other.
+ * Fails closed (protects every image) when either lookup fails.
+ */
+export function buildUnifiedHeldImagePredicate(nodeId: number): (imageId: string) => boolean {
+  const serviceHeld = ServiceUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
+  const stackHeld = StackUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
+  if (serviceHeld === null || stackHeld === null) {
+    return () => true;
+  }
+  return (imageId: string) => serviceHeld.has(imageId) || stackHeld.has(imageId);
+}