mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-04 16:07:55 +00:00
feat(recovery): make rollback-recovery image lifecycle visible and controllable (#1753)
* feat(recovery): make rollback-recovery image lifecycle visible and controllable GitHub discussion #1751 asked why Sencho creates sencho-rb/<id>/<service>:hold images during automatic updates and how to clean them up. That surfaced a real safety bug alongside the missing visibility: the manual single-image delete route did not consult the held-image predicate every other deletion path already honors, so a user could delete a rollback-protected image straight through the Images tab and silently break automatic recovery for that update. A short/truncated id also bypassed the predicate's full-id lookup. Fixes: - POST /images/delete now resolves the submitted id to its canonical form and checks the unified held-image predicate before deleting, returning 409 IMAGE_HELD_FOR_ROLLBACK for a protected image. - The Images tab no longer mislabels a protected image as plain "Unused"; a fully-synthetic hold image is kept out of the generic inventory entirely and surfaced instead in a new Resources -> Rollback tab, with an additive "Rollback protected" badge for images that still carry a normal tag too. New capability: - Two settings (Deploy Guardrails): superseded-generation retention (days, replaces a hardcoded 7) and a cap on retained generations per stack. - A new Resources -> Rollback tab lists every generation (stack, short id, state, retention) with an admin-gated manual release action, including releasing the current generation with an explicit warning that automatic rollback becomes unavailable until the next successful update. Release is a single atomic, server-revalidated transition so a stale UI read can never release a row that has since become ineligible. Also consolidated three near-duplicate implementations of the held-image predicate (two of which relied on a require() of a sibling .ts file that silently failed to resolve under the test runner and was never actually exercised by a real test before this change) into one shared module. Known follow-up, not fixed here: an orphaned sencho-rb tag whose recovery row no longer exists (DB restore, node re-add) is invisible in both the Images and Rollback tabs with no UI path to reclaim it. * fix(audit): add summary mapping for rollback generation release * fix(security): sanitize prune target in log sinks and cover release RBAC Closes two open js/log-injection findings on the system prune route by applying the same inline sanitizeForLog barrier the rest of the file already uses. The prune target is validated against an enum by parsePruneTargets before reaching these sinks, so the findings were false positives, but the barrier is cheap and removes the standing alerts on a file this change already touches. Also wraps the generation id in the release log line for consistency with the stack name beside it. Adds coverage for gaps a QA pass identified: - Release endpoint refuses a viewer and a deployer (Admin-only), leaving the generation and its artifacts untouched. - Viewer can still read the generations list, matching the sibling Resources routes. - The predicate the prune routes build reports full-stack rollback holds, not just service-scoped ones, and re-reads per call so a hold taken between plan and delete still gates the delete. - After releasing the current generation, no rollback point is claimed for the stack through any consumer of the current-generation lookup.
This commit is contained in:
Anso
@@ -181,6 +181,10 @@ vi.mock('../services/MeshService', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/recoveryHeldImages', () => ({
|
||||
buildUnifiedHeldImagePredicate: mockBuildUnifiedHeldImagePredicate,
|
||||
}));
|
||||
|
||||
vi.mock('../services/StackUpdateRecoveryService', () => ({
|
||||
StackUpdateRecoveryService: {
|
||||
getInstance: () => ({
|
||||
@@ -191,7 +195,6 @@ vi.mock('../services/StackUpdateRecoveryService', () => ({
|
||||
markImmediateVerified: mockMarkImmediateVerified,
|
||||
abandon: mockAbandon,
|
||||
compensateWithCandidate: mockCompensateWithCandidate,
|
||||
buildUnifiedHeldImagePredicate: mockBuildUnifiedHeldImagePredicate,
|
||||
get: mockGetRecovery,
|
||||
linkGateOrRetain: vi.fn(),
|
||||
}),
|
||||
|
||||
@@ -60,6 +60,8 @@ describe('DeployedStackDeletionService ready transaction', () => {
|
||||
updated_at: now,
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
released_at: null,
|
||||
released_by: null,
|
||||
};
|
||||
db().insertStackUpdateRecoveryGeneration(gen);
|
||||
const svc: ServiceUpdateRecoveryRow = {
|
||||
|
||||
@@ -60,6 +60,8 @@ vi.mock('util', () => ({
|
||||
|
||||
import DockerController, { selectMainWebPort, parseExitCode, isContainerFailed } from '../services/DockerController';
|
||||
import { CacheService } from '../services/CacheService';
|
||||
import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
import fs from 'fs/promises';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
@@ -529,6 +531,9 @@ describe('DockerController - getClassifiedResources', () => {
|
||||
beforeEach(() => {
|
||||
CacheService.getInstance().invalidate('project-name-map');
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('classifies managed and unmanaged images', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
@@ -616,6 +621,94 @@ describe('DockerController - getClassifiedResources', () => {
|
||||
expect(result.volumes.find(v => v.Name === 'my-stack_data')!.managedStatus).toBe('managed');
|
||||
expect(result.volumes.find(v => v.Name === 'random_vol')!.managedStatus).toBe('unmanaged');
|
||||
});
|
||||
|
||||
it('excludes an image whose only tag is a synthetic sencho-rb rollback hold', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{ Id: 'img-hold-only', RepoTags: ['sencho-rb/abc123456789/web:hold'], Size: 50, Containers: 0 },
|
||||
{ Id: 'img-normal', RepoTags: ['nginx:latest'], Size: 100, Containers: 0 },
|
||||
]);
|
||||
mockDocker.listContainers.mockResolvedValue([]);
|
||||
mockDocker.listVolumes.mockResolvedValue({ Volumes: [] });
|
||||
mockDocker.listNetworks.mockResolvedValue([]);
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const result = await dc.getClassifiedResources(['my-stack']);
|
||||
|
||||
expect(result.images.find(i => i.Id === 'img-hold-only')).toBeUndefined();
|
||||
expect(result.images.find(i => i.Id === 'img-normal')).toBeDefined();
|
||||
});
|
||||
|
||||
it('keeps an image visible when it carries both a normal tag and a sencho-rb hold tag', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{ Id: 'img-multi-tag', RepoTags: ['myregistry/app:1.4', 'sencho-rb/abc123456789/app:hold'], Size: 100, Containers: 0 },
|
||||
]);
|
||||
mockDocker.listContainers.mockResolvedValue([]);
|
||||
mockDocker.listVolumes.mockResolvedValue({ Volumes: [] });
|
||||
mockDocker.listNetworks.mockResolvedValue([]);
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const result = await dc.getClassifiedResources(['my-stack']);
|
||||
|
||||
const img = result.images.find(i => i.Id === 'img-multi-tag');
|
||||
expect(img).toBeDefined();
|
||||
expect(img!.RepoTags).toEqual(['myregistry/app:1.4', 'sencho-rb/abc123456789/app:hold']);
|
||||
});
|
||||
|
||||
it('marks an image rollbackProtected with kind "stack" when StackUpdateRecoveryService holds it', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{ Id: 'img-stack-held', RepoTags: ['myregistry/app:1.4', 'sencho-rb/abc123456789/app:hold'], Size: 100, Containers: 0 },
|
||||
]);
|
||||
mockDocker.listContainers.mockResolvedValue([]);
|
||||
mockDocker.listVolumes.mockResolvedValue({ Volumes: [] });
|
||||
mockDocker.listNetworks.mockResolvedValue([]);
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set(['img-stack-held']));
|
||||
vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set());
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const result = await dc.getClassifiedResources(['my-stack']);
|
||||
|
||||
const img = result.images.find(i => i.Id === 'img-stack-held');
|
||||
expect(img?.rollbackProtected).toBe(true);
|
||||
expect(img?.rollbackProtectionKind).toBe('stack');
|
||||
});
|
||||
|
||||
it('marks an image rollbackProtected with kind "service" when only ServiceUpdateRecoveryService holds it', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{ Id: 'img-service-held', RepoTags: ['myregistry/app:1.4'], Size: 100, Containers: 0 },
|
||||
]);
|
||||
mockDocker.listContainers.mockResolvedValue([]);
|
||||
mockDocker.listVolumes.mockResolvedValue({ Volumes: [] });
|
||||
mockDocker.listNetworks.mockResolvedValue([]);
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set());
|
||||
vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set(['img-service-held']));
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const result = await dc.getClassifiedResources(['my-stack']);
|
||||
|
||||
const img = result.images.find(i => i.Id === 'img-service-held');
|
||||
expect(img?.rollbackProtected).toBe(true);
|
||||
expect(img?.rollbackProtectionKind).toBe('service');
|
||||
});
|
||||
|
||||
it('fails closed (marks every image rollbackProtected) when a held-image lookup fails', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{ Id: 'img-unrelated', RepoTags: ['myregistry/app:1.4'], Size: 100, Containers: 0 },
|
||||
]);
|
||||
mockDocker.listContainers.mockResolvedValue([]);
|
||||
mockDocker.listVolumes.mockResolvedValue({ Volumes: [] });
|
||||
mockDocker.listNetworks.mockResolvedValue([]);
|
||||
// getHeldImageIds returns null when its own DB lookup fails (already logs
|
||||
// internally); the badge must fail the same direction as the delete guard
|
||||
// (recoveryHeldImages.ts), not the opposite.
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(null);
|
||||
vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set());
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const result = await dc.getClassifiedResources(['my-stack']);
|
||||
|
||||
const img = result.images.find(i => i.Id === 'img-unrelated');
|
||||
expect(img?.rollbackProtected).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ── pruneManagedOnly / estimateManagedReclaim (images) ─────────────────
|
||||
@@ -1102,6 +1195,40 @@ describe('DockerController - inspectImage', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// --- resolveImageId --------------------------------------------------------------
|
||||
|
||||
describe('DockerController - resolveImageId', () => {
|
||||
it('returns the canonical full Id from docker.getImage(id).inspect()', async () => {
|
||||
mockDocker.getImage.mockReturnValue({
|
||||
inspect: vi.fn().mockResolvedValue({ Id: 'sha256:' + 'a'.repeat(64) }),
|
||||
});
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const result = await dc.resolveImageId('a'.repeat(12));
|
||||
|
||||
expect(result).toBe('sha256:' + 'a'.repeat(64));
|
||||
expect(mockDocker.getImage).toHaveBeenCalledWith('a'.repeat(12));
|
||||
});
|
||||
|
||||
it('returns null on a 404 from Docker', async () => {
|
||||
mockDocker.getImage.mockReturnValue({
|
||||
inspect: vi.fn().mockRejectedValue(Object.assign(new Error('No such image'), { statusCode: 404 })),
|
||||
});
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
expect(await dc.resolveImageId('missing')).toBeNull();
|
||||
});
|
||||
|
||||
it('rethrows a non-404 Docker error', async () => {
|
||||
mockDocker.getImage.mockReturnValue({
|
||||
inspect: vi.fn().mockRejectedValue(Object.assign(new Error('docker daemon unreachable'), { statusCode: 500 })),
|
||||
});
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
await expect(dc.resolveImageId('sha256:abc')).rejects.toThrow('docker daemon unreachable');
|
||||
});
|
||||
});
|
||||
|
||||
// --- label / image inspection for the label inventory --------------------------
|
||||
|
||||
describe('DockerController - inspectImageLabels', () => {
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* Unit tests for the unified held-image predicate's fail-closed composition:
|
||||
* a lookup failure on either underlying service must protect every image,
|
||||
* not just the ones the other service happens to hold.
|
||||
*/
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
import { buildUnifiedHeldImagePredicate } from '../services/recoveryHeldImages';
|
||||
import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('buildUnifiedHeldImagePredicate', () => {
|
||||
it('holds an image present in either service\'s held set', () => {
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set(['sha256:stack-held']));
|
||||
vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set(['sha256:service-held']));
|
||||
|
||||
const predicate = buildUnifiedHeldImagePredicate(1);
|
||||
|
||||
expect(predicate('sha256:stack-held')).toBe(true);
|
||||
expect(predicate('sha256:service-held')).toBe(true);
|
||||
expect(predicate('sha256:unrelated')).toBe(false);
|
||||
});
|
||||
|
||||
it('fails closed (protects every image) when StackUpdateRecoveryService.getHeldImageIds returns null', () => {
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(null);
|
||||
vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set());
|
||||
|
||||
const predicate = buildUnifiedHeldImagePredicate(1);
|
||||
|
||||
expect(predicate('sha256:anything')).toBe(true);
|
||||
});
|
||||
|
||||
it('fails closed (protects every image) when ServiceUpdateRecoveryService.getHeldImageIds returns null', () => {
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set());
|
||||
vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(null);
|
||||
|
||||
const predicate = buildUnifiedHeldImagePredicate(1);
|
||||
|
||||
expect(predicate('sha256:anything')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The prune routes (/system/prune/plan, /system/prune/system) build their
|
||||
* predicate via ServiceUpdateRecoveryService.buildHeldImagePredicate, not the
|
||||
* module function directly. That method delegates to the shared module, so a
|
||||
* full-stack rollback hold must gate prune too, not just service-scoped holds.
|
||||
*/
|
||||
describe('ServiceUpdateRecoveryService.buildHeldImagePredicate (the prune-path entry point)', () => {
|
||||
it('protects a full-stack rollback hold, not just service-scoped holds', () => {
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set(['sha256:stack-held']));
|
||||
vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set());
|
||||
|
||||
const predicate = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(1);
|
||||
|
||||
expect(predicate('sha256:stack-held')).toBe(true);
|
||||
expect(predicate('sha256:unrelated')).toBe(false);
|
||||
});
|
||||
|
||||
it('re-reads the held set on every call so a hold taken after plan time still gates the delete', () => {
|
||||
const stackSpy = vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set());
|
||||
vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set());
|
||||
|
||||
const predicate = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(1);
|
||||
expect(predicate('sha256:late-hold')).toBe(false);
|
||||
|
||||
// A generation is captured between plan and delete.
|
||||
stackSpy.mockReturnValue(new Set(['sha256:late-hold']));
|
||||
expect(predicate('sha256:late-hold')).toBe(true);
|
||||
});
|
||||
|
||||
it('fails closed on the prune path when a held lookup fails', () => {
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(null);
|
||||
vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set());
|
||||
|
||||
const predicate = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(1);
|
||||
|
||||
expect(predicate('sha256:anything')).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,455 @@
|
||||
/**
|
||||
* Real-DB tests for the rollback-generation retention/cap/release lifecycle:
|
||||
* DatabaseService's retention/cap/release SQL, StackUpdateRecoveryService's
|
||||
* cap enforcement and releaseGeneration orchestration, and the
|
||||
* GET/POST /api/system/rollback/generations routes. Docker is stubbed
|
||||
* (no real daemon); the DB is real via setupTestDb() so the SQL under test
|
||||
* (atomic release UPDATE, retention/cap queries) runs for real.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, afterEach, beforeEach, vi } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import bcrypt from 'bcrypt';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
|
||||
import type { StackUpdateRecoveryGenerationRow, HealthGateRunRow } from '../services/DatabaseService';
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let authHeader: string;
|
||||
let viewerCookie: string;
|
||||
let deployerCookie: string;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let StackUpdateRecoveryService: typeof import('../services/StackUpdateRecoveryService').StackUpdateRecoveryService;
|
||||
let DockerController: typeof import('../services/DockerController').default;
|
||||
|
||||
const mockRemove = vi.fn().mockResolvedValue(undefined);
|
||||
const mockGetImage = vi.fn(() => ({ remove: mockRemove }));
|
||||
|
||||
const NODE = 1;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ app } = await import('../index'));
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'));
|
||||
({ default: DockerController } = await import('../services/DockerController'));
|
||||
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
|
||||
authHeader = `Bearer ${token}`;
|
||||
|
||||
// Non-admin personas for the RBAC block below. Release is requireAdmin
|
||||
// (a host-destructive Docker operation); the list endpoint is stack:read.
|
||||
const db = DatabaseService.getInstance();
|
||||
for (const [role, pw] of [['viewer', 'vwpass'], ['deployer', 'dppass']] as const) {
|
||||
const hash = await bcrypt.hash(pw, 1);
|
||||
db.addUser({ username: `rb-${role}`, password_hash: hash, role });
|
||||
const res = await request(app).post('/api/auth/login').send({ username: `rb-${role}`, password: pw });
|
||||
const cookies = res.headers['set-cookie'] as string | string[];
|
||||
const c = Array.isArray(cookies) ? cookies[0] : cookies;
|
||||
if (role === 'viewer') viewerCookie = c;
|
||||
else deployerCookie = c;
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(() => cleanupTestDb(tmpDir));
|
||||
|
||||
beforeEach(() => {
|
||||
mockRemove.mockClear().mockResolvedValue(undefined);
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDocker: () => ({ getImage: mockGetImage }),
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
const db = DatabaseService.getInstance();
|
||||
// Restore the shipped defaults so a test that tunes retention/cap does not
|
||||
// leak that value into the next one.
|
||||
db.updateGlobalSetting('recovery_retention_days', '7');
|
||||
db.updateGlobalSetting('recovery_max_generations', '0');
|
||||
db.getDb().prepare('DELETE FROM stack_update_recovery_generations').run();
|
||||
db.getDb().prepare('DELETE FROM health_gate_runs').run();
|
||||
});
|
||||
|
||||
function makeRow(overrides: Partial<StackUpdateRecoveryGenerationRow> = {}): StackUpdateRecoveryGenerationRow {
|
||||
const id = overrides.id ?? randomUUID();
|
||||
const now = Date.now();
|
||||
return {
|
||||
id,
|
||||
node_id: NODE,
|
||||
stack_name: 'my-stack',
|
||||
status: 'active',
|
||||
phase: 'immediate_verified',
|
||||
is_current: 1,
|
||||
backup_slot_id: null,
|
||||
override_path: null,
|
||||
services_json: JSON.stringify([{
|
||||
serviceName: 'web',
|
||||
scale: 1,
|
||||
hasBuild: false,
|
||||
declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag',
|
||||
replicas: [{
|
||||
containerId: 'c1',
|
||||
imageId: `sha256:${id.replace(/-/g, '').padEnd(64, '0').slice(0, 64)}`,
|
||||
repoDigest: null,
|
||||
state: 'running',
|
||||
rollbackTag: `sencho-rb/${id.replace(/-/g, '').slice(0, 12)}/web:hold`,
|
||||
}],
|
||||
}]),
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: null,
|
||||
operation_lease_expires_at: null,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
released_at: null,
|
||||
released_by: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function insertRow(overrides: Partial<StackUpdateRecoveryGenerationRow> = {}): StackUpdateRecoveryGenerationRow {
|
||||
const row = makeRow(overrides);
|
||||
DatabaseService.getInstance().insertStackUpdateRecoveryGeneration(row);
|
||||
return row;
|
||||
}
|
||||
|
||||
function insertHealthGate(overrides: Partial<HealthGateRunRow> = {}): HealthGateRunRow {
|
||||
const run: HealthGateRunRow = {
|
||||
id: randomUUID(),
|
||||
node_id: NODE,
|
||||
stack_name: 'my-stack',
|
||||
trigger_action: 'update',
|
||||
status: 'observing',
|
||||
reason: null,
|
||||
window_seconds: 90,
|
||||
containers_json: '[]',
|
||||
started_at: Date.now(),
|
||||
ended_at: null,
|
||||
created_by: null,
|
||||
target_scope: 'stack',
|
||||
service_name: null,
|
||||
failure_source: null,
|
||||
...overrides,
|
||||
};
|
||||
DatabaseService.getInstance().insertHealthGateRun(run);
|
||||
return run;
|
||||
}
|
||||
|
||||
function imageIdOf(row: StackUpdateRecoveryGenerationRow): string {
|
||||
const parsed = JSON.parse(row.services_json);
|
||||
return parsed[0].replicas[0].imageId as string;
|
||||
}
|
||||
|
||||
describe('recovery_retention_days wired into casHandoffGeneration', () => {
|
||||
it('uses a configured retention value instead of the hardcoded 7 days', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
db.updateGlobalSetting('recovery_retention_days', '2');
|
||||
const current = insertRow({ status: 'active', is_current: 1 });
|
||||
const candidate = insertRow({
|
||||
id: randomUUID(),
|
||||
status: 'candidate',
|
||||
phase: 'acquired',
|
||||
is_current: 0,
|
||||
stack_name: current.stack_name,
|
||||
});
|
||||
const ok = db.casHandoffGeneration(candidate.id, NODE, current.stack_name);
|
||||
expect(ok).toBe(true);
|
||||
|
||||
const superseded = db.getStackUpdateRecoveryGeneration(current.id)!;
|
||||
expect(superseded.status).toBe('superseded');
|
||||
const expiresInDays = (superseded.artifact_expires_at! - Date.now()) / (24 * 60 * 60 * 1000);
|
||||
expect(expiresInDays).toBeGreaterThan(1.9);
|
||||
expect(expiresInDays).toBeLessThan(2.1);
|
||||
});
|
||||
|
||||
it('reflects a retention-days change made between two consecutive handoffs, not the value at the time of the first', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
db.updateGlobalSetting('recovery_retention_days', '2');
|
||||
const stackA = insertRow({ stack_name: 'stack-a', status: 'active', is_current: 1 });
|
||||
const candidateA = insertRow({
|
||||
id: randomUUID(), status: 'candidate', phase: 'acquired', is_current: 0, stack_name: stackA.stack_name,
|
||||
});
|
||||
expect(db.casHandoffGeneration(candidateA.id, NODE, stackA.stack_name)).toBe(true);
|
||||
const supersededA = db.getStackUpdateRecoveryGeneration(stackA.id)!;
|
||||
const daysA = (supersededA.artifact_expires_at! - Date.now()) / (24 * 60 * 60 * 1000);
|
||||
expect(daysA).toBeGreaterThan(1.9);
|
||||
expect(daysA).toBeLessThan(2.1);
|
||||
|
||||
// Change the setting without restarting anything, then handoff a
|
||||
// different stack: its expiry must reflect the new value, not a value
|
||||
// cached from the first call.
|
||||
db.updateGlobalSetting('recovery_retention_days', '5');
|
||||
const stackB = insertRow({ stack_name: 'stack-b', status: 'active', is_current: 1 });
|
||||
const candidateB = insertRow({
|
||||
id: randomUUID(), status: 'candidate', phase: 'acquired', is_current: 0, stack_name: stackB.stack_name,
|
||||
});
|
||||
expect(db.casHandoffGeneration(candidateB.id, NODE, stackB.stack_name)).toBe(true);
|
||||
const supersededB = db.getStackUpdateRecoveryGeneration(stackB.id)!;
|
||||
const daysB = (supersededB.artifact_expires_at! - Date.now()) / (24 * 60 * 60 * 1000);
|
||||
expect(daysB).toBeGreaterThan(4.9);
|
||||
expect(daysB).toBeLessThan(5.1);
|
||||
|
||||
// The earlier write is not retroactively touched by the later setting change.
|
||||
const supersededAAfter = db.getStackUpdateRecoveryGeneration(stackA.id)!;
|
||||
expect(supersededAAfter.artifact_expires_at).toBe(supersededA.artifact_expires_at);
|
||||
});
|
||||
});
|
||||
|
||||
describe('recovery_max_generations cap enforcement (reconcileIncomplete)', () => {
|
||||
it('retains current + (cap - 1) superseded generations; forces the rest to expire now', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
db.updateGlobalSetting('recovery_max_generations', '2');
|
||||
const stackName = 'capped-stack';
|
||||
insertRow({ stack_name: stackName, status: 'active', is_current: 1 });
|
||||
const superseded: StackUpdateRecoveryGenerationRow[] = [];
|
||||
for (let i = 0; i < 3; i++) {
|
||||
superseded.push(insertRow({
|
||||
id: randomUUID(),
|
||||
stack_name: stackName,
|
||||
status: 'superseded',
|
||||
is_current: 0,
|
||||
artifact_expires_at: Date.now() + 6 * 24 * 60 * 60 * 1000,
|
||||
created_at: Date.now() - (3 - i) * 60_000,
|
||||
}));
|
||||
}
|
||||
|
||||
const svc = StackUpdateRecoveryService.getInstance();
|
||||
svc.start();
|
||||
await svc.reconcileIncomplete();
|
||||
svc.stop();
|
||||
|
||||
// cap=2 => current (1) + 1 superseded kept; the other 2 superseded get
|
||||
// artifact_expires_at pulled to now and their artifacts retired.
|
||||
const rows = superseded.map((r) => db.getStackUpdateRecoveryGeneration(r.id)!);
|
||||
const stillRetained = rows.filter((r) => !r.artifacts_retired);
|
||||
expect(stillRetained.length).toBe(1);
|
||||
// Keeps the newest superseded row.
|
||||
expect(stillRetained[0].id).toBe(superseded[2].id);
|
||||
});
|
||||
|
||||
it('never touches a recovery_required generation', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
db.updateGlobalSetting('recovery_max_generations', '1');
|
||||
const stackName = 'stuck-stack';
|
||||
insertRow({ stack_name: stackName, status: 'active', is_current: 1 });
|
||||
const stuck = insertRow({
|
||||
id: randomUUID(),
|
||||
stack_name: stackName,
|
||||
status: 'recovery_required',
|
||||
is_current: 0,
|
||||
});
|
||||
|
||||
const svc = StackUpdateRecoveryService.getInstance();
|
||||
svc.start();
|
||||
await svc.reconcileIncomplete();
|
||||
svc.stop();
|
||||
|
||||
const after = db.getStackUpdateRecoveryGeneration(stuck.id)!;
|
||||
expect(after.artifacts_retired).toBe(0);
|
||||
expect(after.artifact_expires_at).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('StackUpdateRecoveryService.releaseGeneration', () => {
|
||||
it('release on the current generation clears the held-image set for its image', async () => {
|
||||
const row = insertRow({ status: 'active', is_current: 1 });
|
||||
const svc = StackUpdateRecoveryService.getInstance();
|
||||
|
||||
expect(svc.getHeldImageIds(NODE)?.has(imageIdOf(row))).toBe(true);
|
||||
|
||||
const result = await svc.releaseGeneration(row.id, 'tester');
|
||||
expect(result.ok).toBe(true);
|
||||
expect(svc.getHeldImageIds(NODE)?.has(imageIdOf(row))).toBe(false);
|
||||
|
||||
const after = DatabaseService.getInstance().getStackUpdateRecoveryGeneration(row.id)!;
|
||||
expect(after.is_current).toBe(0);
|
||||
expect(after.released_at).not.toBeNull();
|
||||
});
|
||||
|
||||
it('after releasing the current generation, no rollback point is claimed for the stack (D05)', async () => {
|
||||
const row = insertRow({ status: 'active', is_current: 1 });
|
||||
const svc = StackUpdateRecoveryService.getInstance();
|
||||
|
||||
expect(svc.getCurrent(NODE, row.stack_name)).toBeDefined();
|
||||
|
||||
const result = await svc.releaseGeneration(row.id, 'tester');
|
||||
expect(result.ok).toBe(true);
|
||||
|
||||
// Both consumers of the current-generation lookup filter on is_current = 1,
|
||||
// which release clears, so a released row can never be offered as a live
|
||||
// rollback target by a later failed update.
|
||||
expect(svc.getCurrent(NODE, row.stack_name)).toBeUndefined();
|
||||
expect(svc.isRestoredCurrentPinActive(NODE, row.stack_name)).toBe(false);
|
||||
|
||||
// It is also gone from the list endpoint's supported-status projection.
|
||||
const res = await request(app)
|
||||
.get('/api/system/rollback/generations')
|
||||
.set('Authorization', authHeader);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.some((g: { id: string }) => g.id === row.id)).toBe(false);
|
||||
});
|
||||
|
||||
it('release on a restored_current generation clears the service-update pin', async () => {
|
||||
const row = insertRow({ status: 'restored_current', is_current: 1 });
|
||||
const svc = StackUpdateRecoveryService.getInstance();
|
||||
|
||||
expect(svc.isRestoredCurrentPinActive(NODE, row.stack_name)).toBe(true);
|
||||
const result = await svc.releaseGeneration(row.id, 'tester');
|
||||
expect(result.ok).toBe(true);
|
||||
expect(svc.isRestoredCurrentPinActive(NODE, row.stack_name)).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects release while the linked health gate is observing', async () => {
|
||||
const gate = insertHealthGate({ status: 'observing' });
|
||||
const row = insertRow({ status: 'active', is_current: 1, health_gate_id: gate.id, stack_name: gate.stack_name });
|
||||
const svc = StackUpdateRecoveryService.getInstance();
|
||||
|
||||
const result = await svc.releaseGeneration(row.id, 'tester');
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.reason).toBe('not_eligible');
|
||||
|
||||
const after = DatabaseService.getInstance().getStackUpdateRecoveryGeneration(row.id)!;
|
||||
expect(after.released_at).toBeNull();
|
||||
expect(after.is_current).toBe(1);
|
||||
});
|
||||
|
||||
it('allows release once the linked health gate has passed', async () => {
|
||||
const gate = insertHealthGate({ status: 'passed' });
|
||||
const row = insertRow({ status: 'active', is_current: 1, health_gate_id: gate.id, stack_name: gate.stack_name });
|
||||
const svc = StackUpdateRecoveryService.getInstance();
|
||||
|
||||
const result = await svc.releaseGeneration(row.id, 'tester');
|
||||
expect(result.ok).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects release when the row has already moved to recovery_required (race)', async () => {
|
||||
const row = insertRow({ status: 'recovery_required', is_current: 1 });
|
||||
const svc = StackUpdateRecoveryService.getInstance();
|
||||
|
||||
const result = await svc.releaseGeneration(row.id, 'tester');
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.reason).toBe('not_eligible');
|
||||
|
||||
const after = DatabaseService.getInstance().getStackUpdateRecoveryGeneration(row.id)!;
|
||||
expect(after.artifacts_retired).toBe(0);
|
||||
});
|
||||
|
||||
it('rejects a second release of an already-released generation', async () => {
|
||||
const row = insertRow({ status: 'active', is_current: 1 });
|
||||
const svc = StackUpdateRecoveryService.getInstance();
|
||||
|
||||
const first = await svc.releaseGeneration(row.id, 'tester');
|
||||
expect(first.ok).toBe(true);
|
||||
const second = await svc.releaseGeneration(row.id, 'tester');
|
||||
expect(second.ok).toBe(false);
|
||||
if (!second.ok) expect(second.reason).toBe('already_released');
|
||||
});
|
||||
|
||||
it('leaves artifacts_retired at 0 (retryable) when Docker tag removal fails', async () => {
|
||||
mockRemove.mockRejectedValueOnce(Object.assign(new Error('docker busy'), { statusCode: 500 }));
|
||||
const row = insertRow({ status: 'active', is_current: 1 });
|
||||
const svc = StackUpdateRecoveryService.getInstance();
|
||||
|
||||
const result = await svc.releaseGeneration(row.id, 'tester');
|
||||
expect(result.ok).toBe(true);
|
||||
if (result.ok) expect(result.artifactsCleaned).toBe(false);
|
||||
|
||||
const after = DatabaseService.getInstance().getStackUpdateRecoveryGeneration(row.id)!;
|
||||
expect(after.released_at).not.toBeNull();
|
||||
expect(after.artifacts_retired).toBe(0);
|
||||
|
||||
// The reconcile sweep retries a released-but-uncleaned row immediately.
|
||||
mockRemove.mockResolvedValue(undefined);
|
||||
svc.start();
|
||||
await svc.reconcileIncomplete();
|
||||
svc.stop();
|
||||
const retried = DatabaseService.getInstance().getStackUpdateRecoveryGeneration(row.id)!;
|
||||
expect(retried.artifacts_retired).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET/POST /api/system/rollback/generations', () => {
|
||||
it('lists generations for the requesting node with a releasable flag', async () => {
|
||||
const row = insertRow({ status: 'active', is_current: 1 });
|
||||
const res = await request(app)
|
||||
.get('/api/system/rollback/generations')
|
||||
.set('Authorization', authHeader);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const found = res.body.find((g: { id: string }) => g.id === row.id);
|
||||
expect(found).toBeDefined();
|
||||
expect(found.stackName).toBe(row.stack_name);
|
||||
expect(found.isCurrent).toBe(true);
|
||||
expect(found.releasable).toBe(true);
|
||||
});
|
||||
|
||||
it('releases a generation and returns success', async () => {
|
||||
const row = insertRow({ status: 'superseded', is_current: 0 });
|
||||
const res = await request(app)
|
||||
.post(`/api/system/rollback/generations/${row.id}/release`)
|
||||
.set('Authorization', authHeader);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.success).toBe(true);
|
||||
});
|
||||
|
||||
it('404s releasing a generation that belongs to a different node', async () => {
|
||||
const row = insertRow({ status: 'superseded', is_current: 0, node_id: 999 });
|
||||
const res = await request(app)
|
||||
.post(`/api/system/rollback/generations/${row.id}/release`)
|
||||
.set('Authorization', authHeader);
|
||||
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
|
||||
it('409s releasing an ineligible generation', async () => {
|
||||
const row = insertRow({ status: 'recovery_required', is_current: 1 });
|
||||
const res = await request(app)
|
||||
.post(`/api/system/rollback/generations/${row.id}/release`)
|
||||
.set('Authorization', authHeader);
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.code).toBe('NOT_ELIGIBLE');
|
||||
});
|
||||
});
|
||||
|
||||
describe('RBAC on the rollback-generation routes', () => {
|
||||
it('refuses a viewer POST to the release endpoint and leaves the generation intact', async () => {
|
||||
const row = insertRow({ status: 'superseded', is_current: 0 });
|
||||
const res = await request(app)
|
||||
.post(`/api/system/rollback/generations/${row.id}/release`)
|
||||
.set('Cookie', viewerCookie);
|
||||
|
||||
expect(res.status).toBe(403);
|
||||
const after = DatabaseService.getInstance().getStackUpdateRecoveryGeneration(row.id)!;
|
||||
expect(after.released_at).toBeNull();
|
||||
expect(after.artifacts_retired).toBe(0);
|
||||
expect(mockRemove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses a deployer POST to the release endpoint (release is Admin-only)', async () => {
|
||||
const row = insertRow({ status: 'superseded', is_current: 0 });
|
||||
const res = await request(app)
|
||||
.post(`/api/system/rollback/generations/${row.id}/release`)
|
||||
.set('Cookie', deployerCookie);
|
||||
|
||||
expect(res.status).toBe(403);
|
||||
const after = DatabaseService.getInstance().getStackUpdateRecoveryGeneration(row.id)!;
|
||||
expect(after.released_at).toBeNull();
|
||||
expect(mockRemove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('allows a viewer to read the generations list (stack:read, matching sibling Resources routes)', async () => {
|
||||
const row = insertRow({ status: 'superseded', is_current: 0 });
|
||||
const res = await request(app)
|
||||
.get('/api/system/rollback/generations')
|
||||
.set('Cookie', viewerCookie);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.some((g: { id: string }) => g.id === row.id)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -241,6 +241,88 @@ describe('prune_on_update (auto-prune after updates)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('recovery_retention_days (superseded rollback generation retention)', () => {
|
||||
it('defaults to 7 days in a freshly seeded database', () => {
|
||||
expect(DatabaseService.getInstance().getGlobalSettings().recovery_retention_days).toBe('7');
|
||||
});
|
||||
|
||||
it('is exposed through the settings GET projection', async () => {
|
||||
const res = await request(app).get('/api/settings').set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.recovery_retention_days).toBeDefined();
|
||||
});
|
||||
|
||||
it('accepts a well-formed write and persists it', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/settings')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ key: 'recovery_retention_days', value: '14' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(DatabaseService.getInstance().getGlobalSettings().recovery_retention_days).toBe('14');
|
||||
DatabaseService.getInstance().updateGlobalSetting('recovery_retention_days', '7');
|
||||
});
|
||||
|
||||
it('rejects an out-of-range value (400) and does not write it', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/settings')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ key: 'recovery_retention_days', value: '91' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toBe('Validation failed');
|
||||
expect(DatabaseService.getInstance().getGlobalSettings().recovery_retention_days).not.toBe('91');
|
||||
});
|
||||
|
||||
it('rejects a non-numeric value (400) and does not write it', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/settings')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ key: 'recovery_retention_days', value: 'banana' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toBe('Validation failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('recovery_max_generations (cap on retained rollback generations per stack)', () => {
|
||||
it('defaults to 0 (unlimited) in a freshly seeded database', () => {
|
||||
expect(DatabaseService.getInstance().getGlobalSettings().recovery_max_generations).toBe('0');
|
||||
});
|
||||
|
||||
it('is exposed through the settings GET projection', async () => {
|
||||
const res = await request(app).get('/api/settings').set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.recovery_max_generations).toBeDefined();
|
||||
});
|
||||
|
||||
it('accepts a well-formed write and persists it', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/settings')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ key: 'recovery_max_generations', value: '3' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(DatabaseService.getInstance().getGlobalSettings().recovery_max_generations).toBe('3');
|
||||
DatabaseService.getInstance().updateGlobalSetting('recovery_max_generations', '0');
|
||||
});
|
||||
|
||||
it('rejects a negative value (400) and does not write it', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/settings')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ key: 'recovery_max_generations', value: '-1' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toBe('Validation failed');
|
||||
expect(DatabaseService.getInstance().getGlobalSettings().recovery_max_generations).not.toBe('-1');
|
||||
});
|
||||
|
||||
it('rejects an out-of-range value (400) and does not write it', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/settings')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ key: 'recovery_max_generations', value: '51' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toBe('Validation failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('session_sliding_refresh (keep active sessions alive)', () => {
|
||||
it('defaults to ON in a freshly seeded database', () => {
|
||||
expect(DatabaseService.getInstance().getGlobalSettings().session_sliding_refresh).toBe('1');
|
||||
|
||||
@@ -152,6 +152,8 @@ describe('StackUpdateRecoveryService', () => {
|
||||
updated_at: Date.now(),
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
released_at: null,
|
||||
released_by: null,
|
||||
};
|
||||
|
||||
vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row);
|
||||
@@ -193,6 +195,8 @@ describe('StackUpdateRecoveryService', () => {
|
||||
updated_at: Date.now(),
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
released_at: null,
|
||||
released_by: null,
|
||||
};
|
||||
vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row);
|
||||
const update = vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration')
|
||||
@@ -354,6 +358,8 @@ describe('StackUpdateRecoveryService', () => {
|
||||
updated_at: Date.now(),
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
released_at: null,
|
||||
released_by: null,
|
||||
};
|
||||
mockRemove.mockRejectedValueOnce(Object.assign(new Error('docker busy'), { statusCode: 500 }));
|
||||
const markRetired = vi.spyOn(DatabaseService.prototype, 'markStackUpdateRecoveryArtifactsRetired')
|
||||
|
||||
@@ -13,6 +13,8 @@ let app: import('express').Express;
|
||||
let authHeader: string;
|
||||
let SelfIdentityService: typeof import('../services/SelfIdentityService').default;
|
||||
let DockerController: typeof import('../services/DockerController').default;
|
||||
let ServiceUpdateRecoveryService: typeof import('../services/ServiceUpdateRecoveryService').ServiceUpdateRecoveryService;
|
||||
let StackUpdateRecoveryService: typeof import('../services/StackUpdateRecoveryService').StackUpdateRecoveryService;
|
||||
|
||||
const SELF_IMAGE = 'a'.repeat(64);
|
||||
const SELF_NETWORK = 'b'.repeat(64);
|
||||
@@ -26,6 +28,8 @@ beforeAll(async () => {
|
||||
({ app } = await import('../index'));
|
||||
({ default: SelfIdentityService } = await import('../services/SelfIdentityService'));
|
||||
({ default: DockerController } = await import('../services/DockerController'));
|
||||
({ ServiceUpdateRecoveryService } = await import('../services/ServiceUpdateRecoveryService'));
|
||||
({ StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'));
|
||||
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
|
||||
authHeader = `Bearer ${token}`;
|
||||
});
|
||||
@@ -48,6 +52,10 @@ function stubSelfIdentity(opts: { imageId?: string; networkId?: string; containe
|
||||
function stubDockerControllerNoops() {
|
||||
const fake = {
|
||||
removeImage: vi.fn().mockResolvedValue(undefined),
|
||||
// Identity resolver by default: canonicalId === the submitted id, so
|
||||
// existing removeImage(id) assertions keep working. Tests that need
|
||||
// short-id canonicalization override this per-test.
|
||||
resolveImageId: vi.fn().mockImplementation(async (id: string) => id),
|
||||
removeNetwork: vi.fn().mockResolvedValue(undefined),
|
||||
removeVolume: vi.fn().mockResolvedValue(undefined),
|
||||
removeContainers: vi.fn().mockResolvedValue([]),
|
||||
@@ -209,3 +217,71 @@ describe('Self-protection in dev mode (SelfIdentityService empty)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('Held-image protection on /api/system/images/delete', () => {
|
||||
it('refuses to delete a rollback-held image with 409 IMAGE_HELD_FOR_ROLLBACK', async () => {
|
||||
stubSelfIdentity({});
|
||||
const docker = stubDockerControllerNoops();
|
||||
const heldId = 'sha256:' + OTHER_IMAGE;
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set([heldId]));
|
||||
vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set());
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/system/images/delete')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ id: heldId });
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.code).toBe('IMAGE_HELD_FOR_ROLLBACK');
|
||||
expect(docker.removeImage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('still deletes an unrelated image when a held predicate is active', async () => {
|
||||
stubSelfIdentity({});
|
||||
const docker = stubDockerControllerNoops();
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds')
|
||||
.mockReturnValue(new Set(['sha256:' + 'z'.repeat(64)]));
|
||||
vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set());
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/system/images/delete')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ id: OTHER_IMAGE });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(docker.removeImage).toHaveBeenCalledWith(OTHER_IMAGE);
|
||||
});
|
||||
|
||||
it('canonicalizes a short/truncated id before checking the held predicate, closing the bypass', async () => {
|
||||
stubSelfIdentity({});
|
||||
const docker = stubDockerControllerNoops();
|
||||
const shortId = OTHER_IMAGE.slice(0, 12);
|
||||
const canonicalId = 'sha256:' + OTHER_IMAGE;
|
||||
docker.resolveImageId.mockImplementation(async (id: string) => (id === shortId ? canonicalId : id));
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set([canonicalId]));
|
||||
vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set());
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/system/images/delete')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ id: shortId });
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.code).toBe('IMAGE_HELD_FOR_ROLLBACK');
|
||||
expect(docker.removeImage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns 404 when the image no longer exists', async () => {
|
||||
stubSelfIdentity({});
|
||||
const docker = stubDockerControllerNoops();
|
||||
docker.resolveImageId.mockResolvedValue(null);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/system/images/delete')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ id: OTHER_IMAGE });
|
||||
|
||||
expect(res.status).toBe(404);
|
||||
expect(docker.removeImage).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -27,6 +27,8 @@ export const SETTING_WRITE_PERMISSIONS: Record<string, PermissionAction> = {
|
||||
env_block_deploy_on_missing_required: 'node:manage',
|
||||
auto_create_missing_external_networks: 'node:manage',
|
||||
notification_dispatch_retries: 'node:manage',
|
||||
recovery_retention_days: 'node:manage',
|
||||
recovery_max_generations: 'node:manage',
|
||||
developer_mode: 'system:settings',
|
||||
metrics_retention_hours: 'system:settings',
|
||||
log_retention_days: 'system:settings',
|
||||
@@ -114,6 +116,8 @@ const SettingsPatchSchema = z.object({
|
||||
env_block_deploy_on_missing_required: z.enum(['0', '1']),
|
||||
auto_create_missing_external_networks: z.enum(['0', '1']),
|
||||
image_update_sidebar_indicators: z.enum(['0', '1']),
|
||||
recovery_retention_days: z.coerce.number().int().min(1).max(90).transform(String),
|
||||
recovery_max_generations: z.coerce.number().int().min(0).max(50).transform(String),
|
||||
// Strict: do not use bare z.coerce.number() (null/false/'' become 0; true becomes 1).
|
||||
notification_dispatch_retries: z.unknown().superRefine((v, ctx) => {
|
||||
if (parseNotificationDispatchRetries(v) === null) {
|
||||
|
||||
@@ -9,6 +9,9 @@ import DockerController, {
|
||||
import { isPruneTarget } from '../services/prunePlan';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
import { StackUpdateRecoveryService, shortGenerationId } from '../services/StackUpdateRecoveryService';
|
||||
import { buildUnifiedHeldImagePredicate } from '../services/recoveryHeldImages';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import SelfIdentityService from '../services/SelfIdentityService';
|
||||
import { requireAdmin } from '../middleware/tierGates';
|
||||
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
|
||||
@@ -262,7 +265,7 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
|
||||
});
|
||||
}
|
||||
const target = targets[0];
|
||||
console.log(`[Resources] System prune: ${target} (scope: ${pruneScope})`);
|
||||
console.log(`[Resources] System prune: ${sanitizeForLog(target)} (scope: ${pruneScope})`);
|
||||
const pruneStartedAt = Date.now();
|
||||
let result: { success: boolean; reclaimedBytes: number };
|
||||
if (pruneScope === 'managed' && target !== 'containers') {
|
||||
@@ -280,7 +283,7 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
|
||||
result = await dockerController.pruneSystem(target, undefined, isImageHeld);
|
||||
}
|
||||
|
||||
console.log(`[Resources] System prune completed: ${target}, reclaimed ${result.reclaimedBytes} bytes`);
|
||||
console.log(`[Resources] System prune completed: ${sanitizeForLog(target)}, reclaimed ${result.reclaimedBytes} bytes`);
|
||||
if (isDebugEnabled()) {
|
||||
console.debug('[Resources:debug] System prune', {
|
||||
target, scope: pruneScope, ms: Date.now() - pruneStartedAt, reclaimedBytes: result.reclaimedBytes,
|
||||
@@ -459,9 +462,23 @@ systemMaintenanceRouter.post('/images/delete', async (req: Request, res: Respons
|
||||
return res.status(400).json({ error: 'Invalid image ID format' });
|
||||
}
|
||||
if (rejectIfSelf('image', id, res)) return;
|
||||
console.log(`[Resources] Delete image: ${hexId.substring(0, 12)}`);
|
||||
const dockerController = DockerController.getInstance(req.nodeId);
|
||||
await dockerController.removeImage(id);
|
||||
// Resolve to the canonical full image ID before the held-image check: the
|
||||
// submitted id can be a short/truncated form (isValidDockerResourceId
|
||||
// accepts 12-64 hex chars), which a full-64-char held-set lookup would miss.
|
||||
const canonicalId = await dockerController.resolveImageId(id);
|
||||
if (!canonicalId) {
|
||||
return res.status(404).json({ error: 'Image not found' });
|
||||
}
|
||||
const isImageHeld = buildUnifiedHeldImagePredicate(req.nodeId);
|
||||
if (isImageHeld(canonicalId)) {
|
||||
return res.status(409).json({
|
||||
error: 'Image is held for a pending update rollback and cannot be deleted manually. It is removed automatically once the rollback window expires, or can be released from Resources → Rollback.',
|
||||
code: 'IMAGE_HELD_FOR_ROLLBACK',
|
||||
});
|
||||
}
|
||||
console.log(`[Resources] Delete image: ${hexId.substring(0, 12)}`);
|
||||
await dockerController.removeImage(canonicalId);
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
res.json({ success: true, message: 'Image deleted' });
|
||||
} catch (error: unknown) {
|
||||
@@ -470,6 +487,79 @@ systemMaintenanceRouter.post('/images/delete', async (req: Request, res: Respons
|
||||
}
|
||||
});
|
||||
|
||||
// Full-stack rollback generations (the sencho-rb/<id>/<service>:hold images).
|
||||
// Global read under stack:read, matching the rest of the Docker resource
|
||||
// inventory on this page (/system/resources, /system/images); release is
|
||||
// requireAdmin, matching every other host-destructive Docker action here.
|
||||
systemMaintenanceRouter.get('/rollback/generations', async (req: Request, res: Response) => {
|
||||
if (!requirePermission(req, res, 'stack:read')) return;
|
||||
try {
|
||||
const service = StackUpdateRecoveryService.getInstance();
|
||||
const rows = DatabaseService.getInstance()
|
||||
.listStackUpdateRecoveryGenerationsForNode(req.nodeId)
|
||||
.filter((row) => row.artifacts_retired === 0
|
||||
&& (row.status === 'active' || row.status === 'restored_current'
|
||||
|| row.status === 'superseded' || row.status === 'recovery_required'));
|
||||
res.json(rows.map((row) => ({
|
||||
id: row.id,
|
||||
shortId: shortGenerationId(row.id),
|
||||
stackName: row.stack_name,
|
||||
status: row.status,
|
||||
isCurrent: row.is_current === 1,
|
||||
phase: row.phase,
|
||||
createdAt: row.created_at,
|
||||
artifactExpiresAt: row.artifact_expires_at,
|
||||
releasable: service.isReleaseEligible(row),
|
||||
})));
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch rollback generations:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch rollback generations' });
|
||||
}
|
||||
});
|
||||
|
||||
systemMaintenanceRouter.post('/rollback/generations/:id/release', async (req: Request, res: Response) => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
try {
|
||||
const id = req.params.id as string;
|
||||
const service = StackUpdateRecoveryService.getInstance();
|
||||
const row = service.get(id);
|
||||
if (!row || row.node_id !== req.nodeId) {
|
||||
return res.status(404).json({ error: 'Rollback generation not found' });
|
||||
}
|
||||
const result = await service.releaseGeneration(id, req.user?.username ?? null);
|
||||
if (!result.ok) {
|
||||
switch (result.reason) {
|
||||
case 'not_found':
|
||||
return res.status(404).json({ error: 'Rollback generation not found' });
|
||||
case 'already_released':
|
||||
return res.status(409).json({
|
||||
error: 'Rollback protection was already released for this generation.',
|
||||
code: 'ALREADY_RELEASED',
|
||||
});
|
||||
case 'not_eligible':
|
||||
return res.status(409).json({
|
||||
error: 'This rollback generation cannot be released right now (it may be observing a health gate, mid-recovery, or already in progress).',
|
||||
code: 'NOT_ELIGIBLE',
|
||||
});
|
||||
default: {
|
||||
const _exhaustive: never = result.reason;
|
||||
throw new Error(`Unhandled release reason: ${_exhaustive}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`[Resources] Released rollback generation ${sanitizeForLog(shortGenerationId(id))} for ${sanitizeForLog(result.row.stack_name)}`);
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
res.json({
|
||||
success: true,
|
||||
message: result.artifactsCleaned ? 'Rollback protection released' : 'Rollback protection released; cleanup will finish shortly',
|
||||
artifactsCleaned: result.artifactsCleaned,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to release rollback generation:', error);
|
||||
res.status(500).json({ error: 'Failed to release rollback generation' });
|
||||
}
|
||||
});
|
||||
|
||||
systemMaintenanceRouter.post('/volumes/delete', async (req: Request, res: Response) => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
try {
|
||||
|
||||
@@ -30,6 +30,7 @@ import {
|
||||
MissingExternalNetworksError,
|
||||
type DeployInvocationContext,
|
||||
} from './network/missingExternalNetworksError';
|
||||
import { buildUnifiedHeldImagePredicate } from './recoveryHeldImages';
|
||||
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
|
||||
import type { NotificationCategory } from './NotificationService';
|
||||
|
||||
@@ -1019,7 +1020,7 @@ export class ComposeService {
|
||||
try {
|
||||
const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1';
|
||||
if (pruneOnUpdate) {
|
||||
const isImageHeld = recoverySvc.buildUnifiedHeldImagePredicate(this.nodeId);
|
||||
const isImageHeld = buildUnifiedHeldImagePredicate(this.nodeId);
|
||||
const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld);
|
||||
const reclaimed = result.reclaimedBytes > 0
|
||||
? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB`
|
||||
|
||||
@@ -71,6 +71,8 @@ export interface StackUpdateDetail {
|
||||
}
|
||||
|
||||
const SERVICES_JSON_VERSION = 1;
|
||||
const DEFAULT_RECOVERY_RETENTION_DAYS = 7;
|
||||
const DEFAULT_RECOVERY_MAX_GENERATIONS = 0;
|
||||
|
||||
function isStackServiceStatus(value: unknown): value is StackServiceStatus {
|
||||
if (!value || typeof value !== 'object') return false;
|
||||
@@ -253,6 +255,9 @@ export interface StackUpdateRecoveryGenerationRow {
|
||||
updated_at: number;
|
||||
created_by: string | null;
|
||||
artifacts_retired: number;
|
||||
/** Set when an operator manually released rollback protection early (see releaseStackUpdateRecoveryGeneration). */
|
||||
released_at: number | null;
|
||||
released_by: string | null;
|
||||
}
|
||||
|
||||
/** Durable cleanup tombstone for stack/node deletion artifact sweep. */
|
||||
@@ -1898,6 +1903,12 @@ export class DatabaseService {
|
||||
`);
|
||||
|
||||
maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0');
|
||||
// Manual release (operator gave up rollback protection early). Additive
|
||||
// columns rather than a new `status` enum value, since `status` carries a
|
||||
// CHECK constraint that would need the heavier table-rebuild migration
|
||||
// pattern used for health_gate_runs below.
|
||||
maybeAddCol('stack_update_recovery_generations', 'released_at', 'INTEGER');
|
||||
maybeAddCol('stack_update_recovery_generations', 'released_by', 'TEXT');
|
||||
maybeAddCol('stack_update_cleanup_pending', 'required_blueprint_id', 'INTEGER');
|
||||
|
||||
// Distributed API model columns
|
||||
@@ -2040,6 +2051,11 @@ export class DatabaseService {
|
||||
stmt.run('reclaim_hero', '0');
|
||||
stmt.run('health_gate_enabled', '1');
|
||||
stmt.run('health_gate_window_seconds', '90');
|
||||
// Superseded-generation retention (days) and a per-stack cap on total
|
||||
// retained generations (0 = unlimited). Never applies to the current
|
||||
// generation, which stays protected until superseded or released.
|
||||
stmt.run('recovery_retention_days', '7');
|
||||
stmt.run('recovery_max_generations', '0');
|
||||
stmt.run('image_update_check_interval_minutes', '120');
|
||||
stmt.run('image_update_check_mode', 'interval');
|
||||
stmt.run('image_update_check_cron', '');
|
||||
@@ -4213,16 +4229,39 @@ export class DatabaseService {
|
||||
return result.changes === 1;
|
||||
}
|
||||
|
||||
/** Days a superseded generation's Docker/FS artifacts are retained before automatic cleanup. Never applies to the current generation. */
|
||||
public getRecoveryRetentionDays(): number {
|
||||
try {
|
||||
const raw = parseInt(this.getGlobalSettings()['recovery_retention_days'] ?? '', 10);
|
||||
return Number.isFinite(raw) && raw > 0 ? Math.min(raw, 90) : DEFAULT_RECOVERY_RETENTION_DAYS;
|
||||
} catch (e) {
|
||||
console.warn('[DatabaseService] recovery_retention_days read failed; using default:', (e as Error).message);
|
||||
return DEFAULT_RECOVERY_RETENTION_DAYS;
|
||||
}
|
||||
}
|
||||
|
||||
/** Total generations retained per stack, current included (0 = unlimited). */
|
||||
public getRecoveryMaxGenerations(): number {
|
||||
try {
|
||||
const raw = parseInt(this.getGlobalSettings()['recovery_max_generations'] ?? '', 10);
|
||||
return Number.isFinite(raw) && raw >= 0 ? Math.min(raw, 50) : DEFAULT_RECOVERY_MAX_GENERATIONS;
|
||||
} catch (e) {
|
||||
console.warn('[DatabaseService] recovery_max_generations read failed; using default:', (e as Error).message);
|
||||
return DEFAULT_RECOVERY_MAX_GENERATIONS;
|
||||
}
|
||||
}
|
||||
|
||||
public casHandoffGeneration(candidateId: string, nodeId: number, stackName: string): boolean {
|
||||
const handoff = this.db.transaction(() => {
|
||||
const candidate = this.getStackUpdateRecoveryGeneration(candidateId);
|
||||
if (!candidate || candidate.node_id !== nodeId || candidate.stack_name !== stackName) return false;
|
||||
if (candidate.status !== 'candidate' || candidate.phase !== 'acquired') return false;
|
||||
const retentionMs = this.getRecoveryRetentionDays() * 24 * 60 * 60 * 1000;
|
||||
this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations
|
||||
SET status = 'superseded', is_current = 0, artifact_expires_at = ?, updated_at = ?
|
||||
WHERE node_id = ? AND stack_name = ? AND is_current = 1 AND id != ?`
|
||||
).run(Date.now() + 7 * 24 * 60 * 60 * 1000, Date.now(), nodeId, stackName, candidateId);
|
||||
).run(Date.now() + retentionMs, Date.now(), nodeId, stackName, candidateId);
|
||||
const result = this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations
|
||||
SET status = 'active', is_current = 1, phase = 'handoff_committed', updated_at = ?
|
||||
@@ -4234,18 +4273,41 @@ export class DatabaseService {
|
||||
}
|
||||
|
||||
|
||||
/** Generations whose Docker/FS artifacts can be retired (not actively held). */
|
||||
/**
|
||||
* Generations whose Docker/FS artifacts can be retired (not actively held).
|
||||
* A manually released row (released_at set) is swept immediately regardless
|
||||
* of its expiry timers; a naturally abandoned/superseded row still waits out
|
||||
* artifact_expires_at / gate_retain_until.
|
||||
*/
|
||||
public listStackUpdateRecoveryGenerationsForArtifactRetirement(now: number): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
`SELECT * FROM stack_update_recovery_generations
|
||||
WHERE artifacts_retired = 0
|
||||
AND is_current = 0
|
||||
AND status IN ('abandoned', 'superseded')
|
||||
AND (artifact_expires_at IS NULL OR artifact_expires_at <= ?)
|
||||
AND (gate_retain_until IS NULL OR gate_retain_until <= ?)`
|
||||
AND (
|
||||
released_at IS NOT NULL
|
||||
OR (
|
||||
status IN ('abandoned', 'superseded')
|
||||
AND (artifact_expires_at IS NULL OR artifact_expires_at <= ?)
|
||||
AND (gate_retain_until IS NULL OR gate_retain_until <= ?)
|
||||
)
|
||||
)`
|
||||
).all(now, now) as StackUpdateRecoveryGenerationRow[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Superseded, not-yet-retired, not-released generations across every node
|
||||
* for cap enforcement (mirrors the other reconcile-sweep list methods,
|
||||
* which are also unscoped by node), newest first per (node_id, stack_name).
|
||||
*/
|
||||
public listActiveSupersededGenerations(): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
`SELECT * FROM stack_update_recovery_generations
|
||||
WHERE status = 'superseded' AND artifacts_retired = 0 AND released_at IS NULL
|
||||
ORDER BY node_id, stack_name, created_at DESC, id DESC`
|
||||
).all() as StackUpdateRecoveryGenerationRow[];
|
||||
}
|
||||
|
||||
public markStackUpdateRecoveryArtifactsRetired(id: string): boolean {
|
||||
const result = this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations
|
||||
@@ -4266,6 +4328,33 @@ export class DatabaseService {
|
||||
return result.changes === 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Operator-initiated release of rollback protection. A single conditional
|
||||
* UPDATE both revalidates eligibility and performs the transition
|
||||
* atomically, so a stale caller can never release a row that has since
|
||||
* become ineligible (e.g. it started a health gate observation, or moved
|
||||
* to recovery_required). Only clears is_current/timestamps; Docker tag and
|
||||
* override-file cleanup is the caller's job via retireGenerationArtifacts,
|
||||
* matching how abandon() already separates the DB transition from cleanup.
|
||||
*/
|
||||
public releaseStackUpdateRecoveryGeneration(id: string, releasedBy: string | null): boolean {
|
||||
const now = Date.now();
|
||||
const result = this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations
|
||||
SET released_at = ?, released_by = ?, is_current = 0, updated_at = ?
|
||||
WHERE id = ?
|
||||
AND released_at IS NULL
|
||||
AND artifacts_retired = 0
|
||||
AND phase = 'immediate_verified'
|
||||
AND status IN ('active', 'restored_current', 'superseded')
|
||||
AND (health_gate_id IS NULL OR NOT EXISTS (
|
||||
SELECT 1 FROM health_gate_runs g
|
||||
WHERE g.id = stack_update_recovery_generations.health_gate_id AND g.status = 'observing'
|
||||
))`
|
||||
).run(now, releasedBy, now, id);
|
||||
return result.changes === 1;
|
||||
}
|
||||
|
||||
/** Pre-handoff candidates whose operation lease has expired. */
|
||||
public listStaleStackUpdateRecoveryCandidates(now: number): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
@@ -4303,6 +4392,7 @@ export class DatabaseService {
|
||||
const rows = this.db.prepare(
|
||||
`SELECT services_json FROM stack_update_recovery_generations
|
||||
WHERE node_id = ?
|
||||
AND released_at IS NULL
|
||||
AND status IN ('candidate','active','restored_current','recovery_required')
|
||||
AND (artifact_expires_at IS NULL OR artifact_expires_at > ? OR gate_retain_until > ? OR is_current = 1)`
|
||||
).all(nodeId, now, now) as Array<{ services_json: string }>;
|
||||
@@ -4438,7 +4528,7 @@ export class DatabaseService {
|
||||
const categories = [
|
||||
'deploy_success', 'deploy_failure', 'stack_started', 'stack_stopped', 'stack_restarted',
|
||||
'image_update_applied', 'update_started', 'health_gate_passed', 'health_gate_failed',
|
||||
'network_auto_created',
|
||||
'network_auto_created', 'rollback_generation_released',
|
||||
];
|
||||
const placeholders = categories.map(() => '?').join(', ');
|
||||
const sql = `
|
||||
|
||||
@@ -143,6 +143,9 @@ export interface ClassifiedImage {
|
||||
managedBy: string | null;
|
||||
managedStatus: 'managed' | 'unmanaged' | 'unused';
|
||||
isSencho: boolean;
|
||||
/** True when a StackUpdateRecoveryService/ServiceUpdateRecoveryService hold protects this image from pruning. Additive: does not change managedStatus semantics. */
|
||||
rollbackProtected: boolean;
|
||||
rollbackProtectionKind?: 'stack' | 'service';
|
||||
}
|
||||
|
||||
export interface PortInUseInfo {
|
||||
@@ -561,23 +564,52 @@ class DockerController {
|
||||
|
||||
const selfIdentity = SelfIdentityService.getInstance();
|
||||
|
||||
const images: ClassifiedImage[] = this.validateApiData<any[]>(rawImages).map((img: any) => {
|
||||
const usedByStacks = [...(imageToStacks.get(img.Id) ?? [])].sort((a, b) => a.localeCompare(b));
|
||||
const managedBy = usedByStacks[0] ?? null;
|
||||
const managedStatus: ClassifiedImage['managedStatus'] =
|
||||
img.Containers === 0 ? 'unused' :
|
||||
managedBy ? 'managed' : 'unmanaged';
|
||||
return {
|
||||
Id: img.Id,
|
||||
RepoTags: img.RepoTags ?? [],
|
||||
Size: img.Size ?? 0,
|
||||
Containers: img.Containers ?? 0,
|
||||
usedByStacks,
|
||||
managedBy,
|
||||
managedStatus,
|
||||
isSencho: selfIdentity.isOwnImage(img.Id),
|
||||
};
|
||||
});
|
||||
// Dynamic (async) imports avoid a static cycle: StackUpdateRecoveryService
|
||||
// imports DockerController directly, and ServiceUpdateRecoveryService
|
||||
// reaches it transitively through ComposeService. It must be `await import`
|
||||
// rather than require(), which does not resolve under Vitest's loader.
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
const { ServiceUpdateRecoveryService } = await import('./ServiceUpdateRecoveryService');
|
||||
const stackHeld = StackUpdateRecoveryService.getInstance().getHeldImageIds(this.nodeId);
|
||||
const serviceHeld = ServiceUpdateRecoveryService.getInstance().getHeldImageIds(this.nodeId);
|
||||
// A null lookup means "held state unknown" (the DB read failed); treat it
|
||||
// as held so the badge never disagrees with the delete guard, which fails
|
||||
// the same way (recoveryHeldImages.ts's buildUnifiedHeldImagePredicate).
|
||||
const rollbackKind = (imageId: string): ClassifiedImage['rollbackProtectionKind'] => {
|
||||
if (stackHeld === null || stackHeld.has(imageId)) return 'stack';
|
||||
if (serviceHeld === null || serviceHeld.has(imageId)) return 'service';
|
||||
return undefined;
|
||||
};
|
||||
|
||||
// Only hide an image from the generic inventory when every visible tag is
|
||||
// a synthetic sencho-rb hold tag; an image that also carries a normal
|
||||
// registry tag stays visible here (with the badge below) so the generic
|
||||
// inventory stays complete. Its generation still surfaces in the Rollback tab.
|
||||
const isFullySyntheticHoldImage = (repoTags: string[]): boolean =>
|
||||
repoTags.length > 0 && repoTags.every((tag) => tag.startsWith('sencho-rb/'));
|
||||
|
||||
const images: ClassifiedImage[] = this.validateApiData<any[]>(rawImages)
|
||||
.map((img: any) => {
|
||||
const usedByStacks = [...(imageToStacks.get(img.Id) ?? [])].sort((a, b) => a.localeCompare(b));
|
||||
const managedBy = usedByStacks[0] ?? null;
|
||||
const managedStatus: ClassifiedImage['managedStatus'] =
|
||||
img.Containers === 0 ? 'unused' :
|
||||
managedBy ? 'managed' : 'unmanaged';
|
||||
const rollbackProtectionKind = rollbackKind(img.Id);
|
||||
return {
|
||||
Id: img.Id,
|
||||
RepoTags: img.RepoTags ?? [],
|
||||
Size: img.Size ?? 0,
|
||||
Containers: img.Containers ?? 0,
|
||||
usedByStacks,
|
||||
managedBy,
|
||||
managedStatus,
|
||||
isSencho: selfIdentity.isOwnImage(img.Id),
|
||||
rollbackProtected: rollbackProtectionKind !== undefined,
|
||||
rollbackProtectionKind,
|
||||
};
|
||||
})
|
||||
.filter((img) => !isFullySyntheticHoldImage(img.RepoTags));
|
||||
|
||||
const volumes: ClassifiedVolume[] = rawVolumes.map((vol: any) => {
|
||||
const stack = DockerController.resolveProjectLabel(vol.Labels?.['com.docker.compose.project'], knownSet, projectToStack);
|
||||
@@ -1504,6 +1536,23 @@ class DockerController {
|
||||
return { inspect, history };
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve any valid Docker image reference (full ID, short ID, digest, or
|
||||
* tag) to its canonical full sha256 ID. isValidDockerResourceId accepts
|
||||
* short IDs down to 12 hex chars, which a held-image-id set lookup (always
|
||||
* keyed on the full 64-char form) would miss without this resolve step.
|
||||
* Returns null when the image does not exist.
|
||||
*/
|
||||
public async resolveImageId(id: string): Promise<string | null> {
|
||||
try {
|
||||
const info = await this.docker.getImage(id).inspect();
|
||||
return info.Id;
|
||||
} catch (error) {
|
||||
if ((error as { statusCode?: number })?.statusCode === 404) return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public async removeVolume(name: string) {
|
||||
const volume = this.docker.getVolume(name);
|
||||
await volume.remove({ force: true });
|
||||
|
||||
@@ -48,6 +48,9 @@ export type NotificationCategory =
|
||||
| 'update_started'
|
||||
| 'health_gate_passed'
|
||||
| 'health_gate_failed'
|
||||
// Manual rollback-generation release (Resources → Rollback). History-only
|
||||
// for the same reason as the drift pair above.
|
||||
| 'rollback_generation_released'
|
||||
// Automatic external-network creation during deploy. History-only.
|
||||
| 'network_auto_created'
|
||||
| 'node_update_available'
|
||||
@@ -67,7 +70,7 @@ export const ALL_SUPPRESSIBLE_CATEGORIES: readonly NotificationCategory[] = [
|
||||
...ALL_NOTIFICATION_CATEGORIES,
|
||||
'drift_detected', 'drift_resolved',
|
||||
'update_started', 'health_gate_passed', 'health_gate_failed',
|
||||
'network_auto_created',
|
||||
'network_auto_created', 'rollback_generation_released',
|
||||
];
|
||||
|
||||
/** Webhook timeout: 10 seconds per external dispatch call. */
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { DatabaseService, type ServiceUpdateRecoveryRow } from './DatabaseService';
|
||||
import { getComposeCommandTimeoutMs } from './ComposeService';
|
||||
import { buildUnifiedHeldImagePredicate } from './recoveryHeldImages';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
|
||||
const SWEEP_INTERVAL_MS = 5 * 60_000;
|
||||
@@ -229,26 +230,13 @@ export class ServiceUpdateRecoveryService {
|
||||
/**
|
||||
* A predicate a pruner can call immediately before deleting each candidate
|
||||
* image. Re-reads the held set on every call (rather than snapshotting it
|
||||
* once) so a snapshot that becomes eligible between plan and delete is
|
||||
* still honored. When the held set cannot be read, returns true for every
|
||||
* id so prune skips deletes (fail closed).
|
||||
* once, unlike recoveryHeldImages.buildUnifiedHeldImagePredicate) so a
|
||||
* generation that becomes eligible between plan and delete is still
|
||||
* honored. When the held set cannot be read, returns true for every id so
|
||||
* prune skips deletes (fail closed).
|
||||
*/
|
||||
public buildHeldImagePredicate(nodeId: number): (imageId: string) => boolean {
|
||||
return (imageId: string) => {
|
||||
const held = this.getHeldImageIds(nodeId);
|
||||
if (held === null) return true;
|
||||
if (held.has(imageId)) return true;
|
||||
try {
|
||||
// Dynamic import avoids a static cycle with StackUpdateRecoveryService.
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const { StackUpdateRecoveryService } = require('./StackUpdateRecoveryService') as typeof import('./StackUpdateRecoveryService');
|
||||
const stackHeld = StackUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
|
||||
if (stackHeld === null) return true;
|
||||
return stackHeld.has(imageId);
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
return (imageId: string) => buildUnifiedHeldImagePredicate(nodeId)(imageId);
|
||||
}
|
||||
|
||||
private nextClaimExpiry(now: number): number {
|
||||
|
||||
@@ -70,9 +70,13 @@ function sanitizeServiceSlug(name: string): string {
|
||||
return name.replace(/[^a-zA-Z0-9._-]/g, '-').toLowerCase() || 'svc';
|
||||
}
|
||||
|
||||
/** Same short form used in the opaque rollback tag, so the UI's "Generation" label matches the Docker tag. */
|
||||
export function shortGenerationId(generationId: string): string {
|
||||
return generationId.replace(/-/g, '').slice(0, 12);
|
||||
}
|
||||
|
||||
function opaqueRollbackTag(generationId: string, serviceName: string): string {
|
||||
const short = generationId.replace(/-/g, '').slice(0, 12);
|
||||
return `sencho-rb/${short}/${sanitizeServiceSlug(serviceName)}:hold`;
|
||||
return `sencho-rb/${shortGenerationId(generationId)}/${sanitizeServiceSlug(serviceName)}:hold`;
|
||||
}
|
||||
|
||||
function parseServicesJson(raw: string): StackRecoveryServiceCapture[] {
|
||||
@@ -284,6 +288,8 @@ export class StackUpdateRecoveryService {
|
||||
updated_at: now,
|
||||
created_by: createdBy,
|
||||
artifacts_retired: 0,
|
||||
released_at: null,
|
||||
released_by: null,
|
||||
};
|
||||
DatabaseService.getInstance().insertStackUpdateRecoveryGeneration(row);
|
||||
return row;
|
||||
@@ -329,7 +335,7 @@ export class StackUpdateRecoveryService {
|
||||
throw new Error('Stack directory escapes compose base');
|
||||
}
|
||||
|
||||
const short = generationId.replace(/-/g, '').slice(0, 12);
|
||||
const short = shortGenerationId(generationId);
|
||||
if (!/^[a-f0-9]{12}$/i.test(short)) {
|
||||
throw new Error('Invalid recovery generation id');
|
||||
}
|
||||
@@ -398,6 +404,74 @@ export class StackUpdateRecoveryService {
|
||||
return ok;
|
||||
}
|
||||
|
||||
/**
|
||||
* Informational mirror of releaseStackUpdateRecoveryGeneration's WHERE
|
||||
* clause, for the list endpoint to grey out a row it already knows is
|
||||
* ineligible. Not authoritative: releaseGeneration revalidates for real.
|
||||
*/
|
||||
public isReleaseEligible(row: StackUpdateRecoveryGenerationRow): boolean {
|
||||
if (row.released_at !== null || row.artifacts_retired !== 0) return false;
|
||||
if (row.phase !== 'immediate_verified') return false;
|
||||
if (!['active', 'restored_current', 'superseded'].includes(row.status)) return false;
|
||||
if (row.health_gate_id) {
|
||||
const gate = DatabaseService.getInstance().getHealthGateRun(row.node_id, row.stack_name, row.health_gate_id);
|
||||
if (gate?.status === 'observing') return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Operator-initiated release of rollback protection, current generation
|
||||
* included. The DB transition (releaseStackUpdateRecoveryGeneration)
|
||||
* atomically revalidates eligibility and clears is_current, which is what
|
||||
* stops getCurrent()/isRestoredCurrentPinActive() from reporting a released
|
||||
* row as the live rollback point. Docker tag + override cleanup reuses the
|
||||
* same idempotent retireGenerationArtifacts() that abandon() already relies
|
||||
* on, so a mid-cleanup Docker failure leaves artifacts_retired at 0 and is
|
||||
* retried by the next reconcileIncomplete() sweep rather than silently
|
||||
* "succeeding" in the UI.
|
||||
*/
|
||||
public async releaseGeneration(
|
||||
id: string,
|
||||
releasedBy: string | null,
|
||||
): Promise<
|
||||
| { ok: true; row: StackUpdateRecoveryGenerationRow; artifactsCleaned: boolean }
|
||||
| { ok: false; reason: 'not_found' | 'already_released' | 'not_eligible' }
|
||||
> {
|
||||
const before = this.get(id);
|
||||
if (!before) return { ok: false, reason: 'not_found' };
|
||||
if (before.released_at !== null) return { ok: false, reason: 'already_released' };
|
||||
|
||||
const released = DatabaseService.getInstance().releaseStackUpdateRecoveryGeneration(id, releasedBy);
|
||||
if (!released) return { ok: false, reason: 'not_eligible' };
|
||||
|
||||
const row = this.get(id);
|
||||
if (!row) return { ok: false, reason: 'not_found' };
|
||||
const artifactsCleaned = await this.retireGenerationArtifacts(row);
|
||||
|
||||
const wasCurrent = before.is_current === 1;
|
||||
try {
|
||||
DatabaseService.getInstance().addNotificationHistory(row.node_id, {
|
||||
level: wasCurrent ? 'warning' : 'info',
|
||||
category: 'rollback_generation_released',
|
||||
message: wasCurrent
|
||||
? `${row.stack_name}: current rollback protection released. Automatic rollback is unavailable until the next successful full-stack update.`
|
||||
: `${row.stack_name}: rollback protection released for generation ${shortGenerationId(row.id)}.`,
|
||||
timestamp: Date.now(),
|
||||
stack_name: row.stack_name,
|
||||
actor_username: releasedBy,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Failed to record release activity for %s:',
|
||||
sanitizeForLog(id),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
|
||||
return { ok: true, row, artifactsCleaned };
|
||||
}
|
||||
|
||||
public linkHealthGate(id: string, healthGateId: string): void {
|
||||
DatabaseService.getInstance().linkStackUpdateRecoveryHealthGate(id, healthGateId);
|
||||
}
|
||||
@@ -460,22 +534,6 @@ export class StackUpdateRecoveryService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unified held-image predicate: service-scoped + full-stack holds.
|
||||
* Fail closed (skip prune) when either lookup fails.
|
||||
*/
|
||||
public buildUnifiedHeldImagePredicate(nodeId: number): (imageId: string) => boolean {
|
||||
// Dynamic require avoids a static cycle with ServiceUpdateRecoveryService.
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const { ServiceUpdateRecoveryService } = require('./ServiceUpdateRecoveryService') as typeof import('./ServiceUpdateRecoveryService');
|
||||
const serviceHeld = ServiceUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
|
||||
const stackHeld = this.getHeldImageIds(nodeId);
|
||||
if (serviceHeld === null || stackHeld === null) {
|
||||
return () => true;
|
||||
}
|
||||
return (imageId: string) => serviceHeld.has(imageId) || stackHeld.has(imageId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Post-handoff compensation: restore files + pinned up, then probe before
|
||||
* reporting restored_current / immediate_verified.
|
||||
@@ -655,7 +713,20 @@ export class StackUpdateRecoveryService {
|
||||
}
|
||||
}
|
||||
if (!tagsOk || !overrideOk) return false;
|
||||
DatabaseService.getInstance().markStackUpdateRecoveryArtifactsRetired(row.id);
|
||||
try {
|
||||
DatabaseService.getInstance().markStackUpdateRecoveryArtifactsRetired(row.id);
|
||||
} catch (error) {
|
||||
// Tags/override are already gone at this point; a DB write failure here
|
||||
// must not surface as "release/abandon failed" to the caller (the
|
||||
// mutation it asked for already happened). Leave artifacts_retired at 0
|
||||
// so the next reconcileIncomplete() sweep retries the DB write alone.
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Failed to mark artifacts retired for %s: %s',
|
||||
sanitizeForLog(row.id),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -680,16 +751,38 @@ export class StackUpdateRecoveryService {
|
||||
});
|
||||
flagged += 1;
|
||||
}
|
||||
let capped = 0;
|
||||
const maxGenerations = db.getRecoveryMaxGenerations();
|
||||
if (maxGenerations > 0) {
|
||||
// The current generation always counts as one of the cap, so the
|
||||
// superseded budget is one less; it can never itself be evicted here.
|
||||
const supersededBudget = Math.max(0, maxGenerations - 1);
|
||||
const byStack = new Map<string, StackUpdateRecoveryGenerationRow[]>();
|
||||
for (const row of db.listActiveSupersededGenerations()) {
|
||||
const key = `${row.node_id}:${row.stack_name}`;
|
||||
const list = byStack.get(key) ?? [];
|
||||
list.push(row);
|
||||
byStack.set(key, list);
|
||||
}
|
||||
for (const rows of byStack.values()) {
|
||||
for (const row of rows.slice(supersededBudget)) {
|
||||
if (row.artifact_expires_at === null || row.artifact_expires_at > now) {
|
||||
db.updateStackUpdateRecoveryGeneration(row.id, { artifact_expires_at: now });
|
||||
capped += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let retired = 0;
|
||||
for (const row of db.listStackUpdateRecoveryGenerationsForArtifactRetirement(now)) {
|
||||
// Never retire an active/current or recovery_required hold target.
|
||||
if (row.is_current === 1 || row.status === 'recovery_required') continue;
|
||||
if (await this.retireGenerationArtifacts(row)) retired += 1;
|
||||
}
|
||||
if (abandoned > 0 || flagged > 0 || retired > 0) {
|
||||
if (abandoned > 0 || flagged > 0 || capped > 0 || retired > 0) {
|
||||
console.log(
|
||||
`[StackUpdateRecovery] Reconciled ${abandoned} stale candidate(s), `
|
||||
+ `${flagged} stuck generation(s), retired ${retired} artifact set(s)`,
|
||||
+ `${flagged} stuck generation(s), ${capped} generation(s) over cap, retired ${retired} artifact set(s)`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService';
|
||||
import { StackUpdateRecoveryService } from './StackUpdateRecoveryService';
|
||||
|
||||
/**
|
||||
* Unified held-image predicate: service-scoped + full-stack rollback holds.
|
||||
* Lives in its own module (rather than on either service) so both can be
|
||||
* imported here statically without a cycle -- ServiceUpdateRecoveryService
|
||||
* and StackUpdateRecoveryService intentionally do not import each other.
|
||||
* Fails closed (protects every image) when either lookup fails.
|
||||
*/
|
||||
export function buildUnifiedHeldImagePredicate(nodeId: number): (imageId: string) => boolean {
|
||||
const serviceHeld = ServiceUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
|
||||
const stackHeld = StackUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
|
||||
if (serviceHeld === null || stackHeld === null) {
|
||||
return () => true;
|
||||
}
|
||||
return (imageId: string) => serviceHeld.has(imageId) || stackHeld.has(imageId);
|
||||
}
|
||||
@@ -40,6 +40,7 @@ export const AUDIT_ROUTE_SUMMARIES: Record<string, string> = {
|
||||
'POST /system/images/delete': 'Deleted images',
|
||||
'POST /system/volumes/delete': 'Deleted volumes',
|
||||
'POST /system/networks/delete': 'Deleted networks',
|
||||
'POST /system/rollback/generations/*/release': 'Released rollback protection',
|
||||
'POST /system/networks': 'Created network',
|
||||
'POST /system/console-token': 'Generated console token',
|
||||
'POST /system/reapply-compose': 'Triggered compose reapply',
|
||||
|
||||
@@ -101,6 +101,21 @@ The Stack Dossier carries a **Rollback readiness** section that answers one ques
|
||||
<img src="/images/health-gated-updates/dossier-rollback-readiness.png" alt="Rollback readiness section in the Stack Dossier showing the overall state chip and the six rows: Previous compose file, Previous env file, Previous image tag, Last successful deploy, Healthchecks, and the Application data row marked not covered" />
|
||||
</Frame>
|
||||
|
||||
## Automatic rollback images
|
||||
|
||||
Before a full-stack update runs, Sencho captures the running image of every service as an opaque, uniquely named copy so it can automatically restore the prior state if the update or its health gate fails. These copies exist in Docker as `sencho-rb/<generation>/<service>:hold`, but they are Sencho-internal recovery state, not part of your image inventory: they are kept out of **Resources → Images** and listed instead in **Resources → Rollback**. If a captured image still carries its original registry tag alongside the hold tag (a compose file pinned to an immutable tag, for example), it stays visible in the Images tab too, badged **Rollback protected** instead of the usual unused label, since it is held on purpose rather than left behind by accident.
|
||||
|
||||
Each capture is one **rollback generation**. The generation currently backing a stack's live deployment is retained for as long as it is current; once a newer update supersedes it, it is retained for a configurable window before Sencho cleans it up automatically. A stack updated repeatedly in a short span can have more than one superseded generation in that window at once.
|
||||
|
||||
**Resources → Rollback** lists every generation on the node: the stack it belongs to, a short generation id, whether it is the current protection or a superseded one awaiting cleanup, and roughly when it clears. An admin can release a generation's protection early from that list, including the current one, which immediately frees its image but means Sencho cannot automatically roll that stack back until its next successful full-stack update; the confirmation dialog says so before you proceed. A generation that is mid-recovery or still being observed by a health gate cannot be released until that finishes.
|
||||
|
||||
Because these images are deliberately held, deleting one directly (by id, including through the API) is refused. Release the generation from **Resources → Rollback** instead, or leave it to clear on its own.
|
||||
|
||||
Two settings under **Settings > Infrastructure > Stacks > Deploy Guardrails** control the automatic cleanup:
|
||||
|
||||
- **Superseded rollback retention** sets how many days a superseded generation is kept before its image is cleaned up. The current generation is unaffected by this window; it stays protected until it is superseded or manually released. Default 7 days.
|
||||
- **Maximum retained rollback generations per stack** caps how many generations a stack keeps at once, current generation included, so the oldest superseded generations beyond the cap are cleaned up ahead of the retention window. 0 (default) leaves the count unlimited and relies on the retention window alone.
|
||||
|
||||
## Classified failures
|
||||
|
||||
When a deploy or update fails, Sencho classifies the failure from the compose output and shows the cause with a suggested next step in the recovery panel: an image pull failure, a missing environment variable, a host port conflict, a missing bind-mount path, a permission problem, a crashed container, a failed healthcheck, an unavailable dependency, an unreachable node or Docker daemon, or an invalid compose file. The classification also lands in **Copy details**, so a bug report carries the cause, not just the raw output.
|
||||
@@ -126,4 +141,10 @@ When a deploy or update fails, Sencho classifies the failure from the compose ou
|
||||
<Accordion title="Updates from the sidebar menu now show a dialog first">
|
||||
The sidebar's per-stack **Update** action runs the same path as the editor toolbar, so it shows the same readiness dialog and deploy progress. One click on **Update now** proceeds. On nodes that do not advertise the capability, updates run directly without the dialog.
|
||||
</Accordion>
|
||||
<Accordion title="Why do I see sencho-rb/... images in docker images on the host">
|
||||
Those are automatic rollback images: an opaque copy of a service's prior image, held so Sencho can restore it if a full-stack update fails. They are not leftovers. Sencho keeps them out of **Resources → Images** on purpose (they are recovery state, not image inventory) and lists them in **Resources → Rollback** instead, showing which stack and generation each one belongs to and how soon it clears on its own. If one still carries a normal registry tag too, it also stays visible in the Images tab with a **Rollback protected** badge.
|
||||
</Accordion>
|
||||
<Accordion title="Deleting a rollback-protected image fails">
|
||||
That failure is intentional: the image is protected by an active or recently superseded rollback generation. Open **Resources → Rollback**, find the matching generation, and use **Release** there if you are sure you do not need it. Releasing the current generation means Sencho cannot automatically roll that stack back until its next successful full-stack update.
|
||||
</Accordion>
|
||||
</AccordionGroup>
|
||||
|
||||
@@ -29,6 +29,8 @@ import { cn } from '@/lib/utils';
|
||||
import { ReclaimHero } from './resources/ReclaimHero';
|
||||
import { FootprintTreemap } from './resources/FootprintTreemap';
|
||||
import { ImageDetailsSheet } from './resources/ImageDetailsSheet';
|
||||
import { RollbackGenerationsTab, type RollbackGeneration } from './resources/RollbackGenerationsTab';
|
||||
import { TableSkeleton } from './resources/TableSkeleton';
|
||||
import { VolumeBrowserSheet } from './resources/VolumeBrowserSheet';
|
||||
import { VolumeNameLabel } from './resources/VolumeNameLabel';
|
||||
import { useTableSort } from '@/hooks/useTableSort';
|
||||
@@ -59,6 +61,9 @@ interface DockerImage {
|
||||
managedBy: string | null;
|
||||
managedStatus: 'managed' | 'unmanaged' | 'unused';
|
||||
isSencho: boolean;
|
||||
/** True when a rollback hold protects this image from pruning; additive, independent of managedStatus. */
|
||||
rollbackProtected: boolean;
|
||||
rollbackProtectionKind?: 'stack' | 'service';
|
||||
}
|
||||
|
||||
interface DockerVolume {
|
||||
@@ -272,6 +277,26 @@ function SenchoBadge() {
|
||||
);
|
||||
}
|
||||
|
||||
function RollbackProtectedBadge({ kind }: { kind?: 'stack' | 'service' }) {
|
||||
return (
|
||||
<TooltipProvider>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Badge variant="outline" className="text-[10px] h-5 gap-1 border-brand/40 text-brand">
|
||||
<ShieldCheck className="w-3 h-3" strokeWidth={2} />
|
||||
Rollback protected
|
||||
</Badge>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
{kind === 'stack'
|
||||
? 'Held as a full-stack rollback point. See Resources → Rollback.'
|
||||
: 'Held for a pending per-service update rollback.'}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Severity Badge ─────────────────────────────────────────────────────────────
|
||||
|
||||
// ── Quick Clean Prune Button ───────────────────────────────────────────────────
|
||||
@@ -327,24 +352,6 @@ function PruneButton({ target, icon, label, accentClass, onManaged, onAll }: Pru
|
||||
);
|
||||
}
|
||||
|
||||
// ── Table Skeleton ─────────────────────────────────────────────────────────────
|
||||
|
||||
function TableSkeleton({ cols, rows = 5 }: { cols: number; rows?: number }) {
|
||||
return (
|
||||
<TableBody>
|
||||
{Array.from({ length: rows }).map((_, r) => (
|
||||
<TableRow key={r} className="animate-in fade-in-0" style={{ animationDelay: `${r * 40}ms` }}>
|
||||
{Array.from({ length: cols }).map((_, c) => (
|
||||
<TableCell key={c}>
|
||||
<Skeleton className={cn('h-4', c === 0 ? 'w-24' : c === 1 ? 'w-48' : 'w-16')} />
|
||||
</TableCell>
|
||||
))}
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
);
|
||||
}
|
||||
|
||||
// Stable comparator maps for the resource tables (module scope so useTableSort
|
||||
// does not re-sort on every render). Mirrors the Security Images sort standard.
|
||||
const IMAGE_COMPARATORS: Record<'repo' | 'size' | 'status', (a: DockerImage, b: DockerImage) => number> = {
|
||||
@@ -366,7 +373,7 @@ interface ResourcesViewProps {
|
||||
|
||||
export default function ResourcesView({ headerActions }: ResourcesViewProps = {}) {
|
||||
const isMobile = useIsMobile();
|
||||
const [resourceTab, setResourceTab] = useState<'images' | 'volumes' | 'unmanaged'>('images');
|
||||
const [resourceTab, setResourceTab] = useState<'images' | 'volumes' | 'unmanaged' | 'rollback'>('images');
|
||||
const { isAdmin, can } = useAuth();
|
||||
const canReadResources = can('stack:read');
|
||||
const canDeployResources = can('stack:deploy');
|
||||
@@ -377,6 +384,7 @@ export default function ResourcesView({ headerActions }: ResourcesViewProps = {}
|
||||
const [volumes, setVolumes] = useState<DockerVolume[]>([]);
|
||||
const [networks, setNetworks] = useState<DockerNetwork[]>([]);
|
||||
const [orphans, setOrphans] = useState<Record<string, UnmanagedContainer[]>>({});
|
||||
const [rollbackGenerations, setRollbackGenerations] = useState<RollbackGeneration[]>([]);
|
||||
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [isActioning, setIsActioning] = useState(false);
|
||||
@@ -438,12 +446,13 @@ export default function ResourcesView({ headerActions }: ResourcesViewProps = {}
|
||||
const generation = ++fetchGenerationRef.current;
|
||||
setIsLoading(true);
|
||||
try {
|
||||
const [usageRes, resourcesRes, orphansRes, summariesRes, settingsRes] = await Promise.all([
|
||||
const [usageRes, resourcesRes, orphansRes, summariesRes, settingsRes, rollbackRes] = await Promise.all([
|
||||
apiFetch('/system/docker-df'),
|
||||
apiFetch('/system/resources'),
|
||||
apiFetch('/system/orphans'),
|
||||
apiFetch('/security/image-summaries').catch(() => null),
|
||||
apiFetch('/settings').catch(() => null),
|
||||
apiFetch('/system/rollback/generations').catch(() => null),
|
||||
]);
|
||||
|
||||
// Resolve every body before the staleness check so a stale
|
||||
@@ -453,6 +462,7 @@ export default function ResourcesView({ headerActions }: ResourcesViewProps = {}
|
||||
const orphansData = orphansRes.ok ? await orphansRes.json() : null;
|
||||
const summariesData = summariesRes && summariesRes.ok ? await summariesRes.json() : null;
|
||||
const settingsData = settingsRes && settingsRes.ok ? await settingsRes.json() : null;
|
||||
const rollbackData = rollbackRes && rollbackRes.ok ? await rollbackRes.json() : null;
|
||||
|
||||
if (fetchGenerationRef.current !== generation) return;
|
||||
|
||||
@@ -471,6 +481,7 @@ export default function ResourcesView({ headerActions }: ResourcesViewProps = {}
|
||||
setSelectedOrphans([]);
|
||||
}
|
||||
if (summariesData) setScanSummaries(summariesData);
|
||||
setRollbackGenerations(Array.isArray(rollbackData) ? rollbackData : []);
|
||||
} catch (err) {
|
||||
if (fetchGenerationRef.current !== generation) return;
|
||||
console.error('Failed to fetch data', err);
|
||||
@@ -928,6 +939,7 @@ export default function ResourcesView({ headerActions }: ResourcesViewProps = {}
|
||||
{ value: 'images', label: 'Images', count: images.length },
|
||||
{ value: 'volumes', label: 'Volumes', count: volumes.length },
|
||||
{ value: 'unmanaged', label: 'Unmanaged', count: totalOrphansCount },
|
||||
{ value: 'rollback', label: 'Rollback', count: rollbackGenerations.length },
|
||||
]}
|
||||
/>
|
||||
) : (
|
||||
@@ -950,6 +962,12 @@ export default function ResourcesView({ headerActions }: ResourcesViewProps = {}
|
||||
<span className="ml-1.5 text-[10px] text-stat-subtitle tabular-nums">{totalOrphansCount}</span>
|
||||
</TabsTrigger>
|
||||
</TabsHighlightItem>
|
||||
<TabsHighlightItem value="rollback">
|
||||
<TabsTrigger value="rollback" className="relative">
|
||||
Rollback
|
||||
<span className="ml-1.5 text-[10px] text-stat-subtitle tabular-nums">{rollbackGenerations.length}</span>
|
||||
</TabsTrigger>
|
||||
</TabsHighlightItem>
|
||||
</TabsHighlight>
|
||||
</TabsList>
|
||||
</div>
|
||||
@@ -1050,6 +1068,7 @@ export default function ResourcesView({ headerActions }: ResourcesViewProps = {}
|
||||
) : undefined}
|
||||
/>
|
||||
{img.isSencho && <SenchoBadge />}
|
||||
{img.rollbackProtected && <RollbackProtectedBadge kind={img.rollbackProtectionKind} />}
|
||||
{(() => {
|
||||
const tag = img.RepoTags?.[0];
|
||||
const summary = tag ? scanSummaries[tag] : undefined;
|
||||
@@ -1351,6 +1370,17 @@ export default function ResourcesView({ headerActions }: ResourcesViewProps = {}
|
||||
)}
|
||||
</div>
|
||||
</TabsContent>
|
||||
|
||||
{/* Rollback */}
|
||||
<TabsContent value="rollback" className="m-0 border-0 p-0 animate-in fade-in-0 duration-200">
|
||||
<RollbackGenerationsTab
|
||||
generations={rollbackGenerations}
|
||||
isLoading={isLoading}
|
||||
isAdmin={isAdmin}
|
||||
nodeId={activeNode?.id}
|
||||
onReleased={fetchAllData}
|
||||
/>
|
||||
</TabsContent>
|
||||
</div>
|
||||
</Tabs>
|
||||
</>
|
||||
|
||||
@@ -474,4 +474,41 @@ describe('ResourcesView', () => {
|
||||
await screen.findByText('off-img:latest');
|
||||
expect(screen.queryByTestId('reclaim-hero')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('badges a rollback-protected image without changing its managed/unused status', async () => {
|
||||
mockedFetch.mockImplementation((url: string) => {
|
||||
if (url === '/system/resources') {
|
||||
return Promise.resolve(jsonResponse({
|
||||
images: [{ ...image('nginx:1.25'), managedStatus: 'unused', rollbackProtected: true, rollbackProtectionKind: 'stack' }],
|
||||
volumes: [],
|
||||
networks: [],
|
||||
}));
|
||||
}
|
||||
return Promise.resolve(jsonResponse({}));
|
||||
});
|
||||
|
||||
render(<ResourcesView />);
|
||||
await screen.findByText('nginx:1.25');
|
||||
expect(screen.getByText('Rollback protected')).toBeInTheDocument();
|
||||
expect(screen.getByText('Unused')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows rollback generations in the Rollback tab, admin-gated release button included', async () => {
|
||||
mockedFetch.mockImplementation((url: string) => {
|
||||
if (url === '/system/rollback/generations') {
|
||||
return Promise.resolve(jsonResponse([
|
||||
{ id: 'gen-1', shortId: 'abc123456789', stackName: 'seerr', status: 'active', isCurrent: true, phase: 'immediate_verified', createdAt: Date.now(), artifactExpiresAt: null, releasable: true },
|
||||
]));
|
||||
}
|
||||
return Promise.resolve(jsonResponse({}));
|
||||
});
|
||||
|
||||
render(<ResourcesView />);
|
||||
await userEvent.click(await screen.findByRole('tab', { name: /rollback/i }));
|
||||
|
||||
expect(await screen.findByText('seerr')).toBeInTheDocument();
|
||||
expect(screen.getByText('abc123456789')).toBeInTheDocument();
|
||||
expect(screen.getByText('Current')).toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: /release rollback protection/i })).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -66,6 +66,7 @@ export type NotificationCategory =
|
||||
| 'update_started'
|
||||
| 'health_gate_passed'
|
||||
| 'health_gate_failed'
|
||||
| 'rollback_generation_released'
|
||||
| 'node_update_available'
|
||||
| 'system';
|
||||
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import { useState } from 'react';
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { ScrollArea } from '@/components/ui/scroll-area';
|
||||
import { ConfirmModal } from '@/components/ui/modal';
|
||||
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip';
|
||||
import { Unlock } from 'lucide-react';
|
||||
import { apiFetch } from '@/lib/api';
|
||||
import { toast } from '@/components/ui/toast-store';
|
||||
import { SENCHO_OPEN_STACK_EVENT, type SenchoOpenStackDetail } from '@/lib/events';
|
||||
import { TableSkeleton } from './TableSkeleton';
|
||||
|
||||
export interface RollbackGeneration {
|
||||
id: string;
|
||||
shortId: string;
|
||||
stackName: string;
|
||||
status: 'active' | 'restored_current' | 'superseded' | 'recovery_required';
|
||||
isCurrent: boolean;
|
||||
phase: string;
|
||||
createdAt: number;
|
||||
artifactExpiresAt: number | null;
|
||||
/** Best-effort UI hint only; the server revalidates eligibility on release. */
|
||||
releasable: boolean;
|
||||
}
|
||||
|
||||
interface RollbackGenerationsTabProps {
|
||||
generations: RollbackGeneration[];
|
||||
isLoading: boolean;
|
||||
isAdmin: boolean;
|
||||
nodeId?: number;
|
||||
/** Refetches the Resources page's data after a successful release. */
|
||||
onReleased: () => void | Promise<void>;
|
||||
}
|
||||
|
||||
function formatExpiry(gen: RollbackGeneration): string {
|
||||
if (gen.isCurrent) return 'Protected while current';
|
||||
if (gen.status === 'recovery_required') return 'Recovery required';
|
||||
if (gen.artifactExpiresAt === null) return 'Pending';
|
||||
const days = (gen.artifactExpiresAt - Date.now()) / (24 * 60 * 60 * 1000);
|
||||
if (days <= 0) return 'Expiring now';
|
||||
if (days < 1) return `Expires in ${Math.max(1, Math.round(days * 24))}h`;
|
||||
return `Expires in ${Math.round(days)}d`;
|
||||
}
|
||||
|
||||
function StateBadge({ gen }: { gen: RollbackGeneration }) {
|
||||
switch (gen.status) {
|
||||
case 'recovery_required':
|
||||
return <Badge variant="destructive" className="text-[10px] h-5">Recovery required</Badge>;
|
||||
case 'superseded':
|
||||
return <Badge variant="secondary" className="text-[10px] h-5">Superseded</Badge>;
|
||||
case 'active':
|
||||
case 'restored_current':
|
||||
return gen.isCurrent
|
||||
? <Badge variant="default" className="text-[10px] h-5">Current</Badge>
|
||||
: <Badge variant="secondary" className="text-[10px] h-5">Superseded</Badge>;
|
||||
default: {
|
||||
const unhandled: never = gen.status;
|
||||
return <Badge variant="secondary" className="text-[10px] h-5">{String(unhandled)}</Badge>;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-stack rollback generations (the sencho-rb/<id>/<service>:hold images
|
||||
* StackUpdateRecoveryService creates). Kept in its own tab rather than the
|
||||
* generic Images list: this is durable recovery state with its own lifecycle
|
||||
* (stack, generation, retention, release), not ordinary Docker image inventory.
|
||||
*/
|
||||
export function RollbackGenerationsTab({ generations, isLoading, isAdmin, nodeId, onReleased }: RollbackGenerationsTabProps) {
|
||||
const [confirmRelease, setConfirmRelease] = useState<RollbackGeneration | null>(null);
|
||||
const [isReleasing, setIsReleasing] = useState(false);
|
||||
|
||||
const handleRelease = async () => {
|
||||
if (!confirmRelease) return;
|
||||
setIsReleasing(true);
|
||||
const loadingId = toast.loading(`Releasing rollback protection for ${confirmRelease.shortId}...`);
|
||||
try {
|
||||
const res = await apiFetch(`/system/rollback/generations/${confirmRelease.id}/release`, { method: 'POST' });
|
||||
const data = await res.json().catch(() => null);
|
||||
if (!res.ok) {
|
||||
throw new Error(data?.error || 'Failed to release rollback protection');
|
||||
}
|
||||
toast.success(data?.message || 'Rollback protection released');
|
||||
await onReleased();
|
||||
} catch (error) {
|
||||
const err = error as Record<string, unknown>;
|
||||
toast.error(String(err?.message || 'Failed to release rollback protection'));
|
||||
} finally {
|
||||
toast.dismiss(loadingId);
|
||||
setIsReleasing(false);
|
||||
setConfirmRelease(null);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<p className="mb-3 text-sm leading-relaxed text-stat-subtitle">
|
||||
Rollback-protected images from full-stack updates. Each generation is kept so a failed update can be
|
||||
automatically rolled back, and clears on its own once it is superseded and its retention window
|
||||
passes (configurable under Settings → Infrastructure → Stacks → Deploy Guardrails).
|
||||
</p>
|
||||
<div className="rounded-lg border border-card-border border-t-card-border-top bg-card shadow-card-bevel overflow-hidden">
|
||||
<ScrollArea className="h-[62vh] max-md:h-auto">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Stack</TableHead>
|
||||
<TableHead>Generation</TableHead>
|
||||
<TableHead>State</TableHead>
|
||||
<TableHead>Retention</TableHead>
|
||||
<TableHead className="text-right">Actions</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
{isLoading ? <TableSkeleton cols={5} /> : (
|
||||
<TableBody>
|
||||
{generations.length === 0 ? (
|
||||
<TableRow>
|
||||
<TableCell colSpan={5} className="text-center py-8 text-muted-foreground text-sm">
|
||||
No rollback-protected generations on this node.
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
) : generations.map((gen, i) => (
|
||||
<TableRow
|
||||
key={gen.id}
|
||||
className="animate-in fade-in-0 duration-200 hover:bg-muted/30 transition-colors"
|
||||
style={{ animationDelay: `${Math.min(i * 20, 200)}ms` }}
|
||||
>
|
||||
<TableCell className="font-medium">
|
||||
<button
|
||||
type="button"
|
||||
disabled={nodeId === undefined}
|
||||
className="hover:underline underline-offset-2 disabled:no-underline disabled:cursor-default"
|
||||
onClick={() => nodeId !== undefined && window.dispatchEvent(
|
||||
new CustomEvent<SenchoOpenStackDetail>(SENCHO_OPEN_STACK_EVENT, { detail: { nodeId, stackName: gen.stackName } }),
|
||||
)}
|
||||
>
|
||||
{gen.stackName}
|
||||
</button>
|
||||
</TableCell>
|
||||
<TableCell className="font-mono text-xs text-muted-foreground">{gen.shortId}</TableCell>
|
||||
<TableCell><StateBadge gen={gen} /></TableCell>
|
||||
<TableCell className="text-xs text-stat-subtitle">{formatExpiry(gen)}</TableCell>
|
||||
<TableCell className="text-right">
|
||||
{isAdmin && (
|
||||
<TooltipProvider>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-7 w-7 text-muted-foreground hover:text-destructive transition-colors"
|
||||
disabled={!gen.releasable}
|
||||
onClick={() => setConfirmRelease(gen)}
|
||||
aria-label={`Release rollback protection for ${gen.shortId}`}
|
||||
>
|
||||
<Unlock className="w-3.5 h-3.5" strokeWidth={1.5} />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
{gen.releasable
|
||||
? 'Release rollback protection'
|
||||
: 'Not releasable right now (mid-recovery or observing a health gate)'}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
)}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
)}
|
||||
</Table>
|
||||
</ScrollArea>
|
||||
</div>
|
||||
|
||||
<ConfirmModal
|
||||
open={!!confirmRelease}
|
||||
onOpenChange={(open) => !open && setConfirmRelease(null)}
|
||||
variant="destructive"
|
||||
kicker="ROLLBACK · RELEASE · IRREVERSIBLE"
|
||||
title={`Release rollback protection for ${confirmRelease?.stackName ?? ''}`}
|
||||
confirmLabel={isReleasing ? 'Releasing...' : 'Release'}
|
||||
confirming={isReleasing}
|
||||
onConfirm={handleRelease}
|
||||
>
|
||||
<p className="text-sm text-stat-subtitle">
|
||||
{confirmRelease?.isCurrent ? (
|
||||
<>
|
||||
This is <span className="font-medium text-stat-value">{confirmRelease?.stackName}</span>'s
|
||||
current rollback point. Releasing it now means Sencho will not be able to automatically
|
||||
roll this stack back until its next successful full-stack update.
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
Permanently removes the held rollback image for generation{' '}
|
||||
<span className="font-mono font-medium text-stat-value">{confirmRelease?.shortId}</span>{' '}
|
||||
ahead of its normal retention window.
|
||||
</>
|
||||
)}
|
||||
</p>
|
||||
</ConfirmModal>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { TableBody, TableRow, TableCell } from '@/components/ui/table';
|
||||
import { Skeleton } from '@/components/ui/skeleton';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
/** Shared loading placeholder for the Resources page's tabbed tables (Images, Volumes, Rollback). */
|
||||
export function TableSkeleton({ cols, rows = 5 }: { cols: number; rows?: number }) {
|
||||
return (
|
||||
<TableBody>
|
||||
{Array.from({ length: rows }).map((_, r) => (
|
||||
<TableRow key={r} className="animate-in fade-in-0" style={{ animationDelay: `${r * 40}ms` }}>
|
||||
{Array.from({ length: cols }).map((_, c) => (
|
||||
<TableCell key={c}>
|
||||
<Skeleton className={cn('h-4', c === 0 ? 'w-24' : c === 1 ? 'w-48' : 'w-16')} />
|
||||
</TableCell>
|
||||
))}
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { render, screen, waitFor } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { RollbackGenerationsTab, type RollbackGeneration } from '../RollbackGenerationsTab';
|
||||
import { toast } from '@/components/ui/toast-store';
|
||||
|
||||
const apiFetch = vi.fn();
|
||||
vi.mock('@/lib/api', () => ({ apiFetch: (...args: unknown[]) => apiFetch(...args) }));
|
||||
|
||||
vi.mock('@/components/ui/toast-store', () => ({
|
||||
toast: {
|
||||
error: vi.fn(),
|
||||
success: vi.fn(),
|
||||
loading: vi.fn(() => 'toast-id'),
|
||||
dismiss: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
function generation(overrides: Partial<RollbackGeneration> = {}): RollbackGeneration {
|
||||
return {
|
||||
id: 'gen-1',
|
||||
shortId: 'abc123456789',
|
||||
stackName: 'seerr',
|
||||
status: 'superseded',
|
||||
isCurrent: false,
|
||||
phase: 'immediate_verified',
|
||||
createdAt: Date.now(),
|
||||
artifactExpiresAt: Date.now() + 3 * 24 * 60 * 60 * 1000,
|
||||
releasable: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
apiFetch.mockReset();
|
||||
(toast.success as ReturnType<typeof vi.fn>).mockReset();
|
||||
(toast.error as ReturnType<typeof vi.fn>).mockReset();
|
||||
});
|
||||
|
||||
describe('RollbackGenerationsTab', () => {
|
||||
it('shows superseded-generation confirm copy (not the current-generation warning) for a non-current release', async () => {
|
||||
const onReleased = vi.fn();
|
||||
render(<RollbackGenerationsTab generations={[generation({ isCurrent: false })]} isLoading={false} isAdmin onReleased={onReleased} />);
|
||||
|
||||
await userEvent.click(screen.getByRole('button', { name: /release rollback protection/i }));
|
||||
|
||||
expect(await screen.findByText(/Permanently removes the held rollback image/i)).toBeInTheDocument();
|
||||
expect(screen.queryByText(/Automatic rollback is unavailable until/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows the current-generation warning copy when releasing the current generation', async () => {
|
||||
const onReleased = vi.fn();
|
||||
render(<RollbackGenerationsTab generations={[generation({ isCurrent: true, status: 'active' })]} isLoading={false} isAdmin onReleased={onReleased} />);
|
||||
|
||||
await userEvent.click(screen.getByRole('button', { name: /release rollback protection/i }));
|
||||
|
||||
expect(await screen.findByText(/Sencho will not be able to automatically/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('confirming release POSTs to the release endpoint and calls onReleased on success', async () => {
|
||||
apiFetch.mockResolvedValue({ ok: true, json: async () => ({ success: true, message: 'Rollback protection released', artifactsCleaned: true }) });
|
||||
const onReleased = vi.fn();
|
||||
render(<RollbackGenerationsTab generations={[generation()]} isLoading={false} isAdmin onReleased={onReleased} />);
|
||||
|
||||
await userEvent.click(screen.getByRole('button', { name: /release rollback protection/i }));
|
||||
await userEvent.click(await screen.findByRole('button', { name: 'Release' }));
|
||||
|
||||
await waitFor(() => expect(apiFetch).toHaveBeenCalledWith('/system/rollback/generations/gen-1/release', { method: 'POST' }));
|
||||
await waitFor(() => expect(onReleased).toHaveBeenCalled());
|
||||
expect(toast.success).toHaveBeenCalledWith('Rollback protection released');
|
||||
});
|
||||
|
||||
it('surfaces the backend partial-cleanup message distinctly from a full release', async () => {
|
||||
apiFetch.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ success: true, message: 'Rollback protection released; cleanup will finish shortly', artifactsCleaned: false }),
|
||||
});
|
||||
const onReleased = vi.fn();
|
||||
render(<RollbackGenerationsTab generations={[generation()]} isLoading={false} isAdmin onReleased={onReleased} />);
|
||||
|
||||
await userEvent.click(screen.getByRole('button', { name: /release rollback protection/i }));
|
||||
await userEvent.click(await screen.findByRole('button', { name: 'Release' }));
|
||||
|
||||
await waitFor(() => expect(toast.success).toHaveBeenCalledWith('Rollback protection released; cleanup will finish shortly'));
|
||||
});
|
||||
|
||||
it('surfaces the server error via toast and closes the modal without a lingering Releasing state on failure', async () => {
|
||||
apiFetch.mockResolvedValue({ ok: false, json: async () => ({ error: 'This rollback generation cannot be released right now (it may be observing a health gate, mid-recovery, or already in progress).', code: 'NOT_ELIGIBLE' }) });
|
||||
const onReleased = vi.fn();
|
||||
render(<RollbackGenerationsTab generations={[generation()]} isLoading={false} isAdmin onReleased={onReleased} />);
|
||||
|
||||
await userEvent.click(screen.getByRole('button', { name: /release rollback protection/i }));
|
||||
await userEvent.click(await screen.findByRole('button', { name: 'Release' }));
|
||||
|
||||
await waitFor(() => expect(toast.error).toHaveBeenCalledWith(expect.stringContaining('cannot be released right now')));
|
||||
expect(onReleased).not.toHaveBeenCalled();
|
||||
// Modal closes (confirm button no longer present) rather than staying stuck mid-action.
|
||||
await waitFor(() => expect(screen.queryByRole('button', { name: 'Release' })).not.toBeInTheDocument());
|
||||
});
|
||||
|
||||
it('hides the Release action for a non-admin', () => {
|
||||
render(<RollbackGenerationsTab generations={[generation()]} isLoading={false} isAdmin={false} onReleased={vi.fn()} />);
|
||||
expect(screen.queryByRole('button', { name: /release rollback protection/i })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('disables the Release action when the generation is not releasable', () => {
|
||||
render(<RollbackGenerationsTab generations={[generation({ releasable: false })]} isLoading={false} isAdmin onReleased={vi.fn()} />);
|
||||
expect(screen.getByRole('button', { name: /release rollback protection/i })).toBeDisabled();
|
||||
});
|
||||
|
||||
it('renders an empty state when there are no generations', () => {
|
||||
render(<RollbackGenerationsTab generations={[]} isLoading={false} isAdmin onReleased={vi.fn()} />);
|
||||
expect(screen.getByText(/No rollback-protected generations on this node/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows a loading skeleton instead of the empty state while the initial fetch is in flight', () => {
|
||||
render(<RollbackGenerationsTab generations={[]} isLoading={true} isAdmin onReleased={vi.fn()} />);
|
||||
expect(screen.queryByText(/No rollback-protected generations on this node/i)).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
@@ -32,11 +32,13 @@ interface StacksSectionProps {
|
||||
onDirtyChange?: (dirty: boolean) => void;
|
||||
}
|
||||
|
||||
type GuardrailFields = Pick<PatchableSettings, 'health_gate_enabled' | 'health_gate_window_seconds' | 'env_block_deploy_on_missing_required' | 'auto_create_missing_external_networks'>;
|
||||
type GuardrailFields = Pick<PatchableSettings, 'health_gate_enabled' | 'health_gate_window_seconds' | 'recovery_retention_days' | 'recovery_max_generations' | 'env_block_deploy_on_missing_required' | 'auto_create_missing_external_networks'>;
|
||||
|
||||
const DEFAULT_GUARDRAILS: GuardrailFields = {
|
||||
health_gate_enabled: DEFAULT_SETTINGS.health_gate_enabled,
|
||||
health_gate_window_seconds: DEFAULT_SETTINGS.health_gate_window_seconds,
|
||||
recovery_retention_days: DEFAULT_SETTINGS.recovery_retention_days,
|
||||
recovery_max_generations: DEFAULT_SETTINGS.recovery_max_generations,
|
||||
env_block_deploy_on_missing_required: DEFAULT_SETTINGS.env_block_deploy_on_missing_required,
|
||||
auto_create_missing_external_networks: DEFAULT_SETTINGS.auto_create_missing_external_networks,
|
||||
};
|
||||
@@ -92,6 +94,8 @@ export function StacksSection({ onDirtyChange }: StacksSectionProps) {
|
||||
const safe: GuardrailFields = {
|
||||
health_gate_enabled: (nodeData.health_gate_enabled as '0' | '1') ?? DEFAULT_SETTINGS.health_gate_enabled,
|
||||
health_gate_window_seconds: nodeData.health_gate_window_seconds ?? DEFAULT_SETTINGS.health_gate_window_seconds,
|
||||
recovery_retention_days: nodeData.recovery_retention_days ?? DEFAULT_SETTINGS.recovery_retention_days,
|
||||
recovery_max_generations: nodeData.recovery_max_generations ?? DEFAULT_SETTINGS.recovery_max_generations,
|
||||
env_block_deploy_on_missing_required: (nodeData.env_block_deploy_on_missing_required as '0' | '1') ?? DEFAULT_SETTINGS.env_block_deploy_on_missing_required,
|
||||
auto_create_missing_external_networks: (nodeData.auto_create_missing_external_networks as '0' | '1') ?? DEFAULT_SETTINGS.auto_create_missing_external_networks,
|
||||
};
|
||||
@@ -219,6 +223,30 @@ export function StacksSection({ onDirtyChange }: StacksSectionProps) {
|
||||
max={600}
|
||||
/>
|
||||
</SettingsField>
|
||||
<SettingsField
|
||||
label="Superseded rollback retention"
|
||||
helper="Days an older rollback generation is retained after a newer update supersedes it, before its held image is cleaned up automatically. The current generation stays protected until it is superseded or manually released from Resources → Rollback. Default 7 days."
|
||||
>
|
||||
<NumberChip
|
||||
value={settings.recovery_retention_days || '7'}
|
||||
onChange={(v) => onGuardrailChange('recovery_retention_days', v)}
|
||||
suffix="d"
|
||||
min={1}
|
||||
max={90}
|
||||
/>
|
||||
</SettingsField>
|
||||
<SettingsField
|
||||
label="Maximum retained rollback generations per stack"
|
||||
helper="Caps how many rollback generations a stack keeps at once, current generation included (so 1 keeps only the current, 2 keeps the current plus one superseded). The oldest superseded generations beyond the cap are cleaned up early, ahead of the retention window above. 0 = unlimited (retention window only)."
|
||||
>
|
||||
<NumberChip
|
||||
value={settings.recovery_max_generations || '0'}
|
||||
onChange={(v) => onGuardrailChange('recovery_max_generations', v)}
|
||||
suffix="generations"
|
||||
min={0}
|
||||
max={50}
|
||||
/>
|
||||
</SettingsField>
|
||||
<SettingsField
|
||||
label="Block deploy on missing required env vars"
|
||||
helper="When on, a deploy or update is refused before it starts if a required ${VAR:?message} variable is unset or empty, so the stack fails fast with a clear message instead of mid-deploy. Off by default."
|
||||
|
||||
@@ -122,6 +122,8 @@ describe('split section save payloads', () => {
|
||||
'env_block_deploy_on_missing_required',
|
||||
'health_gate_enabled',
|
||||
'health_gate_window_seconds',
|
||||
'recovery_max_generations',
|
||||
'recovery_retention_days',
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -131,7 +131,7 @@ export const SETTINGS_ITEMS: readonly SettingsItemMeta[] = [
|
||||
group: 'infrastructure',
|
||||
label: 'Stacks',
|
||||
description: 'Stack editor, lifecycle workflow preferences, and deploy guardrails.',
|
||||
keywords: ['stack', 'compose', 'deploy', 'guardrail', 'health gate', 'observation', 'env', 'required variable', 'progress', 'modal', 'inline', 'diff', 'preview', 'save', 'editor', 'workflow'],
|
||||
keywords: ['stack', 'compose', 'deploy', 'guardrail', 'health gate', 'observation', 'env', 'required variable', 'progress', 'modal', 'inline', 'diff', 'preview', 'save', 'editor', 'workflow', 'rollback', 'retention', 'generation'],
|
||||
tier: null,
|
||||
scope: 'node',
|
||||
},
|
||||
|
||||
@@ -19,6 +19,8 @@ export interface PatchableSettings {
|
||||
snapshot_documentation?: '0' | '1';
|
||||
health_gate_enabled?: '0' | '1';
|
||||
health_gate_window_seconds?: string;
|
||||
recovery_retention_days?: string;
|
||||
recovery_max_generations?: string;
|
||||
env_block_deploy_on_missing_required?: '0' | '1';
|
||||
auto_create_missing_external_networks?: '0' | '1';
|
||||
image_update_sidebar_indicators?: '0' | '1';
|
||||
@@ -47,6 +49,8 @@ export const DEFAULT_SETTINGS: PatchableSettings = {
|
||||
snapshot_documentation: '0',
|
||||
health_gate_enabled: '1',
|
||||
health_gate_window_seconds: '90',
|
||||
recovery_retention_days: '7',
|
||||
recovery_max_generations: '0',
|
||||
env_block_deploy_on_missing_required: '0',
|
||||
auto_create_missing_external_networks: '0',
|
||||
image_update_sidebar_indicators: '1',
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import {
|
||||
Rocket, RefreshCcw, CircleStop, Play, ArrowUp, Activity, Loader2, AlertCircle,
|
||||
TriangleAlert, CircleCheck, HeartPulse, HeartCrack, ArrowDownToLine,
|
||||
TriangleAlert, CircleCheck, HeartPulse, HeartCrack, ArrowDownToLine, Unlock,
|
||||
} from 'lucide-react';
|
||||
import type { LucideIcon } from 'lucide-react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
@@ -49,6 +49,7 @@ const CATEGORY_ICON: Record<string, LucideIcon> = {
|
||||
update_started: ArrowUp,
|
||||
health_gate_passed: HeartPulse,
|
||||
health_gate_failed: HeartCrack,
|
||||
rollback_generation_released: Unlock,
|
||||
};
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
@@ -17,6 +17,7 @@ export const CATEGORY_LABELS: Record<NotificationCategory, string> = {
|
||||
update_started: 'Update started',
|
||||
health_gate_passed: 'Health gate passed',
|
||||
health_gate_failed: 'Health gate failed',
|
||||
rollback_generation_released: 'Rollback protection released',
|
||||
node_update_available: 'Node update',
|
||||
system: 'System',
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user