feat(fleet): reapply Compose configuration without a version update (#1716)

* feat(fleet): reapply Compose configuration without a version update

Add a distinct Fleet Reapply configuration path so Compose-managed nodes can recreate Sencho from the current on-disk project when already up to date, without pulling or rewriting the image reference.

* fix(fleet): confirm remote reapply and close concurrent tracker race

Require confirmation for remote compose reapply, and lock dispatch before the remote POST so a second request cannot overwrite a successful in-flight tracker.

* fix(ui): icon-only Reapply control so Up to date badge can breathe

Collapse the Node updates Reapply label into a tooltip so the status pill no longer wraps in the Status column.

* feat(editor): Save & Reapply self-stack via fleet compose reapply (#1726)

* feat(editor): Save & Reapply self-stack via fleet compose reapply

Eligible admins can apply on-disk Compose edits to Sencho's own stack from the editor using the same confirm, dispatch, and reconnect path as Fleet Node Updates.

* fix(editor): gate Save & Reapply label to self-stack only

Ordinary stacks were labeled Save & Reapply whenever the node was
reapply-eligible. Require the selected file to be the self-stack for the
toolbar label and diff confirm CTA.

* fix(ui): move compose diff action label helper out of dialog module

Keep ComposeDiffPreviewDialog component-only so react-refresh Fast Refresh
lint passes after the Save and reapply stacked merge.
This commit is contained in:
Anso
2026-07-28 14:26:46 -04:00
committed by GitHub
parent 543e4ef256
commit b0b423b234
42 changed files with 1770 additions and 168 deletions
@@ -73,6 +73,7 @@ function setTracker(over: Partial<import('../services/FleetUpdateTrackerService'
previousVersion: '0.83.0',
previousProcessStart: 1,
wasOffline: false,
operationKind: 'update',
...over,
});
}
@@ -370,6 +371,7 @@ describe('forced-recheck throttle', () => {
FleetUpdateTrackerService.getInstance().set(proxyNodeId, {
status: 'failed', startedAt: Date.now(), previousVersion: null,
previousProcessStart: null, wasOffline: false, resolvedAt: Date.now(), error: 'boom',
operationKind: 'update',
});
const second = await request(app)
@@ -413,3 +415,170 @@ describe('GET /api/fleet/update-status/release-notes', () => {
expect(res.body.htmlUrl).toBeNull();
});
});
describe('compose reapply status and concurrency', () => {
it('exposes canReapplyCompose for local when SelfUpdateService is available', async () => {
vi.spyOn(SelfUpdateService.getInstance(), 'isAvailable').mockReturnValue(true);
mockCompareTargetFetch();
const res = await request(app).get('/api/fleet/update-status').set('Authorization', adminAuth);
expect(res.status).toBe(200);
const local = res.body.nodes.find((n: { type: string }) => n.type === 'local');
expect(local.canReapplyCompose).toBe(true);
});
it('sets canReapplyCompose false for a remote without self-update capability', async () => {
mockMeta(ONLINE({ capabilities: ['stacks'] }));
mockCompareTargetFetch();
const res = await request(app).get('/api/fleet/update-status').set('Authorization', adminAuth);
expect(res.status).toBe(200);
const remote = res.body.nodes.find((n: { nodeId: number }) => n.nodeId === proxyNodeId);
expect(remote.canReapplyCompose).toBe(false);
});
it('resolves a reapply tracker via startedAt change without requiring a version bump', async () => {
setTracker({
operationKind: 'reapply_configuration',
previousVersion: '0.83.0',
previousProcessStart: 1,
startedAt: Date.now() - RECENT_MS,
});
mockMeta(ONLINE({ version: '0.83.0', startedAt: 2 }));
mockCompareTargetFetch();
expect(await getStatus()).toBe('completed');
});
it('does not complete a reapply tracker via signal 4 when version is already current', async () => {
setTracker({
operationKind: 'reapply_configuration',
previousVersion: '0.99.0',
previousProcessStart: 1,
startedAt: Date.now() - 20_000,
});
// Node already at compare target; signal 4 would false-complete an update,
// but must not for reapply while startedAt is unchanged.
mockMeta(ONLINE({ version: '0.99.0', startedAt: 1 }));
mockCompareTargetFetch();
expect(await getStatus()).toBe('updating');
});
it('returns 409 when reapply is requested while an update tracker is in flight', async () => {
setTracker({ operationKind: 'update' });
mockTarget();
const res = await request(app)
.post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`)
.set('Authorization', adminAuth);
expect(res.status).toBe(409);
expect(res.body?.error).toMatch(/already in progress/i);
});
it('dispatches remote reapply to /api/system/reapply-compose without updateBlocked gating', async () => {
mockTarget();
mockMeta(ONLINE({ updateBlocked: true, imagePinKind: 'digest', imageChannel: 'community' }));
let reapplyUrl: string | null = null;
vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
const url = String(input);
try {
if (new URL(url).hostname === 'api.github.com') {
return new Response(JSON.stringify({ tag_name: 'v0.99.0' }), { status: 200 });
}
} catch { /* fall through */ }
reapplyUrl = url;
return new Response(JSON.stringify({ message: 'ok' }), { status: 202 });
});
const res = await request(app)
.post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`)
.set('Authorization', adminAuth);
expect(res.status).toBe(202);
expect(reapplyUrl).toContain('/api/system/reapply-compose');
expect(FleetUpdateTrackerService.getInstance().get(proxyNodeId)?.operationKind)
.toBe('reapply_configuration');
});
it('reserves the tracker before remote dispatch so a concurrent reapply gets 409 without overwriting success', async () => {
mockTarget();
// Hold meta so the first request sits in the dispatch set before the
// pollable tracker exists; the second must still 409 on that lock.
let releaseMeta!: (value: RemoteMeta) => void;
const metaHeld = new Promise<RemoteMeta>((resolve) => { releaseMeta = resolve; });
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockImplementation(async () => metaHeld);
let releaseRemote!: (value: Response) => void;
const remoteHeld = new Promise<Response>((resolve) => { releaseRemote = resolve; });
vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
const url = String(input);
try {
if (new URL(url).hostname === 'api.github.com') {
return new Response(JSON.stringify({ tag_name: 'v0.99.0' }), { status: 200 });
}
} catch { /* fall through */ }
if (url.includes('/api/system/reapply-compose')) {
return remoteHeld;
}
return new Response('{}', { status: 200 });
});
// Supertest is lazy until the thenable is consumed; start the request now.
const firstPromise = request(app)
.post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`)
.set('Authorization', adminAuth)
.then((res) => res);
await vi.waitFor(() => {
expect(NodeRegistry.getInstance().fetchMetaForNode).toHaveBeenCalled();
});
const secondDuringMeta = await request(app)
.post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`)
.set('Authorization', adminAuth);
expect(secondDuringMeta.status).toBe(409);
expect(secondDuringMeta.body?.error).toMatch(/already in progress/i);
releaseMeta(ONLINE());
await vi.waitFor(() => {
expect(FleetUpdateTrackerService.getInstance().get(proxyNodeId)?.status).toBe('updating');
});
const secondDuringPost = await request(app)
.post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`)
.set('Authorization', adminAuth);
expect(secondDuringPost.status).toBe(409);
releaseRemote(new Response(JSON.stringify({ message: 'ok' }), { status: 202 }));
const first = await firstPromise;
expect(first.status).toBe(202);
const tracker = FleetUpdateTrackerService.getInstance().get(proxyNodeId);
expect(tracker?.status).toBe('updating');
expect(tracker?.operationKind).toBe('reapply_configuration');
expect(tracker?.previousVersion).toBe('0.83.0');
expect(tracker?.previousProcessStart).toBe(1);
expect(tracker?.error).toBeUndefined();
});
it('marks a reserved remote reapply as failed when the peer rejects, without leaving a false updating row for a second request', async () => {
mockTarget();
mockMeta(ONLINE());
vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
const url = String(input);
try {
if (new URL(url).hostname === 'api.github.com') {
return new Response(JSON.stringify({ tag_name: 'v0.99.0' }), { status: 200 });
}
} catch { /* fall through */ }
return new Response(JSON.stringify({
error: 'An image operation is already in progress.',
code: 'IMAGE_OPERATION_IN_FLIGHT',
}), { status: 409 });
});
const res = await request(app)
.post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`)
.set('Authorization', adminAuth);
expect(res.status).toBe(502);
const tracker = FleetUpdateTrackerService.getInstance().get(proxyNodeId);
expect(tracker?.status).toBe('failed');
expect(tracker?.error).toMatch(/already in progress/i);
});
});
@@ -14,6 +14,7 @@ function mk(over: Partial<UpdateTracker>): UpdateTracker {
previousVersion: null,
previousProcessStart: null,
wasOffline: false,
operationKind: 'update',
...over,
};
}
@@ -270,4 +270,46 @@ describe('ImageOperationService', () => {
expect(changed).not.toBe(baseline);
});
it('executeClaimedComposeReapply transitions to recreating and watches helper exit before trigger', async () => {
const callOrder: string[] = [];
vi.spyOn(SelfUpdateService.getInstance(), 'getResolvedComposeImageForUpdate').mockResolvedValue(null);
vi.spyOn(SelfUpdateService.getInstance(), 'getComposeServiceName').mockReturnValue('sencho');
vi.spyOn(SelfUpdateService.getInstance(), 'onceHelperExit').mockImplementation(() => {
callOrder.push('watch');
});
vi.spyOn(SelfUpdateService.getInstance(), 'triggerComposeReapply').mockImplementation(async () => {
callOrder.push('trigger');
});
vi.spyOn(SelfUpdateService.getInstance(), 'getLastError').mockReturnValue(null);
const service = ImageOperationService.getInstance();
const claim = await service.claimComposeReapply();
expect(claim).toEqual({ ok: true });
const result = await service.executeClaimedComposeReapply();
const current = await service.getCurrentOperation();
expect(result).toEqual({ ok: true });
expect(current?.kind).toBe('compose_reapply');
expect(current?.state).toBe('recreating');
expect(callOrder).toEqual(['watch', 'trigger']);
});
it('reconcileOnStartup resolves compose_reapply via marker-only success without pin match', async () => {
const service = ImageOperationService.getInstance();
vi.spyOn(SelfUpdateService.getInstance(), 'getResolvedComposeImageForUpdate').mockResolvedValue(null);
vi.spyOn(SelfUpdateService.getInstance(), 'getComposeServiceName').mockReturnValue('sencho');
const claim = await service.claimComposeReapply();
expect(claim).toEqual({ ok: true });
const current = await service.getCurrentOperation();
expect(current?.kind).toBe('compose_reapply');
expect(current?.targetImageRef).toBeNull();
const markerPath = path.join(tmpDir, `image-op-success-${current!.operationId}.json`);
await fs.writeFile(markerPath, JSON.stringify({ ok: true, operationId: current!.operationId }), 'utf8');
await service.reconcileOnStartup();
const resolved = await service.getCurrentOperation();
expect(resolved?.state).toBe('succeeded');
});
});
@@ -21,6 +21,7 @@ import {
} from '../helpers/selfUpdateCompose';
import {
buildComposeReadArgs,
buildComposeConfigValidateArgs,
buildSelfUpdateComposeCmd,
buildSelfUpdateRunArgs,
shQuote,
@@ -302,3 +303,22 @@ describe('buildSelfUpdateRunArgs (repinWritable branch)', () => {
expect(args).not.toContain('/opt/sencho:/opt/sencho:rw');
});
});
describe('buildComposeConfigValidateArgs', () => {
it('runs compose config in a throwaway helper with the working dir mounted read-only', () => {
const args = buildComposeConfigValidateArgs({
workingDir: '/opt/sencho',
imageName: 'saelix/sencho:1.0.0',
configFiles: 'docker-compose.yml,/opt/sencho/override.yml',
hostBindMounts: [{ source: '/etc/sencho', destination: '/etc/sencho' }],
});
expect(args).toContain('/opt/sencho:/opt/sencho:ro');
expect(args).toContain('/var/run/docker.sock:/var/run/docker.sock');
expect(args).toContain('/etc/sencho:/etc/sencho:ro');
const cmd = args[args.length - 1];
expect(cmd).toContain('docker compose');
expect(cmd).toContain('config');
expect(cmd).toContain(shQuote('docker-compose.yml'));
expect(cmd).toContain(shQuote('/opt/sencho/override.yml'));
});
});
+200 -25
View File
@@ -62,6 +62,8 @@ import { PROXY_TIER_HEADER, deployProvenanceHeaders } from '../services/license-
import { LicenseService } from '../services/LicenseService';
const updateTracker = FleetUpdateTrackerService.getInstance();
/** Sync lock for remote reapply while meta is fetched (before the pollable tracker exists). */
const remoteReapplyDispatching = new Set<number>();
const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull failure
// Shown in the Node Updates UI when a node's image is pinned in a way Fleet
// cannot repin (digest or an unresolved value). Node-neutral so it reads the
@@ -1114,6 +1116,7 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
let remoteImagePinKind: ImagePinKind | null = null;
let remoteUpdateBlocked = false;
let remoteImageChannel: 'community' | 'hardened' | 'unknown' | null = null;
let remoteCapabilities: string[] = [];
if (node.type === 'local') {
version = gatewayVersion;
} else {
@@ -1125,8 +1128,18 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
remoteImagePinKind = meta.imagePinKind;
remoteUpdateBlocked = meta.updateBlocked;
remoteImageChannel = meta.imageChannel;
remoteCapabilities = meta.capabilities ?? [];
}
const isReapply = tracker?.operationKind === 'reapply_configuration';
const earlyFailMsg = isReapply
? (node.type === 'local'
? 'Local reapply did not complete. The container may not have restarted; check Docker logs on the host.'
: 'Reapply may have failed. The node is still running and its process start time has not changed.')
: (node.type === 'local'
? 'Local update did not complete. The container may not have restarted; check Docker logs on the host.'
: 'Update may have failed. The node is still running and its version has not changed.');
if (tracker?.status === 'updating') {
const elapsed = Date.now() - tracker.startedAt;
@@ -1136,7 +1149,9 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
if (elapsed > UPDATE_TIMEOUT_MS) {
if (debug) console.debug('[Fleet:debug] Node', node.id, 'timed out after', Math.round(elapsed / 1000) + 's');
resolveTerminal(node, tracker, 'timeout', UPDATE_TIMEOUT_MSG);
resolveTerminal(node, tracker, 'timeout', isReapply
? 'Node did not come back online within 5 minutes after reapply.'
: UPDATE_TIMEOUT_MSG);
} else if (node.type === 'remote') {
if (remoteUpdateError) {
if (debug) console.debug('[Fleet:debug] Node', node.id, 'reported pull failure:', remoteUpdateError);
@@ -1146,10 +1161,9 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
if (debug) console.debug('[Fleet:debug] Node', node.id, 'went offline (restarting)');
updateTracker.set(node.id, { ...tracker, wasOffline: true });
}
} else if (isValidVersion(version) && version !== tracker.previousVersion) {
// Signal 1: a valid, different version. A null/unparseable version
// from a transient /api/meta blip is NOT a version change, so it
// must not complete a still-running, same-process node here.
} else if (!isReapply && isValidVersion(version) && version !== tracker.previousVersion) {
// Signal 1: a valid, different version. Skipped for reapply because
// the authored image/version is not expected to change.
if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 1 (version changed):', tracker.previousVersion, '->', version);
updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed'));
} else if (
@@ -1173,20 +1187,20 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 3 (offline then online, startedAt unavailable)');
updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed'));
} else if (
!isReapply &&
elapsed > 15_000 &&
isValidVersion(version) &&
gatewayValid &&
!semver.lt(version, compareVersion!)
) {
// Signal 4: remote is now at or above gateway version (after
// minimum processing time). Catches fast restarts where the 5s
// polling interval misses the offline window and startedAt
// hasn't been observed to change yet.
// minimum processing time). Never used for reapply: an already
// current node would false-complete before the helper runs.
if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 4 (version >= compare target):', version, '>=', compareVersion);
updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed'));
} else if (elapsed > EARLY_FAIL_MS) {
if (debug) console.debug('[Fleet:debug] Node', node.id, 'early fail after', Math.round(elapsed / 1000) + 's - no signals detected');
resolveTerminal(node, tracker, 'failed', 'Update may have failed. The node is still running and its version has not changed.');
resolveTerminal(node, tracker, 'failed', earlyFailMsg);
}
} else if (node.type === 'local') {
// Local node has only two failure signals: an explicit pull/spawn
@@ -1202,7 +1216,7 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
selfUpdate.clearLastError();
} else if (elapsed > EARLY_FAIL_MS) {
if (debug) console.debug('[Fleet:debug] Local node', node.id, 'early fail after', Math.round(elapsed / 1000) + 's');
resolveTerminal(node, tracker, 'failed', 'Local update did not complete. The container may not have restarted; check Docker logs on the host.');
resolveTerminal(node, tracker, 'failed', earlyFailMsg);
}
}
}
@@ -1286,6 +1300,10 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
updateBlocked,
updateBlockedReason,
imageChannel,
operationKind: currentTracker?.operationKind ?? null,
canReapplyCompose: node.type === 'local'
? SelfUpdateService.getInstance().isAvailable()
: remoteOnline && remoteCapabilities.includes('self-update'),
};
}),
);
@@ -1305,6 +1323,8 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
skipActive: false,
skippedVersion: null,
...EMPTY_PIN_STATUS,
operationKind: null,
canReapplyCompose: false,
};
});
@@ -1345,17 +1365,46 @@ fleetRouter.get('/update-status/release-notes', authMiddleware, async (req: Requ
// repin a semver-pinned compose to that release. It is omitted otherwise (never
// sent as null/invalid), and an older remote that predates this field simply
// ignores the extra body key and behaves as before.
function postSystemUpdate(target: { apiUrl: string; apiToken: string }, targetVersion?: string) {
function postSystemEndpoint(
target: { apiUrl: string; apiToken: string },
endpoint: '/api/system/update' | '/api/system/reapply-compose',
body: Record<string, unknown> = {},
) {
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
return fetch(`${target.apiUrl.replace(/\/$/, '')}/api/system/update`, {
return fetch(`${target.apiUrl.replace(/\/$/, '')}${endpoint}`, {
method: 'POST',
headers,
body: JSON.stringify(targetVersion ? { targetVersion } : {}),
body: JSON.stringify(body),
signal: AbortSignal.timeout(10000),
});
}
function postSystemUpdate(target: { apiUrl: string; apiToken: string }, targetVersion?: string) {
return postSystemEndpoint(target, '/api/system/update', targetVersion ? { targetVersion } : {});
}
function postSystemReapplyCompose(target: { apiUrl: string; apiToken: string }) {
return postSystemEndpoint(target, '/api/system/reapply-compose');
}
/** Clear a terminal tracker row, or time out a stale in-flight one. Returns a
* conflict message when another update/reapply is still actively running. */
function beginTrackerOperation(nodeId: number, conflictError: string): string | null {
const existing = updateTracker.get(nodeId);
if (existing?.status === 'updating') {
if (Date.now() - existing.startedAt > UPDATE_TIMEOUT_MS) {
updateTracker.set(nodeId, updateTracker.resolve(existing, 'timeout', UPDATE_TIMEOUT_MSG));
} else {
return conflictError;
}
}
if (existing && (existing.status === 'timeout' || existing.status === 'failed' || existing.status === 'completed')) {
updateTracker.delete(nodeId);
}
return null;
}
function parseRemoteUpdateFailure(payload: unknown): { error: string; code?: string } {
if (!payload || typeof payload !== 'object') {
return { error: 'Remote node rejected update request.' };
@@ -1435,18 +1484,10 @@ fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, r
const requestedTarget = parseRequestedTargetVersion(req, res);
if (requestedTarget === null) return; // invalid supplied value; 400 already sent
const existing = updateTracker.get(nodeId);
if (existing?.status === 'updating') {
if (Date.now() - existing.startedAt > UPDATE_TIMEOUT_MS) {
updateTracker.set(nodeId, updateTracker.resolve(existing, 'timeout', UPDATE_TIMEOUT_MSG));
} else {
res.status(409).json({ error: 'Update already in progress for this node.' });
return;
}
}
// Clear terminal states to allow retry.
if (existing && (existing.status === 'timeout' || existing.status === 'failed' || existing.status === 'completed')) {
updateTracker.delete(nodeId);
const conflict = beginTrackerOperation(nodeId, 'Update already in progress for this node.');
if (conflict) {
res.status(409).json({ error: conflict });
return;
}
console.log('[Fleet] Update triggered for node', node.name, node.type);
@@ -1548,6 +1589,140 @@ fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, r
}
});
fleetRouter.post('/nodes/:nodeId/reapply-compose', authMiddleware, async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
try {
const nodeId = parseIntParam(req, res, 'nodeId', 'node ID');
if (nodeId === null) return;
const db = DatabaseService.getInstance();
const node = db.getNode(nodeId);
if (!node) {
res.status(404).json({ error: 'Node not found' });
return;
}
const conflict = beginTrackerOperation(nodeId, 'An update or reapply is already in progress for this node.');
if (conflict) {
res.status(409).json({ error: conflict });
return;
}
console.log('[Fleet] Compose reapply triggered for node', node.name, node.type);
if (node.type === 'local') {
const selfUpdate = SelfUpdateService.getInstance();
if (!selfUpdate.isAvailable()) {
res.status(503).json({ error: 'Compose reapply unavailable on the local node.' });
return;
}
const claim = await ImageOperationService.getInstance().claimComposeReapply();
if (!claim.ok) {
res.status(409).json({ error: 'An image operation is already in progress.', code: claim.failureCode });
return;
}
updateTracker.set(
nodeId,
updateTracker.create('updating', getSenchoVersion(), null, undefined, undefined, 'reapply_configuration'),
);
res.status(202).json({ message: 'Compose reapply initiated on local node. The server will restart shortly.' });
setTimeout(() => {
ImageOperationService.getInstance().executeClaimedComposeReapply().catch(error => {
console.error('[ImageOperation] Unexpected compose reapply failure:', error);
});
}, 500);
return;
}
// Sync lock before any await so a concurrent reapply gets 409 without
// racing the remote POST. The pollable tracker is created only after meta
// is known (full process identity), immediately before dispatch.
if (remoteReapplyDispatching.has(nodeId)) {
res.status(409).json({ error: 'An update or reapply is already in progress for this node.' });
return;
}
remoteReapplyDispatching.add(nodeId);
const failOwnedTracker = (
error: string,
code?: string,
previousVersion: string | null = null,
previousProcessStart: number | null = null,
) => {
const current = updateTracker.get(nodeId);
if (current?.status !== 'updating' || current.operationKind !== 'reapply_configuration') return;
updateTracker.set(
nodeId,
updateTracker.create('failed', previousVersion, previousProcessStart, error, code, 'reapply_configuration'),
);
};
try {
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) {
const error = formatNoTargetError(node);
res.status(503).json({ error });
return;
}
const meta = await NodeRegistry.getInstance().fetchMetaForNode(node.id);
if (!meta.online) {
const error = 'Remote node is unreachable. Verify the node is running and the API URL is correct.';
res.status(503).json({ error });
return;
}
if (!meta.capabilities.includes('self-update')) {
const error = 'Remote node does not support compose reapply. It may need to be updated manually first.';
res.status(503).json({ error });
return;
}
// Digest pins and updateBlocked are intentional non-gates: reapply never
// repins the image, so blocked update rows remain eligible.
// Reserve before the remote POST so a concurrent reapply still sees
// 'updating' after this request leaves the dispatch set in finally.
updateTracker.set(
nodeId,
updateTracker.create(
'updating',
meta.version,
meta.startedAt,
undefined,
undefined,
'reapply_configuration',
),
);
const response = await postSystemReapplyCompose(target);
if (!response.ok) {
const failure = parseRemoteUpdateFailure(await response.json().catch(() => null));
failOwnedTracker(failure.error, failure.code, meta.version, meta.startedAt);
res.status(502).json(failure);
return;
}
res.status(202).json({ message: `Compose reapply initiated on ${node.name}.` });
} finally {
remoteReapplyDispatching.delete(nodeId);
}
} catch (error) {
console.error('[Fleet] Node compose reapply error:', error);
const errorMsg = getErrorMessage(error, 'Failed to trigger compose reapply.');
const failedNodeId = parseInt(req.params.nodeId as string, 10);
if (!isNaN(failedNodeId)) {
remoteReapplyDispatching.delete(failedNodeId);
const current = updateTracker.get(failedNodeId);
if (current?.status === 'updating' && current.operationKind === 'reapply_configuration') {
updateTracker.set(
failedNodeId,
updateTracker.create('failed', null, null, errorMsg, undefined, 'reapply_configuration'),
);
}
}
res.status(500).json({ error: 'Failed to trigger compose reapply.' });
}
});
fleetRouter.post('/update-all', authMiddleware, async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
try {
+20
View File
@@ -175,3 +175,23 @@ systemUpdateRouter.post('/update', async (req: Request, res: Response): Promise<
});
}, 500);
});
systemUpdateRouter.post('/reapply-compose', async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
const selfUpdate = SelfUpdateService.getInstance();
if (!selfUpdate.isAvailable()) {
res.status(503).json({ error: 'Compose reapply unavailable. Sencho must be deployed via Docker Compose.' });
return;
}
const claim = await ImageOperationService.getInstance().claimComposeReapply();
if (!claim.ok) {
res.status(409).json({ error: 'An image operation is already in progress.', code: claim.failureCode });
return;
}
res.status(202).json({ message: 'Compose reapply initiated. The server will restart shortly.' });
setTimeout(() => {
ImageOperationService.getInstance().executeClaimedComposeReapply().catch(error => {
console.error('[ImageOperation] Unexpected compose reapply failure:', error);
});
}, 500);
});
@@ -1,3 +1,5 @@
export type FleetOperationKind = 'update' | 'reapply_configuration';
export interface UpdateTracker {
status: 'updating' | 'completed' | 'timeout' | 'failed';
startedAt: number;
@@ -11,6 +13,8 @@ export interface UpdateTracker {
wasOffline: boolean;
/** Timestamp when the tracker transitioned to a terminal state (completed/failed/timeout). */
resolvedAt?: number;
/** Distinguishes version updates from compose reapply so poll heuristics stay correct. */
operationKind: FleetOperationKind;
}
export type TerminalStatus = 'completed' | 'failed' | 'timeout';
@@ -61,13 +65,15 @@ export class FleetUpdateTrackerService {
return this.trackers.size;
}
/** Create a new tracker with `startedAt=now` and resolvedAt set if terminal. */
/** Create a new tracker with `startedAt=now` and resolvedAt set if terminal.
* `operationKind` defaults to `'update'` so existing call sites stay unchanged. */
public create(
status: UpdateTracker['status'],
previousVersion: string | null,
previousProcessStart: number | null,
error?: string,
code?: string,
operationKind: FleetOperationKind = 'update',
): UpdateTracker {
const now = Date.now();
return {
@@ -78,6 +84,7 @@ export class FleetUpdateTrackerService {
wasOffline: false,
error,
code,
operationKind,
resolvedAt: status !== 'updating' ? now : undefined,
};
}
+67 -26
View File
@@ -8,7 +8,7 @@ import type { ImagePinKind } from '../helpers/selfUpdateCompose';
import type { LocalRegistryAccess } from './hardenedEntitlementTypes';
import { getAuthToken, httpRequest } from './registry-api';
export type ImageOperationKind = 'switch' | 'update' | 'community_update';
export type ImageOperationKind = 'switch' | 'update' | 'community_update' | 'compose_reapply';
export type ImageOperationState = 'pending_pull' | 'pulling' | 'patching' | 'recreating' | 'succeeded' | 'failed';
type FailureCode = 'self_update_unavailable' | 'entitlement_denied' | 'preflight_mismatch' | 'compose_unavailable' | 'registry_access_unavailable' | 'update_failed' | 'interrupted_by_restart';
@@ -146,32 +146,12 @@ export class ImageOperationService {
public async claimCommunityUpdate(options?: { targetVersion?: string }): Promise<
{ ok: true } | { ok: false; failureCode: 'IMAGE_OPERATION_IN_FLIGHT' }
> {
const selfUpdate = SelfUpdateService.getInstance();
const resolved = await selfUpdate.getResolvedComposeImageForUpdate();
const operation = this.newOperation(
'community_update',
resolved?.imageRef ?? null,
options?.targetVersion ?? null,
resolved?.filePath ?? null,
selfUpdate.getComposeServiceName(),
);
if (!await this.tryClaim(operation)) {
return { ok: false, failureCode: 'IMAGE_OPERATION_IN_FLIGHT' };
}
// Disk non-terminal state is the concurrency lock; clear the in-memory mutex
// so a later claim can observe the persisted pending operation.
this.claimed = false;
return { ok: true };
return this.claimComposeOperation('community_update', options?.targetVersion ?? null);
}
public async executeClaimedCommunityUpdate(options?: { targetVersion?: string }): Promise<{ ok: boolean; failureCode?: string }> {
const operation = await this.getCurrentOperation();
if (!operation || operation.kind !== 'community_update') {
return { ok: false, failureCode: 'update_failed' };
}
if (!['pending_pull', 'pulling', 'patching', 'recreating'].includes(operation.state)) {
return { ok: false, failureCode: 'update_failed' };
}
const operation = await this.getActiveClaimedOperation('community_update');
if (!operation) return { ok: false, failureCode: 'update_failed' };
const selfUpdate = SelfUpdateService.getInstance();
try {
operation.state = 'pulling';
@@ -207,6 +187,37 @@ export class ImageOperationService {
return this.executeClaimedCommunityUpdate(options);
}
public async claimComposeReapply(): Promise<
{ ok: true } | { ok: false; failureCode: 'IMAGE_OPERATION_IN_FLIGHT' }
> {
return this.claimComposeOperation('compose_reapply', null);
}
public async executeClaimedComposeReapply(): Promise<{ ok: boolean; failureCode?: string }> {
const operation = await this.getActiveClaimedOperation('compose_reapply');
if (!operation) return { ok: false, failureCode: 'update_failed' };
const selfUpdate = SelfUpdateService.getInstance();
try {
// No pull/patch for reapply: jump straight to recreating.
operation.state = 'recreating';
await this.persist(operation);
this.watchHelperExit(operation);
await selfUpdate.triggerComposeReapply({
successMarkerFile: this.successMarkerFile(operation),
successMarkerContent: JSON.stringify({ ok: true, operationId: operation.operationId }),
});
if (selfUpdate.getLastError()) {
await this.fail(operation, 'update_failed');
return { ok: false, failureCode: 'update_failed' };
}
return { ok: true };
} catch (error) {
console.error('[ImageOperation] Compose reapply failed:', error);
await this.fail(operation, 'update_failed');
return { ok: false, failureCode: 'update_failed' };
}
}
public async getOperation(operationId: string): Promise<ImageOperation | null> {
const filePath = this.operationFile(operationId);
if (!filePath) return null;
@@ -243,8 +254,9 @@ export class ImageOperationService {
const markerPath = this.successMarkerFile(operation);
for (let elapsed = 0; elapsed < 30_000; elapsed += 1_000) {
const markerOk = await this.isSuccessMarkerForOperation(markerPath, operation.operationId);
if (operation.kind === 'community_update') {
// Community success is the marker alone; floating tags may not equal targetImageRef.
if (operation.kind === 'community_update' || operation.kind === 'compose_reapply') {
// Marker-only success: community updates may leave floating tags that do
// not equal targetImageRef, and reapply never sets a target image at all.
if (markerOk) {
operation.state = 'succeeded';
operation.resolvedAt = new Date().toISOString();
@@ -284,6 +296,35 @@ export class ImageOperationService {
});
}
private async claimComposeOperation(
kind: 'community_update' | 'compose_reapply',
targetImageRef: string | null,
): Promise<{ ok: true } | { ok: false; failureCode: 'IMAGE_OPERATION_IN_FLIGHT' }> {
const selfUpdate = SelfUpdateService.getInstance();
const resolved = await selfUpdate.getResolvedComposeImageForUpdate();
const operation = this.newOperation(
kind,
resolved?.imageRef ?? null,
targetImageRef,
resolved?.filePath ?? null,
selfUpdate.getComposeServiceName(),
);
if (!await this.tryClaim(operation)) {
return { ok: false, failureCode: 'IMAGE_OPERATION_IN_FLIGHT' };
}
// Disk non-terminal state is the concurrency lock; clear the in-memory mutex
// so a later claim can observe the persisted pending operation.
this.claimed = false;
return { ok: true };
}
private async getActiveClaimedOperation(kind: ImageOperationKind): Promise<ImageOperation | null> {
const operation = await this.getCurrentOperation();
if (!operation || operation.kind !== kind) return null;
if (!['pending_pull', 'pulling', 'patching', 'recreating'].includes(operation.state)) return null;
return operation;
}
private newOperation(kind: ImageOperationKind, previousImageRef: string | null, targetImageRef: string | null, composeFilePath: string | null, serviceName: string | null, preflightFingerprint?: string): ImageOperation {
return {
schemaVersion: 1,
+65 -2
View File
@@ -205,6 +205,27 @@ export function buildSelfUpdateRunArgs(
];
}
/**
* Build the argv for a throwaway helper that runs `docker compose … config`
* against the host compose project. Reuses the recreate helper's mount layout
* (socket + working dir + host binds) without mounting /app/data, since
* validation is read-only. Pure and exported for unit testing.
*/
export function buildComposeConfigValidateArgs(
ctx: Pick<ComposeContext, 'workingDir' | 'imageName' | 'hostBindMounts'> & { configFiles: string },
): string[] {
const { workingDir, imageName, hostBindMounts, configFiles } = ctx;
const fFlags = configFiles.split(',').flatMap(f => {
const trimmed = f.trim();
return trimmed ? ['-f', trimmed] : [];
});
const composeCmd = ['docker compose', ...fFlags.map(shQuote), 'config'].join(' ');
return buildSelfUpdateRunArgs(
{ workingDir, imageName, dataDirHost: null, hostBindMounts },
composeCmd,
);
}
class SelfUpdateService {
private static instance: SelfUpdateService;
private canSelfUpdate = false;
@@ -527,6 +548,46 @@ class SelfUpdateService {
this.spawnHelper(env, composeCopy, options?.successMarkerFile, options?.successMarkerContent);
}
/**
* Recreate the Sencho service from the exact current on-disk Compose project
* without pulling or rewriting the image reference. Used by Fleet "Reapply
* configuration". Validates the authored compose via a throwaway helper
* before the last-breath recreate so invalid config fails before shutdown.
*/
async triggerComposeReapply(options?: {
successMarkerFile?: string;
successMarkerContent?: string;
}): Promise<void> {
if (!this.composeContext) return;
const env = this.buildEnv();
this.lastUpdateError = null;
this.pendingHelperExitError = undefined;
try { fs.unlinkSync(UPDATE_ERROR_FILE); } catch { /* absent is the steady state */ }
try { fs.unlinkSync(STAGED_PATCH_FILE); } catch { /* absent is the steady state */ }
const { workingDir, configFiles, imageName, hostBindMounts } = this.composeContext;
console.log('[SelfUpdate] Validating compose configuration before reapply...');
try {
await execFileAsync(
'docker',
buildComposeConfigValidateArgs({ workingDir, imageName, hostBindMounts, configFiles }),
{ env, timeout: 60_000, maxBuffer: 10 * 1024 * 1024 },
);
} catch (error) {
const stderr = (error as { stderr?: Buffer | string })?.stderr?.toString().trim();
const stdout = (error as { stdout?: Buffer | string })?.stdout?.toString().trim();
this.lastUpdateError =
stderr || stdout || (error as Error).message || 'Compose configuration validation failed.';
console.error('[SelfUpdate] Compose reapply validation failed:', this.lastUpdateError);
return;
}
// No pull and no compose rewrite: the authored image ref is authoritative.
// Skip dangling-image prune (nothing was pulled).
this.spawnHelper(env, undefined, options?.successMarkerFile, options?.successMarkerContent, false);
}
/**
* Spawn the "last breath" helper container that recreates Sencho (and, when a
* repin is staged, copies the rewritten compose file onto the host first).
@@ -538,6 +599,7 @@ class SelfUpdateService {
composeCopy?: ComposeCopy,
successMarkerFile?: string,
successMarkerContent?: string,
pruneOnUpdateOverride?: boolean,
): void {
if (!this.composeContext) return;
const { workingDir, configFiles, serviceName, imageName, dataDirHost, hostBindMounts } = this.composeContext;
@@ -551,8 +613,9 @@ class SelfUpdateService {
// Opt-out (default ON): after a clean recreate, prune the dangling image
// layers the pull orphaned. Read fresh so this node honors its own setting.
const stderrTmp = '/tmp/_sencho_err';
const pruneOnUpdate =
DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1';
const pruneOnUpdate = pruneOnUpdateOverride ?? (
DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1'
);
const composeCmd = buildSelfUpdateComposeCmd(
fFlags,
serviceName,
+2
View File
@@ -42,6 +42,7 @@ export const AUDIT_ROUTE_SUMMARIES: Record<string, string> = {
'POST /system/networks/delete': 'Deleted networks',
'POST /system/networks': 'Created network',
'POST /system/console-token': 'Generated console token',
'POST /system/reapply-compose': 'Triggered compose reapply',
// Node management
'POST /nodes': 'Added node',
@@ -91,6 +92,7 @@ export const AUDIT_ROUTE_SUMMARIES: Record<string, string> = {
'DELETE /fleet/snapshots': 'Deleted fleet backup',
'POST /fleet/snapshots/*/restore': 'Restored fleet backup',
'POST /fleet/nodes/*/update': 'Triggered fleet node update',
'POST /fleet/nodes/*/reapply-compose': 'Triggered fleet node compose reapply',
'POST /fleet/update-all': 'Triggered fleet-wide update',
'POST /fleet/role/reanchor': 'Re-anchored fleet replica',
'POST /fleet/role/demote': 'Demoted fleet replica to control',