fix: repin semver compose tags during fleet self-update (#1587)

* fix: repin semver compose tags during fleet self-update

Fleet updates failed when docker-compose.yml pinned a semver tag because recreate reused the on-disk pin. Pull the target image first, rewrite semver pins via the update helper, and block digest or unresolved pins with fast 409s.

* fix: update OFFLINE_META shape in capability and node-registry meta tests
This commit is contained in:
Anso
2026-07-07 14:40:50 -04:00
committed by GitHub
parent dbe230eef3
commit e12602091a
31 changed files with 1460 additions and 74 deletions
@@ -61,6 +61,8 @@ describe('fetchRemoteMeta Authorization header', () => {
startedAt: null,
updateError: null,
online: false,
imagePinKind: null,
updateBlocked: false,
});
});
});
@@ -233,9 +233,11 @@ describe('GET /api/fleet/update-status (pilot-agent)', () => {
startedAt: 1700000000,
updateError: null,
online: true,
imagePinKind: null,
updateBlocked: false,
};
}
return { version: null, capabilities: [], startedAt: null, updateError: null, online: false };
return { version: null, capabilities: [], startedAt: null, updateError: null, online: false, imagePinKind: null, updateBlocked: false };
});
const res = await request(app).get('/api/fleet/update-status').set('Authorization', authHeader);
@@ -254,6 +256,8 @@ describe('GET /api/fleet/update-status (pilot-agent)', () => {
startedAt: null,
updateError: null,
online: false,
imagePinKind: null,
updateBlocked: false,
});
const res = await request(app).get('/api/fleet/update-status').set('Authorization', authHeader);
@@ -36,6 +36,8 @@ const META_ONLINE_OUTDATED: RemoteMeta = {
startedAt: 1,
updateError: null,
online: true,
imagePinKind: null,
updateBlocked: false,
};
const META_OFFLINE: RemoteMeta = {
@@ -44,6 +46,8 @@ const META_OFFLINE: RemoteMeta = {
startedAt: null,
updateError: null,
online: false,
imagePinKind: null,
updateBlocked: false,
};
const META_NO_SELF_UPDATE: RemoteMeta = {
@@ -251,7 +255,7 @@ describe('GET /api/fleet/update-status (remote-meta cache invalidation)', () =>
mockTargetForPilot();
mockCompareTargetFetch();
// Remote now reports a different version than before the update (signal 1).
mockMeta({ version: '0.99.0', capabilities: ['stacks'], startedAt: 2, updateError: null, online: true });
mockMeta({ version: '0.99.0', capabilities: ['stacks'], startedAt: 2, updateError: null, online: true, imagePinKind: null, updateBlocked: false });
const tracker = FleetUpdateTrackerService.getInstance();
tracker.set(proxyNodeId, tracker.create('updating', '0.83.0', null));
@@ -268,7 +272,7 @@ describe('GET /api/fleet/update-status (remote-meta cache invalidation)', () =>
it('does not drop the cache on a steady-state completed poll (transition guard)', async () => {
mockTargetForPilot();
mockCompareTargetFetch();
mockMeta({ version: '0.99.0', capabilities: ['stacks'], startedAt: 2, updateError: null, online: true });
mockMeta({ version: '0.99.0', capabilities: ['stacks'], startedAt: 2, updateError: null, online: true, imagePinKind: null, updateBlocked: false });
// Already completed before this poll: no transition, so no invalidation.
const tracker = FleetUpdateTrackerService.getInstance();
@@ -39,6 +39,8 @@ const ONLINE = (over: Partial<RemoteMeta> = {}): RemoteMeta => ({
startedAt: 1,
updateError: null,
online: true,
imagePinKind: null,
updateBlocked: false,
...over,
});
@@ -54,6 +54,8 @@ describe('NodeRegistry.fetchMetaForNode', () => {
startedAt: null,
updateError: null,
online: false,
imagePinKind: null,
updateBlocked: false,
});
expect(axiosSpy).not.toHaveBeenCalled();
db.deleteNode(nodeId);
@@ -25,7 +25,7 @@ afterAll(() => cleanupTestDb(tmpDir));
afterEach(() => vi.restoreAllMocks());
const ONLINE = { startedAt: null, updateError: null, online: true } as const;
const ONLINE = { startedAt: null, updateError: null, online: true, imagePinKind: null, updateBlocked: false } as const;
const capable: RemoteMeta = { version: '0.93.0', capabilities: ['fleet', 'cross-node-rbac'], ...ONLINE };
const incapable: RemoteMeta = { version: '0.92.0', capabilities: ['fleet', 'labels'], ...ONLINE };
@@ -42,7 +42,7 @@ describe('remoteSupportsCrossNodeRbac', () => {
it('fails closed when the remote is offline (empty capabilities)', async () => {
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode')
.mockResolvedValue({ version: null, capabilities: [], startedAt: null, updateError: null, online: false });
.mockResolvedValue({ version: null, capabilities: [], startedAt: null, updateError: null, online: false, imagePinKind: null, updateBlocked: false });
expect(await remoteSupportsCrossNodeRbac(NODE_ID)).toBe(false);
});
@@ -50,7 +50,7 @@ describe('remoteSupportsCrossNodeRbac', () => {
// A 0.0.0-dev image reports version null (non-semver) but is reachable and
// genuinely advertises the capability; it must not be wrongly denied.
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode')
.mockResolvedValue({ version: null, capabilities: ['fleet', 'cross-node-rbac'], startedAt: null, updateError: null, online: true });
.mockResolvedValue({ version: null, capabilities: ['fleet', 'cross-node-rbac'], startedAt: null, updateError: null, online: true, imagePinKind: null, updateBlocked: false });
expect(await remoteSupportsCrossNodeRbac(NODE_ID)).toBe(true);
});
@@ -0,0 +1,304 @@
/**
* Pure-unit coverage for the compose image handling that powers pinned
* self-update (GitHub issue: a version-pinned compose image could never update
* because recreate reused the running image). Split across the pure helper
* module and the three self-update argv builders that carry the repin handoff:
* - classifyImagePin / buildTargetImageRef / isValidImageRef,
* - resolveServiceImageFromContents (reverse -f precedence),
* - patchComposeServiceImage (comment-preserving round-trip),
* - buildComposeReadArgs (throwaway cat container),
* - the composeCopy branch of buildSelfUpdateComposeCmd,
* - the repinWritable branch of buildSelfUpdateRunArgs.
*/
import { describe, expect, it } from 'vitest';
import {
classifyImagePin,
buildTargetImageRef,
isRepinBlocked,
isValidImageRef,
resolveServiceImageFromContents,
patchComposeServiceImage,
} from '../helpers/selfUpdateCompose';
import {
buildComposeReadArgs,
buildSelfUpdateComposeCmd,
buildSelfUpdateRunArgs,
shQuote,
} from '../services/SelfUpdateService';
describe('isRepinBlocked', () => {
it('blocks digest and unknown pins only', () => {
expect(isRepinBlocked('digest')).toBe(true);
expect(isRepinBlocked('unknown')).toBe(true);
expect(isRepinBlocked('semver')).toBe(false);
expect(isRepinBlocked('floating')).toBe(false);
});
});
describe('classifyImagePin', () => {
it('classifies an explicit semver tag (with and without v prefix) as semver', () => {
expect(classifyImagePin('saelix/sencho:0.93.3')).toBe('semver');
expect(classifyImagePin('saelix/sencho:v1.2.3')).toBe('semver');
expect(classifyImagePin('ghcr.io/studio-saelix/sencho:1.0.0-rc.1')).toBe('semver');
});
it('classifies latest, implicit, and other moving tags as floating', () => {
expect(classifyImagePin('saelix/sencho:latest')).toBe('floating');
expect(classifyImagePin('saelix/sencho')).toBe('floating'); // implicit latest
expect(classifyImagePin('saelix/sencho:dev')).toBe('floating');
expect(classifyImagePin('saelix/sencho:edge')).toBe('floating');
});
it('does not mistake a registry port for a tag', () => {
// The colon before the last slash is a registry port, not a tag separator,
// so an untagged image on a custom-port registry is floating, not semver.
expect(classifyImagePin('registry.example.com:5000/sencho')).toBe('floating');
expect(classifyImagePin('registry.example.com:5000/sencho:0.93.3')).toBe('semver');
});
it('classifies a digest pin as digest', () => {
expect(classifyImagePin('saelix/sencho@sha256:abc123')).toBe('digest');
expect(classifyImagePin('saelix/sencho:0.93.3@sha256:abc123')).toBe('digest');
});
it('classifies interpolated, empty, or blank references as unknown', () => {
expect(classifyImagePin('saelix/sencho:${SENCHO_TAG}')).toBe('unknown');
expect(classifyImagePin('${SENCHO_IMAGE}')).toBe('unknown');
expect(classifyImagePin('')).toBe('unknown');
expect(classifyImagePin(' ')).toBe('unknown');
});
});
describe('buildTargetImageRef', () => {
it('swaps only the tag and keeps the registry and repository', () => {
expect(buildTargetImageRef('saelix/sencho:0.93.3', '0.94.0')).toBe('saelix/sencho:0.94.0');
expect(buildTargetImageRef('ghcr.io/studio-saelix/sencho:1.0.0', '1.1.0')).toBe(
'ghcr.io/studio-saelix/sencho:1.1.0',
);
});
it('preserves a v prefix on the current tag', () => {
expect(buildTargetImageRef('saelix/sencho:v0.93.3', '0.94.0')).toBe('saelix/sencho:v0.94.0');
});
it('preserves a -dev repository variant (only the tag is replaced)', () => {
expect(buildTargetImageRef('ghcr.io/studio-saelix/sencho-dev:0.93.3', '0.94.0')).toBe(
'ghcr.io/studio-saelix/sencho-dev:0.94.0',
);
});
it('drops any digest suffix on the current ref', () => {
expect(buildTargetImageRef('saelix/sencho:0.93.3@sha256:abc123', '0.94.0')).toBe(
'saelix/sencho:0.94.0',
);
});
it('adds a tag when the current ref has none', () => {
expect(buildTargetImageRef('saelix/sencho', '0.94.0')).toBe('saelix/sencho:0.94.0');
expect(buildTargetImageRef('registry.example.com:5000/sencho', '0.94.0')).toBe(
'registry.example.com:5000/sencho:0.94.0',
);
});
});
describe('isValidImageRef', () => {
it('accepts plausible registry/repo:tag and digest references', () => {
expect(isValidImageRef('saelix/sencho:0.93.3')).toBe(true);
expect(isValidImageRef('ghcr.io/studio-saelix/sencho-dev:v1.2.3')).toBe(true);
expect(isValidImageRef('registry.example.com:5000/sencho@sha256:abc')).toBe(true);
});
it('rejects empty, whitespace, and control-character references', () => {
expect(isValidImageRef('')).toBe(false);
expect(isValidImageRef('saelix/sencho :0.93.3')).toBe(false);
expect(isValidImageRef('saelix/sencho\n')).toBe(false);
});
it('rejects references with shell metacharacters or a leading separator', () => {
expect(isValidImageRef('saelix/sencho;rm -rf /')).toBe(false);
expect(isValidImageRef('$(touch pwned)')).toBe(false);
expect(isValidImageRef(':leadingcolon')).toBe(false);
});
it('rejects an implausibly long reference', () => {
expect(isValidImageRef('a'.repeat(513))).toBe(false);
});
});
describe('resolveServiceImageFromContents', () => {
const base = 'services:\n sencho:\n image: saelix/sencho:0.93.3\n';
it('returns the service image and classifies its pin', () => {
const resolved = resolveServiceImageFromContents([{ filePath: '/c/base.yml', content: base }], 'sencho');
expect(resolved).toEqual({
filePath: '/c/base.yml',
imageRef: 'saelix/sencho:0.93.3',
fileContent: base,
pinKind: 'semver',
});
});
it('lets a later -f override win (reverse precedence)', () => {
// Docker Compose merges later -f files over earlier ones, so the resolver
// must scan in reverse and return the highest-precedence declaration.
const override = 'services:\n sencho:\n image: saelix/sencho:latest\n';
const resolved = resolveServiceImageFromContents(
[
{ filePath: '/c/base.yml', content: base },
{ filePath: '/c/override.yml', content: override },
],
'sencho',
);
expect(resolved?.filePath).toBe('/c/override.yml');
expect(resolved?.imageRef).toBe('saelix/sencho:latest');
expect(resolved?.pinKind).toBe('floating');
});
it('falls back to the base file when the override does not set the image', () => {
// A common override only tweaks ports/env; it must not shadow the base image.
const override = 'services:\n sencho:\n ports:\n - "1852:1852"\n';
const resolved = resolveServiceImageFromContents(
[
{ filePath: '/c/base.yml', content: base },
{ filePath: '/c/override.yml', content: override },
],
'sencho',
);
expect(resolved?.filePath).toBe('/c/base.yml');
expect(resolved?.imageRef).toBe('saelix/sencho:0.93.3');
});
it('skips a malformed override and resolves from the next readable file', () => {
const malformed = 'services:\n sencho:\n image: : : bad';
const resolved = resolveServiceImageFromContents(
[
{ filePath: '/c/base.yml', content: base },
{ filePath: '/c/broken.yml', content: malformed },
],
'sencho',
);
expect(resolved?.filePath).toBe('/c/base.yml');
});
it('returns null when no file declares a string image for the service', () => {
const noImage = 'services:\n sencho:\n build: .\n';
expect(resolveServiceImageFromContents([{ filePath: '/c/x.yml', content: noImage }], 'sencho')).toBeNull();
expect(resolveServiceImageFromContents([{ filePath: '/c/x.yml', content: base }], 'other')).toBeNull();
});
});
describe('patchComposeServiceImage', () => {
it('rewrites only the target service image and preserves comments and other keys', () => {
const content = [
'# Sencho self-hosted',
'services:',
' sencho:',
' image: saelix/sencho:0.93.3 # pinned',
' ports:',
' - "1852:1852"',
' db:',
' image: postgres:16',
'',
].join('\n');
const patched = patchComposeServiceImage(content, 'sencho', 'saelix/sencho:0.94.0');
expect(patched).toContain('image: saelix/sencho:0.94.0');
expect(patched).not.toContain('saelix/sencho:0.93.3');
// Comments, sibling service, and unrelated keys survive the round-trip.
expect(patched).toContain('# Sencho self-hosted');
expect(patched).toContain('image: postgres:16');
expect(patched).toContain('- "1852:1852"');
});
it('throws when the service has no image to patch (never silently no-ops)', () => {
const content = 'services:\n sencho:\n build: .\n';
expect(() => patchComposeServiceImage(content, 'sencho', 'saelix/sencho:0.94.0')).toThrow(/no image/i);
});
});
describe('buildComposeReadArgs', () => {
it('emits a throwaway root cat container that mounts the working dir read-only', () => {
const args = buildComposeReadArgs('/opt/sencho', 'saelix/sencho:0.93.3', '/opt/sencho/docker-compose.yml');
expect(args).toEqual([
'run', '--rm',
'--user', 'root',
'--entrypoint', 'cat',
'-v', '/opt/sencho:/opt/sencho:ro',
'-w', '/opt/sencho',
'saelix/sencho:0.93.3',
'/opt/sencho/docker-compose.yml',
]);
});
it('keeps operator paths as discrete argv data (execFile spawns no shell)', () => {
const args = buildComposeReadArgs('/srv/$(touch pwned)', 'img', '/srv/$(touch pwned)/compose.yml');
expect(args).toContain('/srv/$(touch pwned):/srv/$(touch pwned):ro');
expect(args[args.length - 1]).toBe('/srv/$(touch pwned)/compose.yml');
});
});
describe('buildSelfUpdateComposeCmd (repin copy branch)', () => {
const fFlags = ['-f', '/opt/sencho/docker-compose.yml'];
const stderrTmp = '/tmp/_sencho_err';
const errorFile = '/app/data/.sencho-update-error';
it('copies the staged compose file onto the host before recreate', () => {
const cmd = buildSelfUpdateComposeCmd(fFlags, 'sencho', stderrTmp, errorFile, false, {
stagedPath: '/app/data/.sencho-compose-patch',
targetPath: '/opt/sencho/docker-compose.yml',
});
// The copy runs before the recreate so a failed write never half-applies.
expect(cmd.indexOf('cp ')).toBeLessThan(cmd.indexOf('up -d --force-recreate'));
expect(cmd).toContain(
`cp ${shQuote('/app/data/.sencho-compose-patch')} ${shQuote('/opt/sencho/docker-compose.yml')}`,
);
});
it('aborts before recreate when the copy fails and records the error', () => {
const cmd = buildSelfUpdateComposeCmd(fFlags, 'sencho', stderrTmp, errorFile, false, {
stagedPath: '/app/data/.sencho-compose-patch',
targetPath: '/opt/sencho/docker-compose.yml',
});
// A failed copy writes the error file and exits 1 before any recreate.
expect(cmd).toContain('exit 1');
expect(cmd.indexOf('exit 1')).toBeLessThan(cmd.indexOf('up -d --force-recreate'));
expect(cmd).toContain('Failed to write the updated compose file');
});
it('shell-quotes the copy paths so metacharacters cannot break the command', () => {
const cmd = buildSelfUpdateComposeCmd(fFlags, 'sencho', stderrTmp, errorFile, false, {
stagedPath: '/app/data/x; rm -rf /',
targetPath: '/opt/sencho/y; echo pwned',
});
expect(cmd).toContain(shQuote('/app/data/x; rm -rf /'));
expect(cmd).toContain(shQuote('/opt/sencho/y; echo pwned'));
expect(cmd).not.toContain('cp /app/data/x; rm -rf /');
});
it('omits the copy step entirely when no composeCopy is supplied', () => {
const cmd = buildSelfUpdateComposeCmd(fFlags, 'sencho', stderrTmp, errorFile, false);
expect(cmd).not.toContain('cp ');
expect(cmd).not.toContain('Failed to write the updated compose file');
});
});
describe('buildSelfUpdateRunArgs (repinWritable branch)', () => {
const COMPOSE = 'COMPOSE_CMD';
it('mounts the working dir read-write only when a repin is staged', () => {
const args = buildSelfUpdateRunArgs(
{ workingDir: '/opt/sencho', imageName: 'img', dataDirHost: '/opt/sencho/data', hostBindMounts: [], repinWritable: true },
COMPOSE,
);
expect(args).toContain('/opt/sencho:/opt/sencho:rw');
expect(args).not.toContain('/opt/sencho:/opt/sencho:ro');
});
it('keeps the working dir read-only when no repin is staged (minimal write scope)', () => {
const args = buildSelfUpdateRunArgs(
{ workingDir: '/opt/sencho', imageName: 'img', dataDirHost: '/opt/sencho/data', hostBindMounts: [], repinWritable: false },
COMPOSE,
);
expect(args).toContain('/opt/sencho:/opt/sencho:ro');
expect(args).not.toContain('/opt/sencho:/opt/sencho:rw');
});
});
@@ -0,0 +1,183 @@
/**
* Route coverage for pinned-compose self-update: targetVersion validation,
* synchronous preflight (409 before 202), and the safe public /api/meta pin
* subset (imagePinKind + updateBlocked, never composeImageRef).
*/
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import { buildTargetImageRef } from '../helpers/selfUpdateCompose';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
let tmpDir: string;
let app: import('express').Express;
let adminAuth: string;
let localNodeId: number;
let SelfUpdateService: typeof import('../services/SelfUpdateService').default;
let FleetUpdateTrackerService: typeof import('../services/FleetUpdateTrackerService').FleetUpdateTrackerService;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
function mockSelfUpdateAvailable(over: {
pinInfo?: { pinKind: 'semver' | 'floating' | 'digest' | 'unknown'; composeImageRef: string; filePath: string } | null;
preflight?: { ok: true } | { ok: false; reason: string };
available?: boolean;
} = {}) {
const svc = SelfUpdateService.getInstance();
vi.spyOn(svc, 'isAvailable').mockReturnValue(over.available ?? true);
vi.spyOn(svc, 'getPinInfo').mockResolvedValue(over.pinInfo ?? null);
vi.spyOn(svc, 'canSelfUpdateTarget').mockResolvedValue(over.preflight ?? { ok: true });
vi.spyOn(svc, 'triggerUpdate').mockResolvedValue(undefined);
vi.spyOn(svc, 'getLastError').mockReturnValue(null);
}
function mockCompareTargetFetch() {
vi.spyOn(globalThis, 'fetch').mockImplementation(async () =>
new Response(JSON.stringify({ tag_name: 'v0.99.0' }), {
status: 200,
headers: { 'content-type': 'application/json' },
}),
);
}
beforeAll(async () => {
tmpDir = await setupTestDb();
({ app } = await import('../index'));
SelfUpdateService = (await import('../services/SelfUpdateService')).default;
({ FleetUpdateTrackerService } = await import('../services/FleetUpdateTrackerService'));
({ DatabaseService } = await import('../services/DatabaseService'));
localNodeId = DatabaseService.getInstance().getNodes().find(n => n.type === 'local')!.id;
adminAuth = `Bearer ${jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' })}`;
});
afterAll(() => cleanupTestDb(tmpDir));
afterEach(() => {
vi.restoreAllMocks();
const tracker = FleetUpdateTrackerService.getInstance();
for (const [id] of tracker.entries()) tracker.delete(id);
});
describe('POST /api/system/update', () => {
it('returns 400 for a supplied but invalid targetVersion', async () => {
mockSelfUpdateAvailable();
const res = await request(app)
.post('/api/system/update')
.set('Authorization', adminAuth)
.send({ targetVersion: 'not-a-version' });
expect(res.status).toBe(400);
expect(res.body?.error).toMatch(/invalid target version/i);
});
it('returns 409 when preflight blocks a digest or unknown pin', async () => {
mockSelfUpdateAvailable({
preflight: { ok: false, reason: 'Cannot repin digest pin.' },
});
const res = await request(app)
.post('/api/system/update')
.set('Authorization', adminAuth)
.send({ targetVersion: '0.99.0' });
expect(res.status).toBe(409);
expect(res.body?.code).toBe('update_blocked');
expect(res.body?.error).toMatch(/cannot repin/i);
});
it('returns 202 and schedules the update when preflight passes', async () => {
const triggerSpy = vi.spyOn(SelfUpdateService.getInstance(), 'triggerUpdate').mockResolvedValue(undefined);
mockSelfUpdateAvailable({ preflight: { ok: true } });
const res = await request(app)
.post('/api/system/update')
.set('Authorization', adminAuth)
.send({ targetVersion: '0.99.0' });
expect(res.status).toBe(202);
expect(res.body?.message).toMatch(/restart/i);
// triggerUpdate runs on res finish + delay; flush the microtask queue.
await new Promise(r => setTimeout(r, 600));
expect(triggerSpy).toHaveBeenCalledWith({ targetVersion: '0.99.0' });
});
});
describe('GET /api/meta pin subset', () => {
it('exposes imagePinKind and updateBlocked but never composeImageRef', async () => {
mockSelfUpdateAvailable({
pinInfo: { pinKind: 'semver', composeImageRef: 'saelix/sencho:0.93.3', filePath: '/opt/sencho/docker-compose.yml' },
});
const res = await request(app).get('/api/meta');
expect(res.status).toBe(200);
expect(res.body.imagePinKind).toBe('semver');
expect(res.body.updateBlocked).toBe(false);
expect(res.body).not.toHaveProperty('composeImageRef');
expect(res.body).not.toHaveProperty('targetImageRef');
});
it('reports updateBlocked=true for a digest pin', async () => {
mockSelfUpdateAvailable({
pinInfo: { pinKind: 'digest', composeImageRef: 'saelix/sencho@sha256:abc', filePath: '/opt/sencho/docker-compose.yml' },
});
const res = await request(app).get('/api/meta');
expect(res.body.imagePinKind).toBe('digest');
expect(res.body.updateBlocked).toBe(true);
});
});
describe('POST /api/fleet/nodes/:nodeId/update (local preflight)', () => {
it('returns 409 before 202 when the local pin cannot be repinned', async () => {
mockCompareTargetFetch();
mockSelfUpdateAvailable({
preflight: { ok: false, reason: 'Blocked by digest pin.' },
});
const res = await request(app)
.post(`/api/fleet/nodes/${localNodeId}/update`)
.set('Authorization', adminAuth)
.send({ targetVersion: '0.99.0' });
expect(res.status).toBe(409);
expect(res.body?.code).toBe('update_blocked');
});
});
describe('GET /api/fleet/update-status pin projection', () => {
it('includes compose pin fields for the local node on the authenticated route', async () => {
mockCompareTargetFetch();
mockSelfUpdateAvailable({
pinInfo: { pinKind: 'semver', composeImageRef: 'saelix/sencho:0.83.0', filePath: '/opt/sencho/docker-compose.yml' },
});
const res = await request(app)
.get('/api/fleet/update-status')
.set('Authorization', adminAuth);
expect(res.status).toBe(200);
const local = res.body.nodes.find((n: { nodeId: number }) => n.nodeId === localNodeId);
expect(local.imagePinKind).toBe('semver');
expect(local.composeImageRef).toBe('saelix/sencho:0.83.0');
expect(local.targetImageRef).toBe(buildTargetImageRef(local.composeImageRef, local.latestVersion));
expect(local.updateBlocked).toBe(false);
});
it('marks the local node updateBlocked when the pin is unknown', async () => {
mockCompareTargetFetch();
mockSelfUpdateAvailable({
pinInfo: { pinKind: 'unknown', composeImageRef: 'saelix/sencho:${TAG}', filePath: '/opt/sencho/docker-compose.yml' },
});
const res = await request(app)
.get('/api/fleet/update-status')
.set('Authorization', adminAuth);
const local = res.body.nodes.find((n: { nodeId: number }) => n.nodeId === localNodeId);
expect(local.updateBlocked).toBe(true);
expect(local.updateBlockedReason).toMatch(/cannot update automatically/i);
});
});
+137
View File
@@ -0,0 +1,137 @@
import { parse, parseDocument } from 'yaml';
import semver from 'semver';
/**
* How the Sencho service's compose `image:` is pinned. Drives the Fleet
* self-update behavior:
* - `semver` : rewrite the tag to the target release, then recreate.
* - `floating` : pull the compose-declared ref (e.g. `:latest`), no rewrite.
* - `digest` : `@sha256:...` pin; blocked (needs a manual digest change).
* - `unknown` : interpolated (`${VAR}`), unparseable, or the service/image is
* absent; blocked, because the tag cannot be resolved safely.
*/
export type ImagePinKind = 'floating' | 'semver' | 'digest' | 'unknown';
/** True when Fleet cannot repin or resolve the compose image automatically. */
export function isRepinBlocked(pinKind: ImagePinKind): boolean {
return pinKind === 'digest' || pinKind === 'unknown';
}
export interface ResolvedComposeImage {
/** Absolute path of the highest-precedence compose file that sets the image. */
filePath: string;
/** The verbatim compose-declared image reference. */
imageRef: string;
/** The full contents of that file, so the caller can patch it in place. */
fileContent: string;
pinKind: ImagePinKind;
}
/**
* Classify how an image reference is pinned. Aligned with the preflight
* `usesLatestTag` rule: a digest wins, an interpolated value is unknown, a tag
* that parses as semver is a pin, and everything else (implicit latest and
* non-semver moving tags) is floating.
*/
export function classifyImagePin(imageRef: string): ImagePinKind {
const ref = imageRef?.trim();
if (!ref) return 'unknown';
// Compose variable interpolation: the real tag is only known at deploy time.
if (ref.includes('${') || ref.includes('$(')) return 'unknown';
if (ref.includes('@sha256:') || ref.startsWith('sha256:')) return 'digest';
const lastSlash = ref.lastIndexOf('/');
const lastColon = ref.lastIndexOf(':');
// A colon after the last slash is a tag separator; before it is a registry port.
const tag = lastColon > lastSlash ? ref.slice(lastColon + 1) : '';
if (!tag) return 'floating'; // no explicit tag -> implicit latest
if (tag === 'latest') return 'floating';
if (semver.valid(tag.replace(/^v/, ''))) return 'semver';
return 'floating'; // other moving tag (dev, stable, edge, ...)
}
/**
* Build the target image reference for a semver-pinned install: keep the
* registry and repository (including any `-dev` variant), swap only the tag to
* the target version. Any digest suffix on the current ref is dropped. A `v`
* prefix on the current tag is preserved so `:v1.2.3` stays `:v...`.
*/
export function buildTargetImageRef(currentRef: string, targetVersion: string): string {
const ref = currentRef.split('@')[0];
const lastSlash = ref.lastIndexOf('/');
const lastColon = ref.lastIndexOf(':');
const hasTag = lastColon > lastSlash;
const currentTag = hasTag ? ref.slice(lastColon + 1) : '';
const base = hasTag ? ref.slice(0, lastColon) : ref;
const prefix = /^v\d/.test(currentTag) ? 'v' : '';
return `${base}:${prefix}${targetVersion}`;
}
/**
* Conservative image-reference validation before `docker pull`. execFile runs
* docker with no shell, so this is defense in depth, not the only guard: it
* fails closed on whitespace, control characters, or an implausibly long value.
*/
export function isValidImageRef(ref: string): boolean {
if (!ref || ref.length > 512) return false;
if (/\s/.test(ref)) return false;
return /^[a-zA-Z0-9][a-zA-Z0-9._\-/:@]*$/.test(ref);
}
/**
* Resolve the Sencho service's image from already-read compose file contents.
* Files are given in compose `-f` order; they are scanned in REVERSE so the
* highest-precedence override that explicitly sets the service image wins,
* matching how Docker Compose merges later `-f` files over earlier ones.
* Returns null when no file declares a string image for the service.
*/
export function resolveServiceImageFromContents(
files: ReadonlyArray<{ filePath: string; content: string }>,
serviceName: string,
): ResolvedComposeImage | null {
for (let i = files.length - 1; i >= 0; i--) {
const { filePath, content } = files[i];
let parsed: unknown;
try {
parsed = parse(content);
} catch {
continue; // a malformed override cannot own the image; try the next file
}
const image = extractServiceImage(parsed, serviceName);
if (image) {
return { filePath, imageRef: image, fileContent: content, pinKind: classifyImagePin(image) };
}
}
return null;
}
/** Read `services.<name>.image` as a trimmed string, or null if absent/non-string. */
function extractServiceImage(parsed: unknown, serviceName: string): string | null {
if (!isRecord(parsed)) return null;
const services = parsed.services;
if (!isRecord(services)) return null;
const service = services[serviceName];
if (!isRecord(service)) return null;
const image = service.image;
if (typeof image !== 'string') return null;
const trimmed = image.trim();
return trimmed || null;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
/**
* Rewrite only the target service's `image:` value, preserving comments,
* formatting, and every other node via a document round-trip. Throws when the
* service has no image to patch, so a caller never silently no-ops.
*/
export function patchComposeServiceImage(content: string, serviceName: string, newImageRef: string): string {
const doc = parseDocument(content);
if (!doc.hasIn(['services', serviceName, 'image'])) {
throw new Error(`Service "${serviceName}" has no image to patch`);
}
doc.setIn(['services', serviceName, 'image'], newImageRef);
return String(doc);
}
+102 -7
View File
@@ -14,12 +14,12 @@ import { getHostMemory } from '../helpers/hostMemory';
import { FileSystemService } from '../services/FileSystemService';
import { ComposeService } from '../services/ComposeService';
import { StackOpLockService } from '../services/StackOpLockService';
import SelfUpdateService from '../services/SelfUpdateService';
import SelfUpdateService, { type PinInfo } from '../services/SelfUpdateService';
import { getSenchoVersion, isValidVersion } from '../services/CapabilityRegistry';
import { authMiddleware } from '../middleware/auth';
import { requirePaid, requireAdmin, requireNodeProxy } from '../middleware/tierGates';
import { requirePermission } from '../middleware/permissions';
import { scheduleLocalUpdate } from './license';
import { respondSelfUpdatePreflight, scheduleLocalUpdate } from './license';
import { runPolicyGate, assertPolicyGateAllows, buildPolicyGateOptions } from '../helpers/policyGate';
import { remoteSupportsCrossNodeRbac } from '../helpers/remoteCapabilities';
import { captureLocalNodeFiles, captureRemoteNodeFiles, buildSnapshotDocumentation, pickDossierFields, dossierHasContent, type SnapshotNodeData, type SnapshotDocumentation } from '../utils/snapshot-capture';
@@ -28,6 +28,8 @@ import { isValidStackName } from '../utils/validation';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
import { parseIntParam } from '../utils/parseIntParam';
import { parseRequestedTargetVersion, pickCompareTarget } from '../utils/targetVersion';
import { buildTargetImageRef, isRepinBlocked, type ImagePinKind } from '../helpers/selfUpdateCompose';
import { withTimeout, TimeoutError } from '../utils/withTimeout';
// Mirror the system-maintenance route timeout so fleet's local-node prune
@@ -54,6 +56,39 @@ import { LicenseService } from '../services/LicenseService';
const updateTracker = FleetUpdateTrackerService.getInstance();
const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull failure
// Shown in the Node Updates UI when a node's image is pinned in a way Fleet
// cannot repin (digest or an unresolved value). Node-neutral so it reads the
// same for the local node and a remote row.
const REPIN_BLOCKED_REASON =
'This node pins its Sencho image to a digest or a value Fleet cannot resolve, so it cannot update automatically. Change the image tag in its compose file, then update.';
const EMPTY_PIN_STATUS = {
imagePinKind: null,
composeImageRef: null,
targetImageRef: null,
updateBlocked: false,
updateBlockedReason: null,
} as const;
function localPinStatusFields(
pin: PinInfo,
compareVersion: string | null,
compareValid: boolean,
blockedReason: string,
) {
const updateBlocked = isRepinBlocked(pin.pinKind);
const compareTarget = pickCompareTarget(compareVersion, compareValid);
return {
imagePinKind: pin.pinKind,
composeImageRef: pin.composeImageRef,
targetImageRef:
pin.pinKind === 'semver' && compareTarget
? buildTargetImageRef(pin.composeImageRef, compareTarget)
: null,
updateBlocked,
updateBlockedReason: updateBlocked ? blockedReason : null,
};
}
// Throttle the forced latest-version refresh so a caller cannot loop the recheck
// endpoint to hammer GitHub / Docker Hub. The 30-minute cache still serves reads
// between forced refreshes; this only bounds how often we bypass it.
@@ -222,6 +257,12 @@ async function getCompareTarget(gatewayVersion: string | null) {
return result;
}
async function resolveUpdateTarget(requested?: string): Promise<string | undefined> {
if (requested !== undefined) return requested;
const { compareVersion, compareValid } = await getCompareTarget(getSenchoVersion());
return pickCompareTarget(compareVersion, compareValid);
}
async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
try {
const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(node.id));
@@ -1073,6 +1114,8 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
let remoteStartedAt: number | null = null;
let remoteUpdateError: string | null = null;
let remoteOnline = false;
let remoteImagePinKind: ImagePinKind | null = null;
let remoteUpdateBlocked = false;
if (node.type === 'local') {
version = gatewayVersion;
} else {
@@ -1081,6 +1124,8 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
remoteStartedAt = meta.startedAt;
remoteUpdateError = meta.updateError;
remoteOnline = meta.online;
remoteImagePinKind = meta.imagePinKind;
remoteUpdateBlocked = meta.updateBlocked;
}
if (tracker?.status === 'updating') {
@@ -1202,6 +1247,27 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
skippedVersion = skipRow.skippedVersion;
}
// Image-pin metadata. The local row gets full detail (this route is
// hub-only and authenticated); remote rows get only the safe subset
// (pinKind + blocked flag) carried on the remote's /api/meta. A full
// remote composeImageRef would need a new authenticated remote endpoint
// and is deliberately out of scope.
let imagePinKind: ImagePinKind | null = null;
let composeImageRef: string | null = null;
let targetImageRef: string | null = null;
let updateBlocked = false;
let updateBlockedReason: string | null = null;
if (node.type === 'local') {
const pin = await SelfUpdateService.getInstance().getPinInfo();
if (pin) {
({ imagePinKind, composeImageRef, targetImageRef, updateBlocked, updateBlockedReason } =
localPinStatusFields(pin, compareVersion, compareValid, REPIN_BLOCKED_REASON));
}
} else {
imagePinKind = remoteImagePinKind;
updateBlocked = remoteUpdateBlocked;
}
return {
nodeId: node.id,
name: node.name,
@@ -1213,6 +1279,11 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
error: currentTracker?.error ?? null,
skipActive,
skippedVersion,
imagePinKind,
composeImageRef,
targetImageRef,
updateBlocked,
updateBlockedReason,
};
}),
);
@@ -1231,6 +1302,7 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
error: null,
skipActive: false,
skippedVersion: null,
...EMPTY_PIN_STATUS,
};
});
@@ -1266,12 +1338,18 @@ fleetRouter.get('/update-status/release-notes', authMiddleware, async (req: Requ
// Pilot loopback targets carry an empty apiToken because the tunnel bridge
// re-injects admin auth; sending a malformed `Bearer ` header would 401 on
// the pilot's local Express. Omit the header in that case.
function postSystemUpdate(target: { apiUrl: string; apiToken: string }) {
//
// targetVersion is forwarded only when it is a valid semver so the remote can
// repin a semver-pinned compose to that release. It is omitted otherwise (never
// sent as null/invalid), and an older remote that predates this field simply
// ignores the extra body key and behaves as before.
function postSystemUpdate(target: { apiUrl: string; apiToken: string }, targetVersion?: string) {
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
return fetch(`${target.apiUrl.replace(/\/$/, '')}/api/system/update`, {
method: 'POST',
headers,
body: JSON.stringify(targetVersion ? { targetVersion } : {}),
signal: AbortSignal.timeout(10000),
});
}
@@ -1339,6 +1417,9 @@ fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, r
return;
}
const requestedTarget = parseRequestedTargetVersion(req, res);
if (requestedTarget === null) return; // invalid supplied value; 400 already sent
const existing = updateTracker.get(nodeId);
if (existing?.status === 'updating') {
if (Date.now() - existing.startedAt > UPDATE_TIMEOUT_MS) {
@@ -1359,12 +1440,15 @@ fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, r
}
if (node.type === 'local') {
if (!SelfUpdateService.getInstance().isAvailable()) {
const selfUpdate = SelfUpdateService.getInstance();
if (!selfUpdate.isAvailable()) {
res.status(503).json({ error: 'Self-update unavailable on the local node.' });
return;
}
const resolvedTarget = await resolveUpdateTarget(requestedTarget);
if (!respondSelfUpdatePreflight(res, await selfUpdate.canSelfUpdateTarget(resolvedTarget))) return;
updateTracker.set(nodeId, updateTracker.create('updating', getSenchoVersion(), null));
scheduleLocalUpdate(res, 'Update initiated on local node. The server will restart shortly.');
scheduleLocalUpdate(res, 'Update initiated on local node. The server will restart shortly.', resolvedTarget);
return;
}
@@ -1386,8 +1470,13 @@ fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, r
res.status(503).json({ error: 'Remote node does not support self-update. It may need to be updated manually first.' });
return;
}
if (meta.updateBlocked) {
res.status(409).json({ error: REPIN_BLOCKED_REASON, code: 'update_blocked' });
return;
}
const response = await postSystemUpdate(target);
const resolvedTarget = await resolveUpdateTarget(requestedTarget);
const response = await postSystemUpdate(target, resolvedTarget);
if (!response.ok) {
const err = await response.json().catch(() => ({}));
@@ -1417,6 +1506,9 @@ fleetRouter.post('/update-all', authMiddleware, async (req: Request, res: Respon
const nodes = db.getNodes();
const gatewayVersion = getSenchoVersion();
const { compareVersion, compareValid } = await getCompareTarget(gatewayVersion);
// Forward the compare target so each remote repins a semver pin to it; omit
// when there is no valid target so a remote falls back to legacy behavior.
const updateAllTarget = pickCompareTarget(compareVersion, compareValid);
const debug = isDebugEnabled();
console.log('[Fleet] Update-all triggered,', nodes.length, 'nodes registered');
@@ -1451,10 +1543,13 @@ fleetRouter.post('/update-all', authMiddleware, async (req: Request, res: Respon
if (!meta.capabilities.includes('self-update')) {
return { name: node.name, triggered: false };
}
if (meta.updateBlocked) {
return { name: node.name, triggered: false };
}
if (isValidVersion(meta.version) && compareValid && !semver.lt(meta.version, compareVersion!)) {
return { name: node.name, triggered: false };
}
const response = await postSystemUpdate(target);
const response = await postSystemUpdate(target, updateAllTarget);
if (response.ok) {
updateTracker.set(node.id, updateTracker.create('updating', meta.version, meta.startedAt));
return { name: node.name, triggered: true };
+27 -5
View File
@@ -3,6 +3,8 @@ import { LicenseService } from '../services/LicenseService';
import SelfUpdateService from '../services/SelfUpdateService';
import { requireAdmin } from '../middleware/tierGates';
import { rejectApiTokenScope } from '../middleware/apiTokenScope';
import { parseRequestedTargetVersion } from '../utils/targetVersion';
import type { SelfUpdatePreflight } from '../services/SelfUpdateService';
const LICENSE_SCOPE_MESSAGE = 'API tokens cannot manage licenses.';
@@ -83,15 +85,29 @@ licenseRouter.get('/billing-portal', async (_req: Request, res: Response): Promi
* Respond 202, then trigger the "last breath" self-update after flush.
* Exported because the fleet "update this node" route reuses the same
* response shape + post-flush trigger for local-node self-updates.
*
* `targetVersion` (the Fleet compare target) drives the pinned-image repin;
* when omitted, triggerUpdate keeps the legacy behavior of pulling the running
* image and recreating from the on-disk compose.
*/
export function scheduleLocalUpdate(res: Response, message: string): void {
/** Send 409 when preflight fails; returns false so the route can early-return. */
export function respondSelfUpdatePreflight(
res: Response,
preflight: SelfUpdatePreflight,
): preflight is { ok: true } {
if (preflight.ok) return true;
res.status(409).json({ error: preflight.reason, code: 'update_blocked' });
return false;
}
export function scheduleLocalUpdate(res: Response, message: string, targetVersion?: string): void {
res.status(202).json({ message });
res.on('finish', () => {
setTimeout(() => {
// Defense in depth: triggerUpdate records its own errors into
// lastUpdateError; guard against an unexpected throw becoming an
// unhandled rejection.
SelfUpdateService.getInstance().triggerUpdate().catch((err) => {
SelfUpdateService.getInstance().triggerUpdate({ targetVersion }).catch((err) => {
console.error('[SelfUpdate] Unexpected error during triggerUpdate:', err);
});
}, 500);
@@ -100,11 +116,17 @@ export function scheduleLocalUpdate(res: Response, message: string): void {
export const systemUpdateRouter = Router();
systemUpdateRouter.post('/update', (req: Request, res: Response): void => {
systemUpdateRouter.post('/update', async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
if (!SelfUpdateService.getInstance().isAvailable()) {
const selfUpdate = SelfUpdateService.getInstance();
if (!selfUpdate.isAvailable()) {
res.status(503).json({ error: 'Self-update unavailable. Sencho must be deployed via Docker Compose.' });
return;
}
scheduleLocalUpdate(res, 'Update initiated. The server will restart shortly.');
const targetVersion = parseRequestedTargetVersion(req, res);
if (targetVersion === null) return; // invalid supplied value; 400 already sent
// Fail fast on a pin we cannot repin (digest/unknown) so the caller gets a
// 409 instead of a 202 that would later fail after the reconnect overlay.
if (!respondSelfUpdatePreflight(res, await selfUpdate.canSelfUpdateTarget(targetVersion))) return;
scheduleLocalUpdate(res, 'Update initiated. The server will restart shortly.', targetVersion);
});
+13 -2
View File
@@ -1,5 +1,6 @@
import { Router, type Request, type Response } from 'express';
import { getActiveCapabilities, getSenchoVersion } from '../services/CapabilityRegistry';
import { isRepinBlocked } from '../helpers/selfUpdateCompose';
import { MeshService } from '../services/MeshService';
import SelfUpdateService from '../services/SelfUpdateService';
@@ -28,13 +29,23 @@ metaRouter.get('/health', (_req: Request, res: Response): void => {
// Public meta endpoint. Returns this instance's version and supported
// capabilities. No auth required (like /health). Used by remote nodes during
// connection tests.
metaRouter.get('/meta', (_req: Request, res: Response): void => {
const updateError = SelfUpdateService.getInstance().getLastError();
//
// Image-pin fields are intentionally limited to the non-sensitive subset:
// `imagePinKind` (a bounded enum) and `updateBlocked` (a boolean). The full
// composeImageRef is NEVER exposed here because this endpoint is public and the
// ref can carry a private registry/repository name.
metaRouter.get('/meta', async (_req: Request, res: Response): Promise<void> => {
const selfUpdate = SelfUpdateService.getInstance();
const updateError = selfUpdate.getLastError();
const pin = await selfUpdate.getPinInfo({ cacheOnly: true });
const updateBlocked = pin ? isRepinBlocked(pin.pinKind) : false;
res.json({
version: getSenchoVersion(),
capabilities: getActiveCapabilities(),
startedAt: processStartedAt,
experimental: process.env.SENCHO_EXPERIMENTAL === 'true',
...(pin ? { imagePinKind: pin.pinKind } : {}),
updateBlocked,
...(updateError ? { updateError } : {}),
});
});
@@ -4,6 +4,16 @@ import fs from 'fs';
import semver from 'semver';
import { SENCHO_VERSION } from '../generated/version';
import { isDebugEnabled } from '../utils/debug';
import type { ImagePinKind } from '../helpers/selfUpdateCompose';
const IMAGE_PIN_KINDS: readonly ImagePinKind[] = ['floating', 'semver', 'digest', 'unknown'];
/** Coerce an untrusted /api/meta value to a known pin kind, or null. */
function parseImagePinKind(value: unknown): ImagePinKind | null {
return typeof value === 'string' && (IMAGE_PIN_KINDS as readonly string[]).includes(value)
? (value as ImagePinKind)
: null;
}
/**
* Static registry of capabilities supported by THIS Sencho instance.
@@ -96,6 +106,14 @@ export interface RemoteMeta {
updateError: string | null;
/** True when the /api/meta request succeeded (node is reachable). */
online: boolean;
/**
* How the remote pins its Sencho image, when it advertises it. Null for an
* older remote that predates this field or one that could not classify its
* pin. The hub uses only this safe subset (no full image ref) for remote rows.
*/
imagePinKind: ImagePinKind | null;
/** True when the remote reports its update is blocked (digest/unknown pin). */
updateBlocked: boolean;
}
// Runtime capability overrides; services call disableCapability() during init.
@@ -142,6 +160,8 @@ export const OFFLINE_META: RemoteMeta = {
startedAt: null,
updateError: null,
online: false,
imagePinKind: null,
updateBlocked: false,
};
/** Strip any `user:pass@` userinfo from a URL so credentials never reach the logs. */
@@ -164,6 +184,8 @@ export async function fetchRemoteMeta(baseUrl: string, apiToken: string): Promis
startedAt: typeof res.data.startedAt === 'number' ? res.data.startedAt : null,
updateError: typeof res.data.updateError === 'string' ? res.data.updateError : null,
online: true,
imagePinKind: parseImagePinKind(res.data.imagePinKind),
updateBlocked: res.data.updateBlocked === true,
};
if (isDebugEnabled()) {
// Diagnostic aid for "why is this feature gated?": log the resolved version
+247 -29
View File
@@ -5,6 +5,15 @@ import DockerController from './DockerController';
import { DatabaseService } from './DatabaseService';
import { disableCapability } from './CapabilityRegistry';
import { isDebugEnabled } from '../utils/debug';
import {
buildTargetImageRef,
isRepinBlocked,
isValidImageRef,
patchComposeServiceImage,
resolveServiceImageFromContents,
type ImagePinKind,
type ResolvedComposeImage,
} from '../helpers/selfUpdateCompose';
const execFileAsync = promisify(execFile);
@@ -13,6 +22,23 @@ const execFileAsync = promisify(execFile);
// the NEW gateway process (which always mounts /app/data) can reach it.
const UPDATE_ERROR_FILE = '/app/data/.sencho-update-error';
// Rewritten compose file the main process stages under /app/data for the helper
// to copy onto the host compose file. Uses the same /app/data handoff as the
// error file because the main process cannot reach the compose working dir.
const STAGED_PATCH_FILE = '/app/data/.sencho-compose-patch';
// Cap how long a cached pin classification is served to status/meta callers.
// Update DECISIONS always read fresh (fresh: true), so this only bounds how
// stale a displayed pin kind can be after a live compose edit.
const PIN_CACHE_TTL_MS = 5 * 60 * 1000;
// Operator-facing block reasons. Deliberately generic: no host paths, compose
// contents, or registry/repository names (this text can reach the UI).
const UPDATE_BLOCKED_REASON =
'This install pins the Sencho image to a digest or a value Fleet cannot resolve, so it cannot repin automatically. Change the image tag in your compose file, then update again.';
const UPDATE_READ_FAILED_REASON =
'Could not read the Sencho compose file to determine how its image is pinned. Confirm the update helper can reach the compose directory, then try again.';
interface HostMount {
source: string;
destination: string;
@@ -25,6 +51,22 @@ export type DockerMount = {
Destination: string;
};
/** Pin classification exposed to callers (status/meta), without file contents. */
export interface PinInfo {
pinKind: ImagePinKind;
composeImageRef: string;
filePath: string;
}
/** Outcome of the pre-update check the route layer runs before responding. */
export type SelfUpdatePreflight = { ok: true } | { ok: false; reason: string };
/** A staged compose rewrite the helper copies onto the host before recreate. */
export interface ComposeCopy {
stagedPath: string;
targetPath: string;
}
/**
* Find the host-side path Docker resolved for /app/data, regardless of whether
* the operator declared a bind or a named volume. The helper container uses
@@ -48,15 +90,36 @@ export function shQuote(value: string): string {
return `'${value.replace(/'/g, `'\\''`)}'`;
}
/**
* Build the argv for a one-shot helper that reads a single compose config file
* from the host. The main process cannot open the compose working dir, so it
* mounts it read-only into a throwaway container and cats the file out. Pure and
* exported for unit testing. The path is a discrete argv element handed to
* execFile (no shell), so a path with shell metacharacters stays inert.
*/
export function buildComposeReadArgs(workingDir: string, imageName: string, configFilePath: string): string[] {
return [
'run', '--rm',
'--user', 'root',
'--entrypoint', 'cat',
'-v', `${workingDir}:${workingDir}:ro`,
'-w', workingDir,
imageName,
configFilePath,
];
}
/**
* Build the shell command the helper container runs to recreate Sencho. Kept as
* a pure, exported function so the prune-on-update branch is unit-testable.
* a pure, exported function so the prune-on-update and repin-copy branches are
* unit-testable.
*
* Label-derived inputs (serviceName, the fFlags config paths) are shell-quoted
* so they cannot alter the command structure. The recreate writes the error
* file only on failure; the optional dangling prune runs only on success, so
* the two branches never overlap. The prune suppresses its own output and
* `|| true`, so it can never alter $ec or be mistaken for an update error.
* Label-derived inputs (serviceName, the fFlags config paths) and the repin copy
* paths are shell-quoted so they cannot alter the command structure. When a
* composeCopy is present the rewritten compose file is copied onto the host
* BEFORE the recreate; a failed copy aborts before recreate so a half-applied
* update never runs. The recreate writes the error file only on failure; the
* optional dangling prune runs only on success, so the branches never overlap.
*/
export function buildSelfUpdateComposeCmd(
fFlags: string[],
@@ -64,10 +127,15 @@ export function buildSelfUpdateComposeCmd(
stderrTmp: string,
errorFile: string,
pruneOnUpdate: boolean,
composeCopy?: ComposeCopy,
): string {
const recreate = ['docker compose', ...fFlags.map(shQuote), 'up -d --force-recreate', shQuote(serviceName), `2>${stderrTmp}`].join(' ');
const copyStep = composeCopy
? [`cp ${shQuote(composeCopy.stagedPath)} ${shQuote(composeCopy.targetPath)} 2>${stderrTmp} || { { echo "exit=1"; echo "Failed to write the updated compose file"; cat ${stderrTmp}; } > ${errorFile} 2>/dev/null; exit 1; }`]
: [];
return [
'sleep 3',
...copyStep,
recreate,
'ec=$?',
`if [ $ec -ne 0 ]; then { echo "exit=$ec"; cat ${stderrTmp}; } > ${errorFile} 2>/dev/null; fi`,
@@ -97,15 +165,19 @@ interface ComposeContext {
* paths (compose labels / container mounts). They are placed as discrete argv
* elements and handed to execFile, which spawns docker with no shell, so a path
* carrying shell metacharacters stays inert data and never reaches a shell.
*
* workingDir is mounted read-only unless `repinWritable` is set, which happens
* only for a semver repin that must copy the rewritten compose file onto the
* host. Limiting :rw to that case keeps the helper's host write scope minimal.
*/
export function buildSelfUpdateRunArgs(
ctx: Pick<ComposeContext, 'workingDir' | 'imageName' | 'dataDirHost' | 'hostBindMounts'>,
ctx: Pick<ComposeContext, 'workingDir' | 'imageName' | 'dataDirHost' | 'hostBindMounts'> & { repinWritable?: boolean },
composeCmd: string,
): string[] {
const { workingDir, imageName, dataDirHost, hostBindMounts } = ctx;
const { workingDir, imageName, dataDirHost, hostBindMounts, repinWritable } = ctx;
const mountArgs: string[] = [
'-v', '/var/run/docker.sock:/var/run/docker.sock',
'-v', `${workingDir}:${workingDir}:ro`,
'-v', `${workingDir}:${workingDir}:${repinWritable ? 'rw' : 'ro'}`,
];
if (dataDirHost) {
mountArgs.push('-v', `${dataDirHost}:/app/data:rw`);
@@ -133,6 +205,7 @@ class SelfUpdateService {
private canSelfUpdate = false;
private composeContext: ComposeContext | null = null;
private lastUpdateError: string | null = null;
private pinCache: { info: ResolvedComposeImage | null; at: number } | null = null;
public static getInstance(): SelfUpdateService {
if (!SelfUpdateService.instance) {
@@ -177,7 +250,10 @@ class SelfUpdateService {
return;
}
// Read the container's own image name for direct docker pull
// Read the container's own image name for the legacy (no target) pull and
// as a fallback ref. It is captured once here and can go stale if the
// compose file is edited later, so update decisions resolve the compose
// image fresh instead of trusting this value.
const imageName = info.Config?.Image;
if (!imageName) {
console.log('[SelfUpdate] Could not determine container image name');
@@ -195,13 +271,22 @@ class SelfUpdateService {
const dataDirHost = findDataDirHost(rawMounts);
if (!dataDirHost) {
console.log('[SelfUpdate] /app/data mount not found - update error recovery will be unavailable');
console.log('[SelfUpdate] /app/data mount not found - update error recovery and compose repin will be unavailable');
}
this.composeContext = { workingDir, configFiles, serviceName, imageName, dataDirHost, hostBindMounts };
this.canSelfUpdate = true;
console.log(`[SelfUpdate] Ready - service="${serviceName}" image="${imageName}" in ${workingDir}`);
// Warm the pin-info cache in the background for diagnostics and the first
// status/meta read. This is never a source of truth for an update
// decision, which always resolves fresh.
void this.resolveComposeImage(true)
.then(resolved => {
if (resolved) console.log(`[SelfUpdate] Compose image pin: ${resolved.imageRef} (${resolved.pinKind})`);
})
.catch(() => { /* diagnostic only */ });
// Surface any error from a previous failed update attempt (persisted by
// the helper container) so the new process can report it to the user.
this.recoverPreviousError();
@@ -225,6 +310,73 @@ class SelfUpdateService {
this.lastUpdateError = null;
}
/**
* Classify how this instance's compose image is pinned. Serves a short-lived
* cache to frequent status/meta callers; pass `{ fresh: true }` for update
* decisions so a live compose edit is always reflected. Returns null when
* self-update is unavailable or the compose file could not be read.
*/
async getPinInfo(opts?: { fresh?: boolean; cacheOnly?: boolean }): Promise<PinInfo | null> {
const resolved = await this.resolveComposeImage(opts?.fresh ?? false, opts?.cacheOnly ?? false);
if (!resolved) return null;
return { pinKind: resolved.pinKind, composeImageRef: resolved.imageRef, filePath: resolved.filePath };
}
/**
* Preflight the route layer runs before responding, so a blocked update fails
* fast with a 409 instead of returning 202 and stalling the reconnect overlay.
* A missing target version is the legacy pull-current path and makes no repin
* decision. A digest/unknown pin (or an unreadable compose file) is blocked.
*/
async canSelfUpdateTarget(targetVersion?: string): Promise<SelfUpdatePreflight> {
if (!this.canSelfUpdate) {
return { ok: false, reason: 'Self-update is unavailable on this instance.' };
}
if (!targetVersion) return { ok: true };
const info = await this.getPinInfo({ fresh: true });
if (!info) return { ok: false, reason: UPDATE_READ_FAILED_REASON };
if (isRepinBlocked(info.pinKind)) {
return { ok: false, reason: UPDATE_BLOCKED_REASON };
}
return { ok: true };
}
/** Resolve the compose-declared service image fresh from the host, or from a
* short-lived cache. Reads each config file via a throwaway cat container. */
private async resolveComposeImage(fresh: boolean, cacheOnly = false): Promise<ResolvedComposeImage | null> {
if (!this.composeContext) return null;
const now = Date.now();
if (!fresh && this.pinCache && now - this.pinCache.at < PIN_CACHE_TTL_MS) {
return this.pinCache.info;
}
if (cacheOnly) return null;
const { workingDir, configFiles, serviceName, imageName } = this.composeContext;
const env = this.buildEnv();
const debug = isDebugEnabled();
const files: { filePath: string; content: string }[] = [];
for (const raw of configFiles.split(',')) {
const filePath = raw.trim();
if (!filePath) continue;
try {
const { stdout } = await execFileAsync('docker', buildComposeReadArgs(workingDir, imageName, filePath), {
env,
timeout: 30_000,
maxBuffer: 10 * 1024 * 1024,
});
files.push({ filePath, content: stdout });
} catch (error) {
if (debug) console.debug('[SelfUpdate:debug] Could not read compose file', filePath, (error as Error).message);
}
}
const info = resolveServiceImageFromContents(files, serviceName);
this.pinCache = { info, at: now };
return info;
}
private buildEnv(): NodeJS.ProcessEnv {
return { ...process.env, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' };
}
/** Surfaces any error the helper container persisted before the previous
* gateway process died, then deletes the file. */
private recoverPreviousError(): void {
@@ -242,25 +394,63 @@ class SelfUpdateService {
}
}
async triggerUpdate(): Promise<void> {
/**
* Pull the target image, then (for a semver pin) rewrite the compose tag and
* recreate. Pull happens BEFORE any compose mutation, so a failed pull leaves
* the compose file untouched. `targetVersion` comes from the Fleet compare
* target; when omitted this keeps the legacy behavior of pulling the running
* image and recreating from the on-disk compose.
*/
async triggerUpdate(options?: { targetVersion?: string }): Promise<void> {
if (!this.composeContext) return;
const { workingDir, configFiles, serviceName, imageName, dataDirHost, hostBindMounts } = this.composeContext;
const env = { ...process.env, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' };
const env = this.buildEnv();
this.lastUpdateError = null;
try { fs.unlinkSync(UPDATE_ERROR_FILE); } catch { /* absent is the steady state */ }
try { fs.unlinkSync(STAGED_PATCH_FILE); } catch { /* absent is the steady state */ }
// Async pull: a sync execFileSync blocks the event loop, which lets the frontend
// overlay see a false "online" response between the pull finishing and the restart.
const { imageName, serviceName, dataDirHost } = this.composeContext;
const targetVersion = options?.targetVersion;
let pullRef = imageName;
let repin: { resolved: ResolvedComposeImage; ref: string } | null = null;
if (targetVersion) {
const resolved = await this.resolveComposeImage(true);
const pinKind = resolved?.pinKind ?? 'unknown';
// Defense in depth: the route preflight already rejected these, but the
// compose file could change between preflight and this last-breath call.
if (!resolved || isRepinBlocked(pinKind)) {
this.lastUpdateError = resolved ? UPDATE_BLOCKED_REASON : UPDATE_READ_FAILED_REASON;
console.error('[SelfUpdate] Update blocked:', this.lastUpdateError);
return;
}
pullRef = pinKind === 'semver' ? buildTargetImageRef(resolved.imageRef, targetVersion) : resolved.imageRef;
if (!isValidImageRef(pullRef)) {
this.lastUpdateError = 'Aborting update: the computed image reference is invalid.';
console.error('[SelfUpdate] Update blocked:', this.lastUpdateError, pullRef);
return;
}
if (pinKind === 'semver') {
if (!dataDirHost) {
this.lastUpdateError =
'Cannot rewrite the pinned compose image: the data directory needed for the update handoff is not mounted. Change the image tag manually and update again.';
console.error('[SelfUpdate] Repin blocked:', this.lastUpdateError);
return;
}
repin = { resolved, ref: pullRef };
}
}
// Pull FIRST so the compose file is never mutated for an image we could not
// fetch. A sync execFileSync would block the event loop and let the overlay
// see a false "online" between the pull finishing and the restart.
const debug = isDebugEnabled();
const pullStart = Date.now();
console.log(`[SelfUpdate] Pulling latest image: ${imageName}...`);
if (debug) console.debug('[SelfUpdate:debug] Pull context:', { workingDir, configFiles, serviceName, dataDirHost, mountCount: hostBindMounts.length });
console.log(`[SelfUpdate] Pulling image: ${pullRef}...`);
if (debug) console.debug('[SelfUpdate:debug] Pull context:', { targetVersion: targetVersion ?? null, repin: !!repin });
try {
await execFileAsync('docker', ['pull', imageName], {
env,
timeout: 300_000, // 5 min max for pull
});
await execFileAsync('docker', ['pull', pullRef], { env, timeout: 300_000 });
if (debug) console.debug('[SelfUpdate:debug] Pull completed in', Math.round((Date.now() - pullStart) / 1000) + 's');
} catch (error) {
const stderr = (error as { stderr?: Buffer | string })?.stderr?.toString().trim();
@@ -269,11 +459,39 @@ class SelfUpdateService {
return;
}
// The main container cannot access the compose file at its host path,
// so the helper bind-mounts the compose working directory from the host.
// Run attached (no -d): if compose recreate fails before it kills us,
// execFile's callback receives the helper's exit code + stderr directly.
console.log(`[SelfUpdate] Spawning updater container... (last breath)`);
// Rewrite the pinned tag only after a confirmed pull (semver only). The
// rewrite is staged under /app/data; the helper copies it onto the host
// compose file just before recreate.
let composeCopy: ComposeCopy | undefined;
if (repin) {
try {
const patched = patchComposeServiceImage(repin.resolved.fileContent, serviceName, repin.ref);
fs.writeFileSync(STAGED_PATCH_FILE, patched, 'utf8');
composeCopy = { stagedPath: STAGED_PATCH_FILE, targetPath: repin.resolved.filePath };
console.log(`[SelfUpdate] Repinning "${serviceName}" image to ${repin.ref}`);
} catch (error) {
this.lastUpdateError =
`Pulled ${repin.ref} but could not rewrite the compose image tag (${(error as Error).message}). Change the tag manually and update again.`;
console.error('[SelfUpdate] Repin failed:', this.lastUpdateError);
return;
}
}
this.spawnHelper(env, composeCopy);
}
/**
* Spawn the "last breath" helper container that recreates Sencho (and, when a
* repin is staged, copies the rewritten compose file onto the host first).
* Runs attached (no -d): if the recreate fails before it kills us, execFile's
* callback receives the helper's exit code and stderr directly.
*/
private spawnHelper(env: NodeJS.ProcessEnv, composeCopy?: ComposeCopy): void {
if (!this.composeContext) return;
const { workingDir, configFiles, serviceName, imageName, dataDirHost, hostBindMounts } = this.composeContext;
console.log('[SelfUpdate] Spawning updater container... (last breath)');
if (isDebugEnabled()) console.debug('[SelfUpdate:debug] Helper context:', { workingDir, serviceName, dataDirHost, repin: !!composeCopy, mountCount: hostBindMounts.length });
const fFlags = configFiles.split(',').flatMap(f => ['-f', f.trim()]);
// On failure, persist exit code + stderr to UPDATE_ERROR_FILE (host-mounted)
@@ -283,8 +501,8 @@ class SelfUpdateService {
const stderrTmp = '/tmp/_sencho_err';
const pruneOnUpdate =
DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1';
const composeCmd = buildSelfUpdateComposeCmd(fFlags, serviceName, stderrTmp, UPDATE_ERROR_FILE, pruneOnUpdate);
const args = buildSelfUpdateRunArgs({ workingDir, imageName, dataDirHost, hostBindMounts }, composeCmd);
const composeCmd = buildSelfUpdateComposeCmd(fFlags, serviceName, stderrTmp, UPDATE_ERROR_FILE, pruneOnUpdate, composeCopy);
const args = buildSelfUpdateRunArgs({ workingDir, imageName, dataDirHost, hostBindMounts, repinWritable: !!composeCopy }, composeCmd);
// Callback may never fire on success (we die mid-call during recreate);
// that is fine because the restart itself is the success signal.
+29
View File
@@ -0,0 +1,29 @@
import type { Request, Response } from 'express';
import semver from 'semver';
import { isValidVersion } from '../services/CapabilityRegistry';
/** The semver compare target when valid, otherwise undefined (legacy pull-current). */
export function pickCompareTarget(compareVersion: string | null, compareValid: boolean): string | undefined {
if (!compareValid || !isValidVersion(compareVersion)) return undefined;
return compareVersion;
}
/**
* Parse an optional `targetVersion` from a self-update request body.
*
* - Omitted or null -> returns undefined (the caller chooses the default).
* - Present and a valid semver -> returns the normalized version.
* - Present but not a valid semver -> writes a 400 and returns null so the
* caller early-returns. We never silently fall back on a supplied-but-bad
* value, because that would hide a client bug and update to the wrong target.
*/
export function parseRequestedTargetVersion(req: Request, res: Response): string | null | undefined {
const raw = (req.body ?? {})?.targetVersion;
if (raw === undefined || raw === null) return undefined;
const normalized = typeof raw === 'string' ? semver.valid(raw) : null;
if (!normalized || !isValidVersion(normalized) || raw.length > 64) {
res.status(400).json({ error: 'Invalid target version' });
return null;
}
return normalized;
}
+10 -2
View File
@@ -59,21 +59,29 @@ Updating the gateway is special because the dashboard is hosted by the very cont
After **Update & restart** is confirmed:
1. The browser captures the gateway's current boot timestamp from `/api/health`.
2. The server pulls the latest image and spawns a short-lived helper container that runs `docker compose up -d --force-recreate` against the host's compose working directory.
2. The server classifies the compose image pin. For semver pins it pulls the target release, stages a rewritten compose file, and spawns a short-lived helper container that copies the file onto the host and runs `docker compose up -d --force-recreate`.
3. A full-screen **Updating Sencho...** overlay takes over the browser tab. The overlay polls `/api/health` every 3 seconds and keeps the page from reloading until a *new* boot timestamp comes back, even if the API briefly responds during the pull.
4. Once a fresh boot timestamp is reported, the overlay reloads the page on the new version.
If the pull or compose rewrite fails before the container restarts, the overlay dismisses within a few seconds and surfaces the error in a toast instead of waiting for the full reconnect timeout.
If the new container does not come up within 5 minutes, the overlay surfaces an **Update timed out** message with a *Try Reloading* button so the page is never left waiting indefinitely. If the gateway can detect that the update did not even start (for example, the image pull failed before the helper container could spawn) it surfaces a **Failed** badge with the underlying error on the Local card. The badge appears as soon as the helper writes its error file, or by the 3-minute mark at the latest, instead of waiting for the full 5-minute timeout.
<Note>
The self-update helper container inherits all bind mounts from the main Sencho container 1:1. If your `docker-compose.yml` references `env_file`, `configs`, or `secrets` outside the compose working directory, those host paths must be mounted into the Sencho container at the *same container path* as on the host. See [Troubleshooting](/operations/troubleshooting#local-self-update-fails-with-env-file-not-found) if you encounter `env file not found` errors during a local update.
</Note>
## Pinned image tags
Fleet self-update respects how each node compose file declares the Sencho image. Semver pins are rewritten to the target release before recreate. Floating tags such as latest are pulled without changing the compose file. Digest pins and unresolved interpolated values block automatic updates; those rows show a **Pinned** badge instead of an **Update** button.
The image pull always runs before any compose rewrite, so a failed pull never leaves the compose file half-updated.
## What happens during an update
For both local and remote nodes, an update goes through the same three steps:
1. **Pull** the latest `saelix/sencho` image from the registry.
1. **Pull** the target Sencho image from the registry (the compose-declared reference for floating tags, or the repinned semver tag for version pins).
2. **Recreate** the container with the new image via `docker compose up -d --force-recreate`.
3. **Restart** the Sencho process. The node is briefly offline during the swap.
+57 -3
View File
@@ -772,6 +772,17 @@ paths:
items:
type: string
example: ["stacks", "containers", "fleet", "auto-updates", "host-console"]
imagePinKind:
type: string
nullable: true
enum: [floating, semver, digest, unknown]
description: How the Sencho service image is pinned in the operator's compose file. Omitted when self-update is unavailable or the compose file could not be read.
updateBlocked:
type: boolean
description: True when the image is pinned in a way Fleet cannot repin automatically (digest or unresolved value).
updateError:
type: string
description: Error from the last failed self-update attempt, when present.
/api/system/update:
post:
@@ -779,9 +790,21 @@ paths:
tags: [Health]
summary: Trigger self-update
description: |
Instructs this Sencho instance to pull the latest Docker image and recreate its own container.
Instructs this Sencho instance to pull the update image and recreate its own container.
An optional release version in the request body drives semver repinning; floating tags are pulled without rewriting the compose file.
Digest and unresolved pins are rejected with 409 before any pull runs.
Returns 202 immediately; the actual update happens asynchronously after the response is sent.
Requires the instance to be deployed via Docker Compose with the Docker socket mounted.
requestBody:
required: false
content:
application/json:
schema:
type: object
properties:
targetVersion:
type: string
description: Release version to update to (valid semver).
responses:
"202":
description: Update initiated.
@@ -795,6 +818,23 @@ paths:
example: "Update initiated. The server will restart shortly."
"401":
$ref: "#/components/responses/Unauthorized"
"400":
description: Invalid release version in request body.
content:
application/json:
schema:
$ref: "#/components/schemas/Error"
"409":
description: Update blocked because the compose image cannot be repinned automatically.
content:
application/json:
schema:
type: object
properties:
error:
type: string
code:
type: string
"503":
description: Self-update not available (not running in Docker Compose).
content:
@@ -2594,7 +2634,7 @@ paths:
operationId: getFleetUpdateStatus
tags: [Fleet]
summary: Fleet update status
description: Returns version comparison and active update status for all nodes. Compares each node's version against the gateway's version.
description: Returns version comparison and active update status for all nodes. Compares each node's version against the gateway's version. Local rows include compose pin details; remote rows expose only the safe pin kind and blocked flag from each node's public meta.
responses:
"200":
description: Update status for all nodes.
@@ -2631,6 +2671,20 @@ paths:
skippedVersion:
type: string
nullable: true
imagePinKind:
type: string
nullable: true
composeImageRef:
type: string
nullable: true
targetImageRef:
type: string
nullable: true
updateBlocked:
type: boolean
updateBlockedReason:
type: string
nullable: true
"401":
$ref: "#/components/responses/Unauthorized"
"403":
@@ -2766,7 +2820,7 @@ paths:
operationId: triggerFleetUpdateAll
tags: [Fleet]
summary: Update all outdated nodes
description: Triggers self-update on all remote nodes running an older version than the gateway. Local nodes are excluded from bulk updates.
description: Triggers self-update on all remote nodes running an older version than the gateway. Forwards the hub compare target as `targetVersion` so semver-pinned remotes can repin before recreate. Local nodes are excluded from bulk updates.
responses:
"202":
description: Bulk update initiated.
+10
View File
@@ -429,6 +429,16 @@ After adding the missing volume mount, restart Sencho and retry the update.
---
## Fleet shows Pinned and will not update the node
**Symptom:** A node row in **Node updates** shows a **Pinned** badge instead of an **Update** button, or an update attempt returns an error about the image pin.
**Cause:** The node's compose file pins Sencho by digest or uses a variable for the image value. Fleet cannot rewrite those automatically.
**Fix:** Edit the compose file to use a semver tag (for example `saelix/sencho:0.94.0`) or a floating tag such as `latest`, redeploy once if needed, then retry the update from the Node updates sheet.
---
## First remote update always times out on old nodes
**Symptom:** After triggering a remote update on a node running a very old Sencho version (pre-v0.39.3), the node successfully restarts with the new version, but the dashboard shows **Timed out** or **Failed** instead of **Updated**.
+4
View File
@@ -69,6 +69,10 @@ If you prefer to control exactly which version you run, pin the image tag in you
image: saelix/sencho:0.38.0
```
Fleet can update semver pins directly from the Node updates sheet: it rewrites the tag to the selected release before recreating the container. Digest pins and compose values that use variable interpolation must be changed manually before Fleet can update the node.
Check [GitHub Releases](https://github.com/studio-saelix/sencho/releases) for available versions and changelogs.
---
+8
View File
@@ -58,6 +58,10 @@ export function FleetView({ onNavigateToNode, onOpenSettingsSection, onOpenMuteR
const { prefs, updatePrefs } = useFleetPreferences();
const updateStatus = useFleetUpdateStatus();
const overview = useFleetOverview({ prefs, updatePrefs, updateStatuses: updateStatus.updateStatuses });
// The local node's status backs the confirm dialog copy (pin + target ref).
const localUpdateConfirmStatus = updateStatus.localUpdateConfirm !== null
? updateStatus.updateStatuses.find(s => s.nodeId === updateStatus.localUpdateConfirm)
: undefined;
const topology = useTopologyPreferences();
const { exporting, exportDossier } = useFleetDossierExport();
@@ -318,6 +322,10 @@ export function FleetView({ onNavigateToNode, onOpenSettingsSection, onOpenMuteR
open={updateStatus.localUpdateConfirm !== null}
onOpenChange={(open) => { if (!open) updateStatus.setLocalUpdateConfirm(null); }}
onConfirm={updateStatus.confirmLocalUpdate}
imagePinKind={localUpdateConfirmStatus?.imagePinKind}
composeImageRef={localUpdateConfirmStatus?.composeImageRef}
targetImageRef={localUpdateConfirmStatus?.targetImageRef}
targetVersion={localUpdateConfirmStatus?.latestVersion}
/>
{NodeActionModals}
@@ -1,13 +1,22 @@
import { Download } from 'lucide-react';
import { ConfirmModal } from '@/components/ui/modal';
import { formatVersion } from '@/lib/version';
import type { ImagePinKind } from './types';
interface LocalUpdateConfirmDialogProps {
open: boolean;
onOpenChange: (open: boolean) => void;
onConfirm: () => void;
imagePinKind?: ImagePinKind | null;
composeImageRef?: string | null;
targetImageRef?: string | null;
targetVersion?: string | null;
}
export function LocalUpdateConfirmDialog({ open, onOpenChange, onConfirm }: LocalUpdateConfirmDialogProps) {
export function LocalUpdateConfirmDialog({
open, onOpenChange, onConfirm, imagePinKind, composeImageRef, targetImageRef, targetVersion,
}: LocalUpdateConfirmDialogProps) {
const versionLabel = formatVersion(targetVersion) ?? 'the latest release';
return (
<ConfirmModal
open={open}
@@ -22,9 +31,16 @@ export function LocalUpdateConfirmDialog({ open, onOpenChange, onConfirm }: Loca
}
onConfirm={onConfirm}
>
<p className="text-sm text-stat-subtitle">
Pulls the latest Sencho image and restarts the server. The dashboard briefly disconnects and reconnects automatically when the update completes.
</p>
{imagePinKind === 'semver' && composeImageRef && targetImageRef ? (
<p className="text-sm text-stat-subtitle">
This install pins <code className="text-stat-value">{composeImageRef}</code>. Updating rewrites it to{' '}
<code className="text-stat-value">{targetImageRef}</code> and restarts the server. The dashboard briefly disconnects and reconnects automatically when the update completes.
</p>
) : (
<p className="text-sm text-stat-subtitle">
Pulls Sencho {versionLabel} and restarts the server. The dashboard briefly disconnects and reconnects automatically when the update completes.
</p>
)}
</ConfirmModal>
);
}
@@ -26,6 +26,7 @@ import { useLicense } from '@/context/LicenseContext';
import { useNodes, type Node } from '@/context/NodeContext';
import { cordonNode, uncordonNode } from '@/lib/nodesApi';
import { UpdateStatusBadge } from './UpdateStatusBadge';
import { PinnedUpdateBadge } from './PinnedUpdateBadge';
import { StackSection } from './NodeCardStackList';
import type { Label as StackLabel } from '../label-types';
import type { FleetNode, NodeUpdateStatus } from './types';
@@ -227,11 +228,14 @@ export function NodeCard({ node, onNavigate, labelMap, updateStatus, onUpdate, u
onDismiss={isAdmin && onDismissUpdate ? () => onDismissUpdate(node.id) : undefined}
/>
)}
{updateStatus?.updateAvailable && !updateStatus.updateStatus && !updateStatus?.skipActive && (
{updateStatus?.updateAvailable && !updateStatus.updateStatus && !updateStatus?.skipActive && !updateStatus?.updateBlocked && (
<Badge className="text-[10px] px-1.5 py-0 h-4 bg-warning/15 text-warning border-warning/30 shrink-0">
Update available
</Badge>
)}
{updateStatus?.updateBlocked && updateStatus?.updateAvailable && !updateStatus.updateStatus && !updateStatus?.skipActive && (
<PinnedUpdateBadge reason={updateStatus.updateBlockedReason} />
)}
{updateStatus?.skipActive && (
<Badge className="text-[10px] px-1.5 py-0 h-4 bg-muted text-muted-foreground border-card-border/40 shrink-0">
Skipped
@@ -310,7 +314,7 @@ export function NodeCard({ node, onNavigate, labelMap, updateStatus, onUpdate, u
)}
{/* Update button (mutating action: admin only, matches the requireAdmin route guard) */}
{isOnline && updateStatus?.updateAvailable && !updateStatus.updateStatus && !updateStatus?.skipActive && onUpdate && isAdmin && (
{isOnline && updateStatus?.updateAvailable && !updateStatus.updateStatus && !updateStatus?.skipActive && !updateStatus?.updateBlocked && onUpdate && isAdmin && (
<div className="mt-3 pt-3 border-t border-border/50">
<Button
variant="outline"
@@ -13,6 +13,7 @@ import { apiFetch } from '@/lib/api';
import { toast } from '@/components/ui/toast-store';
import { formatVersion, isValidVersion } from '@/lib/version';
import { UpdateStatusBadge } from './UpdateStatusBadge';
import { PinnedUpdateBadge } from './PinnedUpdateBadge';
import type { NodeUpdateStatus } from './types';
interface NodeUpdatesSheetProps {
@@ -434,7 +435,13 @@ export function NodeUpdatesSheet({
Unskip
</Button>
)}
{s.updateAvailable && !s.updateStatus && !s.skipActive && isAdmin && (
{s.updateBlocked && s.updateAvailable && !s.updateStatus && !s.skipActive && (
<PinnedUpdateBadge
reason={s.updateBlockedReason}
className="text-[10px] px-1.5 py-0 h-5 bg-muted text-muted-foreground border-card-border/40"
/>
)}
{s.updateAvailable && !s.updateStatus && !s.skipActive && !s.updateBlocked && isAdmin && (
<Button
variant="outline"
size="sm"
@@ -460,7 +467,7 @@ export function NodeUpdatesSheet({
Skip
</Button>
)}
{s.updateAvailable && !s.updateStatus && !s.skipActive && !isAdmin && (
{s.updateAvailable && !s.updateStatus && !s.skipActive && !s.updateBlocked && !isAdmin && (
<Badge className="text-[10px] px-1.5 py-0 h-5 bg-warning/15 text-warning border-warning/30">
<CircleAlert className="w-2.5 h-2.5 mr-0.5" /> Available
</Badge>
@@ -0,0 +1,19 @@
import { Ban } from 'lucide-react';
import { Badge } from '@/components/ui/badge';
import { PINNED_UPDATE_BLOCKED_FALLBACK } from './types';
interface PinnedUpdateBadgeProps {
reason?: string | null;
className?: string;
}
export function PinnedUpdateBadge({
reason,
className = 'text-[10px] px-1.5 py-0 h-4 bg-muted text-muted-foreground border-card-border/40 shrink-0',
}: PinnedUpdateBadgeProps) {
return (
<Badge className={className} title={reason ?? PINNED_UPDATE_BLOCKED_FALLBACK}>
<Ban className="w-2.5 h-2.5 mr-0.5" strokeWidth={1.5} /> Pinned
</Badge>
);
}
@@ -73,7 +73,7 @@ export function ReconnectingOverlay({ preUpdateStartedAt }: ReconnectingOverlayP
<Loader2 className="w-10 h-10 text-muted-foreground animate-spin mx-auto" strokeWidth={1.5} />
<h2 className="text-lg font-medium">Updating Sencho...</h2>
<p className="text-sm text-muted-foreground max-w-sm">
The server is pulling the latest image and restarting. This page will reload automatically.
The server is pulling the update and restarting. This page will reload automatically.
</p>
<p className="text-xs text-muted-foreground tabular-nums">{elapsed}s elapsed</p>
</>
@@ -0,0 +1,49 @@
import { describe, it, expect, vi } from 'vitest';
import { render, screen } from '@testing-library/react';
vi.mock('@/components/ui/modal', () => ({
ConfirmModal: ({ open, title, children, confirmLabel }: {
open: boolean; title: string; children: React.ReactNode; confirmLabel: React.ReactNode;
}) => open ? (
<div>
<h2>{title}</h2>
{children}
<button type="button">{confirmLabel}</button>
</div>
) : null,
}));
import { LocalUpdateConfirmDialog } from '../LocalUpdateConfirmDialog';
describe('LocalUpdateConfirmDialog', () => {
it('explains semver repinning when compose and target refs are known', () => {
render(
<LocalUpdateConfirmDialog
open
onOpenChange={vi.fn()}
onConfirm={vi.fn()}
imagePinKind="semver"
composeImageRef="saelix/sencho:0.93.3"
targetImageRef="saelix/sencho:0.94.0"
targetVersion="0.94.0"
/>,
);
expect(screen.getByText(/rewrites it to/i)).toBeInTheDocument();
expect(screen.getByText('saelix/sencho:0.93.3')).toBeInTheDocument();
expect(screen.getByText('saelix/sencho:0.94.0')).toBeInTheDocument();
});
it('uses the generic pull copy for a floating tag', () => {
render(
<LocalUpdateConfirmDialog
open
onOpenChange={vi.fn()}
onConfirm={vi.fn()}
imagePinKind="floating"
targetVersion="0.94.0"
/>,
);
expect(screen.getByText(/Pulls Sencho v0\.94\.0/i)).toBeInTheDocument();
expect(screen.queryByText(/rewrites it to/i)).not.toBeInTheDocument();
});
});
@@ -116,10 +116,16 @@ describe('NodeCard', () => {
expect(screen.getByRole('button', { name: /Update/ })).toBeInTheDocument();
});
it('hides the update button for a non-admin but still shows the read-only badge', () => {
useAuthMock.mockReturnValue({ isAdmin: false });
render(<NodeCard {...baseProps(onlineNode())} updateStatus={updateAvailableStatus} onUpdate={vi.fn()} />);
it('hides the update button and shows Pinned when updateBlocked', () => {
useAuthMock.mockReturnValue({ isAdmin: true });
render(
<NodeCard
{...baseProps(onlineNode())}
updateStatus={{ ...updateAvailableStatus, updateBlocked: true, updateBlockedReason: 'Digest pin.' }}
onUpdate={vi.fn()}
/>,
);
expect(screen.getByText('Pinned')).toBeInTheDocument();
expect(screen.queryByRole('button', { name: /Update/ })).not.toBeInTheDocument();
expect(screen.getByText('Update available')).toBeInTheDocument();
});
});
@@ -336,6 +336,17 @@ describe('NodeUpdatesSheet', () => {
expect(toast.info).not.toHaveBeenCalled();
});
it('hides the Update button and shows a Pinned badge when updateBlocked', () => {
const blocked: NodeUpdateStatus = {
...STATUSES[1],
updateBlocked: true,
updateBlockedReason: 'Digest pin blocks automatic update.',
};
render(<NodeUpdatesSheet {...baseProps({ updateStatuses: [STATUSES[0], blocked, STATUSES[2]] })} />);
expect(screen.getByText('Pinned')).toBeInTheDocument();
expect(screen.queryByRole('button', { name: /Update$/ })).not.toBeInTheDocument();
});
it('hides every mutating affordance for a non-admin but keeps the read-only table', () => {
render(<NodeUpdatesSheet {...baseProps({ isAdmin: false })} />);
// Read-only status remains visible
@@ -24,6 +24,11 @@ const STATUSES: NodeUpdateStatus[] = [
{ nodeId: 2, name: 'Edge', type: 'remote', version: '1.0.0', latestVersion: '1.1.0', updateAvailable: true, updateStatus: null },
];
const BLOCKED_STATUS: NodeUpdateStatus = {
nodeId: 3, name: 'Pinned', type: 'remote', version: '1.0.0', latestVersion: '1.1.0',
updateAvailable: true, updateStatus: null, updateBlocked: true, updateBlockedReason: 'Digest pin blocks update.',
};
beforeEach(() => {
apiFetchMock.mockReset();
toastSuccess.mockReset();
@@ -142,6 +147,78 @@ describe('useFleetUpdateStatus', () => {
expect(toastSuccess).toHaveBeenCalledWith(expect.stringContaining('2 nodes'));
});
it('triggerNodeUpdate on a blocked node toasts and does not POST', async () => {
apiFetchMock.mockResolvedValue(okJson({ nodes: [...STATUSES, BLOCKED_STATUS] }));
const { result } = renderHook(() => useFleetUpdateStatus());
await act(async () => { await result.current.fetchUpdateStatus(); });
apiFetchMock.mockClear();
await act(async () => { await result.current.triggerNodeUpdate(3); });
expect(toastError).toHaveBeenCalledWith('Digest pin blocks update.');
expect(apiFetchMock).not.toHaveBeenCalled();
});
it('confirmLocalUpdate forwards targetVersion when latestVersion is valid', async () => {
apiFetchMock.mockResolvedValue(okJson({ nodes: STATUSES }));
const { result } = renderHook(() => useFleetUpdateStatus());
await act(async () => { await result.current.fetchUpdateStatus(); });
await act(async () => { await result.current.triggerNodeUpdate(1); });
expect(result.current.localUpdateConfirm).toBe(1);
apiFetchMock.mockResolvedValue(okJson({ message: 'ok' }));
vi.stubGlobal('fetch', vi.fn(() => Promise.resolve(
new Response(JSON.stringify({ startedAt: 1000 }), { status: 200, headers: { 'Content-Type': 'application/json' } }),
)));
await act(async () => { await result.current.confirmLocalUpdate(); });
expect(apiFetchMock).toHaveBeenCalledWith(
'/fleet/nodes/1/update',
expect.objectContaining({
method: 'POST',
localOnly: true,
body: JSON.stringify({ targetVersion: '1.1.0' }),
}),
);
expect(result.current.reconnecting).toBe(true);
vi.unstubAllGlobals();
});
it('dismisses the reconnecting overlay when the local update resolves failed', async () => {
apiFetchMock.mockResolvedValue(okJson({ nodes: STATUSES }));
const { result } = renderHook(() => useFleetUpdateStatus());
await act(async () => { await result.current.fetchUpdateStatus(); });
vi.useFakeTimers();
try {
vi.stubGlobal('fetch', vi.fn(() => Promise.resolve(
new Response(JSON.stringify({ startedAt: 1000 }), { status: 200, headers: { 'Content-Type': 'application/json' } }),
)));
apiFetchMock.mockResolvedValueOnce(okJson({ message: 'ok' }));
await act(async () => { await result.current.triggerNodeUpdate(1); });
await act(async () => { await result.current.confirmLocalUpdate(); });
expect(result.current.reconnecting).toBe(true);
const failedLocal = {
...STATUSES[0],
updateStatus: 'failed' as const,
error: 'Pull failed',
};
apiFetchMock.mockResolvedValue(okJson({ nodes: [failedLocal, STATUSES[1]] }));
await act(async () => { await vi.advanceTimersByTimeAsync(3000); });
expect(result.current.reconnecting).toBe(false);
expect(toastError).toHaveBeenCalledWith('Pull failed');
} finally {
vi.useRealTimers();
vi.unstubAllGlobals();
}
});
it('checkUpdates opens the modal and toggles the checking flag', async () => {
apiFetchMock.mockResolvedValue(okJson({ nodes: STATUSES }));
const { result } = renderHook(() => useFleetUpdateStatus());
@@ -1,7 +1,30 @@
import { useState, useCallback, useRef } from 'react';
import { useState, useCallback, useRef, useEffect } from 'react';
import { apiFetch } from '@/lib/api';
import { toast } from '@/components/ui/toast-store';
import type { NodeUpdateStatus } from '../types';
import { isValidVersion } from '@/lib/version';
import { PINNED_UPDATE_BLOCKED_FALLBACK, type NodeUpdateStatus } from '../types';
/** POST body for an update trigger: forward the target release when it is a
* valid version so the receiving node can repin a semver pin to it; omit
* otherwise so the backend falls back to its compare target. */
function updateRequestInit(status: NodeUpdateStatus | undefined): RequestInit & { localOnly: true } {
const base = { method: 'POST', localOnly: true } as const;
return isValidVersion(status?.latestVersion)
? { ...base, body: JSON.stringify({ targetVersion: status!.latestVersion }) }
: base;
}
function parseUpdateError(err: Record<string, unknown>, fallback: string): string {
const nested = err?.data as Record<string, unknown> | undefined;
const message = err?.message ?? err?.error ?? nested?.error;
return typeof message === 'string' && message ? message : fallback;
}
function toastIfUpdateBlocked(status: NodeUpdateStatus | undefined): boolean {
if (!status?.updateBlocked) return false;
toast.error(status.updateBlockedReason ?? PINNED_UPDATE_BLOCKED_FALLBACK);
return true;
}
export function useFleetUpdateStatus() {
const [updateStatuses, setUpdateStatuses] = useState<NodeUpdateStatus[]>([]);
@@ -42,6 +65,9 @@ export function useFleetUpdateStatus() {
const triggerNodeUpdate = useCallback(async (nodeId: number) => {
const status = updateStatusesRef.current.find(s => s.nodeId === nodeId);
// A pin we cannot repin (digest/unknown) has no update action; the button
// is disabled upstream, but guard here so a stale click cannot POST.
if (toastIfUpdateBlocked(status)) return;
if (status?.type === 'local') {
setLocalUpdateConfirm(nodeId);
return;
@@ -49,13 +75,13 @@ export function useFleetUpdateStatus() {
setUpdatingNodeId(nodeId);
try {
const res = await apiFetch(`/fleet/nodes/${nodeId}/update`, { method: 'POST', localOnly: true });
const res = await apiFetch(`/fleet/nodes/${nodeId}/update`, updateRequestInit(status));
if (res.ok) {
toast.success(`Update initiated on ${status?.name ?? 'node'}.`);
fetchUpdateStatus();
} else {
const err = await res.json().catch(() => ({}));
toast.error(err?.message || err?.error || err?.data?.error || 'Failed to trigger update.');
toast.error(parseUpdateError(err, 'Failed to trigger update.'));
}
} catch (e: unknown) {
toast.error((e as Error)?.message || 'Something went wrong.');
@@ -68,6 +94,8 @@ export function useFleetUpdateStatus() {
const nodeId = localUpdateConfirm;
setLocalUpdateConfirm(null);
if (!nodeId) return;
const status = updateStatusesRef.current.find(s => s.nodeId === nodeId);
if (toastIfUpdateBlocked(status)) return;
setUpdatingNodeId(nodeId);
try {
@@ -82,13 +110,15 @@ export function useFleetUpdateStatus() {
}
} catch { /* fall back to offline-then-online detection */ }
const res = await apiFetch(`/fleet/nodes/${nodeId}/update`, { method: 'POST', localOnly: true });
const res = await apiFetch(`/fleet/nodes/${nodeId}/update`, updateRequestInit(status));
if (res.ok) {
setPreUpdateStartedAt(bootBefore);
setReconnecting(true);
} else {
// A blocked pin returns 409 fast (before any 202), so the overlay
// never starts here; surface the reason through the toast path.
const err = await res.json().catch(() => ({}));
toast.error(err?.message || err?.error || err?.data?.error || 'Failed to trigger local update.');
toast.error(parseUpdateError(err, 'Failed to trigger local update.'));
}
} catch (e: unknown) {
toast.error((e as Error)?.message || 'Something went wrong.');
@@ -110,7 +140,7 @@ export function useFleetUpdateStatus() {
fetchUpdateStatus();
} else {
const err = await res.json().catch(() => ({}));
toast.error(err?.message || err?.error || err?.data?.error || 'Failed to trigger fleet update.');
toast.error(parseUpdateError(err, 'Failed to trigger fleet update.'));
}
} catch (e: unknown) {
toast.error((e as Error)?.message || 'Something went wrong.');
@@ -137,6 +167,37 @@ export function useFleetUpdateStatus() {
setCheckingUpdates(false);
}, [fetchUpdateStatus]);
// While the reconnect overlay is up, poll the local node's update status.
// A pull/patch failure leaves the old gateway alive (no restart), so the
// overlay's health poll would sit for the full 5-minute timeout. Detecting
// the resolved `failed` status here dismisses the overlay fast and surfaces
// the error, instead of leaving the operator on the spinner. A genuine
// restart makes this endpoint unreachable (caught, keeps polling) and the
// overlay's own health poll reloads the page on success.
useEffect(() => {
if (!reconnecting) return;
const poll = setInterval(async () => {
try {
const res = await apiFetch('/fleet/update-status', { localOnly: true });
if (!res.ok) return;
const data = await res.json();
const nodes: NodeUpdateStatus[] = data.nodes ?? [];
setUpdateStatuses(prev => JSON.stringify(prev) === JSON.stringify(nodes) ? prev : nodes);
const local = nodes.find(s => s.type === 'local');
if (local && (local.updateStatus === 'failed' || local.updateStatus === 'timeout')) {
setReconnecting(false);
setPreUpdateStartedAt(null);
toast.error(local.error || 'Local update failed. The server did not restart.');
}
} catch (error) {
// Expected while the process restarts; the overlay's health poll
// drives the reload on success.
console.warn('[Fleet] Reconnect status poll failed:', error);
}
}, 3000);
return () => clearInterval(poll);
}, [reconnecting]);
return {
updateStatuses,
updatingNodeId,
@@ -31,6 +31,12 @@ export interface FleetNode {
pilot_last_seen?: number | null;
}
export type ImagePinKind = 'floating' | 'semver' | 'digest' | 'unknown';
/** Shown when the backend omits a node-specific block reason. */
export const PINNED_UPDATE_BLOCKED_FALLBACK =
'This node cannot be updated automatically while its image is pinned this way.';
export interface NodeUpdateStatus {
nodeId: number;
name: string;
@@ -42,6 +48,17 @@ export interface NodeUpdateStatus {
error?: string | null;
skipActive?: boolean;
skippedVersion?: string | null;
/** How this node's Sencho image is pinned. Present for the local node and,
* as the safe subset, for remotes that advertise it; null/absent otherwise. */
imagePinKind?: ImagePinKind | null;
/** The compose-declared image ref. Local node only (authenticated route). */
composeImageRef?: string | null;
/** The ref a semver pin will be rewritten to. Local node only. */
targetImageRef?: string | null;
/** True when the pin (digest/unknown) cannot be updated automatically. */
updateBlocked?: boolean;
/** Human-readable block reason. Local node only. */
updateBlockedReason?: string | null;
}
export type ViewMode = 'grid' | 'topology';