fix: repin semver compose tags during fleet self-update (#1587)

* fix: repin semver compose tags during fleet self-update

Fleet updates failed when docker-compose.yml pinned a semver tag because recreate reused the on-disk pin. Pull the target image first, rewrite semver pins via the update helper, and block digest or unresolved pins with fast 409s.

* fix: update OFFLINE_META shape in capability and node-registry meta tests
This commit is contained in:
Anso
2026-07-07 14:40:50 -04:00
committed by GitHub
parent dbe230eef3
commit e12602091a
31 changed files with 1460 additions and 74 deletions
@@ -61,6 +61,8 @@ describe('fetchRemoteMeta Authorization header', () => {
startedAt: null,
updateError: null,
online: false,
imagePinKind: null,
updateBlocked: false,
});
});
});
@@ -233,9 +233,11 @@ describe('GET /api/fleet/update-status (pilot-agent)', () => {
startedAt: 1700000000,
updateError: null,
online: true,
imagePinKind: null,
updateBlocked: false,
};
}
return { version: null, capabilities: [], startedAt: null, updateError: null, online: false };
return { version: null, capabilities: [], startedAt: null, updateError: null, online: false, imagePinKind: null, updateBlocked: false };
});
const res = await request(app).get('/api/fleet/update-status').set('Authorization', authHeader);
@@ -254,6 +256,8 @@ describe('GET /api/fleet/update-status (pilot-agent)', () => {
startedAt: null,
updateError: null,
online: false,
imagePinKind: null,
updateBlocked: false,
});
const res = await request(app).get('/api/fleet/update-status').set('Authorization', authHeader);
@@ -36,6 +36,8 @@ const META_ONLINE_OUTDATED: RemoteMeta = {
startedAt: 1,
updateError: null,
online: true,
imagePinKind: null,
updateBlocked: false,
};
const META_OFFLINE: RemoteMeta = {
@@ -44,6 +46,8 @@ const META_OFFLINE: RemoteMeta = {
startedAt: null,
updateError: null,
online: false,
imagePinKind: null,
updateBlocked: false,
};
const META_NO_SELF_UPDATE: RemoteMeta = {
@@ -251,7 +255,7 @@ describe('GET /api/fleet/update-status (remote-meta cache invalidation)', () =>
mockTargetForPilot();
mockCompareTargetFetch();
// Remote now reports a different version than before the update (signal 1).
mockMeta({ version: '0.99.0', capabilities: ['stacks'], startedAt: 2, updateError: null, online: true });
mockMeta({ version: '0.99.0', capabilities: ['stacks'], startedAt: 2, updateError: null, online: true, imagePinKind: null, updateBlocked: false });
const tracker = FleetUpdateTrackerService.getInstance();
tracker.set(proxyNodeId, tracker.create('updating', '0.83.0', null));
@@ -268,7 +272,7 @@ describe('GET /api/fleet/update-status (remote-meta cache invalidation)', () =>
it('does not drop the cache on a steady-state completed poll (transition guard)', async () => {
mockTargetForPilot();
mockCompareTargetFetch();
mockMeta({ version: '0.99.0', capabilities: ['stacks'], startedAt: 2, updateError: null, online: true });
mockMeta({ version: '0.99.0', capabilities: ['stacks'], startedAt: 2, updateError: null, online: true, imagePinKind: null, updateBlocked: false });
// Already completed before this poll: no transition, so no invalidation.
const tracker = FleetUpdateTrackerService.getInstance();
@@ -39,6 +39,8 @@ const ONLINE = (over: Partial<RemoteMeta> = {}): RemoteMeta => ({
startedAt: 1,
updateError: null,
online: true,
imagePinKind: null,
updateBlocked: false,
...over,
});
@@ -54,6 +54,8 @@ describe('NodeRegistry.fetchMetaForNode', () => {
startedAt: null,
updateError: null,
online: false,
imagePinKind: null,
updateBlocked: false,
});
expect(axiosSpy).not.toHaveBeenCalled();
db.deleteNode(nodeId);
@@ -25,7 +25,7 @@ afterAll(() => cleanupTestDb(tmpDir));
afterEach(() => vi.restoreAllMocks());
const ONLINE = { startedAt: null, updateError: null, online: true } as const;
const ONLINE = { startedAt: null, updateError: null, online: true, imagePinKind: null, updateBlocked: false } as const;
const capable: RemoteMeta = { version: '0.93.0', capabilities: ['fleet', 'cross-node-rbac'], ...ONLINE };
const incapable: RemoteMeta = { version: '0.92.0', capabilities: ['fleet', 'labels'], ...ONLINE };
@@ -42,7 +42,7 @@ describe('remoteSupportsCrossNodeRbac', () => {
it('fails closed when the remote is offline (empty capabilities)', async () => {
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode')
.mockResolvedValue({ version: null, capabilities: [], startedAt: null, updateError: null, online: false });
.mockResolvedValue({ version: null, capabilities: [], startedAt: null, updateError: null, online: false, imagePinKind: null, updateBlocked: false });
expect(await remoteSupportsCrossNodeRbac(NODE_ID)).toBe(false);
});
@@ -50,7 +50,7 @@ describe('remoteSupportsCrossNodeRbac', () => {
// A 0.0.0-dev image reports version null (non-semver) but is reachable and
// genuinely advertises the capability; it must not be wrongly denied.
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode')
.mockResolvedValue({ version: null, capabilities: ['fleet', 'cross-node-rbac'], startedAt: null, updateError: null, online: true });
.mockResolvedValue({ version: null, capabilities: ['fleet', 'cross-node-rbac'], startedAt: null, updateError: null, online: true, imagePinKind: null, updateBlocked: false });
expect(await remoteSupportsCrossNodeRbac(NODE_ID)).toBe(true);
});
@@ -0,0 +1,304 @@
/**
* Pure-unit coverage for the compose image handling that powers pinned
* self-update (GitHub issue: a version-pinned compose image could never update
* because recreate reused the running image). Split across the pure helper
* module and the three self-update argv builders that carry the repin handoff:
* - classifyImagePin / buildTargetImageRef / isValidImageRef,
* - resolveServiceImageFromContents (reverse -f precedence),
* - patchComposeServiceImage (comment-preserving round-trip),
* - buildComposeReadArgs (throwaway cat container),
* - the composeCopy branch of buildSelfUpdateComposeCmd,
* - the repinWritable branch of buildSelfUpdateRunArgs.
*/
import { describe, expect, it } from 'vitest';
import {
classifyImagePin,
buildTargetImageRef,
isRepinBlocked,
isValidImageRef,
resolveServiceImageFromContents,
patchComposeServiceImage,
} from '../helpers/selfUpdateCompose';
import {
buildComposeReadArgs,
buildSelfUpdateComposeCmd,
buildSelfUpdateRunArgs,
shQuote,
} from '../services/SelfUpdateService';
describe('isRepinBlocked', () => {
it('blocks digest and unknown pins only', () => {
expect(isRepinBlocked('digest')).toBe(true);
expect(isRepinBlocked('unknown')).toBe(true);
expect(isRepinBlocked('semver')).toBe(false);
expect(isRepinBlocked('floating')).toBe(false);
});
});
describe('classifyImagePin', () => {
it('classifies an explicit semver tag (with and without v prefix) as semver', () => {
expect(classifyImagePin('saelix/sencho:0.93.3')).toBe('semver');
expect(classifyImagePin('saelix/sencho:v1.2.3')).toBe('semver');
expect(classifyImagePin('ghcr.io/studio-saelix/sencho:1.0.0-rc.1')).toBe('semver');
});
it('classifies latest, implicit, and other moving tags as floating', () => {
expect(classifyImagePin('saelix/sencho:latest')).toBe('floating');
expect(classifyImagePin('saelix/sencho')).toBe('floating'); // implicit latest
expect(classifyImagePin('saelix/sencho:dev')).toBe('floating');
expect(classifyImagePin('saelix/sencho:edge')).toBe('floating');
});
it('does not mistake a registry port for a tag', () => {
// The colon before the last slash is a registry port, not a tag separator,
// so an untagged image on a custom-port registry is floating, not semver.
expect(classifyImagePin('registry.example.com:5000/sencho')).toBe('floating');
expect(classifyImagePin('registry.example.com:5000/sencho:0.93.3')).toBe('semver');
});
it('classifies a digest pin as digest', () => {
expect(classifyImagePin('saelix/sencho@sha256:abc123')).toBe('digest');
expect(classifyImagePin('saelix/sencho:0.93.3@sha256:abc123')).toBe('digest');
});
it('classifies interpolated, empty, or blank references as unknown', () => {
expect(classifyImagePin('saelix/sencho:${SENCHO_TAG}')).toBe('unknown');
expect(classifyImagePin('${SENCHO_IMAGE}')).toBe('unknown');
expect(classifyImagePin('')).toBe('unknown');
expect(classifyImagePin(' ')).toBe('unknown');
});
});
describe('buildTargetImageRef', () => {
it('swaps only the tag and keeps the registry and repository', () => {
expect(buildTargetImageRef('saelix/sencho:0.93.3', '0.94.0')).toBe('saelix/sencho:0.94.0');
expect(buildTargetImageRef('ghcr.io/studio-saelix/sencho:1.0.0', '1.1.0')).toBe(
'ghcr.io/studio-saelix/sencho:1.1.0',
);
});
it('preserves a v prefix on the current tag', () => {
expect(buildTargetImageRef('saelix/sencho:v0.93.3', '0.94.0')).toBe('saelix/sencho:v0.94.0');
});
it('preserves a -dev repository variant (only the tag is replaced)', () => {
expect(buildTargetImageRef('ghcr.io/studio-saelix/sencho-dev:0.93.3', '0.94.0')).toBe(
'ghcr.io/studio-saelix/sencho-dev:0.94.0',
);
});
it('drops any digest suffix on the current ref', () => {
expect(buildTargetImageRef('saelix/sencho:0.93.3@sha256:abc123', '0.94.0')).toBe(
'saelix/sencho:0.94.0',
);
});
it('adds a tag when the current ref has none', () => {
expect(buildTargetImageRef('saelix/sencho', '0.94.0')).toBe('saelix/sencho:0.94.0');
expect(buildTargetImageRef('registry.example.com:5000/sencho', '0.94.0')).toBe(
'registry.example.com:5000/sencho:0.94.0',
);
});
});
describe('isValidImageRef', () => {
it('accepts plausible registry/repo:tag and digest references', () => {
expect(isValidImageRef('saelix/sencho:0.93.3')).toBe(true);
expect(isValidImageRef('ghcr.io/studio-saelix/sencho-dev:v1.2.3')).toBe(true);
expect(isValidImageRef('registry.example.com:5000/sencho@sha256:abc')).toBe(true);
});
it('rejects empty, whitespace, and control-character references', () => {
expect(isValidImageRef('')).toBe(false);
expect(isValidImageRef('saelix/sencho :0.93.3')).toBe(false);
expect(isValidImageRef('saelix/sencho\n')).toBe(false);
});
it('rejects references with shell metacharacters or a leading separator', () => {
expect(isValidImageRef('saelix/sencho;rm -rf /')).toBe(false);
expect(isValidImageRef('$(touch pwned)')).toBe(false);
expect(isValidImageRef(':leadingcolon')).toBe(false);
});
it('rejects an implausibly long reference', () => {
expect(isValidImageRef('a'.repeat(513))).toBe(false);
});
});
describe('resolveServiceImageFromContents', () => {
const base = 'services:\n sencho:\n image: saelix/sencho:0.93.3\n';
it('returns the service image and classifies its pin', () => {
const resolved = resolveServiceImageFromContents([{ filePath: '/c/base.yml', content: base }], 'sencho');
expect(resolved).toEqual({
filePath: '/c/base.yml',
imageRef: 'saelix/sencho:0.93.3',
fileContent: base,
pinKind: 'semver',
});
});
it('lets a later -f override win (reverse precedence)', () => {
// Docker Compose merges later -f files over earlier ones, so the resolver
// must scan in reverse and return the highest-precedence declaration.
const override = 'services:\n sencho:\n image: saelix/sencho:latest\n';
const resolved = resolveServiceImageFromContents(
[
{ filePath: '/c/base.yml', content: base },
{ filePath: '/c/override.yml', content: override },
],
'sencho',
);
expect(resolved?.filePath).toBe('/c/override.yml');
expect(resolved?.imageRef).toBe('saelix/sencho:latest');
expect(resolved?.pinKind).toBe('floating');
});
it('falls back to the base file when the override does not set the image', () => {
// A common override only tweaks ports/env; it must not shadow the base image.
const override = 'services:\n sencho:\n ports:\n - "1852:1852"\n';
const resolved = resolveServiceImageFromContents(
[
{ filePath: '/c/base.yml', content: base },
{ filePath: '/c/override.yml', content: override },
],
'sencho',
);
expect(resolved?.filePath).toBe('/c/base.yml');
expect(resolved?.imageRef).toBe('saelix/sencho:0.93.3');
});
it('skips a malformed override and resolves from the next readable file', () => {
const malformed = 'services:\n sencho:\n image: : : bad';
const resolved = resolveServiceImageFromContents(
[
{ filePath: '/c/base.yml', content: base },
{ filePath: '/c/broken.yml', content: malformed },
],
'sencho',
);
expect(resolved?.filePath).toBe('/c/base.yml');
});
it('returns null when no file declares a string image for the service', () => {
const noImage = 'services:\n sencho:\n build: .\n';
expect(resolveServiceImageFromContents([{ filePath: '/c/x.yml', content: noImage }], 'sencho')).toBeNull();
expect(resolveServiceImageFromContents([{ filePath: '/c/x.yml', content: base }], 'other')).toBeNull();
});
});
describe('patchComposeServiceImage', () => {
it('rewrites only the target service image and preserves comments and other keys', () => {
const content = [
'# Sencho self-hosted',
'services:',
' sencho:',
' image: saelix/sencho:0.93.3 # pinned',
' ports:',
' - "1852:1852"',
' db:',
' image: postgres:16',
'',
].join('\n');
const patched = patchComposeServiceImage(content, 'sencho', 'saelix/sencho:0.94.0');
expect(patched).toContain('image: saelix/sencho:0.94.0');
expect(patched).not.toContain('saelix/sencho:0.93.3');
// Comments, sibling service, and unrelated keys survive the round-trip.
expect(patched).toContain('# Sencho self-hosted');
expect(patched).toContain('image: postgres:16');
expect(patched).toContain('- "1852:1852"');
});
it('throws when the service has no image to patch (never silently no-ops)', () => {
const content = 'services:\n sencho:\n build: .\n';
expect(() => patchComposeServiceImage(content, 'sencho', 'saelix/sencho:0.94.0')).toThrow(/no image/i);
});
});
describe('buildComposeReadArgs', () => {
it('emits a throwaway root cat container that mounts the working dir read-only', () => {
const args = buildComposeReadArgs('/opt/sencho', 'saelix/sencho:0.93.3', '/opt/sencho/docker-compose.yml');
expect(args).toEqual([
'run', '--rm',
'--user', 'root',
'--entrypoint', 'cat',
'-v', '/opt/sencho:/opt/sencho:ro',
'-w', '/opt/sencho',
'saelix/sencho:0.93.3',
'/opt/sencho/docker-compose.yml',
]);
});
it('keeps operator paths as discrete argv data (execFile spawns no shell)', () => {
const args = buildComposeReadArgs('/srv/$(touch pwned)', 'img', '/srv/$(touch pwned)/compose.yml');
expect(args).toContain('/srv/$(touch pwned):/srv/$(touch pwned):ro');
expect(args[args.length - 1]).toBe('/srv/$(touch pwned)/compose.yml');
});
});
describe('buildSelfUpdateComposeCmd (repin copy branch)', () => {
const fFlags = ['-f', '/opt/sencho/docker-compose.yml'];
const stderrTmp = '/tmp/_sencho_err';
const errorFile = '/app/data/.sencho-update-error';
it('copies the staged compose file onto the host before recreate', () => {
const cmd = buildSelfUpdateComposeCmd(fFlags, 'sencho', stderrTmp, errorFile, false, {
stagedPath: '/app/data/.sencho-compose-patch',
targetPath: '/opt/sencho/docker-compose.yml',
});
// The copy runs before the recreate so a failed write never half-applies.
expect(cmd.indexOf('cp ')).toBeLessThan(cmd.indexOf('up -d --force-recreate'));
expect(cmd).toContain(
`cp ${shQuote('/app/data/.sencho-compose-patch')} ${shQuote('/opt/sencho/docker-compose.yml')}`,
);
});
it('aborts before recreate when the copy fails and records the error', () => {
const cmd = buildSelfUpdateComposeCmd(fFlags, 'sencho', stderrTmp, errorFile, false, {
stagedPath: '/app/data/.sencho-compose-patch',
targetPath: '/opt/sencho/docker-compose.yml',
});
// A failed copy writes the error file and exits 1 before any recreate.
expect(cmd).toContain('exit 1');
expect(cmd.indexOf('exit 1')).toBeLessThan(cmd.indexOf('up -d --force-recreate'));
expect(cmd).toContain('Failed to write the updated compose file');
});
it('shell-quotes the copy paths so metacharacters cannot break the command', () => {
const cmd = buildSelfUpdateComposeCmd(fFlags, 'sencho', stderrTmp, errorFile, false, {
stagedPath: '/app/data/x; rm -rf /',
targetPath: '/opt/sencho/y; echo pwned',
});
expect(cmd).toContain(shQuote('/app/data/x; rm -rf /'));
expect(cmd).toContain(shQuote('/opt/sencho/y; echo pwned'));
expect(cmd).not.toContain('cp /app/data/x; rm -rf /');
});
it('omits the copy step entirely when no composeCopy is supplied', () => {
const cmd = buildSelfUpdateComposeCmd(fFlags, 'sencho', stderrTmp, errorFile, false);
expect(cmd).not.toContain('cp ');
expect(cmd).not.toContain('Failed to write the updated compose file');
});
});
describe('buildSelfUpdateRunArgs (repinWritable branch)', () => {
const COMPOSE = 'COMPOSE_CMD';
it('mounts the working dir read-write only when a repin is staged', () => {
const args = buildSelfUpdateRunArgs(
{ workingDir: '/opt/sencho', imageName: 'img', dataDirHost: '/opt/sencho/data', hostBindMounts: [], repinWritable: true },
COMPOSE,
);
expect(args).toContain('/opt/sencho:/opt/sencho:rw');
expect(args).not.toContain('/opt/sencho:/opt/sencho:ro');
});
it('keeps the working dir read-only when no repin is staged (minimal write scope)', () => {
const args = buildSelfUpdateRunArgs(
{ workingDir: '/opt/sencho', imageName: 'img', dataDirHost: '/opt/sencho/data', hostBindMounts: [], repinWritable: false },
COMPOSE,
);
expect(args).toContain('/opt/sencho:/opt/sencho:ro');
expect(args).not.toContain('/opt/sencho:/opt/sencho:rw');
});
});
@@ -0,0 +1,183 @@
/**
* Route coverage for pinned-compose self-update: targetVersion validation,
* synchronous preflight (409 before 202), and the safe public /api/meta pin
* subset (imagePinKind + updateBlocked, never composeImageRef).
*/
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import { buildTargetImageRef } from '../helpers/selfUpdateCompose';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
let tmpDir: string;
let app: import('express').Express;
let adminAuth: string;
let localNodeId: number;
let SelfUpdateService: typeof import('../services/SelfUpdateService').default;
let FleetUpdateTrackerService: typeof import('../services/FleetUpdateTrackerService').FleetUpdateTrackerService;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
function mockSelfUpdateAvailable(over: {
pinInfo?: { pinKind: 'semver' | 'floating' | 'digest' | 'unknown'; composeImageRef: string; filePath: string } | null;
preflight?: { ok: true } | { ok: false; reason: string };
available?: boolean;
} = {}) {
const svc = SelfUpdateService.getInstance();
vi.spyOn(svc, 'isAvailable').mockReturnValue(over.available ?? true);
vi.spyOn(svc, 'getPinInfo').mockResolvedValue(over.pinInfo ?? null);
vi.spyOn(svc, 'canSelfUpdateTarget').mockResolvedValue(over.preflight ?? { ok: true });
vi.spyOn(svc, 'triggerUpdate').mockResolvedValue(undefined);
vi.spyOn(svc, 'getLastError').mockReturnValue(null);
}
function mockCompareTargetFetch() {
vi.spyOn(globalThis, 'fetch').mockImplementation(async () =>
new Response(JSON.stringify({ tag_name: 'v0.99.0' }), {
status: 200,
headers: { 'content-type': 'application/json' },
}),
);
}
beforeAll(async () => {
tmpDir = await setupTestDb();
({ app } = await import('../index'));
SelfUpdateService = (await import('../services/SelfUpdateService')).default;
({ FleetUpdateTrackerService } = await import('../services/FleetUpdateTrackerService'));
({ DatabaseService } = await import('../services/DatabaseService'));
localNodeId = DatabaseService.getInstance().getNodes().find(n => n.type === 'local')!.id;
adminAuth = `Bearer ${jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' })}`;
});
afterAll(() => cleanupTestDb(tmpDir));
afterEach(() => {
vi.restoreAllMocks();
const tracker = FleetUpdateTrackerService.getInstance();
for (const [id] of tracker.entries()) tracker.delete(id);
});
describe('POST /api/system/update', () => {
it('returns 400 for a supplied but invalid targetVersion', async () => {
mockSelfUpdateAvailable();
const res = await request(app)
.post('/api/system/update')
.set('Authorization', adminAuth)
.send({ targetVersion: 'not-a-version' });
expect(res.status).toBe(400);
expect(res.body?.error).toMatch(/invalid target version/i);
});
it('returns 409 when preflight blocks a digest or unknown pin', async () => {
mockSelfUpdateAvailable({
preflight: { ok: false, reason: 'Cannot repin digest pin.' },
});
const res = await request(app)
.post('/api/system/update')
.set('Authorization', adminAuth)
.send({ targetVersion: '0.99.0' });
expect(res.status).toBe(409);
expect(res.body?.code).toBe('update_blocked');
expect(res.body?.error).toMatch(/cannot repin/i);
});
it('returns 202 and schedules the update when preflight passes', async () => {
const triggerSpy = vi.spyOn(SelfUpdateService.getInstance(), 'triggerUpdate').mockResolvedValue(undefined);
mockSelfUpdateAvailable({ preflight: { ok: true } });
const res = await request(app)
.post('/api/system/update')
.set('Authorization', adminAuth)
.send({ targetVersion: '0.99.0' });
expect(res.status).toBe(202);
expect(res.body?.message).toMatch(/restart/i);
// triggerUpdate runs on res finish + delay; flush the microtask queue.
await new Promise(r => setTimeout(r, 600));
expect(triggerSpy).toHaveBeenCalledWith({ targetVersion: '0.99.0' });
});
});
describe('GET /api/meta pin subset', () => {
it('exposes imagePinKind and updateBlocked but never composeImageRef', async () => {
mockSelfUpdateAvailable({
pinInfo: { pinKind: 'semver', composeImageRef: 'saelix/sencho:0.93.3', filePath: '/opt/sencho/docker-compose.yml' },
});
const res = await request(app).get('/api/meta');
expect(res.status).toBe(200);
expect(res.body.imagePinKind).toBe('semver');
expect(res.body.updateBlocked).toBe(false);
expect(res.body).not.toHaveProperty('composeImageRef');
expect(res.body).not.toHaveProperty('targetImageRef');
});
it('reports updateBlocked=true for a digest pin', async () => {
mockSelfUpdateAvailable({
pinInfo: { pinKind: 'digest', composeImageRef: 'saelix/sencho@sha256:abc', filePath: '/opt/sencho/docker-compose.yml' },
});
const res = await request(app).get('/api/meta');
expect(res.body.imagePinKind).toBe('digest');
expect(res.body.updateBlocked).toBe(true);
});
});
describe('POST /api/fleet/nodes/:nodeId/update (local preflight)', () => {
it('returns 409 before 202 when the local pin cannot be repinned', async () => {
mockCompareTargetFetch();
mockSelfUpdateAvailable({
preflight: { ok: false, reason: 'Blocked by digest pin.' },
});
const res = await request(app)
.post(`/api/fleet/nodes/${localNodeId}/update`)
.set('Authorization', adminAuth)
.send({ targetVersion: '0.99.0' });
expect(res.status).toBe(409);
expect(res.body?.code).toBe('update_blocked');
});
});
describe('GET /api/fleet/update-status pin projection', () => {
it('includes compose pin fields for the local node on the authenticated route', async () => {
mockCompareTargetFetch();
mockSelfUpdateAvailable({
pinInfo: { pinKind: 'semver', composeImageRef: 'saelix/sencho:0.83.0', filePath: '/opt/sencho/docker-compose.yml' },
});
const res = await request(app)
.get('/api/fleet/update-status')
.set('Authorization', adminAuth);
expect(res.status).toBe(200);
const local = res.body.nodes.find((n: { nodeId: number }) => n.nodeId === localNodeId);
expect(local.imagePinKind).toBe('semver');
expect(local.composeImageRef).toBe('saelix/sencho:0.83.0');
expect(local.targetImageRef).toBe(buildTargetImageRef(local.composeImageRef, local.latestVersion));
expect(local.updateBlocked).toBe(false);
});
it('marks the local node updateBlocked when the pin is unknown', async () => {
mockCompareTargetFetch();
mockSelfUpdateAvailable({
pinInfo: { pinKind: 'unknown', composeImageRef: 'saelix/sencho:${TAG}', filePath: '/opt/sencho/docker-compose.yml' },
});
const res = await request(app)
.get('/api/fleet/update-status')
.set('Authorization', adminAuth);
const local = res.body.nodes.find((n: { nodeId: number }) => n.nodeId === localNodeId);
expect(local.updateBlocked).toBe(true);
expect(local.updateBlockedReason).toMatch(/cannot update automatically/i);
});
});
+137
View File
@@ -0,0 +1,137 @@
import { parse, parseDocument } from 'yaml';
import semver from 'semver';
/**
* How the Sencho service's compose `image:` is pinned. Drives the Fleet
* self-update behavior:
* - `semver` : rewrite the tag to the target release, then recreate.
* - `floating` : pull the compose-declared ref (e.g. `:latest`), no rewrite.
* - `digest` : `@sha256:...` pin; blocked (needs a manual digest change).
* - `unknown` : interpolated (`${VAR}`), unparseable, or the service/image is
* absent; blocked, because the tag cannot be resolved safely.
*/
export type ImagePinKind = 'floating' | 'semver' | 'digest' | 'unknown';
/** True when Fleet cannot repin or resolve the compose image automatically. */
export function isRepinBlocked(pinKind: ImagePinKind): boolean {
return pinKind === 'digest' || pinKind === 'unknown';
}
export interface ResolvedComposeImage {
/** Absolute path of the highest-precedence compose file that sets the image. */
filePath: string;
/** The verbatim compose-declared image reference. */
imageRef: string;
/** The full contents of that file, so the caller can patch it in place. */
fileContent: string;
pinKind: ImagePinKind;
}
/**
* Classify how an image reference is pinned. Aligned with the preflight
* `usesLatestTag` rule: a digest wins, an interpolated value is unknown, a tag
* that parses as semver is a pin, and everything else (implicit latest and
* non-semver moving tags) is floating.
*/
export function classifyImagePin(imageRef: string): ImagePinKind {
const ref = imageRef?.trim();
if (!ref) return 'unknown';
// Compose variable interpolation: the real tag is only known at deploy time.
if (ref.includes('${') || ref.includes('$(')) return 'unknown';
if (ref.includes('@sha256:') || ref.startsWith('sha256:')) return 'digest';
const lastSlash = ref.lastIndexOf('/');
const lastColon = ref.lastIndexOf(':');
// A colon after the last slash is a tag separator; before it is a registry port.
const tag = lastColon > lastSlash ? ref.slice(lastColon + 1) : '';
if (!tag) return 'floating'; // no explicit tag -> implicit latest
if (tag === 'latest') return 'floating';
if (semver.valid(tag.replace(/^v/, ''))) return 'semver';
return 'floating'; // other moving tag (dev, stable, edge, ...)
}
/**
* Build the target image reference for a semver-pinned install: keep the
* registry and repository (including any `-dev` variant), swap only the tag to
* the target version. Any digest suffix on the current ref is dropped. A `v`
* prefix on the current tag is preserved so `:v1.2.3` stays `:v...`.
*/
export function buildTargetImageRef(currentRef: string, targetVersion: string): string {
const ref = currentRef.split('@')[0];
const lastSlash = ref.lastIndexOf('/');
const lastColon = ref.lastIndexOf(':');
const hasTag = lastColon > lastSlash;
const currentTag = hasTag ? ref.slice(lastColon + 1) : '';
const base = hasTag ? ref.slice(0, lastColon) : ref;
const prefix = /^v\d/.test(currentTag) ? 'v' : '';
return `${base}:${prefix}${targetVersion}`;
}
/**
* Conservative image-reference validation before `docker pull`. execFile runs
* docker with no shell, so this is defense in depth, not the only guard: it
* fails closed on whitespace, control characters, or an implausibly long value.
*/
export function isValidImageRef(ref: string): boolean {
if (!ref || ref.length > 512) return false;
if (/\s/.test(ref)) return false;
return /^[a-zA-Z0-9][a-zA-Z0-9._\-/:@]*$/.test(ref);
}
/**
* Resolve the Sencho service's image from already-read compose file contents.
* Files are given in compose `-f` order; they are scanned in REVERSE so the
* highest-precedence override that explicitly sets the service image wins,
* matching how Docker Compose merges later `-f` files over earlier ones.
* Returns null when no file declares a string image for the service.
*/
export function resolveServiceImageFromContents(
files: ReadonlyArray<{ filePath: string; content: string }>,
serviceName: string,
): ResolvedComposeImage | null {
for (let i = files.length - 1; i >= 0; i--) {
const { filePath, content } = files[i];
let parsed: unknown;
try {
parsed = parse(content);
} catch {
continue; // a malformed override cannot own the image; try the next file
}
const image = extractServiceImage(parsed, serviceName);
if (image) {
return { filePath, imageRef: image, fileContent: content, pinKind: classifyImagePin(image) };
}
}
return null;
}
/** Read `services.<name>.image` as a trimmed string, or null if absent/non-string. */
function extractServiceImage(parsed: unknown, serviceName: string): string | null {
if (!isRecord(parsed)) return null;
const services = parsed.services;
if (!isRecord(services)) return null;
const service = services[serviceName];
if (!isRecord(service)) return null;
const image = service.image;
if (typeof image !== 'string') return null;
const trimmed = image.trim();
return trimmed || null;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
/**
* Rewrite only the target service's `image:` value, preserving comments,
* formatting, and every other node via a document round-trip. Throws when the
* service has no image to patch, so a caller never silently no-ops.
*/
export function patchComposeServiceImage(content: string, serviceName: string, newImageRef: string): string {
const doc = parseDocument(content);
if (!doc.hasIn(['services', serviceName, 'image'])) {
throw new Error(`Service "${serviceName}" has no image to patch`);
}
doc.setIn(['services', serviceName, 'image'], newImageRef);
return String(doc);
}
+102 -7
View File
@@ -14,12 +14,12 @@ import { getHostMemory } from '../helpers/hostMemory';
import { FileSystemService } from '../services/FileSystemService';
import { ComposeService } from '../services/ComposeService';
import { StackOpLockService } from '../services/StackOpLockService';
import SelfUpdateService from '../services/SelfUpdateService';
import SelfUpdateService, { type PinInfo } from '../services/SelfUpdateService';
import { getSenchoVersion, isValidVersion } from '../services/CapabilityRegistry';
import { authMiddleware } from '../middleware/auth';
import { requirePaid, requireAdmin, requireNodeProxy } from '../middleware/tierGates';
import { requirePermission } from '../middleware/permissions';
import { scheduleLocalUpdate } from './license';
import { respondSelfUpdatePreflight, scheduleLocalUpdate } from './license';
import { runPolicyGate, assertPolicyGateAllows, buildPolicyGateOptions } from '../helpers/policyGate';
import { remoteSupportsCrossNodeRbac } from '../helpers/remoteCapabilities';
import { captureLocalNodeFiles, captureRemoteNodeFiles, buildSnapshotDocumentation, pickDossierFields, dossierHasContent, type SnapshotNodeData, type SnapshotDocumentation } from '../utils/snapshot-capture';
@@ -28,6 +28,8 @@ import { isValidStackName } from '../utils/validation';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
import { parseIntParam } from '../utils/parseIntParam';
import { parseRequestedTargetVersion, pickCompareTarget } from '../utils/targetVersion';
import { buildTargetImageRef, isRepinBlocked, type ImagePinKind } from '../helpers/selfUpdateCompose';
import { withTimeout, TimeoutError } from '../utils/withTimeout';
// Mirror the system-maintenance route timeout so fleet's local-node prune
@@ -54,6 +56,39 @@ import { LicenseService } from '../services/LicenseService';
const updateTracker = FleetUpdateTrackerService.getInstance();
const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull failure
// Shown in the Node Updates UI when a node's image is pinned in a way Fleet
// cannot repin (digest or an unresolved value). Node-neutral so it reads the
// same for the local node and a remote row.
const REPIN_BLOCKED_REASON =
'This node pins its Sencho image to a digest or a value Fleet cannot resolve, so it cannot update automatically. Change the image tag in its compose file, then update.';
const EMPTY_PIN_STATUS = {
imagePinKind: null,
composeImageRef: null,
targetImageRef: null,
updateBlocked: false,
updateBlockedReason: null,
} as const;
function localPinStatusFields(
pin: PinInfo,
compareVersion: string | null,
compareValid: boolean,
blockedReason: string,
) {
const updateBlocked = isRepinBlocked(pin.pinKind);
const compareTarget = pickCompareTarget(compareVersion, compareValid);
return {
imagePinKind: pin.pinKind,
composeImageRef: pin.composeImageRef,
targetImageRef:
pin.pinKind === 'semver' && compareTarget
? buildTargetImageRef(pin.composeImageRef, compareTarget)
: null,
updateBlocked,
updateBlockedReason: updateBlocked ? blockedReason : null,
};
}
// Throttle the forced latest-version refresh so a caller cannot loop the recheck
// endpoint to hammer GitHub / Docker Hub. The 30-minute cache still serves reads
// between forced refreshes; this only bounds how often we bypass it.
@@ -222,6 +257,12 @@ async function getCompareTarget(gatewayVersion: string | null) {
return result;
}
async function resolveUpdateTarget(requested?: string): Promise<string | undefined> {
if (requested !== undefined) return requested;
const { compareVersion, compareValid } = await getCompareTarget(getSenchoVersion());
return pickCompareTarget(compareVersion, compareValid);
}
async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
try {
const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(node.id));
@@ -1073,6 +1114,8 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
let remoteStartedAt: number | null = null;
let remoteUpdateError: string | null = null;
let remoteOnline = false;
let remoteImagePinKind: ImagePinKind | null = null;
let remoteUpdateBlocked = false;
if (node.type === 'local') {
version = gatewayVersion;
} else {
@@ -1081,6 +1124,8 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
remoteStartedAt = meta.startedAt;
remoteUpdateError = meta.updateError;
remoteOnline = meta.online;
remoteImagePinKind = meta.imagePinKind;
remoteUpdateBlocked = meta.updateBlocked;
}
if (tracker?.status === 'updating') {
@@ -1202,6 +1247,27 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
skippedVersion = skipRow.skippedVersion;
}
// Image-pin metadata. The local row gets full detail (this route is
// hub-only and authenticated); remote rows get only the safe subset
// (pinKind + blocked flag) carried on the remote's /api/meta. A full
// remote composeImageRef would need a new authenticated remote endpoint
// and is deliberately out of scope.
let imagePinKind: ImagePinKind | null = null;
let composeImageRef: string | null = null;
let targetImageRef: string | null = null;
let updateBlocked = false;
let updateBlockedReason: string | null = null;
if (node.type === 'local') {
const pin = await SelfUpdateService.getInstance().getPinInfo();
if (pin) {
({ imagePinKind, composeImageRef, targetImageRef, updateBlocked, updateBlockedReason } =
localPinStatusFields(pin, compareVersion, compareValid, REPIN_BLOCKED_REASON));
}
} else {
imagePinKind = remoteImagePinKind;
updateBlocked = remoteUpdateBlocked;
}
return {
nodeId: node.id,
name: node.name,
@@ -1213,6 +1279,11 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
error: currentTracker?.error ?? null,
skipActive,
skippedVersion,
imagePinKind,
composeImageRef,
targetImageRef,
updateBlocked,
updateBlockedReason,
};
}),
);
@@ -1231,6 +1302,7 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
error: null,
skipActive: false,
skippedVersion: null,
...EMPTY_PIN_STATUS,
};
});
@@ -1266,12 +1338,18 @@ fleetRouter.get('/update-status/release-notes', authMiddleware, async (req: Requ
// Pilot loopback targets carry an empty apiToken because the tunnel bridge
// re-injects admin auth; sending a malformed `Bearer ` header would 401 on
// the pilot's local Express. Omit the header in that case.
function postSystemUpdate(target: { apiUrl: string; apiToken: string }) {
//
// targetVersion is forwarded only when it is a valid semver so the remote can
// repin a semver-pinned compose to that release. It is omitted otherwise (never
// sent as null/invalid), and an older remote that predates this field simply
// ignores the extra body key and behaves as before.
function postSystemUpdate(target: { apiUrl: string; apiToken: string }, targetVersion?: string) {
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
return fetch(`${target.apiUrl.replace(/\/$/, '')}/api/system/update`, {
method: 'POST',
headers,
body: JSON.stringify(targetVersion ? { targetVersion } : {}),
signal: AbortSignal.timeout(10000),
});
}
@@ -1339,6 +1417,9 @@ fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, r
return;
}
const requestedTarget = parseRequestedTargetVersion(req, res);
if (requestedTarget === null) return; // invalid supplied value; 400 already sent
const existing = updateTracker.get(nodeId);
if (existing?.status === 'updating') {
if (Date.now() - existing.startedAt > UPDATE_TIMEOUT_MS) {
@@ -1359,12 +1440,15 @@ fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, r
}
if (node.type === 'local') {
if (!SelfUpdateService.getInstance().isAvailable()) {
const selfUpdate = SelfUpdateService.getInstance();
if (!selfUpdate.isAvailable()) {
res.status(503).json({ error: 'Self-update unavailable on the local node.' });
return;
}
const resolvedTarget = await resolveUpdateTarget(requestedTarget);
if (!respondSelfUpdatePreflight(res, await selfUpdate.canSelfUpdateTarget(resolvedTarget))) return;
updateTracker.set(nodeId, updateTracker.create('updating', getSenchoVersion(), null));
scheduleLocalUpdate(res, 'Update initiated on local node. The server will restart shortly.');
scheduleLocalUpdate(res, 'Update initiated on local node. The server will restart shortly.', resolvedTarget);
return;
}
@@ -1386,8 +1470,13 @@ fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, r
res.status(503).json({ error: 'Remote node does not support self-update. It may need to be updated manually first.' });
return;
}
if (meta.updateBlocked) {
res.status(409).json({ error: REPIN_BLOCKED_REASON, code: 'update_blocked' });
return;
}
const response = await postSystemUpdate(target);
const resolvedTarget = await resolveUpdateTarget(requestedTarget);
const response = await postSystemUpdate(target, resolvedTarget);
if (!response.ok) {
const err = await response.json().catch(() => ({}));
@@ -1417,6 +1506,9 @@ fleetRouter.post('/update-all', authMiddleware, async (req: Request, res: Respon
const nodes = db.getNodes();
const gatewayVersion = getSenchoVersion();
const { compareVersion, compareValid } = await getCompareTarget(gatewayVersion);
// Forward the compare target so each remote repins a semver pin to it; omit
// when there is no valid target so a remote falls back to legacy behavior.
const updateAllTarget = pickCompareTarget(compareVersion, compareValid);
const debug = isDebugEnabled();
console.log('[Fleet] Update-all triggered,', nodes.length, 'nodes registered');
@@ -1451,10 +1543,13 @@ fleetRouter.post('/update-all', authMiddleware, async (req: Request, res: Respon
if (!meta.capabilities.includes('self-update')) {
return { name: node.name, triggered: false };
}
if (meta.updateBlocked) {
return { name: node.name, triggered: false };
}
if (isValidVersion(meta.version) && compareValid && !semver.lt(meta.version, compareVersion!)) {
return { name: node.name, triggered: false };
}
const response = await postSystemUpdate(target);
const response = await postSystemUpdate(target, updateAllTarget);
if (response.ok) {
updateTracker.set(node.id, updateTracker.create('updating', meta.version, meta.startedAt));
return { name: node.name, triggered: true };
+27 -5
View File
@@ -3,6 +3,8 @@ import { LicenseService } from '../services/LicenseService';
import SelfUpdateService from '../services/SelfUpdateService';
import { requireAdmin } from '../middleware/tierGates';
import { rejectApiTokenScope } from '../middleware/apiTokenScope';
import { parseRequestedTargetVersion } from '../utils/targetVersion';
import type { SelfUpdatePreflight } from '../services/SelfUpdateService';
const LICENSE_SCOPE_MESSAGE = 'API tokens cannot manage licenses.';
@@ -83,15 +85,29 @@ licenseRouter.get('/billing-portal', async (_req: Request, res: Response): Promi
* Respond 202, then trigger the "last breath" self-update after flush.
* Exported because the fleet "update this node" route reuses the same
* response shape + post-flush trigger for local-node self-updates.
*
* `targetVersion` (the Fleet compare target) drives the pinned-image repin;
* when omitted, triggerUpdate keeps the legacy behavior of pulling the running
* image and recreating from the on-disk compose.
*/
export function scheduleLocalUpdate(res: Response, message: string): void {
/** Send 409 when preflight fails; returns false so the route can early-return. */
export function respondSelfUpdatePreflight(
res: Response,
preflight: SelfUpdatePreflight,
): preflight is { ok: true } {
if (preflight.ok) return true;
res.status(409).json({ error: preflight.reason, code: 'update_blocked' });
return false;
}
export function scheduleLocalUpdate(res: Response, message: string, targetVersion?: string): void {
res.status(202).json({ message });
res.on('finish', () => {
setTimeout(() => {
// Defense in depth: triggerUpdate records its own errors into
// lastUpdateError; guard against an unexpected throw becoming an
// unhandled rejection.
SelfUpdateService.getInstance().triggerUpdate().catch((err) => {
SelfUpdateService.getInstance().triggerUpdate({ targetVersion }).catch((err) => {
console.error('[SelfUpdate] Unexpected error during triggerUpdate:', err);
});
}, 500);
@@ -100,11 +116,17 @@ export function scheduleLocalUpdate(res: Response, message: string): void {
export const systemUpdateRouter = Router();
systemUpdateRouter.post('/update', (req: Request, res: Response): void => {
systemUpdateRouter.post('/update', async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
if (!SelfUpdateService.getInstance().isAvailable()) {
const selfUpdate = SelfUpdateService.getInstance();
if (!selfUpdate.isAvailable()) {
res.status(503).json({ error: 'Self-update unavailable. Sencho must be deployed via Docker Compose.' });
return;
}
scheduleLocalUpdate(res, 'Update initiated. The server will restart shortly.');
const targetVersion = parseRequestedTargetVersion(req, res);
if (targetVersion === null) return; // invalid supplied value; 400 already sent
// Fail fast on a pin we cannot repin (digest/unknown) so the caller gets a
// 409 instead of a 202 that would later fail after the reconnect overlay.
if (!respondSelfUpdatePreflight(res, await selfUpdate.canSelfUpdateTarget(targetVersion))) return;
scheduleLocalUpdate(res, 'Update initiated. The server will restart shortly.', targetVersion);
});
+13 -2
View File
@@ -1,5 +1,6 @@
import { Router, type Request, type Response } from 'express';
import { getActiveCapabilities, getSenchoVersion } from '../services/CapabilityRegistry';
import { isRepinBlocked } from '../helpers/selfUpdateCompose';
import { MeshService } from '../services/MeshService';
import SelfUpdateService from '../services/SelfUpdateService';
@@ -28,13 +29,23 @@ metaRouter.get('/health', (_req: Request, res: Response): void => {
// Public meta endpoint. Returns this instance's version and supported
// capabilities. No auth required (like /health). Used by remote nodes during
// connection tests.
metaRouter.get('/meta', (_req: Request, res: Response): void => {
const updateError = SelfUpdateService.getInstance().getLastError();
//
// Image-pin fields are intentionally limited to the non-sensitive subset:
// `imagePinKind` (a bounded enum) and `updateBlocked` (a boolean). The full
// composeImageRef is NEVER exposed here because this endpoint is public and the
// ref can carry a private registry/repository name.
metaRouter.get('/meta', async (_req: Request, res: Response): Promise<void> => {
const selfUpdate = SelfUpdateService.getInstance();
const updateError = selfUpdate.getLastError();
const pin = await selfUpdate.getPinInfo({ cacheOnly: true });
const updateBlocked = pin ? isRepinBlocked(pin.pinKind) : false;
res.json({
version: getSenchoVersion(),
capabilities: getActiveCapabilities(),
startedAt: processStartedAt,
experimental: process.env.SENCHO_EXPERIMENTAL === 'true',
...(pin ? { imagePinKind: pin.pinKind } : {}),
updateBlocked,
...(updateError ? { updateError } : {}),
});
});
@@ -4,6 +4,16 @@ import fs from 'fs';
import semver from 'semver';
import { SENCHO_VERSION } from '../generated/version';
import { isDebugEnabled } from '../utils/debug';
import type { ImagePinKind } from '../helpers/selfUpdateCompose';
const IMAGE_PIN_KINDS: readonly ImagePinKind[] = ['floating', 'semver', 'digest', 'unknown'];
/** Coerce an untrusted /api/meta value to a known pin kind, or null. */
function parseImagePinKind(value: unknown): ImagePinKind | null {
return typeof value === 'string' && (IMAGE_PIN_KINDS as readonly string[]).includes(value)
? (value as ImagePinKind)
: null;
}
/**
* Static registry of capabilities supported by THIS Sencho instance.
@@ -96,6 +106,14 @@ export interface RemoteMeta {
updateError: string | null;
/** True when the /api/meta request succeeded (node is reachable). */
online: boolean;
/**
* How the remote pins its Sencho image, when it advertises it. Null for an
* older remote that predates this field or one that could not classify its
* pin. The hub uses only this safe subset (no full image ref) for remote rows.
*/
imagePinKind: ImagePinKind | null;
/** True when the remote reports its update is blocked (digest/unknown pin). */
updateBlocked: boolean;
}
// Runtime capability overrides; services call disableCapability() during init.
@@ -142,6 +160,8 @@ export const OFFLINE_META: RemoteMeta = {
startedAt: null,
updateError: null,
online: false,
imagePinKind: null,
updateBlocked: false,
};
/** Strip any `user:pass@` userinfo from a URL so credentials never reach the logs. */
@@ -164,6 +184,8 @@ export async function fetchRemoteMeta(baseUrl: string, apiToken: string): Promis
startedAt: typeof res.data.startedAt === 'number' ? res.data.startedAt : null,
updateError: typeof res.data.updateError === 'string' ? res.data.updateError : null,
online: true,
imagePinKind: parseImagePinKind(res.data.imagePinKind),
updateBlocked: res.data.updateBlocked === true,
};
if (isDebugEnabled()) {
// Diagnostic aid for "why is this feature gated?": log the resolved version
+247 -29
View File
@@ -5,6 +5,15 @@ import DockerController from './DockerController';
import { DatabaseService } from './DatabaseService';
import { disableCapability } from './CapabilityRegistry';
import { isDebugEnabled } from '../utils/debug';
import {
buildTargetImageRef,
isRepinBlocked,
isValidImageRef,
patchComposeServiceImage,
resolveServiceImageFromContents,
type ImagePinKind,
type ResolvedComposeImage,
} from '../helpers/selfUpdateCompose';
const execFileAsync = promisify(execFile);
@@ -13,6 +22,23 @@ const execFileAsync = promisify(execFile);
// the NEW gateway process (which always mounts /app/data) can reach it.
const UPDATE_ERROR_FILE = '/app/data/.sencho-update-error';
// Rewritten compose file the main process stages under /app/data for the helper
// to copy onto the host compose file. Uses the same /app/data handoff as the
// error file because the main process cannot reach the compose working dir.
const STAGED_PATCH_FILE = '/app/data/.sencho-compose-patch';
// Cap how long a cached pin classification is served to status/meta callers.
// Update DECISIONS always read fresh (fresh: true), so this only bounds how
// stale a displayed pin kind can be after a live compose edit.
const PIN_CACHE_TTL_MS = 5 * 60 * 1000;
// Operator-facing block reasons. Deliberately generic: no host paths, compose
// contents, or registry/repository names (this text can reach the UI).
const UPDATE_BLOCKED_REASON =
'This install pins the Sencho image to a digest or a value Fleet cannot resolve, so it cannot repin automatically. Change the image tag in your compose file, then update again.';
const UPDATE_READ_FAILED_REASON =
'Could not read the Sencho compose file to determine how its image is pinned. Confirm the update helper can reach the compose directory, then try again.';
interface HostMount {
source: string;
destination: string;
@@ -25,6 +51,22 @@ export type DockerMount = {
Destination: string;
};
/** Pin classification exposed to callers (status/meta), without file contents. */
export interface PinInfo {
pinKind: ImagePinKind;
composeImageRef: string;
filePath: string;
}
/** Outcome of the pre-update check the route layer runs before responding. */
export type SelfUpdatePreflight = { ok: true } | { ok: false; reason: string };
/** A staged compose rewrite the helper copies onto the host before recreate. */
export interface ComposeCopy {
stagedPath: string;
targetPath: string;
}
/**
* Find the host-side path Docker resolved for /app/data, regardless of whether
* the operator declared a bind or a named volume. The helper container uses
@@ -48,15 +90,36 @@ export function shQuote(value: string): string {
return `'${value.replace(/'/g, `'\\''`)}'`;
}
/**
* Build the argv for a one-shot helper that reads a single compose config file
* from the host. The main process cannot open the compose working dir, so it
* mounts it read-only into a throwaway container and cats the file out. Pure and
* exported for unit testing. The path is a discrete argv element handed to
* execFile (no shell), so a path with shell metacharacters stays inert.
*/
export function buildComposeReadArgs(workingDir: string, imageName: string, configFilePath: string): string[] {
return [
'run', '--rm',
'--user', 'root',
'--entrypoint', 'cat',
'-v', `${workingDir}:${workingDir}:ro`,
'-w', workingDir,
imageName,
configFilePath,
];
}
/**
* Build the shell command the helper container runs to recreate Sencho. Kept as
* a pure, exported function so the prune-on-update branch is unit-testable.
* a pure, exported function so the prune-on-update and repin-copy branches are
* unit-testable.
*
* Label-derived inputs (serviceName, the fFlags config paths) are shell-quoted
* so they cannot alter the command structure. The recreate writes the error
* file only on failure; the optional dangling prune runs only on success, so
* the two branches never overlap. The prune suppresses its own output and
* `|| true`, so it can never alter $ec or be mistaken for an update error.
* Label-derived inputs (serviceName, the fFlags config paths) and the repin copy
* paths are shell-quoted so they cannot alter the command structure. When a
* composeCopy is present the rewritten compose file is copied onto the host
* BEFORE the recreate; a failed copy aborts before recreate so a half-applied
* update never runs. The recreate writes the error file only on failure; the
* optional dangling prune runs only on success, so the branches never overlap.
*/
export function buildSelfUpdateComposeCmd(
fFlags: string[],
@@ -64,10 +127,15 @@ export function buildSelfUpdateComposeCmd(
stderrTmp: string,
errorFile: string,
pruneOnUpdate: boolean,
composeCopy?: ComposeCopy,
): string {
const recreate = ['docker compose', ...fFlags.map(shQuote), 'up -d --force-recreate', shQuote(serviceName), `2>${stderrTmp}`].join(' ');
const copyStep = composeCopy
? [`cp ${shQuote(composeCopy.stagedPath)} ${shQuote(composeCopy.targetPath)} 2>${stderrTmp} || { { echo "exit=1"; echo "Failed to write the updated compose file"; cat ${stderrTmp}; } > ${errorFile} 2>/dev/null; exit 1; }`]
: [];
return [
'sleep 3',
...copyStep,
recreate,
'ec=$?',
`if [ $ec -ne 0 ]; then { echo "exit=$ec"; cat ${stderrTmp}; } > ${errorFile} 2>/dev/null; fi`,
@@ -97,15 +165,19 @@ interface ComposeContext {
* paths (compose labels / container mounts). They are placed as discrete argv
* elements and handed to execFile, which spawns docker with no shell, so a path
* carrying shell metacharacters stays inert data and never reaches a shell.
*
* workingDir is mounted read-only unless `repinWritable` is set, which happens
* only for a semver repin that must copy the rewritten compose file onto the
* host. Limiting :rw to that case keeps the helper's host write scope minimal.
*/
export function buildSelfUpdateRunArgs(
ctx: Pick<ComposeContext, 'workingDir' | 'imageName' | 'dataDirHost' | 'hostBindMounts'>,
ctx: Pick<ComposeContext, 'workingDir' | 'imageName' | 'dataDirHost' | 'hostBindMounts'> & { repinWritable?: boolean },
composeCmd: string,
): string[] {
const { workingDir, imageName, dataDirHost, hostBindMounts } = ctx;
const { workingDir, imageName, dataDirHost, hostBindMounts, repinWritable } = ctx;
const mountArgs: string[] = [
'-v', '/var/run/docker.sock:/var/run/docker.sock',
'-v', `${workingDir}:${workingDir}:ro`,
'-v', `${workingDir}:${workingDir}:${repinWritable ? 'rw' : 'ro'}`,
];
if (dataDirHost) {
mountArgs.push('-v', `${dataDirHost}:/app/data:rw`);
@@ -133,6 +205,7 @@ class SelfUpdateService {
private canSelfUpdate = false;
private composeContext: ComposeContext | null = null;
private lastUpdateError: string | null = null;
private pinCache: { info: ResolvedComposeImage | null; at: number } | null = null;
public static getInstance(): SelfUpdateService {
if (!SelfUpdateService.instance) {
@@ -177,7 +250,10 @@ class SelfUpdateService {
return;
}
// Read the container's own image name for direct docker pull
// Read the container's own image name for the legacy (no target) pull and
// as a fallback ref. It is captured once here and can go stale if the
// compose file is edited later, so update decisions resolve the compose
// image fresh instead of trusting this value.
const imageName = info.Config?.Image;
if (!imageName) {
console.log('[SelfUpdate] Could not determine container image name');
@@ -195,13 +271,22 @@ class SelfUpdateService {
const dataDirHost = findDataDirHost(rawMounts);
if (!dataDirHost) {
console.log('[SelfUpdate] /app/data mount not found - update error recovery will be unavailable');
console.log('[SelfUpdate] /app/data mount not found - update error recovery and compose repin will be unavailable');
}
this.composeContext = { workingDir, configFiles, serviceName, imageName, dataDirHost, hostBindMounts };
this.canSelfUpdate = true;
console.log(`[SelfUpdate] Ready - service="${serviceName}" image="${imageName}" in ${workingDir}`);
// Warm the pin-info cache in the background for diagnostics and the first
// status/meta read. This is never a source of truth for an update
// decision, which always resolves fresh.
void this.resolveComposeImage(true)
.then(resolved => {
if (resolved) console.log(`[SelfUpdate] Compose image pin: ${resolved.imageRef} (${resolved.pinKind})`);
})
.catch(() => { /* diagnostic only */ });
// Surface any error from a previous failed update attempt (persisted by
// the helper container) so the new process can report it to the user.
this.recoverPreviousError();
@@ -225,6 +310,73 @@ class SelfUpdateService {
this.lastUpdateError = null;
}
/**
* Classify how this instance's compose image is pinned. Serves a short-lived
* cache to frequent status/meta callers; pass `{ fresh: true }` for update
* decisions so a live compose edit is always reflected. Returns null when
* self-update is unavailable or the compose file could not be read.
*/
async getPinInfo(opts?: { fresh?: boolean; cacheOnly?: boolean }): Promise<PinInfo | null> {
const resolved = await this.resolveComposeImage(opts?.fresh ?? false, opts?.cacheOnly ?? false);
if (!resolved) return null;
return { pinKind: resolved.pinKind, composeImageRef: resolved.imageRef, filePath: resolved.filePath };
}
/**
* Preflight the route layer runs before responding, so a blocked update fails
* fast with a 409 instead of returning 202 and stalling the reconnect overlay.
* A missing target version is the legacy pull-current path and makes no repin
* decision. A digest/unknown pin (or an unreadable compose file) is blocked.
*/
async canSelfUpdateTarget(targetVersion?: string): Promise<SelfUpdatePreflight> {
if (!this.canSelfUpdate) {
return { ok: false, reason: 'Self-update is unavailable on this instance.' };
}
if (!targetVersion) return { ok: true };
const info = await this.getPinInfo({ fresh: true });
if (!info) return { ok: false, reason: UPDATE_READ_FAILED_REASON };
if (isRepinBlocked(info.pinKind)) {
return { ok: false, reason: UPDATE_BLOCKED_REASON };
}
return { ok: true };
}
/** Resolve the compose-declared service image fresh from the host, or from a
* short-lived cache. Reads each config file via a throwaway cat container. */
private async resolveComposeImage(fresh: boolean, cacheOnly = false): Promise<ResolvedComposeImage | null> {
if (!this.composeContext) return null;
const now = Date.now();
if (!fresh && this.pinCache && now - this.pinCache.at < PIN_CACHE_TTL_MS) {
return this.pinCache.info;
}
if (cacheOnly) return null;
const { workingDir, configFiles, serviceName, imageName } = this.composeContext;
const env = this.buildEnv();
const debug = isDebugEnabled();
const files: { filePath: string; content: string }[] = [];
for (const raw of configFiles.split(',')) {
const filePath = raw.trim();
if (!filePath) continue;
try {
const { stdout } = await execFileAsync('docker', buildComposeReadArgs(workingDir, imageName, filePath), {
env,
timeout: 30_000,
maxBuffer: 10 * 1024 * 1024,
});
files.push({ filePath, content: stdout });
} catch (error) {
if (debug) console.debug('[SelfUpdate:debug] Could not read compose file', filePath, (error as Error).message);
}
}
const info = resolveServiceImageFromContents(files, serviceName);
this.pinCache = { info, at: now };
return info;
}
private buildEnv(): NodeJS.ProcessEnv {
return { ...process.env, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' };
}
/** Surfaces any error the helper container persisted before the previous
* gateway process died, then deletes the file. */
private recoverPreviousError(): void {
@@ -242,25 +394,63 @@ class SelfUpdateService {
}
}
async triggerUpdate(): Promise<void> {
/**
* Pull the target image, then (for a semver pin) rewrite the compose tag and
* recreate. Pull happens BEFORE any compose mutation, so a failed pull leaves
* the compose file untouched. `targetVersion` comes from the Fleet compare
* target; when omitted this keeps the legacy behavior of pulling the running
* image and recreating from the on-disk compose.
*/
async triggerUpdate(options?: { targetVersion?: string }): Promise<void> {
if (!this.composeContext) return;
const { workingDir, configFiles, serviceName, imageName, dataDirHost, hostBindMounts } = this.composeContext;
const env = { ...process.env, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' };
const env = this.buildEnv();
this.lastUpdateError = null;
try { fs.unlinkSync(UPDATE_ERROR_FILE); } catch { /* absent is the steady state */ }
try { fs.unlinkSync(STAGED_PATCH_FILE); } catch { /* absent is the steady state */ }
// Async pull: a sync execFileSync blocks the event loop, which lets the frontend
// overlay see a false "online" response between the pull finishing and the restart.
const { imageName, serviceName, dataDirHost } = this.composeContext;
const targetVersion = options?.targetVersion;
let pullRef = imageName;
let repin: { resolved: ResolvedComposeImage; ref: string } | null = null;
if (targetVersion) {
const resolved = await this.resolveComposeImage(true);
const pinKind = resolved?.pinKind ?? 'unknown';
// Defense in depth: the route preflight already rejected these, but the
// compose file could change between preflight and this last-breath call.
if (!resolved || isRepinBlocked(pinKind)) {
this.lastUpdateError = resolved ? UPDATE_BLOCKED_REASON : UPDATE_READ_FAILED_REASON;
console.error('[SelfUpdate] Update blocked:', this.lastUpdateError);
return;
}
pullRef = pinKind === 'semver' ? buildTargetImageRef(resolved.imageRef, targetVersion) : resolved.imageRef;
if (!isValidImageRef(pullRef)) {
this.lastUpdateError = 'Aborting update: the computed image reference is invalid.';
console.error('[SelfUpdate] Update blocked:', this.lastUpdateError, pullRef);
return;
}
if (pinKind === 'semver') {
if (!dataDirHost) {
this.lastUpdateError =
'Cannot rewrite the pinned compose image: the data directory needed for the update handoff is not mounted. Change the image tag manually and update again.';
console.error('[SelfUpdate] Repin blocked:', this.lastUpdateError);
return;
}
repin = { resolved, ref: pullRef };
}
}
// Pull FIRST so the compose file is never mutated for an image we could not
// fetch. A sync execFileSync would block the event loop and let the overlay
// see a false "online" between the pull finishing and the restart.
const debug = isDebugEnabled();
const pullStart = Date.now();
console.log(`[SelfUpdate] Pulling latest image: ${imageName}...`);
if (debug) console.debug('[SelfUpdate:debug] Pull context:', { workingDir, configFiles, serviceName, dataDirHost, mountCount: hostBindMounts.length });
console.log(`[SelfUpdate] Pulling image: ${pullRef}...`);
if (debug) console.debug('[SelfUpdate:debug] Pull context:', { targetVersion: targetVersion ?? null, repin: !!repin });
try {
await execFileAsync('docker', ['pull', imageName], {
env,
timeout: 300_000, // 5 min max for pull
});
await execFileAsync('docker', ['pull', pullRef], { env, timeout: 300_000 });
if (debug) console.debug('[SelfUpdate:debug] Pull completed in', Math.round((Date.now() - pullStart) / 1000) + 's');
} catch (error) {
const stderr = (error as { stderr?: Buffer | string })?.stderr?.toString().trim();
@@ -269,11 +459,39 @@ class SelfUpdateService {
return;
}
// The main container cannot access the compose file at its host path,
// so the helper bind-mounts the compose working directory from the host.
// Run attached (no -d): if compose recreate fails before it kills us,
// execFile's callback receives the helper's exit code + stderr directly.
console.log(`[SelfUpdate] Spawning updater container... (last breath)`);
// Rewrite the pinned tag only after a confirmed pull (semver only). The
// rewrite is staged under /app/data; the helper copies it onto the host
// compose file just before recreate.
let composeCopy: ComposeCopy | undefined;
if (repin) {
try {
const patched = patchComposeServiceImage(repin.resolved.fileContent, serviceName, repin.ref);
fs.writeFileSync(STAGED_PATCH_FILE, patched, 'utf8');
composeCopy = { stagedPath: STAGED_PATCH_FILE, targetPath: repin.resolved.filePath };
console.log(`[SelfUpdate] Repinning "${serviceName}" image to ${repin.ref}`);
} catch (error) {
this.lastUpdateError =
`Pulled ${repin.ref} but could not rewrite the compose image tag (${(error as Error).message}). Change the tag manually and update again.`;
console.error('[SelfUpdate] Repin failed:', this.lastUpdateError);
return;
}
}
this.spawnHelper(env, composeCopy);
}
/**
* Spawn the "last breath" helper container that recreates Sencho (and, when a
* repin is staged, copies the rewritten compose file onto the host first).
* Runs attached (no -d): if the recreate fails before it kills us, execFile's
* callback receives the helper's exit code and stderr directly.
*/
private spawnHelper(env: NodeJS.ProcessEnv, composeCopy?: ComposeCopy): void {
if (!this.composeContext) return;
const { workingDir, configFiles, serviceName, imageName, dataDirHost, hostBindMounts } = this.composeContext;
console.log('[SelfUpdate] Spawning updater container... (last breath)');
if (isDebugEnabled()) console.debug('[SelfUpdate:debug] Helper context:', { workingDir, serviceName, dataDirHost, repin: !!composeCopy, mountCount: hostBindMounts.length });
const fFlags = configFiles.split(',').flatMap(f => ['-f', f.trim()]);
// On failure, persist exit code + stderr to UPDATE_ERROR_FILE (host-mounted)
@@ -283,8 +501,8 @@ class SelfUpdateService {
const stderrTmp = '/tmp/_sencho_err';
const pruneOnUpdate =
DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1';
const composeCmd = buildSelfUpdateComposeCmd(fFlags, serviceName, stderrTmp, UPDATE_ERROR_FILE, pruneOnUpdate);
const args = buildSelfUpdateRunArgs({ workingDir, imageName, dataDirHost, hostBindMounts }, composeCmd);
const composeCmd = buildSelfUpdateComposeCmd(fFlags, serviceName, stderrTmp, UPDATE_ERROR_FILE, pruneOnUpdate, composeCopy);
const args = buildSelfUpdateRunArgs({ workingDir, imageName, dataDirHost, hostBindMounts, repinWritable: !!composeCopy }, composeCmd);
// Callback may never fire on success (we die mid-call during recreate);
// that is fine because the restart itself is the success signal.
+29
View File
@@ -0,0 +1,29 @@
import type { Request, Response } from 'express';
import semver from 'semver';
import { isValidVersion } from '../services/CapabilityRegistry';
/** The semver compare target when valid, otherwise undefined (legacy pull-current). */
export function pickCompareTarget(compareVersion: string | null, compareValid: boolean): string | undefined {
if (!compareValid || !isValidVersion(compareVersion)) return undefined;
return compareVersion;
}
/**
* Parse an optional `targetVersion` from a self-update request body.
*
* - Omitted or null -> returns undefined (the caller chooses the default).
* - Present and a valid semver -> returns the normalized version.
* - Present but not a valid semver -> writes a 400 and returns null so the
* caller early-returns. We never silently fall back on a supplied-but-bad
* value, because that would hide a client bug and update to the wrong target.
*/
export function parseRequestedTargetVersion(req: Request, res: Response): string | null | undefined {
const raw = (req.body ?? {})?.targetVersion;
if (raw === undefined || raw === null) return undefined;
const normalized = typeof raw === 'string' ? semver.valid(raw) : null;
if (!normalized || !isValidVersion(normalized) || raw.length > 64) {
res.status(400).json({ error: 'Invalid target version' });
return null;
}
return normalized;
}