mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-18 06:23:18 +00:00
feat(fleet): add remote node update management (#353)
Add the ability to check for outdated nodes and trigger over-the-air updates from Fleet View. Nodes self-update by pulling the latest Docker image and recreating their container via the "last breath" pattern. Backend: - SelfUpdateService: self-container identification via HOSTNAME + Docker Compose labels, triggers pull + force-recreate - CapabilityRegistry: runtime capability disabling via disableCapability() - POST /api/system/update (202 + deferred self-update) - GET /api/fleet/update-status (version comparison across fleet) - POST /api/fleet/nodes/:nodeId/update (single node) - POST /api/fleet/update-all (bulk remote update) - In-memory update tracker with 5-min timeout Frontend: - Node Updates modal with summary stats, search filter, table layout, per-node Update buttons, and bulk Update All - Version badges and update-available indicators on node cards - ReconnectingOverlay for local node updates (polls /api/health) - 5s fast-poll when any node is actively updating - UpdateStatusBadge shared component for consistent badge rendering Requires Skipper (Pro) tier. Nodes must be deployed via Docker Compose with Docker socket access.
This commit is contained in:
Anso
+211
-2
@@ -30,7 +30,9 @@ import { WebhookService } from './services/WebhookService';
|
||||
import { SSOService } from './services/SSOService';
|
||||
import { SchedulerService } from './services/SchedulerService';
|
||||
import { RegistryService } from './services/RegistryService';
|
||||
import { CAPABILITIES, getSenchoVersion, fetchRemoteMeta } from './services/CapabilityRegistry';
|
||||
import { CAPABILITIES, getSenchoVersion, fetchRemoteMeta, getActiveCapabilities } from './services/CapabilityRegistry';
|
||||
import SelfUpdateService from './services/SelfUpdateService';
|
||||
import semver from 'semver';
|
||||
import { CronExpressionParser } from 'cron-parser';
|
||||
import { isValidStackName, isValidRemoteUrl, isPathWithinBase } from './utils/validation';
|
||||
import YAML from 'yaml';
|
||||
@@ -323,7 +325,7 @@ app.get('/api/health', (_req: Request, res: Response): void => {
|
||||
// Public meta endpoint - returns this instance's version and supported capabilities.
|
||||
// No auth required (like /health). Used by remote nodes during connection tests.
|
||||
app.get('/api/meta', (_req: Request, res: Response): void => {
|
||||
res.json({ version: getSenchoVersion(), capabilities: CAPABILITIES });
|
||||
res.json({ version: getSenchoVersion(), capabilities: getActiveCapabilities() });
|
||||
});
|
||||
|
||||
// Auth Routes (no authentication required)
|
||||
@@ -1032,8 +1034,36 @@ app.post('/api/license/validate', async (_req: Request, res: Response): Promise<
|
||||
}
|
||||
});
|
||||
|
||||
// --- Self-Update ---
|
||||
|
||||
/** Respond 202 and trigger the "last breath" self-update after the response flushes. */
|
||||
function scheduleLocalUpdate(res: Response, message: string): void {
|
||||
res.status(202).json({ message });
|
||||
res.on('finish', () => {
|
||||
setTimeout(() => SelfUpdateService.getInstance().triggerUpdate(), 500);
|
||||
});
|
||||
}
|
||||
|
||||
app.post('/api/system/update', (_req: Request, res: Response): void => {
|
||||
if (!SelfUpdateService.getInstance().isAvailable()) {
|
||||
res.status(503).json({ error: 'Self-update unavailable. Sencho must be deployed via Docker Compose.' });
|
||||
return;
|
||||
}
|
||||
scheduleLocalUpdate(res, 'Update initiated. The server will restart shortly.');
|
||||
});
|
||||
|
||||
// --- Fleet Overview (local-only, aggregates all nodes) ---
|
||||
|
||||
// In-memory tracker for remote node updates (transient — lost on gateway restart)
|
||||
interface UpdateTracker {
|
||||
status: 'updating' | 'completed' | 'timeout' | 'failed';
|
||||
startedAt: number;
|
||||
previousVersion: string | null;
|
||||
error?: string;
|
||||
}
|
||||
const updateTracker = new Map<number, UpdateTracker>();
|
||||
const UPDATE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
|
||||
|
||||
interface FleetNodeOverview {
|
||||
id: number;
|
||||
name: string;
|
||||
@@ -1171,6 +1201,182 @@ app.get('/api/fleet/node/:nodeId/stacks/:stackName/containers', async (req: Requ
|
||||
}
|
||||
});
|
||||
|
||||
// Fleet Update Status — returns version comparison and active update status for all nodes
|
||||
app.get('/api/fleet/update-status', async (_req: Request, res: Response): Promise<void> => {
|
||||
if (!requirePro(_req, res)) return;
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const nodes = db.getNodes();
|
||||
const gatewayVersion = getSenchoVersion();
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
nodes.map(async (node) => {
|
||||
const tracker = updateTracker.get(node.id);
|
||||
|
||||
let version: string | null = null;
|
||||
if (node.type === 'local') {
|
||||
version = gatewayVersion;
|
||||
} else if (node.api_url && node.api_token) {
|
||||
const meta = await fetchRemoteMeta(node.api_url, node.api_token);
|
||||
version = meta.version;
|
||||
}
|
||||
|
||||
// For nodes actively updating, check if they've come back with a new version
|
||||
if (tracker?.status === 'updating') {
|
||||
if (Date.now() - tracker.startedAt > UPDATE_TIMEOUT_MS) {
|
||||
updateTracker.set(node.id, { ...tracker, status: 'timeout' });
|
||||
} else if (node.type === 'remote' && version && version !== tracker.previousVersion) {
|
||||
updateTracker.set(node.id, { ...tracker, status: 'completed' });
|
||||
}
|
||||
}
|
||||
|
||||
const currentTracker = updateTracker.get(node.id);
|
||||
return {
|
||||
nodeId: node.id,
|
||||
name: node.name,
|
||||
type: node.type,
|
||||
version,
|
||||
latestVersion: gatewayVersion,
|
||||
updateAvailable: version === null
|
||||
? (node.type === 'remote') // Remote node without /api/meta is pre-capability-negotiation — definitely outdated
|
||||
: (version !== gatewayVersion && !!semver.valid(version) && semver.lt(version, gatewayVersion)),
|
||||
updateStatus: currentTracker?.status ?? null,
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
const nodeStatuses = results.map((r, i) => {
|
||||
if (r.status === 'fulfilled') return r.value;
|
||||
return {
|
||||
nodeId: nodes[i].id,
|
||||
name: nodes[i].name,
|
||||
type: nodes[i].type,
|
||||
version: null,
|
||||
latestVersion: gatewayVersion,
|
||||
updateAvailable: false,
|
||||
updateStatus: null,
|
||||
};
|
||||
});
|
||||
|
||||
res.json({ nodes: nodeStatuses });
|
||||
} catch (error) {
|
||||
console.error('[Fleet] Update status error:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch update status' });
|
||||
}
|
||||
});
|
||||
|
||||
// Trigger update on a specific node
|
||||
app.post('/api/fleet/nodes/:nodeId/update', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requirePro(req, res)) return;
|
||||
try {
|
||||
const nodeId = parseInt(req.params.nodeId as string, 10);
|
||||
const db = DatabaseService.getInstance();
|
||||
const node = db.getNode(nodeId);
|
||||
if (!node) {
|
||||
res.status(404).json({ error: 'Node not found' });
|
||||
return;
|
||||
}
|
||||
|
||||
const existing = updateTracker.get(nodeId);
|
||||
if (existing?.status === 'updating') {
|
||||
res.status(409).json({ error: 'Update already in progress for this node.' });
|
||||
return;
|
||||
}
|
||||
|
||||
if (node.type === 'local') {
|
||||
if (!SelfUpdateService.getInstance().isAvailable()) {
|
||||
res.status(503).json({ error: 'Self-update unavailable on the local node.' });
|
||||
return;
|
||||
}
|
||||
updateTracker.set(nodeId, { status: 'updating', startedAt: Date.now(), previousVersion: getSenchoVersion() });
|
||||
scheduleLocalUpdate(res, 'Update initiated on local node. The server will restart shortly.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Remote node
|
||||
if (!node.api_url || !node.api_token) {
|
||||
res.status(503).json({ error: 'Remote node not configured.' });
|
||||
return;
|
||||
}
|
||||
|
||||
// Check remote capabilities
|
||||
const meta = await fetchRemoteMeta(node.api_url, node.api_token);
|
||||
if (!meta.capabilities.includes('self-update')) {
|
||||
res.status(503).json({ error: 'Remote node does not support self-update. It may need to be updated manually first.' });
|
||||
return;
|
||||
}
|
||||
|
||||
// Trigger remote update
|
||||
const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/system/update`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${node.api_token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const err = await response.json().catch(() => ({}));
|
||||
res.status(502).json({ error: (err as Record<string, string>)?.error || 'Remote node rejected update request.' });
|
||||
return;
|
||||
}
|
||||
|
||||
updateTracker.set(nodeId, { status: 'updating', startedAt: Date.now(), previousVersion: meta.version });
|
||||
res.status(202).json({ message: `Update initiated on ${node.name}.` });
|
||||
} catch (error) {
|
||||
console.error('[Fleet] Node update error:', error);
|
||||
res.status(500).json({ error: 'Failed to trigger node update.' });
|
||||
}
|
||||
});
|
||||
|
||||
// Trigger update on all outdated nodes
|
||||
app.post('/api/fleet/update-all', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requirePro(req, res)) return;
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const nodes = db.getNodes();
|
||||
const gatewayVersion = getSenchoVersion();
|
||||
|
||||
// Filter to eligible candidates, then trigger all in parallel
|
||||
const candidates = nodes.filter(node => {
|
||||
if (node.type === 'local') return false;
|
||||
if (updateTracker.get(node.id)?.status === 'updating') return false;
|
||||
if (!node.api_url || !node.api_token) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
const results = await Promise.allSettled(candidates.map(async (node) => {
|
||||
const meta = await fetchRemoteMeta(node.api_url!, node.api_token!);
|
||||
if (!meta.version || !semver.valid(meta.version) || !semver.lt(meta.version, gatewayVersion) || !meta.capabilities.includes('self-update')) {
|
||||
return { name: node.name, triggered: false };
|
||||
}
|
||||
const response = await fetch(`${node.api_url!.replace(/\/$/, '')}/api/system/update`, {
|
||||
method: 'POST',
|
||||
headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' },
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (response.ok) {
|
||||
updateTracker.set(node.id, { status: 'updating', startedAt: Date.now(), previousVersion: meta.version });
|
||||
return { name: node.name, triggered: true };
|
||||
}
|
||||
return { name: node.name, triggered: false };
|
||||
}));
|
||||
|
||||
const updating: string[] = [];
|
||||
const skipped = nodes.filter(n => !candidates.includes(n)).map(n => n.name);
|
||||
for (const r of results) {
|
||||
const val = r.status === 'fulfilled' ? r.value : { name: 'unknown', triggered: false };
|
||||
(val.triggered ? updating : skipped).push(val.name);
|
||||
}
|
||||
|
||||
res.status(202).json({ updating, skipped });
|
||||
} catch (error) {
|
||||
console.error('[Fleet] Update all error:', error);
|
||||
res.status(500).json({ error: 'Failed to trigger fleet update.' });
|
||||
}
|
||||
});
|
||||
|
||||
async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
|
||||
try {
|
||||
const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(node.id));
|
||||
@@ -5262,6 +5468,9 @@ async function startServer() {
|
||||
// Initialize License Service (starts trial on first boot, periodic validation)
|
||||
LicenseService.getInstance().initialize();
|
||||
|
||||
// Detect whether this instance can self-update (Docker Compose container inspection)
|
||||
await SelfUpdateService.getInstance().initialize();
|
||||
|
||||
// Start Background Watchdog
|
||||
MonitorService.getInstance().start();
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ export const CAPABILITIES = [
|
||||
'api-tokens',
|
||||
'users',
|
||||
'registries',
|
||||
'self-update',
|
||||
] as const;
|
||||
|
||||
export type Capability = (typeof CAPABILITIES)[number];
|
||||
@@ -41,6 +42,19 @@ export interface RemoteMeta {
|
||||
capabilities: string[];
|
||||
}
|
||||
|
||||
// Runtime capability overrides — services call disableCapability() during init
|
||||
const disabledCapabilities = new Set<Capability>();
|
||||
|
||||
export function disableCapability(c: Capability): void {
|
||||
disabledCapabilities.add(c);
|
||||
}
|
||||
|
||||
/** Returns capabilities this instance actually supports at runtime. */
|
||||
export function getActiveCapabilities(): readonly string[] {
|
||||
if (disabledCapabilities.size === 0) return CAPABILITIES;
|
||||
return CAPABILITIES.filter(c => !disabledCapabilities.has(c));
|
||||
}
|
||||
|
||||
/** Fetch /api/meta from a remote Sencho instance. Returns empty data on failure. */
|
||||
export async function fetchRemoteMeta(baseUrl: string, apiToken: string): Promise<RemoteMeta> {
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import { execSync, exec } from 'child_process';
|
||||
import DockerController from './DockerController';
|
||||
import { disableCapability } from './CapabilityRegistry';
|
||||
|
||||
interface ComposeContext {
|
||||
workingDir: string;
|
||||
configFiles: string;
|
||||
serviceName: string;
|
||||
}
|
||||
|
||||
class SelfUpdateService {
|
||||
private static instance: SelfUpdateService;
|
||||
private canSelfUpdate = false;
|
||||
private composeContext: ComposeContext | null = null;
|
||||
|
||||
public static getInstance(): SelfUpdateService {
|
||||
if (!SelfUpdateService.instance) {
|
||||
SelfUpdateService.instance = new SelfUpdateService();
|
||||
}
|
||||
return SelfUpdateService.instance;
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
const hostname = process.env.HOSTNAME;
|
||||
if (!hostname) {
|
||||
console.log('[SelfUpdate] HOSTNAME not set — self-update unavailable (not running in Docker?)');
|
||||
disableCapability('self-update');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const docker = DockerController.getInstance().getDocker();
|
||||
const container = docker.getContainer(hostname);
|
||||
const info = await container.inspect();
|
||||
const labels = info.Config?.Labels ?? {};
|
||||
|
||||
const workingDir = labels['com.docker.compose.project.working_dir'];
|
||||
const configFiles = labels['com.docker.compose.project.config_files'];
|
||||
const serviceName = labels['com.docker.compose.service'];
|
||||
|
||||
if (!workingDir || !configFiles || !serviceName) {
|
||||
console.log('[SelfUpdate] Container lacks Docker Compose labels — self-update unavailable');
|
||||
disableCapability('self-update');
|
||||
return;
|
||||
}
|
||||
|
||||
this.composeContext = { workingDir, configFiles, serviceName };
|
||||
this.canSelfUpdate = true;
|
||||
console.log(`[SelfUpdate] Ready — service="${serviceName}" in ${workingDir}`);
|
||||
} catch (error) {
|
||||
console.log('[SelfUpdate] Could not inspect own container — self-update unavailable:', (error as Error).message);
|
||||
disableCapability('self-update');
|
||||
}
|
||||
}
|
||||
|
||||
isAvailable(): boolean {
|
||||
return this.canSelfUpdate;
|
||||
}
|
||||
|
||||
triggerUpdate(): void {
|
||||
if (!this.composeContext) return;
|
||||
const { workingDir, configFiles, serviceName } = this.composeContext;
|
||||
const env = { ...process.env, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' };
|
||||
|
||||
console.log(`[SelfUpdate] Pulling latest image for ${serviceName}...`);
|
||||
try {
|
||||
execSync(`docker compose -f ${configFiles} pull ${serviceName}`, {
|
||||
cwd: workingDir,
|
||||
env,
|
||||
stdio: 'pipe',
|
||||
timeout: 300_000, // 5 min max for pull
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[SelfUpdate] Pull failed:', (error as Error).message);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[SelfUpdate] Recreating container for ${serviceName}... (last breath)`);
|
||||
exec(`docker compose -f ${configFiles} up -d --force-recreate ${serviceName}`, {
|
||||
cwd: workingDir,
|
||||
env,
|
||||
});
|
||||
// Process will be killed by Docker during recreate — no code runs after this
|
||||
}
|
||||
}
|
||||
|
||||
export default SelfUpdateService;
|
||||
Reference in New Issue
Block a user