feat: make image-update check cadence configurable and visible (#1377)

* feat: make image-update check cadence configurable and visible

The background image-update scanner polled registries on a hardcoded
6-hour interval, with no way to see when it last ran or when the next
run was due. Operators testing updates read this as auto-update being
unreliable: a manual update checks the registry immediately and applies,
so the slow background scan rarely raised the "update available"
notification before the stack was already current.

Backend:
- ImageUpdateService reads image_update_check_interval_minutes (15-1440,
  default 120) and drives a single generation-guarded self-rescheduling
  timer with 10% per-run jitter so fleet nodes do not poll in lockstep.
  restartPolling() applies a new interval live, with no restart, and
  cannot leave a duplicate timer when a save lands mid-scan.
- GET /api/image-updates/status now returns checking, intervalMinutes,
  lastCheckedAt, nextCheckAt, and the manual-cooldown fields. New
  admin-only PUT /api/image-updates/interval persists the setting and
  reschedules.

Frontend:
- New Settings > Automation > Image update checks section to choose the
  interval (read-only for non-admins; admin enforced on the backend).
- The Auto-Update readiness view shows last-checked, next-check, and a
  ticking manual-recheck cooldown, and the copy distinguishes registry
  detection from scheduled auto-update execution.

Adds backend unit and route tests and frontend component tests, and
updates the auto-update documentation.

* fix: drop stale image-update status response in the readiness strip

loadCadence() ran on mount and again after a Recheck with no request
token, so a slow initial /image-updates/status response could resolve
after the recheck-triggered one and overwrite the fresh cooldown with
stale data, or set state after the view unmounted. Guard setCadence with
a monotonic token mirroring loadReadiness, and bump it on unmount. Adds a
regression test for the out-of-order resolution.
This commit is contained in:
Anso
2026-06-15 20:06:13 -04:00
committed by GitHub
parent 02c3b006eb
commit 058cf8f2c7
16 changed files with 890 additions and 29 deletions
@@ -2,7 +2,7 @@
* Unit tests for ImageUpdateService: image ref parsing, compose extraction,
* env file loading, checkImage digest comparison, and rate limiting.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
// ── Hoisted mocks ──────────────────────────────────────────────────────
@@ -12,7 +12,7 @@ const {
mockGetSystemState, mockSetSystemState, mockAddNotificationHistory,
mockDispatchAlert,
mockGetStacks, mockGetStackContent, mockGetEnvContent, mockEnvExists,
mockGetAllContainers,
mockGetAllContainers, mockGetGlobalSettings,
} = vi.hoisted(() => ({
mockGetAuthForRegistry: vi.fn().mockResolvedValue(null),
mockGetStackUpdateStatus: vi.fn().mockReturnValue({}),
@@ -27,6 +27,7 @@ const {
mockGetEnvContent: vi.fn().mockRejectedValue(new Error('no env')),
mockEnvExists: vi.fn().mockResolvedValue(false),
mockGetAllContainers: vi.fn().mockResolvedValue([]),
mockGetGlobalSettings: vi.fn().mockReturnValue({ developer_mode: '0' }),
}));
vi.mock('../services/RegistryService', () => ({
@@ -40,7 +41,7 @@ vi.mock('../services/RegistryService', () => ({
vi.mock('../services/DatabaseService', () => ({
DatabaseService: {
getInstance: () => ({
getGlobalSettings: () => ({ developer_mode: '0' }),
getGlobalSettings: mockGetGlobalSettings,
getNodes: () => [],
upsertStackUpdateStatus: mockUpsertStackUpdateStatus,
getStackUpdateStatus: mockGetStackUpdateStatus,
@@ -696,6 +697,177 @@ describe('ImageUpdateService - stop() cancels startup timeout', () => {
});
});
// ── Configurable interval, status, and reschedule ───────────────────────
describe('ImageUpdateService - configurable interval & status', () => {
beforeEach(() => {
vi.clearAllMocks();
(ImageUpdateService as any).instance = undefined;
mockGetGlobalSettings.mockReturnValue({ developer_mode: '0' });
});
afterEach(() => {
vi.useRealTimers();
});
function deferred() {
let resolve!: () => void;
const promise = new Promise<void>((r) => { resolve = r; });
return { promise, resolve };
}
it('reports the default 120-minute interval before start() runs', () => {
const service = ImageUpdateService.getInstance();
const status = service.getStatus();
expect(status.intervalMinutes).toBe(120);
expect(status.checking).toBe(false);
expect(status.lastCheckedAt).toBeNull();
expect(status.nextCheckAt).toBeNull();
expect(status.manualCooldownMinutes).toBe(2);
expect(status.manualCooldownRemainingMs).toBe(0);
});
it('reads the configured interval from settings', () => {
mockGetGlobalSettings.mockReturnValue({ image_update_check_interval_minutes: '30' });
const service = ImageUpdateService.getInstance();
service.configureFromSettings();
expect(service.getStatus().intervalMinutes).toBe(30);
});
it('clamps an interval below the minimum to 15', () => {
mockGetGlobalSettings.mockReturnValue({ image_update_check_interval_minutes: '5' });
const service = ImageUpdateService.getInstance();
service.configureFromSettings();
expect(service.getStatus().intervalMinutes).toBe(15);
});
it('clamps an interval above the maximum to 1440', () => {
mockGetGlobalSettings.mockReturnValue({ image_update_check_interval_minutes: '5000' });
const service = ImageUpdateService.getInstance();
service.configureFromSettings();
expect(service.getStatus().intervalMinutes).toBe(1440);
});
it('falls back to the default for a malformed or non-integer value', () => {
const service = ImageUpdateService.getInstance();
const badValues: (string | undefined)[] = ['15abc', '30.5', '', undefined];
for (const bad of badValues) {
mockGetGlobalSettings.mockReturnValue(bad === undefined ? {} : { image_update_check_interval_minutes: bad });
service.configureFromSettings();
expect(service.getStatus().intervalMinutes).toBe(120);
}
});
it('stamps lastCheckedAt when a manual refresh runs', async () => {
const service = ImageUpdateService.getInstance();
// getNodes() returns [] in the shared mock, so check() completes immediately.
expect(service.getStatus().lastCheckedAt).toBeNull();
const triggered = service.triggerManualRefresh();
expect(triggered).toBe(true);
await Promise.resolve();
await Promise.resolve();
expect(service.getStatus().lastCheckedAt).not.toBeNull();
});
it('applies ±10% jitter that actually reaches both endpoints', () => {
mockGetGlobalSettings.mockReturnValue({ image_update_check_interval_minutes: '60' });
const service = ImageUpdateService.getInstance();
service.configureFromSettings();
const interval = 60 * 60 * 1000;
// random=0 must reach the low edge (90%), proving jitter is applied and not
// collapsed to the bare interval.
const low = vi.spyOn(Math, 'random').mockReturnValue(0);
expect((service as any).nextDelayMs()).toBe(Math.round(interval * 0.9));
low.mockRestore();
const mid = vi.spyOn(Math, 'random').mockReturnValue(0.5);
expect((service as any).nextDelayMs()).toBe(interval);
mid.mockRestore();
// random→1 must reach the high edge (≈110%).
const high = vi.spyOn(Math, 'random').mockReturnValue(0.999);
const hi = (service as any).nextDelayMs() as number;
expect(hi).toBeGreaterThan(interval);
expect(hi).toBeGreaterThanOrEqual(Math.round(interval * 1.09));
expect(hi).toBeLessThanOrEqual(Math.round(interval * 1.1));
high.mockRestore();
});
it('reports the manual-refresh cooldown remaining and clears it after the window', () => {
vi.useFakeTimers();
const service = ImageUpdateService.getInstance();
expect(service.getManualCooldownRemainingMs()).toBe(0);
service.triggerManualRefresh();
const remaining = service.getManualCooldownRemainingMs();
expect(remaining).toBeGreaterThan(0);
expect(remaining).toBeLessThanOrEqual(2 * 60 * 1000);
vi.advanceTimersByTime(2 * 60 * 1000);
expect(service.getManualCooldownRemainingMs()).toBe(0);
});
it('stop() after start() clears the timer and nulls nextCheckAt without firing a check', () => {
vi.useFakeTimers();
const service = ImageUpdateService.getInstance();
const checkSpy = vi.spyOn(service as any, 'check').mockResolvedValue(undefined);
service.start();
expect(service.getStatus().nextCheckAt).not.toBeNull();
expect(vi.getTimerCount()).toBe(1);
service.stop();
expect(vi.getTimerCount()).toBe(0);
expect(service.getStatus().nextCheckAt).toBeNull();
// Past the old startup delay: the cleared timer + bumped generation mean no
// check fires on a stopped service.
vi.advanceTimersByTime(5 * 60 * 1000);
expect(checkSpy).not.toHaveBeenCalled();
checkSpy.mockRestore();
});
it('restartPolling() while stopped reconfigures the interval but arms no timer', () => {
vi.useFakeTimers();
mockGetGlobalSettings.mockReturnValue({ image_update_check_interval_minutes: '45' });
const service = ImageUpdateService.getInstance();
// Never started: polling is false, so it reconfigures without arming.
service.restartPolling();
expect(service.getStatus().intervalMinutes).toBe(45);
expect(service.getStatus().nextCheckAt).toBeNull();
expect((service as any).timer).toBeNull();
expect(vi.getTimerCount()).toBe(0);
});
it('restartPolling() during an in-flight tick leaves exactly one timer', async () => {
vi.useFakeTimers();
const service = ImageUpdateService.getInstance();
const d = deferred();
const checkSpy = vi.spyOn(service as any, 'check').mockReturnValue(d.promise);
service.start();
expect(vi.getTimerCount()).toBe(1);
// Fire the startup tick: it invokes check() (our pending deferred) and does
// not re-arm until check resolves.
vi.advanceTimersByTime(2 * 60 * 1000);
expect(checkSpy).toHaveBeenCalledTimes(1);
expect(vi.getTimerCount()).toBe(0);
// A settings save lands mid-scan: it arms a fresh timer.
service.restartPolling();
expect(vi.getTimerCount()).toBe(1);
// The original tick resolves; its generation is now stale, so it must not
// re-arm a second timer.
d.resolve();
await d.promise;
await Promise.resolve();
expect(vi.getTimerCount()).toBe(1);
service.stop();
checkSpy.mockRestore();
});
});
// ── Stale stack pruning ─────────────────────────────────────────────────
describe('ImageUpdateService - stale stack pruning', () => {
@@ -72,10 +72,55 @@ describe('GET /api/image-updates/status', () => {
expect(res.status).toBe(401);
});
it('returns a checking flag', async () => {
it('returns the enriched status payload', async () => {
const res = await request(app).get('/api/image-updates/status').set('Cookie', adminCookie);
expect(res.status).toBe(200);
expect(typeof res.body.checking).toBe('boolean');
// start() never runs in route tests, so the interval reflects the seeded
// default (120) via the field initializer rather than NaN.
expect(res.body.intervalMinutes).toBe(120);
expect(res.body.manualCooldownMinutes).toBe(2);
expect(typeof res.body.manualCooldownRemainingMs).toBe('number');
expect('lastCheckedAt' in res.body).toBe(true);
expect('nextCheckAt' in res.body).toBe(true);
});
});
describe('PUT /api/image-updates/interval', () => {
it('rejects unauthenticated requests with 401', async () => {
const res = await request(app).put('/api/image-updates/interval').send({ minutes: 30 });
expect(res.status).toBe(401);
});
it('rejects non-admin users with 403', async () => {
const res = await request(app).put('/api/image-updates/interval').set('Cookie', viewerCookie).send({ minutes: 30 });
expect(res.status).toBe(403);
});
it('rejects an interval below the minimum', async () => {
const res = await request(app).put('/api/image-updates/interval').set('Cookie', adminCookie).send({ minutes: 5 });
expect(res.status).toBe(400);
});
it('rejects an interval above the maximum', async () => {
const res = await request(app).put('/api/image-updates/interval').set('Cookie', adminCookie).send({ minutes: 5000 });
expect(res.status).toBe(400);
});
it('rejects a non-integer interval', async () => {
const res = await request(app).put('/api/image-updates/interval').set('Cookie', adminCookie).send({ minutes: 'soon' });
expect(res.status).toBe(400);
});
it('persists a valid interval and returns the enriched status', async () => {
const res = await request(app).put('/api/image-updates/interval').set('Cookie', adminCookie).send({ minutes: 30 });
expect(res.status).toBe(200);
expect(res.body.intervalMinutes).toBe(30);
// The value is persisted to global_settings...
expect(DatabaseService.getInstance().getGlobalSettings().image_update_check_interval_minutes).toBe('30');
// ...and a follow-up status read reflects the rescheduled cadence.
const statusRes = await request(app).get('/api/image-updates/status').set('Cookie', adminCookie);
expect(statusRes.body.intervalMinutes).toBe(30);
});
});
+26 -1
View File
@@ -1,4 +1,5 @@
import { Router, type Request, type Response } from 'express';
import { z } from 'zod';
import DockerController from '../services/DockerController';
import { DatabaseService } from '../services/DatabaseService';
import { NodeRegistry } from '../services/NodeRegistry';
@@ -50,7 +51,31 @@ imageUpdatesRouter.post('/refresh', authMiddleware, (req: Request, res: Response
});
imageUpdatesRouter.get('/status', authMiddleware, (_req: Request, res: Response): void => {
res.json({ checking: ImageUpdateService.getInstance().isChecking() });
res.json(ImageUpdateService.getInstance().getStatus());
});
// Min/max mirror ImageUpdateService's clamp; the service is the authority and
// re-clamps on read, so this is the user-facing validation boundary.
const IntervalPatchSchema = z.object({
minutes: z.coerce.number().int().min(15).max(1440),
});
imageUpdatesRouter.put('/interval', authMiddleware, (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
const parsed = IntervalPatchSchema.safeParse(req.body);
if (!parsed.success) {
res.status(400).json({ error: 'minutes must be an integer between 15 and 1440' });
return;
}
try {
DatabaseService.getInstance().updateGlobalSetting('image_update_check_interval_minutes', String(parsed.data.minutes));
// Reschedule the live timer so the new cadence takes effect without a restart.
ImageUpdateService.getInstance().restartPolling();
res.json(ImageUpdateService.getInstance().getStatus());
} catch (error) {
console.error('Failed to update image-update interval:', error);
res.status(500).json({ error: 'Failed to update interval' });
}
});
imageUpdatesRouter.get('/fleet', authMiddleware, async (req: Request, res: Response): Promise<void> => {
+1
View File
@@ -1463,6 +1463,7 @@ export class DatabaseService {
stmt.run('reclaim_hero', '1');
stmt.run('health_gate_enabled', '1');
stmt.run('health_gate_window_seconds', '90');
stmt.run('image_update_check_interval_minutes', '120');
// Seed the default local node if none exists
const nodeCount = (this.db.prepare('SELECT COUNT(*) as count FROM nodes').get() as any)?.count || 0;
+148 -14
View File
@@ -19,6 +19,24 @@ export interface ImageCheckResult {
error?: string;
}
/**
* Snapshot of the scanner returned by GET /api/image-updates/status.
* Units differ by field: `intervalMinutes` / `manualCooldownMinutes` are
* minutes, `manualCooldownRemainingMs` is milliseconds. `manualCooldownMinutes`
* is the fixed cooldown ceiling; `manualCooldownRemainingMs` is the live
* remaining time (0 when a manual refresh is allowed). `lastCheckedAt` /
* `nextCheckAt` are epoch-ms or null ("never checked" / "not scheduled");
* `nextCheckAt` is meaningless while `checking` is true.
*/
export interface ImageUpdateStatus {
checking: boolean;
intervalMinutes: number;
lastCheckedAt: number | null;
nextCheckAt: number | null;
manualCooldownMinutes: number;
manualCooldownRemainingMs: number;
}
// ─── Compose file helpers ────────────────────────────────────────────────────
export function loadDotEnv(content: string): Record<string, string> {
@@ -91,14 +109,31 @@ export function extractImagesFromCompose(
export class ImageUpdateService {
private static instance: ImageUpdateService;
private intervalId: NodeJS.Timeout | null = null;
private startupTimeoutId: NodeJS.Timeout | null = null;
private static readonly MIN_INTERVAL_MINUTES = 15;
private static readonly MAX_INTERVAL_MINUTES = 1440; // 24 hours
private static readonly DEFAULT_INTERVAL_MINUTES = 120; // 2 hours
private static readonly INTERVAL_SETTING_KEY = 'image_update_check_interval_minutes';
private static readonly JITTER_FRACTION = 0.1; // ±10% so a fleet does not poll in lockstep
private static readonly STARTUP_DELAY_MS = 2 * 60 * 1000; // 2 min after boot
// A single self-rescheduling timer (replacing the old setInterval): it lets
// us know nextCheckAt precisely, apply per-run jitter, and reschedule on a
// settings change without ever leaving two timers running.
private timer: NodeJS.Timeout | null = null;
private polling = false;
// Bumped by stop()/restartPolling(); a tick whose captured generation no
// longer matches must not re-arm. This is what stops a settings save that
// lands mid-scan from racing the in-flight tick into a duplicate timer.
private scheduleGeneration = 0;
private isRunning = false;
private checkStartedAt = 0;
private lastManualRefreshAt = 0;
private static readonly INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours
private static readonly STARTUP_DELAY_MS = 2 * 60 * 1000; // 2 min after boot
private lastCheckedAt: number | null = null; // when the last scan body started
private nextCheckAt: number | null = null;
// Initialized at declaration so getStatus() never reports NaN before start()
// or configureFromSettings() has run (e.g. route tests that skip startServer).
private intervalMs = ImageUpdateService.DEFAULT_INTERVAL_MINUTES * 60 * 1000;
private static readonly MANUAL_COOLDOWN_MS = 2 * 60 * 1000; // 2 min between manual triggers
private static readonly INTER_IMAGE_DELAY_MS = 300; // be polite to registries
private static readonly CHECK_TIMEOUT_MS = 5 * 60 * 1000; // threshold for the "running long" skip warning
@@ -118,20 +153,99 @@ export class ImageUpdateService {
}
public start() {
if (this.intervalId) return;
this.startupTimeoutId = setTimeout(() => this.check(), ImageUpdateService.STARTUP_DELAY_MS);
this.intervalId = setInterval(() => this.check(), ImageUpdateService.INTERVAL_MS);
if (this.timer) return;
this.polling = true;
this.configureFromSettings();
// Preserve the existing 2-minute post-boot delay before the first check.
this.armNext(ImageUpdateService.STARTUP_DELAY_MS);
}
public stop() {
if (this.startupTimeoutId) {
clearTimeout(this.startupTimeoutId);
this.startupTimeoutId = null;
this.scheduleGeneration++;
if (this.timer) {
clearTimeout(this.timer);
this.timer = null;
}
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = null;
this.polling = false;
this.nextCheckAt = null;
}
/**
* Re-read the configured interval and reschedule the next check at the new
* cadence without restarting Sencho. Safe to call repeatedly: it always
* clears the existing timer first and only arms a new one while polling, so
* it never stacks timers and is a no-op (beyond reconfiguring intervalMs)
* when the service is stopped or was never started.
*/
public restartPolling(): void {
this.scheduleGeneration++;
if (this.timer) {
clearTimeout(this.timer);
this.timer = null;
}
this.configureFromSettings();
if (this.polling) {
this.armNext(this.nextDelayMs());
} else {
this.nextCheckAt = null;
}
}
/**
* Reads image_update_check_interval_minutes into intervalMs, clamped to
* [15, 1440], falling back to the 2-hour default on a missing, blank,
* malformed, or unreadable value.
*/
public configureFromSettings(): void {
this.intervalMs = ImageUpdateService.resolveIntervalMinutes() * 60 * 1000;
}
private static resolveIntervalMinutes(): number {
const fallback = ImageUpdateService.DEFAULT_INTERVAL_MINUTES;
try {
const raw = DatabaseService.getInstance().getGlobalSettings()[ImageUpdateService.INTERVAL_SETTING_KEY];
// Treat missing/blank as unset; Number('') is 0, which would clamp to
// the minimum rather than fall back to the default.
if (raw == null || String(raw).trim() === '') return fallback;
// Number() (not parseInt) so a malformed value like "15abc" is
// rejected to the default rather than silently accepted as 15.
const parsed = Number(raw);
if (!Number.isInteger(parsed)) return fallback;
return Math.min(
ImageUpdateService.MAX_INTERVAL_MINUTES,
Math.max(ImageUpdateService.MIN_INTERVAL_MINUTES, parsed),
);
} catch (e) {
console.warn('[ImageUpdateService] Could not read interval setting; using default:', getErrorMessage(e, String(e)));
return fallback;
}
}
private armNext(delayMs: number): void {
this.nextCheckAt = Date.now() + delayMs;
const gen = this.scheduleGeneration;
this.timer = setTimeout(() => { void this.tick(gen); }, delayMs);
}
private async tick(gen: number): Promise<void> {
if (!this.polling || gen !== this.scheduleGeneration) return;
try {
await this.check();
} finally {
// Only the tick whose generation is still current re-arms. A
// restartPolling()/stop() that landed during the await bumped the
// generation and already rescheduled or cleared, so a stale tick
// bailing here is what keeps exactly one timer alive.
if (this.polling && gen === this.scheduleGeneration) {
this.armNext(this.nextDelayMs());
}
}
}
/** intervalMs with ±10% jitter so multiple nodes do not hit registries together. */
private nextDelayMs(): number {
const jitter = this.intervalMs * ImageUpdateService.JITTER_FRACTION;
return Math.round(this.intervalMs - jitter + Math.random() * 2 * jitter);
}
/**
@@ -153,6 +267,22 @@ export class ImageUpdateService {
return this.isRunning;
}
/** Milliseconds left on the manual-refresh cooldown; 0 when a refresh is allowed. */
public getManualCooldownRemainingMs(): number {
return Math.max(0, this.lastManualRefreshAt + ImageUpdateService.MANUAL_COOLDOWN_MS - Date.now());
}
public getStatus(): ImageUpdateStatus {
return {
checking: this.isRunning,
intervalMinutes: Math.round(this.intervalMs / (60 * 1000)),
lastCheckedAt: this.lastCheckedAt,
nextCheckAt: this.nextCheckAt,
manualCooldownMinutes: ImageUpdateService.manualCooldownMinutes,
manualCooldownRemainingMs: this.getManualCooldownRemainingMs(),
};
}
// ─── Core check ──────────────────────────────────────────────────────────
private async check() {
@@ -176,6 +306,10 @@ export class ImageUpdateService {
}
this.isRunning = true;
this.checkStartedAt = Date.now();
// Stamp last-checked here, in the shared scan path, so a manual Recheck
// updates it too. A skipped concurrent trigger returns above this line,
// so it never bumps the timestamp.
this.lastCheckedAt = this.checkStartedAt;
console.log('[ImageUpdateService] Starting image update check...');
try {