feat: add node-scoped opt-out for image update detection (#1715)

* feat: add node-scoped opt-out for image update detection

Operators who use an external update authority can disable Sencho registry
polling per node without losing explicit stack Update, pull, or redeploy.

* test: fix mocks and lint for image-update checks opt-out

Scheduler tests need isChecksEnabled on the ImageUpdateService mock, and the UpdatesSection older-node fixture must not leave an unused binding.

* fix: gate update-preview and recheck when detection is off

Anatomy was still calling stack update-preview (and contacting registries)
while checks were disabled. Short-circuit those routes and skip recheckStack
writes so disabled nodes stay quiet until detection is re-enabled.
This commit is contained in:
Anso
2026-07-28 10:10:04 -04:00
committed by GitHub
parent e175db8e62
commit fa503ddf27
23 changed files with 722 additions and 80 deletions
+9
View File
@@ -2022,6 +2022,9 @@ export class DatabaseService {
stmt.run('image_update_check_mode', 'interval');
stmt.run('image_update_check_cron', '');
stmt.run('image_update_sidebar_indicators', '1');
// Opt-out for background registry polling. Default on so upgrades keep
// current behavior; missing key is also treated as enabled at read time.
stmt.run('image_update_checks_enabled', '1');
stmt.run('notification_dispatch_retries', '0');
stmt.run('env_block_deploy_on_missing_required', '0');
stmt.run('auto_create_missing_external_networks', '0');
@@ -5005,6 +5008,12 @@ export class DatabaseService {
return result.changes;
}
/** Deletes every update row for a node. Returns deleted row count. */
public clearAllStackUpdateStatus(nodeId: number): number {
const result = this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ?').run(nodeId);
return result.changes;
}
// --- Stack Scan Attempts ---
//
// Tracks the latest post-deploy scan attempt per (nodeId, stackName) so
+91 -8
View File
@@ -14,6 +14,7 @@ import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
import { buildEffectiveServiceModel } from './effectiveServiceModel';
import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
const BACKFILL_KEY = 'image_update_notifications_backfilled';
@@ -75,6 +76,8 @@ export function normalizeImageCheckStatus(r: ImageCheckResult): PreviewImageChec
* `nextCheckAt` is meaningless while `checking` is true.
* `mode` is the active scheduling mode; `cronExpression` is the 5-field
* expression when mode is 'cron', null otherwise or when unconfigured.
* `enabled` is whether background image-update detection is armed; always
* present on current nodes, optional on the wire for older remotes.
*/
export interface ImageUpdateStatus {
checking: boolean;
@@ -86,6 +89,7 @@ export interface ImageUpdateStatus {
mode: 'interval' | 'cron';
cronExpression: string | null;
sidebarIndicators: boolean;
enabled: boolean;
}
// ─── Compose file helpers ────────────────────────────────────────────────────
@@ -410,6 +414,7 @@ export class ImageUpdateService {
private static readonly INTERVAL_SETTING_KEY = 'image_update_check_interval_minutes';
private static readonly MODE_SETTING_KEY = 'image_update_check_mode';
private static readonly CRON_SETTING_KEY = 'image_update_check_cron';
private static readonly ENABLED_SETTING_KEY = 'image_update_checks_enabled';
private static readonly JITTER_FRACTION = 0.1; // ±10% so a fleet does not poll in lockstep
private static readonly STARTUP_DELAY_MS = 2 * 60 * 1000; // 2 min after boot
@@ -452,8 +457,15 @@ export class ImageUpdateService {
public start() {
if (this.timer) return;
this.polling = true;
this.configureFromSettings();
if (!ImageUpdateService.isChecksEnabled()) {
// Detection opted out: stay stopped across restarts so a boot does
// not re-arm registry polling until the setting is turned back on.
this.polling = false;
this.nextCheckAt = null;
return;
}
this.polling = true;
// Interval mode keeps the 2-minute post-boot delay before the first check.
// Cron mode honors its schedule: arm at the next cron fire time so a restart
// never triggers an out-of-cadence check (e.g. a weekly cron must not run on
@@ -476,7 +488,8 @@ export class ImageUpdateService {
* cadence without restarting Sencho. Safe to call repeatedly: it always
* clears the existing timer first and only arms a new one while polling, so
* it never stacks timers and is a no-op (beyond reconfiguring intervalMs)
* when the service is stopped or was never started.
* when the service is stopped or was never started. When checks are
* disabled, clears nextCheckAt and does not arm.
*/
public restartPolling(): void {
this.scheduleGeneration++;
@@ -485,13 +498,65 @@ export class ImageUpdateService {
this.timer = null;
}
this.configureFromSettings();
if (this.polling) {
if (this.polling && ImageUpdateService.isChecksEnabled()) {
this.armNext(this.nextDelayMs());
} else {
this.nextCheckAt = null;
}
}
/**
* Whether background image-update detection is enabled. Missing or blank
* keys default to enabled so upgrades and pre-seed races keep polling.
*/
public static isChecksEnabled(): boolean {
try {
const raw = DatabaseService.getInstance().getGlobalSettings()[ImageUpdateService.ENABLED_SETTING_KEY];
if (raw == null || String(raw).trim() === '') return true;
return raw === '1';
} catch (e) {
console.warn('[ImageUpdateService] Could not read checks-enabled setting; treating as enabled:', getErrorMessage(e, String(e)));
return true;
}
}
/**
* Persist the checks-enabled setting and apply the live transition: stop
* + clear findings when turning off; start (or re-arm) when turning on.
* Safe under repeated toggles (scheduleGeneration bump via stop/start).
*/
public applyChecksEnabled(enabled: boolean): ImageUpdateStatus {
const db = DatabaseService.getInstance();
db.updateGlobalSetting(ImageUpdateService.ENABLED_SETTING_KEY, enabled ? '1' : '0');
if (!enabled) {
this.stop();
// Scanner only writes rows for local nodes. Use the local default
// node ID, not req.nodeId (which may be a remote active node).
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
db.clearAllStackUpdateStatus(localNodeId);
invalidateFleetUpdateCache();
NotificationService.getInstance().broadcastEvent({
type: 'state-invalidate',
scope: 'image-updates',
nodeId: localNodeId,
action: 'checks-disabled',
ts: Date.now(),
});
return this.getStatus();
}
// Re-enable: arm a fresh schedule. start() is a no-op if a timer already
// exists; when we were fully stopped, start() arms. When somehow still
// marked polling without a timer, restartPolling re-arms.
if (!this.timer) {
this.start();
} else {
this.restartPolling();
}
return this.getStatus();
}
/**
* Reads image_update_check_interval_minutes into intervalMs, clamped to
* [15, 1440], falling back to the 2-hour default on a missing, blank,
@@ -600,11 +665,16 @@ export class ImageUpdateService {
}
/**
* Triggers a check immediately, unless one is already running or the
* manual cooldown (MANUAL_COOLDOWN_MS) has not elapsed.
* Returns false if rate-limited, true if a check was started.
* Triggers a check immediately, unless detection is disabled, one is already
* running, or the manual cooldown (MANUAL_COOLDOWN_MS) has not elapsed.
* Returns false if rate-limited or disabled, true if a check was started.
* Callers that need to distinguish disabled from rate-limited must check
* isChecksEnabled() first.
*/
public triggerManualRefresh(): boolean {
if (!ImageUpdateService.isChecksEnabled()) {
return false;
}
const now = Date.now();
if (now - this.lastManualRefreshAt < ImageUpdateService.MANUAL_COOLDOWN_MS) {
return false;
@@ -624,6 +694,7 @@ export class ImageUpdateService {
}
public getStatus(): ImageUpdateStatus {
const enabled = ImageUpdateService.isChecksEnabled();
let sidebarIndicators = false;
try {
const settings = DatabaseService.getInstance().getGlobalSettings();
@@ -632,21 +703,28 @@ export class ImageUpdateService {
console.warn('[ImageUpdateService] Failed to read sidebar indicator setting:', e);
}
return {
checking: this.isRunning,
checking: enabled ? this.isRunning : false,
intervalMinutes: Math.round(this.intervalMs / (60 * 1000)),
lastCheckedAt: this.lastCheckedAt,
nextCheckAt: this.nextCheckAt,
nextCheckAt: enabled ? this.nextCheckAt : null,
manualCooldownMinutes: ImageUpdateService.manualCooldownMinutes,
manualCooldownRemainingMs: this.getManualCooldownRemainingMs(),
mode: this.mode,
cronExpression: this.cronExpression,
sidebarIndicators,
enabled,
};
}
// ─── Core check ──────────────────────────────────────────────────────────
private async check() {
if (!ImageUpdateService.isChecksEnabled()) {
if (isDebugEnabled()) {
console.log('[ImageUpdateService:debug] Checks disabled; skipping scan.');
}
return;
}
// The finally block is the sole owner of isRunning, so a scan that
// overruns can never have its lock released out from under it. A
// previous fixed timer cleared the lock after CHECK_TIMEOUT_MS, which
@@ -908,6 +986,11 @@ export class ImageUpdateService {
* left untouched and a verification_failed result is returned.
*/
public async recheckStack(nodeId: number, stackName: string): Promise<StackRecheckResult> {
// While detection is off, skip registry probes and do not write
// stack_update_status (avoids stale findings after re-enable).
if (!ImageUpdateService.isChecksEnabled()) {
return { outcome: 'cleared', warning: null };
}
const generation = this.reserveStackWriteGeneration(nodeId, stackName);
const db = DatabaseService.getInstance();
const docker = DockerController.getInstance(nodeId);
+4
View File
@@ -1038,6 +1038,10 @@ export class SchedulerService {
imageUpdateService: ImageUpdateService,
isWildcard = false
): Promise<string> {
if (!ImageUpdateService.isChecksEnabled()) {
console.log(`[SchedulerService] Stack "${stackName}": image update detection is disabled; skipped.`);
return `Stack "${stackName}": image update detection is disabled; skipped.`;
}
const containers = await docker.getContainersByStack(stackName);
if (!containers || containers.length === 0) {
if (!isWildcard) {
+14 -3
View File
@@ -3,7 +3,8 @@ import DockerController from './DockerController';
import { DatabaseService } from './DatabaseService';
import { FileSystemService } from './FileSystemService';
import { ComposeDoctorService } from './ComposeDoctorService';
import { UpdatePreviewService, isMovingTag, filterPreviewForService } from './UpdatePreviewService';
import { UpdatePreviewService, isMovingTag, filterPreviewForService, buildDetectionDisabledPreview } from './UpdatePreviewService';
import { ImageUpdateService } from './ImageUpdateService';
import { buildEffectiveServiceModel, type EffectiveServiceModelResult } from './effectiveServiceModel';
import { filterContainersByComposeService } from '../helpers/composeServiceMatch';
import { withTimeout } from '../utils/withTimeout';
@@ -137,6 +138,12 @@ export class UpdateGuardService {
withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness sibling probe'))
: Promise.resolve<ContainerProbe[] | Errored>([]),
this.collect('update preview', stackName, async () => {
// Check inside the thunk so the read stays with the getPreview call.
// Stack GET/POST update-preview use the same isChecksEnabled gate.
if (!ImageUpdateService.isChecksEnabled()) {
const disabled = buildDetectionDisabledPreview(stackName);
return serviceName ? filterPreviewForService(disabled, serviceName) : disabled;
}
const full = await withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness update preview');
return serviceName ? filterPreviewForService(full, serviceName) : full;
}),
@@ -218,8 +225,12 @@ export class UpdateGuardService {
this.collect('backup info', stackName, () => fsSvc.getBackupInfo(stackName)),
this.collect('backup env summary', stackName, () => fsSvc.getBackupEnvSummary(stackName)),
this.collect('stack env presence', stackName, () => fsSvc.envExists(stackName)),
this.collect('update preview', stackName, () =>
withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'rollback readiness update preview')),
this.collect('update preview', stackName, async () => {
if (!ImageUpdateService.isChecksEnabled()) {
return buildDetectionDisabledPreview(stackName);
}
return withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'rollback readiness update preview');
}),
this.collect('activity history', stackName, async () => {
const events = db.getStackActivity(nodeId, stackName, { limit: 50 });
// A successful update is as good a known-good marker as a deploy.
@@ -111,6 +111,11 @@ export interface UpdatePreviewSummary {
verification_failed: boolean;
/** First image check_error when verification_failed; otherwise null. */
verification_error: string | null;
/**
* When true, background detection is disabled on this node and the preview
* was not fetched from registries. Optional for older remotes.
*/
detection_disabled?: boolean;
}
export interface UpdatePreview {
@@ -362,6 +367,33 @@ export function filterPreviewForService(preview: UpdatePreview, serviceName: str
return buildSummary(preview.stack_name, images, buildServices);
}
/** Minimal preview when node-scoped image update detection is disabled. */
export function buildDetectionDisabledPreview(stackName: string): UpdatePreview {
return {
stack_name: stackName,
images: [],
build_services: [],
summary: {
has_update: false,
primary_image: null,
current_tag: null,
next_tag: null,
semver_bump: 'none',
update_kind: 'none',
blocked: false,
blocked_reason: null,
has_build_services: false,
rebuild_available: false,
check_status: 'ok',
verification_failed: false,
verification_error: null,
detection_disabled: true,
},
rollback_target: null,
changelog: null,
};
}
export class UpdatePreviewService {
private static instance: UpdatePreviewService;
@@ -134,6 +134,14 @@ export function updatePreviewSignal(input: UpdatePreviewSummary | Errored, image
if (input === 'error') {
return { ...base, status: 'unknown', affectsVerdict: false, detail: 'The update preview is unavailable.' };
}
if (input.detection_disabled) {
return {
...base,
status: 'unknown',
affectsVerdict: false,
detail: 'Image update detection is disabled for this node.',
};
}
if (input.blocked) {
return {
...base,