gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-08-28 19:27:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(fleet-sync): retry failed pushes and backfill on add-node (#970)

Browse Source 
A control instance now retries fleet-sync pushes that hit a transient
failure and backfills the security state on a freshly registered remote
without waiting for the next policy edit.

New service:
- FleetSyncRetryService (singleton, start/stop) wakes 30s after boot
  and ticks every 5min. For each fleet resource, queries
  getFailedSyncTargets within a 24h window and re-pushes via
  FleetSyncService.pushResourceToNode through the same per-node mutex,
  so a normal fanout in flight serializes naturally with a retry.
- After STALE_THRESHOLD_MS (1h) of continuous failure for a
  previously-working node, dispatches a single warning notification
  per cooldown window. Brand-new nodes that have never succeeded do
  not alert via this path; misconfigured remotes are caught by the
  test-connection affordance at registration time.
- Wired into bootstrap startup/shutdown next to AutoHealService.

Public surface:
- FleetSyncService.pushResourceToNode(node, resource): targeted push
  to one node that re-uses the per-node mutex. Used by the retry
  service and any future targeted-resync flow.
- routes/nodes.ts POST /api/nodes fires pushResourceAsync for both
  resources after a remote-proxy node row commits.

Tuning constants centralized in fleetSyncConstants.ts:
- RETRY_MAX_AGE_MS = 24h
- STALE_THRESHOLD_MS = 1h

Tests:
- 8 vitest cases covering replica skip, retry dispatch, missing-node
  skip, alert-once-per-cooldown across the threshold window, no-alert
  for recent failures, no-alert for brand-new never-succeeded nodes,
  no-alert when the retry itself succeeds, start/stop idempotency.
- Full backend suite: 1781 pass / 5 skipped.
This commit is contained in:
Anso
2026-05-07 13:16:24 -04:00
committed by GitHub
parent 7dde257e1f
commit 33b15d6cba
7 changed files with 389 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/FleetSyncRetryService.ts
+162
View File
@@ -0,0 +1,162 @@
import { DatabaseService, type Node } from './DatabaseService';
import { FleetSyncService, FLEET_RESOURCES } from './FleetSyncService';
import { NotificationService } from './NotificationService';
import { isDebugEnabled } from '../utils/debug';
import { RETRY_MAX_AGE_MS, STALE_THRESHOLD_MS } from './fleetSyncConstants';
const INITIAL_DELAY_MS = 30_000;
const EVAL_INTERVAL_MS = 5 * 60_000;
/**
* Background retry loop for fleet sync.
*
* The control's normal write path fires `pushResource` once per change; if a
* remote was offline at that moment, the failure is recorded on
* `fleet_sync_status` but never re-attempted unless another write happens. In
* practice an operator who sets a policy and walks away leaves the dead remote
* permanently stale. This service ticks every 5 minutes, queries the failed-
* not-yet-succeeded targets within the last 24h, and re-pushes through the
* same per-node mutex used by the write path.
*
* After STALE_THRESHOLD_MS of continuous failure for one (node, resource), we
* dispatch a single warning notification and remember the alert. The next
* recorded success clears the memo so a future failure can alert again.
*/
export class FleetSyncRetryService {
private static instance: FleetSyncRetryService;
private intervalId: NodeJS.Timeout | null = null;
private initialTimer: NodeJS.Timeout | null = null;
private isProcessing = false;
/** Per-target alert memo: `${nodeId}:${resource}` → timestamp of last alert. */
private alertedAt = new Map<string, number>();
private constructor() {}
static getInstance(): FleetSyncRetryService {
if (!FleetSyncRetryService.instance) {
FleetSyncRetryService.instance = new FleetSyncRetryService();
}
return FleetSyncRetryService.instance;
}
start(): void {
this.initialTimer = setTimeout(() => {
void this.evaluate();
this.intervalId = setInterval(() => void this.evaluate(), EVAL_INTERVAL_MS);
}, INITIAL_DELAY_MS);
}
stop(): void {
if (this.initialTimer) {
clearTimeout(this.initialTimer);
this.initialTimer = null;
}
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = null;
}
}
/**
* One pass: find every (node, resource) pair with an unresolved failure
* inside the retry window and re-push to it. Each push goes through the
* FleetSyncService per-node mutex, so a normal fanout in flight will
* serialize naturally.
*/
async evaluate(): Promise<void> {
if (this.isProcessing) return;
this.isProcessing = true;
try {
if (FleetSyncService.getRole() === 'replica') {
if (isDebugEnabled()) {
console.debug('[FleetSyncRetry:debug] Skipping tick: this instance is a replica');
}
return;
}
const db = DatabaseService.getInstance();
for (const resource of FLEET_RESOURCES) {
const failedTargets = db.getFailedSyncTargets(resource, RETRY_MAX_AGE_MS);
for (const target of failedTargets) {
const node = db.getNode(target.node_id);
if (!node) continue;
await this.retryOne(node, resource, target.last_failure_at, target.last_success_at);
}
}
} catch (err) {
console.error('[FleetSyncRetry] evaluate error:', err instanceof Error ? err.message : err);
} finally {
this.isProcessing = false;
}
}
private async retryOne(
node: Node,
resource: typeof FLEET_RESOURCES[number],
lastFailureAt: number | null,
lastSuccessAt: number | null,
): Promise<void> {
if (node.id == null) return;
if (isDebugEnabled()) {
console.debug(
`[FleetSyncRetry:debug] Retrying ${resource} for node "${node.name}" (id=${node.id})`,
);
}
await FleetSyncService.getInstance().pushResourceToNode(
node as Node & { id: number },
resource,
);
// After the retry, re-read status to decide whether to alert. A
// successful retry clears the alert memo via the success path above
// (recordFleetSyncSuccess was called inside the push). A still-failing
// retry might have crossed the staleness threshold.
if (lastFailureAt !== null && this.exceedsStaleThreshold(lastFailureAt, lastSuccessAt)) {
this.maybeAlertStale(node, resource);
}
}
private exceedsStaleThreshold(lastFailureAt: number, lastSuccessAt: number | null): boolean {
// Alert only when a previously-working node has been failing for more
// than the threshold. Brand-new nodes that have never succeeded fall
// through here: misconfigured remotes are caught by the test-connection
// affordance at registration time, not by this background notifier.
if (lastSuccessAt === null) return false;
return lastFailureAt - lastSuccessAt > STALE_THRESHOLD_MS;
}
private maybeAlertStale(node: Node, resource: typeof FLEET_RESOURCES[number]): void {
if (node.id == null) return;
const key = `${node.id}:${resource}`;
const now = Date.now();
const lastAlerted = this.alertedAt.get(key);
if (lastAlerted !== undefined && now - lastAlerted < STALE_THRESHOLD_MS) {
return;
}
// Re-read post-retry status: if the most recent push succeeded, clear
// the memo and skip the alert. The retry above ran inside the per-node
// mutex, so the status is up to date by the time we read it.
// recordFleetSyncSuccess clears last_failure_at to NULL, so a successful
// retry shows as `last_success_at !== null && last_failure_at === null`.
// A retry that landed atop an existing failure shows as both non-null
// with last_success_at >= last_failure_at.
const db = DatabaseService.getInstance();
const fresh = db
.getFleetSyncStatuses()
.find((s) => s.node_id === node.id && s.resource === resource);
if (fresh && fresh.last_success_at !== null
&& (fresh.last_failure_at === null || fresh.last_success_at >= fresh.last_failure_at)) {
this.alertedAt.delete(key);
return;
}
this.alertedAt.set(key, now);
void NotificationService.getInstance()
.dispatchAlert(
'warning',
'system',
`Fleet sync to node "${node.name}" has been failing for over an hour for ${resource}. Check the node's connectivity and API token.`,
)
.catch((err) => {
console.warn('[FleetSyncRetry] Failed to dispatch stale alert:', err);
});
}
}