fix(notifications): version mute replica retractions for soft-cleanup restore (#1703)

* fix(notifications): version mute replica retractions for soft-cleanup restore

Soft cleanup and authoritative delete shared an unversioned permanent
tombstone, so a later hub re-save could not restore scheduled mutes on a
remote. Carry hub-authored kind and source_updated_at on replica DELETE,
allow recoverable recreate when updated_at is newer, keep permanent deletes
fail-closed, and reject stale recoverable DELETEs against newer rows.

* fix(notifications): durable mute retractions across mixed-version fleets

Gate recoverable replica DELETEs on a new capability, durable-queue failures
and incompatible remotes for retry, fan permanent deletes to every known
remote, and return applied vs ignored outcomes on replica writes.
This commit is contained in:
Anso
2026-07-25 23:47:37 -04:00
committed by GitHub
parent 6688da97b1
commit 9859ce60b8
11 changed files with 1601 additions and 157 deletions
+4
View File
@@ -6,6 +6,7 @@ import { AutoHealService } from '../services/AutoHealService';
import { HealthGateService } from '../services/HealthGateService';
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
import { FleetSyncRetryService } from '../services/FleetSyncRetryService';
import { SuppressionRetractionRetryService } from '../services/SuppressionRetractionRetryService';
import { DockerEventManager } from '../services/DockerEventManager';
import { ImageUpdateService } from '../services/ImageUpdateService';
import { SchedulerService } from '../services/SchedulerService';
@@ -37,6 +38,9 @@ export function installShutdownHandlers(server: Server): void {
try { HealthGateService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] HealthGateService cleanup failed:', (e as Error).message); }
try { ServiceUpdateRecoveryService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] ServiceUpdateRecoveryService cleanup failed:', (e as Error).message); }
try { FleetSyncRetryService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] FleetSyncRetryService cleanup failed:', (e as Error).message); }
try { SuppressionRetractionRetryService.getInstance().stop(); } catch (e) {
console.warn('[Shutdown] SuppressionRetractionRetryService cleanup failed:', (e as Error).message);
}
try { DockerEventManager.getInstance().stop(); } catch (e) {
console.warn('[Shutdown] DockerEventManager cleanup failed:', (e as Error).message);
}
+7 -1
View File
@@ -15,6 +15,7 @@ import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryS
import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService';
import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
import { FleetSyncRetryService } from '../services/FleetSyncRetryService';
import { SuppressionRetractionRetryService } from '../services/SuppressionRetractionRetryService';
import { DockerEventManager } from '../services/DockerEventManager';
import TrivyService, { sweepStaleTrivyTempDirs } from '../services/TrivyService';
import { ImageUpdateService } from '../services/ImageUpdateService';
@@ -154,6 +155,7 @@ export async function startServer(server: Server): Promise<void> {
HealthGateService.getInstance().start();
ServiceUpdateRecoveryService.getInstance().start();
FleetSyncRetryService.getInstance().start();
SuppressionRetractionRetryService.getInstance().start();
ImageUpdateService.getInstance().start();
SchedulerService.getInstance().start();
MfaService.getInstance().start();
@@ -166,7 +168,11 @@ export async function startServer(server: Server): Promise<void> {
// Drop the cached /api/meta entry on tunnel reconnect so the next
// /api/nodes/:id/meta refetches fresh capabilities and version through
// the live loopback bridge instead of waiting for the 3-minute TTL.
PilotTunnelManager.getInstance().on('tunnel-up', invalidateRemoteMetaCache);
// Also flush durable mute-replica retractions that waited for this node.
PilotTunnelManager.getInstance().on('tunnel-up', (nodeId: number) => {
invalidateRemoteMetaCache(nodeId);
void SuppressionRetractionRetryService.getInstance().flushNode(nodeId);
});
// Most async initializers still run in parallel. Docker event monitoring
// is sequenced after self identity so it never classifies Sencho's own