mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-09-03 06:07:58 +00:00
fix(notifications): version mute replica retractions for soft-cleanup restore (#1703)
* fix(notifications): version mute replica retractions for soft-cleanup restore Soft cleanup and authoritative delete shared an unversioned permanent tombstone, so a later hub re-save could not restore scheduled mutes on a remote. Carry hub-authored kind and source_updated_at on replica DELETE, allow recoverable recreate when updated_at is newer, keep permanent deletes fail-closed, and reject stale recoverable DELETEs against newer rows. * fix(notifications): durable mute retractions across mixed-version fleets Gate recoverable replica DELETEs on a new capability, durable-queue failures and incompatible remotes for retry, fan permanent deletes to every known remote, and return applied vs ignored outcomes on replica writes.
This commit is contained in:
Anso
@@ -1,9 +1,10 @@
|
||||
import { DatabaseService, type NotificationSuppressionRule, type Node } from '../services/DatabaseService';
|
||||
import { DatabaseService, type NotificationSuppressionRule, type NotificationSuppressionRetraction, type NotificationSuppressionRetractionKind, type Node } from '../services/DatabaseService';
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { LicenseService } from '../services/LicenseService';
|
||||
import { PROXY_TIER_HEADER } from '../services/license-headers';
|
||||
import {
|
||||
NOTIFICATION_SUPPRESSION_SCHEDULE_CAPABILITY,
|
||||
NOTIFICATION_SUPPRESSION_REPLICA_RETRACTION_CAPABILITY,
|
||||
} from '../services/CapabilityRegistry';
|
||||
import { remoteAdvertisesCapability } from './remoteCapabilities';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
@@ -20,6 +21,13 @@ function buildRemoteHeaders(apiToken: string): Record<string, string> {
|
||||
return headers;
|
||||
}
|
||||
|
||||
function retractionFor(
|
||||
kind: NotificationSuppressionRetractionKind,
|
||||
rule: Pick<NotificationSuppressionRule, 'updated_at'>,
|
||||
): NotificationSuppressionRetraction {
|
||||
return { kind, source_updated_at: rule.updated_at };
|
||||
}
|
||||
|
||||
/** Hub node ids that should receive a replica for this rule (before wire identity normalize). */
|
||||
export function replicationTargetIds(rule: NotificationSuppressionRule): number[] {
|
||||
return replicationTargets(rule).map((n) => n.id);
|
||||
@@ -35,46 +43,171 @@ export function replicationTargets(rule: NotificationSuppressionRule): Node[] {
|
||||
return remotes;
|
||||
}
|
||||
|
||||
/** Every registered remote: permanent deletes must reach historical replicas, not only current scope. */
|
||||
export function allRemoteNodes(): Node[] {
|
||||
return DatabaseService.getInstance().getNodes().filter((n) => n.type === 'remote');
|
||||
}
|
||||
|
||||
function replicaPayload(rule: NotificationSuppressionRule): NotificationSuppressionRule {
|
||||
return { ...rule, node_id: null };
|
||||
}
|
||||
|
||||
async function pushRuleToNode(node: Node, rule: NotificationSuppressionRule): Promise<void> {
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!target?.apiUrl) {
|
||||
console.warn(`[SuppressionSync] Skipping node "${node.name}": no proxy target`);
|
||||
return;
|
||||
}
|
||||
const baseUrl = target.apiUrl.replace(/\/$/, '');
|
||||
const res = await fetch(`${baseUrl}/api/notification-suppression-rules/replica`, {
|
||||
method: 'POST',
|
||||
headers: buildRemoteHeaders(target.apiToken),
|
||||
body: JSON.stringify({ rule: replicaPayload(rule) }),
|
||||
signal: AbortSignal.timeout(SYNC_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const body = await res.text().catch(() => '');
|
||||
throw new Error(`HTTP ${res.status}${body ? `: ${body.slice(0, 200)}` : ''}`);
|
||||
type RetractionSupport = 'supported' | 'unsupported' | 'unreachable';
|
||||
|
||||
async function probeRetractionSupport(nodeId: number): Promise<RetractionSupport> {
|
||||
try {
|
||||
const meta = await NodeRegistry.getInstance().fetchMetaForNode(nodeId);
|
||||
// fetchMetaForNode returns OFFLINE_META (online:false, capabilities:[]) on
|
||||
// transport failure instead of throwing; treat that as unreachable.
|
||||
if (meta.online === false) return 'unreachable';
|
||||
return meta.capabilities.includes(NOTIFICATION_SUPPRESSION_REPLICA_RETRACTION_CAPABILITY)
|
||||
? 'supported'
|
||||
: 'unsupported';
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
`[SuppressionSync] Retraction capability probe failed for node ${nodeId}; treating as unreachable:`,
|
||||
getErrorMessage(err, 'unknown'),
|
||||
);
|
||||
return 'unreachable';
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteRuleOnNode(node: Node, ruleId: number): Promise<void> {
|
||||
function enqueuePending(
|
||||
node: Node,
|
||||
ruleId: number,
|
||||
retraction: NotificationSuppressionRetraction,
|
||||
lastError: string,
|
||||
): void {
|
||||
DatabaseService.getInstance().upsertNotificationSuppressionPendingRetraction({
|
||||
rule_id: ruleId,
|
||||
node_id: node.id,
|
||||
kind: retraction.kind,
|
||||
source_updated_at: retraction.source_updated_at,
|
||||
last_error: lastError,
|
||||
});
|
||||
}
|
||||
|
||||
function clearPending(nodeId: number, ruleId: number): void {
|
||||
DatabaseService.getInstance().deleteNotificationSuppressionPendingRetraction(ruleId, nodeId);
|
||||
}
|
||||
|
||||
function resolveRemoteApi(node: Node): { baseUrl: string; apiToken: string } | null {
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!target?.apiUrl) {
|
||||
throw new Error(`no proxy target for node "${node.name}" (id=${node.id})`);
|
||||
if (!target?.apiUrl) return null;
|
||||
return { baseUrl: target.apiUrl.replace(/\/$/, ''), apiToken: target.apiToken };
|
||||
}
|
||||
|
||||
/** Non-2xx (including opaque 404) is always failure; never treat missing routes as applied. */
|
||||
async function throwHttpFailure(res: Response): Promise<never> {
|
||||
const body = await res.text().catch(() => '');
|
||||
throw new Error(`HTTP ${res.status}${body ? `: ${body.slice(0, 200)}` : ''}`);
|
||||
}
|
||||
|
||||
/** Prefer JSON outcome; fall back to applied for pre-outcome remotes (bare 2xx). */
|
||||
async function readOutcome(res: Response): Promise<string> {
|
||||
try {
|
||||
const json = (await res.json()) as { outcome?: string };
|
||||
return typeof json.outcome === 'string' ? json.outcome : 'applied';
|
||||
} catch {
|
||||
return 'applied';
|
||||
}
|
||||
const baseUrl = target.apiUrl.replace(/\/$/, '');
|
||||
const res = await fetch(`${baseUrl}/api/notification-suppression-rules/replica/${ruleId}`, {
|
||||
method: 'DELETE',
|
||||
headers: buildRemoteHeaders(target.apiToken),
|
||||
}
|
||||
|
||||
async function pushRuleToNode(node: Node, rule: NotificationSuppressionRule): Promise<void> {
|
||||
const remote = resolveRemoteApi(node);
|
||||
if (!remote) {
|
||||
console.warn(`[SuppressionSync] Skipping node "${node.name}": no proxy target`);
|
||||
return;
|
||||
}
|
||||
const res = await fetch(`${remote.baseUrl}/api/notification-suppression-rules/replica`, {
|
||||
method: 'POST',
|
||||
headers: buildRemoteHeaders(remote.apiToken),
|
||||
body: JSON.stringify({ rule: replicaPayload(rule) }),
|
||||
signal: AbortSignal.timeout(SYNC_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok && res.status !== 404) {
|
||||
const body = await res.text().catch(() => '');
|
||||
throw new Error(`HTTP ${res.status}${body ? `: ${body.slice(0, 200)}` : ''}`);
|
||||
if (!res.ok) await throwHttpFailure(res);
|
||||
const outcome = await readOutcome(res);
|
||||
if (outcome !== 'applied') {
|
||||
throw new Error(`replica POST outcome=${outcome}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deliver a replica DELETE. On transport failure, durable-queues the retraction.
|
||||
* Clears pending only when the remote reports outcome=applied (not ignored_stale).
|
||||
*/
|
||||
export async function deleteRuleOnNode(
|
||||
node: Node,
|
||||
ruleId: number,
|
||||
retraction: NotificationSuppressionRetraction,
|
||||
): Promise<{ outcome: string }> {
|
||||
const remote = resolveRemoteApi(node);
|
||||
if (!remote) {
|
||||
const err = `no proxy target for node "${node.name}" (id=${node.id})`;
|
||||
enqueuePending(node, ruleId, retraction, err);
|
||||
throw new Error(err);
|
||||
}
|
||||
try {
|
||||
const res = await fetch(`${remote.baseUrl}/api/notification-suppression-rules/replica/${ruleId}`, {
|
||||
method: 'DELETE',
|
||||
headers: buildRemoteHeaders(remote.apiToken),
|
||||
body: JSON.stringify(retraction),
|
||||
signal: AbortSignal.timeout(SYNC_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) await throwHttpFailure(res);
|
||||
const outcome = await readOutcome(res);
|
||||
if (outcome !== 'applied') {
|
||||
// ignored_stale and any other non-applied outcome keep the outbox row.
|
||||
const reason =
|
||||
outcome === 'ignored_stale'
|
||||
? 'remote ignored_stale; rule still present'
|
||||
: `remote DELETE outcome=${outcome}`;
|
||||
enqueuePending(node, ruleId, retraction, reason);
|
||||
return { outcome };
|
||||
}
|
||||
clearPending(node.id, ruleId);
|
||||
return { outcome };
|
||||
} catch (err) {
|
||||
try {
|
||||
enqueuePending(node, ruleId, retraction, getErrorMessage(err, String(err)));
|
||||
} catch (enqueueErr) {
|
||||
console.error(
|
||||
`[SuppressionSync] Failed to durable-queue retract rule=${ruleId} node=${node.id}:`,
|
||||
getErrorMessage(enqueueErr, String(enqueueErr)),
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recoverable DELETE only when the remote advertises versioned retractions.
|
||||
* Otherwise enqueue pending and do not bare-delete on incompatible remotes.
|
||||
* @returns applied when DELETE was accepted; deferred when queued without sending
|
||||
* or when the remote ignored a stale watermark.
|
||||
*/
|
||||
async function deliverRecoverableDelete(
|
||||
node: Node,
|
||||
ruleId: number,
|
||||
retraction: NotificationSuppressionRetraction,
|
||||
context: string,
|
||||
): Promise<'applied' | 'deferred'> {
|
||||
const support = await probeRetractionSupport(node.id);
|
||||
if (support !== 'supported') {
|
||||
const reason =
|
||||
support === 'unsupported'
|
||||
? 'remote lacks notification-suppression-replica-retraction; not sending recoverable DELETE'
|
||||
: 'remote unreachable for retraction capability probe; not sending recoverable DELETE';
|
||||
enqueuePending(node, ruleId, retraction, reason);
|
||||
console.warn(
|
||||
`[SuppressionSync] ${context} on node "${node.name}" (id=${node.id}): ${reason}; queued pending retraction`,
|
||||
);
|
||||
return 'deferred';
|
||||
}
|
||||
const { outcome } = await deleteRuleOnNode(node, ruleId, retraction);
|
||||
return outcome === 'applied' ? 'applied' : 'deferred';
|
||||
}
|
||||
|
||||
async function pushOrCleanupScheduled(node: Node, rule: NotificationSuppressionRule): Promise<void> {
|
||||
const supportsSchedule = await remoteAdvertisesCapability(
|
||||
node.id,
|
||||
@@ -84,36 +217,46 @@ async function pushOrCleanupScheduled(node: Node, rule: NotificationSuppressionR
|
||||
await pushRuleToNode(node, rule);
|
||||
return;
|
||||
}
|
||||
// Probe false means unsupported OR unreachable. Never POST a scheduled rule
|
||||
// through the legacy contract (older remotes would mute all day). Attempt DELETE;
|
||||
// only claim cleanup when DELETE succeeds.
|
||||
// Never POST a scheduled rule through the legacy contract. Soft-cleanup DELETE
|
||||
// requires versioned retraction support so incompatible remotes are not bare-deleted.
|
||||
try {
|
||||
await deleteRuleOnNode(node, rule.id);
|
||||
console.warn(
|
||||
`[SuppressionSync] Scheduled rule ${rule.id} not applied on node "${node.name}" (id=${node.id}): ` +
|
||||
`capability unsupported-or-unreachable; DELETE succeeded and replica was removed`,
|
||||
const result = await deliverRecoverableDelete(
|
||||
node,
|
||||
rule.id,
|
||||
retractionFor('recoverable', rule),
|
||||
`Scheduled rule ${rule.id} not applied`,
|
||||
);
|
||||
if (result === 'applied') {
|
||||
console.warn(
|
||||
`[SuppressionSync] Scheduled rule ${rule.id} not applied on node "${node.name}" (id=${node.id}): ` +
|
||||
`schedule unsupported-or-unreachable; recoverable DELETE applied`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[SuppressionSync] Scheduled rule ${rule.id}: cleanup pending on node "${node.name}" (id=${node.id}); ` +
|
||||
`capability unsupported-or-unreachable and DELETE failed (${getErrorMessage(err, String(err))}). ` +
|
||||
`Prior replica may remain until connectivity returns and the rule is re-saved`,
|
||||
`recoverable DELETE failed (${getErrorMessage(err, String(err))})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanupInvalidScheduleReplica(node: Node, rule: NotificationSuppressionRule): Promise<void> {
|
||||
// Never POST an invalid schedule (would mute all day on remotes that ignore the field).
|
||||
// Attempt DELETE so a prior valid/unscheduled replica cannot keep muting.
|
||||
try {
|
||||
await deleteRuleOnNode(node, rule.id);
|
||||
console.warn(
|
||||
`[SuppressionSync] Corrupt schedule on rule ${rule.id}: replica removed on node "${node.name}" (id=${node.id}); not posting`,
|
||||
const result = await deliverRecoverableDelete(
|
||||
node,
|
||||
rule.id,
|
||||
retractionFor('recoverable', rule),
|
||||
`Corrupt schedule on rule ${rule.id}`,
|
||||
);
|
||||
if (result === 'applied') {
|
||||
console.warn(
|
||||
`[SuppressionSync] Corrupt schedule on rule ${rule.id}: recoverable DELETE applied on node "${node.name}" (id=${node.id}); not posting`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[SuppressionSync] Corrupt schedule on rule ${rule.id}: cleanup pending on node "${node.name}" (id=${node.id}); ` +
|
||||
`DELETE failed (${getErrorMessage(err, String(err))}). Prior replica may remain until connectivity returns`,
|
||||
`DELETE failed (${getErrorMessage(err, String(err))})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -160,13 +303,19 @@ export function syncSuppressionRuleUpdateToFleet(
|
||||
const newIds = new Set(replicationTargetIds(updated));
|
||||
const db = DatabaseService.getInstance();
|
||||
const staleIds = [...oldIds].filter((id) => !newIds.has(id));
|
||||
const staleRetraction = retractionFor('recoverable', updated);
|
||||
|
||||
void Promise.allSettled([
|
||||
...staleIds.map(async (id) => {
|
||||
const node = db.getNode(id);
|
||||
if (!node || node.type !== 'remote') return;
|
||||
try {
|
||||
await deleteRuleOnNode(node, previous.id);
|
||||
await deliverRecoverableDelete(
|
||||
node,
|
||||
previous.id,
|
||||
staleRetraction,
|
||||
`Stale-target retract for rule ${previous.id}`,
|
||||
);
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[SuppressionSync] Failed to delete stale rule ${previous.id} on node "${node.name}":`,
|
||||
@@ -187,14 +336,18 @@ export function syncSuppressionRuleUpdateToFleet(
|
||||
]);
|
||||
}
|
||||
|
||||
/** Best-effort delete of a replicated rule on fleet nodes. */
|
||||
/**
|
||||
* Authoritative delete: fan out permanent retraction to every known remote
|
||||
* (not only current scope), and durable-queue failures for retry.
|
||||
*/
|
||||
export function deleteSuppressionRuleFromFleet(rule: NotificationSuppressionRule): void {
|
||||
const targets = replicationTargets(rule);
|
||||
const targets = allRemoteNodes();
|
||||
if (targets.length === 0) return;
|
||||
const retraction = retractionFor('permanent', rule);
|
||||
void Promise.allSettled(
|
||||
targets.map(async (node) => {
|
||||
try {
|
||||
await deleteRuleOnNode(node, rule.id);
|
||||
await deleteRuleOnNode(node, rule.id, retraction);
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[SuppressionSync] Failed to delete rule ${rule.id} on node "${node.name}":`,
|
||||
@@ -204,3 +357,40 @@ export function deleteSuppressionRuleFromFleet(rule: NotificationSuppressionRule
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry durable pending retractions for one node (tunnel-up / reconnect) or all.
|
||||
* Recoverable rows still require the remote to advertise retraction support.
|
||||
*/
|
||||
export async function flushPendingSuppressionRetractions(nodeId?: number): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const pending = db.listNotificationSuppressionPendingRetractions(nodeId);
|
||||
for (const row of pending) {
|
||||
const node = db.getNode(row.node_id);
|
||||
if (!node || node.type !== 'remote') {
|
||||
db.deleteNotificationSuppressionPendingRetraction(row.rule_id, row.node_id);
|
||||
continue;
|
||||
}
|
||||
const retraction: NotificationSuppressionRetraction = {
|
||||
kind: row.kind,
|
||||
source_updated_at: row.source_updated_at,
|
||||
};
|
||||
try {
|
||||
if (retraction.kind === 'recoverable') {
|
||||
await deliverRecoverableDelete(
|
||||
node,
|
||||
row.rule_id,
|
||||
retraction,
|
||||
`Pending recoverable retract for rule ${row.rule_id}`,
|
||||
);
|
||||
} else {
|
||||
await deleteRuleOnNode(node, row.rule_id, retraction);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[SuppressionSync] Pending retract retry failed rule=${row.rule_id} node=${node.name}:`,
|
||||
getErrorMessage(err, String(err)),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user