mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-26 02:06:49 +00:00
feat(fleet): cross-node bulk label assign with authoritative label discovery (#1389)
* feat(fleet): cross-node bulk label assign with authoritative label discovery Make Fleet Actions > Bulk label assign work across the fleet. Pick a stack label that exists anywhere in the fleet, select stacks on one or more nodes, and the control orchestrates: each target node resolves the label by name, creating it with the same name and color if missing, then adds it to the selected stacks while preserving their existing labels. The local node runs in process; each remote runs its own admin-only local-assign receiver over the node proxy. Per-node failures (unknown node, no proxy target, unreachable, mixed-version remote) degrade that node only and are reported per node in the result. Assignment writes use a transactional INSERT OR IGNORE so the add-preserve path is idempotent and race-free. Also make the shared fleet label discovery authoritative: suggestions, match-preview, and the fleet-stop remote leg now read each node's labels live over the proxy instead of the control database, which does not mirror remote labels. A propagated label therefore appears in, and is stoppable by, Stop-by-label across the fleet, and unreachable nodes are surfaced rather than silently dropped. Fleet Actions runs against the unfiltered node list, so overview filters no longer narrow its scope. The previous node-scoped, replace-by-id bulk-assign endpoint is removed. * fix(fleet): treat malformed remote label responses as per-node failures A 200 response from a remote node whose body is not the expected shape was treated as a benign empty result, so a malformed remote could read as a clean zero-stack assign or a "matched, nothing to stop" no-op and even surface a success toast. Validate the wire shape in the bulk-assign and fleet-stop remote legs and in the authoritative label discovery fan-out; on a malformed body, report the node as a per-node failure with the error attributed to its stacks instead of silently dropping it. * chore: drop accidentally committed temp file
This commit is contained in:
Anso
@@ -39,6 +39,8 @@ import { invalidateNodeCaches, invalidateRemoteMetaCache } from '../helpers/cach
|
||||
import { activeBulkActions } from './labels';
|
||||
import { runLocalLabelStop, isLabelLocalStopResponse, type StackStopResult } from '../helpers/fleetLabelStop';
|
||||
import { collectFleetLabelSummaries } from '../helpers/fleetLabelSummary';
|
||||
import { runLocalLabelAssign, validateLabelTemplate, failAllAssign, type LabelLocalAssignResponse, type AssignNodeResult } from '../helpers/fleetLabelAssign';
|
||||
import { MAX_ASSIGNMENTS } from '../helpers/constants';
|
||||
import { buildLocalConfigurationStatus, type ConfigurationStatus } from './dashboard';
|
||||
import { buildLocalGraph, mergeFleetGraph, isLocalDependencyGraph, type FleetNodeGraphResult } from '../services/DependencyGraphService';
|
||||
import { PROXY_TIER_HEADER } from '../services/license-headers';
|
||||
@@ -1351,6 +1353,133 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res:
|
||||
}
|
||||
});
|
||||
|
||||
// Fleet-wide bulk label assign. Propagates a label template (name + color) to
|
||||
// stacks across one or more nodes: for each target node the label is resolved or
|
||||
// created by name on that node, then assigned to the given stacks preserving
|
||||
// their existing labels (add semantics). Labels are node-local, so the control
|
||||
// never reuses a local label id on a remote; the local node runs in-process and
|
||||
// each remote runs its own `/api/fleet-actions/labels/local-assign` receiver.
|
||||
// Per-node failures (unknown node, no proxy target, unreachable, mixed-version
|
||||
// remote, malformed response) degrade that node only and never discard the rest
|
||||
// of the fan-out.
|
||||
// Tier: requireAdmin (admin-only fleet plumbing; available on every license).
|
||||
fleetRouter.post('/labels/bulk-assign', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
const body = req.body as { label?: unknown; targets?: unknown } | undefined;
|
||||
if (!body || typeof body !== 'object') {
|
||||
res.status(400).json({ error: 'Request body is required' });
|
||||
return;
|
||||
}
|
||||
const validated = validateLabelTemplate(body.label);
|
||||
if (!validated.ok) {
|
||||
res.status(400).json({ error: validated.error });
|
||||
return;
|
||||
}
|
||||
if (!Array.isArray(body.targets) || body.targets.length === 0) {
|
||||
res.status(400).json({ error: 'targets must be a non-empty array' });
|
||||
return;
|
||||
}
|
||||
// Normalize targets: each must name a node and carry a string array of stacks.
|
||||
// Stack names are deduped per node and empty groups are dropped, so the cap
|
||||
// measures real assignments rather than padded input.
|
||||
const targets: { nodeId: number; stackNames: string[] }[] = [];
|
||||
let totalStacks = 0;
|
||||
for (const raw of body.targets as unknown[]) {
|
||||
if (!raw || typeof raw !== 'object') {
|
||||
res.status(400).json({ error: 'each target must be an object' });
|
||||
return;
|
||||
}
|
||||
const { nodeId, stackNames } = raw as { nodeId?: unknown; stackNames?: unknown };
|
||||
if (typeof nodeId !== 'number' || !Number.isInteger(nodeId)) {
|
||||
res.status(400).json({ error: 'target.nodeId must be an integer' });
|
||||
return;
|
||||
}
|
||||
if (!Array.isArray(stackNames) || !stackNames.every(s => typeof s === 'string')) {
|
||||
res.status(400).json({ error: 'target.stackNames must be an array of strings' });
|
||||
return;
|
||||
}
|
||||
const unique = Array.from(new Set(stackNames as string[]));
|
||||
if (unique.length === 0) continue;
|
||||
totalStacks += unique.length;
|
||||
targets.push({ nodeId, stackNames: unique });
|
||||
}
|
||||
if (targets.length === 0) {
|
||||
res.status(400).json({ error: 'no target stacks provided' });
|
||||
return;
|
||||
}
|
||||
if (totalStacks > MAX_ASSIGNMENTS) {
|
||||
res.status(400).json({ error: `targets may not exceed ${MAX_ASSIGNMENTS} stack assignments` });
|
||||
return;
|
||||
}
|
||||
const { template } = validated;
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const nodesById = new Map(db.getNodes().map(n => [n.id, n]));
|
||||
if (isDebugEnabled()) console.debug('[Fleet:debug] bulk-assign:', { label: template.name, targets: targets.length, totalStacks });
|
||||
const results: AssignNodeResult[] = await Promise.all(targets.map(async (target): Promise<AssignNodeResult> => {
|
||||
const node = nodesById.get(target.nodeId);
|
||||
if (!node) {
|
||||
return {
|
||||
nodeId: target.nodeId, nodeName: `Node ${target.nodeId}`, reachable: false, created: false, error: 'Unknown node',
|
||||
stackResults: failAllAssign(target.stackNames, 'Unknown node'),
|
||||
};
|
||||
}
|
||||
if (node.type === 'local') {
|
||||
try {
|
||||
const outcome = await runLocalLabelAssign(node.id, template, target.stackNames);
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: true, created: outcome.created, stackResults: outcome.stackResults };
|
||||
} catch (err) {
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, reachable: true, created: false,
|
||||
stackResults: failAllAssign(target.stackNames, getErrorMessage(err, 'Failed to assign labels')),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!proxyTarget) {
|
||||
const error = formatNoTargetError(node);
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, created: false, error, stackResults: failAllAssign(target.stackNames, error) };
|
||||
}
|
||||
try {
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`;
|
||||
const response = await fetch(`${proxyTarget.apiUrl.replace(/\/$/, '')}/api/fleet-actions/labels/local-assign`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({ label: template, stackNames: target.stackNames }),
|
||||
signal: AbortSignal.timeout(60000),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const err = (await response.json().catch(() => ({}))) as { error?: string };
|
||||
const message = err.error || `Remote returned ${response.status}`;
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, created: false, error: message, stackResults: failAllAssign(target.stackNames, message) };
|
||||
}
|
||||
// A 200 whose body is not the expected { created, results } shape is a
|
||||
// degraded node, not a clean no-op: report it as a per-node failure so a
|
||||
// malformed remote cannot read as a successful zero-stack assign.
|
||||
const remote = (await response.json().catch(() => null)) as Partial<LabelLocalAssignResponse> | null;
|
||||
if (!remote || typeof remote.created !== 'boolean' || !Array.isArray(remote.results)) {
|
||||
const message = 'Remote returned a malformed response';
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, created: false, error: message, stackResults: failAllAssign(target.stackNames, message) };
|
||||
}
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, reachable: true,
|
||||
created: remote.created,
|
||||
stackResults: remote.results,
|
||||
};
|
||||
} catch (err) {
|
||||
const errorMsg = getErrorMessage(err, 'Failed to reach remote node');
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, created: false, error: errorMsg, stackResults: failAllAssign(target.stackNames, errorMsg) };
|
||||
}
|
||||
}));
|
||||
res.json({ results });
|
||||
} catch (error) {
|
||||
console.error('[Fleet] bulk-assign error:', error);
|
||||
res.status(500).json({ error: getErrorMessage(error, 'Failed to run bulk label assign') });
|
||||
}
|
||||
});
|
||||
|
||||
// Fleet-wide Docker prune. Fans out to every node, running per-target prune
|
||||
// (images/volumes/networks) under the chosen scope. Local nodes call
|
||||
// DockerController directly under a per-node bulk-prune lock; remote nodes
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { authMiddleware } from '../middleware/auth';
|
||||
import { requireAdmin, requireBody } from '../middleware/tierGates';
|
||||
import { isValidStackName } from '../utils/validation';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { MAX_ASSIGNMENTS } from '../helpers/constants';
|
||||
import { runLocalLabelStop, type LabelLocalStopResponse } from '../helpers/fleetLabelStop';
|
||||
import { runLocalLabelAssign, validateLabelTemplate, type LabelLocalAssignResponse } from '../helpers/fleetLabelAssign';
|
||||
|
||||
// Per-node fleet-action endpoints. Mounted under `/api/fleet-actions/`, which
|
||||
// is NOT in `PROXY_EXEMPT_PREFIXES`, so when `x-node-id` targets a remote node
|
||||
@@ -14,61 +14,6 @@ import { runLocalLabelStop, type LabelLocalStopResponse } from '../helpers/fleet
|
||||
// because their path must sit behind the `/api/fleet/` proxy-exempt prefix.
|
||||
export const fleetActionsRouter = Router();
|
||||
|
||||
// Hard cap to bound a single bulk-assign request. A node typically has tens of
|
||||
// stacks, not thousands; the cap protects against accidental or malicious
|
||||
// payloads that would force thousands of DB writes in one handler.
|
||||
const MAX_ASSIGNMENTS = 1000;
|
||||
|
||||
// Bulk label assignment for many stacks on a single node. The single-stack
|
||||
// endpoint at `PUT /api/stacks/:stackName/labels` covers one stack at a time;
|
||||
// this wrapper applies the same operation to many stacks atomically per HTTP
|
||||
// request. Tier: requireAdmin (admin-only fleet plumbing). The per-stack
|
||||
// endpoint is Community-tier organization metadata; this multi-stack wrapper
|
||||
// matches the surrounding Fleet Actions surface, which is admin-only but
|
||||
// available on every license.
|
||||
fleetActionsRouter.post(
|
||||
'/labels/bulk-assign',
|
||||
authMiddleware,
|
||||
async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireBody(req, res)) return;
|
||||
const { assignments } = req.body as { assignments?: unknown };
|
||||
if (!Array.isArray(assignments)) {
|
||||
res.status(400).json({ error: 'assignments must be an array' });
|
||||
return;
|
||||
}
|
||||
if (assignments.length > MAX_ASSIGNMENTS) {
|
||||
res.status(400).json({ error: `assignments may not exceed ${MAX_ASSIGNMENTS} entries` });
|
||||
return;
|
||||
}
|
||||
const nodeId = req.nodeId ?? 0;
|
||||
const db = DatabaseService.getInstance();
|
||||
const results: { stackName: string; success: boolean; error?: string }[] = [];
|
||||
for (const entry of assignments as unknown[]) {
|
||||
if (!entry || typeof entry !== 'object') {
|
||||
results.push({ stackName: '', success: false, error: 'Invalid assignment entry' });
|
||||
continue;
|
||||
}
|
||||
const { stackName, labelIds } = entry as { stackName?: unknown; labelIds?: unknown };
|
||||
if (typeof stackName !== 'string' || !isValidStackName(stackName)) {
|
||||
results.push({ stackName: typeof stackName === 'string' ? stackName : '', success: false, error: 'Invalid stack name' });
|
||||
continue;
|
||||
}
|
||||
if (!Array.isArray(labelIds) || !labelIds.every(id => typeof id === 'number')) {
|
||||
results.push({ stackName, success: false, error: 'labelIds must be an array of numbers' });
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
db.setStackLabels(stackName, nodeId, labelIds);
|
||||
results.push({ stackName, success: true });
|
||||
} catch (err) {
|
||||
results.push({ stackName, success: false, error: getErrorMessage(err, 'Failed to set stack labels') });
|
||||
}
|
||||
}
|
||||
res.json({ results });
|
||||
},
|
||||
);
|
||||
|
||||
// Per-node label-matched stop. A control instance calls this on each remote
|
||||
// node during a fleet-wide stop-by-label so the destructive work runs under the
|
||||
// remote's own admin auth and per-node bulk lock. Admin-only and available on
|
||||
@@ -100,3 +45,43 @@ fleetActionsRouter.post(
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Per-node label assign. A control instance calls this on each target node
|
||||
// during a fleet-wide bulk label assign so the label is resolved or created
|
||||
// under the node's own database, by name, and assigned to the given stacks while
|
||||
// preserving their existing labels (add semantics). Admin-only and available on
|
||||
// every license, matching the rest of the Fleet Actions surface. Labels are
|
||||
// node-local, so the control never reuses a local label id on a remote: the
|
||||
// receiver owns label resolution for its own node.
|
||||
fleetActionsRouter.post(
|
||||
'/labels/local-assign',
|
||||
authMiddleware,
|
||||
async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireBody(req, res)) return;
|
||||
const { label, stackNames } = req.body as { label?: unknown; stackNames?: unknown };
|
||||
const validated = validateLabelTemplate(label);
|
||||
if (!validated.ok) {
|
||||
res.status(400).json({ error: validated.error });
|
||||
return;
|
||||
}
|
||||
if (!Array.isArray(stackNames) || !stackNames.every(s => typeof s === 'string')) {
|
||||
res.status(400).json({ error: 'stackNames must be an array of strings' });
|
||||
return;
|
||||
}
|
||||
if (stackNames.length > MAX_ASSIGNMENTS) {
|
||||
res.status(400).json({ error: `stackNames may not exceed ${MAX_ASSIGNMENTS} entries` });
|
||||
return;
|
||||
}
|
||||
const nodeId = req.nodeId ?? 0;
|
||||
try {
|
||||
const outcome = await runLocalLabelAssign(nodeId, validated.template, stackNames as string[]);
|
||||
if (isDebugEnabled()) console.debug('[FleetActions:debug] local-assign:', { nodeId, label: validated.template.name, created: outcome.created, stacks: outcome.stackResults.length });
|
||||
const body: LabelLocalAssignResponse = { created: outcome.created, results: outcome.stackResults };
|
||||
res.json(body);
|
||||
} catch (err) {
|
||||
console.error('[FleetActions] local-assign error:', { nodeId, label: validated.template.name }, err);
|
||||
res.status(500).json({ error: getErrorMessage(err, 'Failed to run local label assign') });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user