feat(stacks): server-side POST /api/stacks/bulk endpoint (#1185)

* feat(stacks): server-side POST /api/stacks/bulk

The frontend's bulk action UI fanned out N parallel POSTs to
/api/stacks/:name/{start,stop,restart,update}. For 30 stacks on a
remote node that was 30 round-trips through the proxy + auth + audit
chain, with no shared mutex and partial-failure UX bolted on the
client.

The new endpoint accepts {action, stackNames} (max 100 names), runs
ops under bounded parallelism (4 concurrent), reuses the per-(nodeId,
stackName) lock from the lifecycle-mutex change so collisions report
stack_op_in_progress as a per-row outcome, and returns
{action, results: [{stackName, ok, error?, code?}, ...]} with a 200
envelope. Per-stack errors are rows, not response codes.

Update action keeps the policy-enforcement check (per-stack, returns
policy_blocked rows) and the post-deploy scan trigger so the
single-stack security contract is preserved. State-invalidate and
image-update notifications fire per successful row so the activity
timeline and image-updates UI reflect bulk operations the same way
as single-stack ones.

Frontend useBulkStackActions swaps the Promise.allSettled fan-out for
a single call; the per-stack toast aggregation moves to reading the
results array. isPaid pre-flight stays in place to avoid a round trip
for Community-tier users on update.

* chore(stacks): dedupe bulk inputs; document tier asymmetry; regression test

Three follow-ups from independent review:

- Dedupe stackNames before scheduling so a payload like ['web','web']
  produces one row, not one ok-row plus one stack_op_in_progress row
  whose presence depended on worker scheduling.
- Add a route-ordering regression test verifying that a stack literally
  named 'bulk' is still reachable via /api/stacks/bulk/restart. Express
  matches the literal /bulk before /:stackName paths only at the
  no-suffix level; the :stackName/restart route still catches it.
- Comment the deliberate tier asymmetry: bulk update is requirePaid;
  single-stack /:stackName/update is open to all tiers. The fan-out
  blast radius is the reason, and it matches the prior frontend gate.

Existing 'policy_blocked per-row' test now uses the real ScanPolicy /
PolicyViolation / PolicyEnforcementResult shapes (the first cut elided
fields tsc strict-checked).
This commit is contained in:
Anso
2026-05-24 15:56:53 -04:00
committed by GitHub
parent 27b8954676
commit 5aedc52737
3 changed files with 552 additions and 24 deletions
@@ -0,0 +1,348 @@
/**
* Integration tests for POST /api/stacks/bulk.
*
* The bulk endpoint replaces the frontend's per-stack fan-out for
* start/stop/restart/update with a single server-side request that runs the
* lifecycle ops under bounded parallelism and returns a per-stack outcome
* map. The endpoint reuses the per-(nodeId, stackName) lock from H-1 so a
* stack that is already busy reports `code: stack_op_in_progress` for that
* row instead of doubling the op.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import request from 'supertest';
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
const {
mockUpdateStack,
mockGetContainersByStack,
mockRestartContainer,
mockStopContainer,
mockStartContainer,
} = vi.hoisted(() => ({
mockUpdateStack: vi.fn(),
mockGetContainersByStack: vi.fn(),
mockRestartContainer: vi.fn(),
mockStopContainer: vi.fn(),
mockStartContainer: vi.fn(),
}));
vi.mock('../services/ComposeService', async () => {
const actual = await vi.importActual<typeof import('../services/ComposeService')>(
'../services/ComposeService',
);
return {
...actual,
ComposeService: {
...actual.ComposeService,
getInstance: () => ({
updateStack: mockUpdateStack,
}),
},
};
});
vi.mock('../services/DockerController', async () => {
const actual = await vi.importActual<typeof import('../services/DockerController')>(
'../services/DockerController',
);
return {
...actual,
default: {
...actual.default,
getInstance: () => ({
getContainersByStack: mockGetContainersByStack,
restartContainer: mockRestartContainer,
stopContainer: mockStopContainer,
startContainer: mockStartContainer,
}),
},
};
});
vi.mock('../services/FileSystemService', () => ({
FileSystemService: {
getInstance: () => ({
getBaseDir: () => '/tmp/compose',
hasComposeFile: vi.fn().mockResolvedValue(true),
}),
},
}));
let tmpDir: string;
let app: import('express').Express;
let authCookie: string;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ app } = await import('../index'));
authCookie = await loginAsTestAdmin(app);
const { NotificationService } = await import('../services/NotificationService');
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
});
afterAll(() => {
vi.restoreAllMocks();
cleanupTestDb(tmpDir);
});
beforeEach(async () => {
mockUpdateStack.mockReset();
mockGetContainersByStack.mockReset();
mockRestartContainer.mockReset();
mockStopContainer.mockReset();
mockStartContainer.mockReset();
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1' }]);
const { StackOpLockService } = await import('../services/StackOpLockService');
StackOpLockService.resetForTests();
});
describe('POST /api/stacks/bulk request validation', () => {
it('rejects an unknown action with 400', async () => {
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'reboot', stackNames: ['web'] });
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/Invalid action/);
});
it('rejects an empty stackNames array', async () => {
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'restart', stackNames: [] });
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/non-empty array/);
});
it('rejects more than 100 stacks per request', async () => {
const stackNames = Array.from({ length: 101 }, (_, i) => `stack-${i}`);
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'restart', stackNames });
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/limited to 100/);
});
it('rejects non-string stackNames entries', async () => {
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'restart', stackNames: ['web', 42, 'api'] });
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/array of strings/);
});
});
describe('POST /api/stacks/bulk execution', () => {
it('restarts three stacks and returns ok results for each', async () => {
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'restart', stackNames: ['web', 'api', 'db'] });
expect(res.status).toBe(200);
expect(res.body.action).toBe('restart');
expect(res.body.results).toHaveLength(3);
expect(res.body.results.every((r: { ok: boolean }) => r.ok)).toBe(true);
expect(mockRestartContainer).toHaveBeenCalledTimes(3);
});
it('reports per-stack failures without short-circuiting other stacks', async () => {
let callCount = 0;
mockRestartContainer.mockImplementation(() => {
callCount += 1;
if (callCount === 2) return Promise.reject(new Error('container crashed'));
return Promise.resolve();
});
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'restart', stackNames: ['web', 'api', 'db'] });
expect(res.status).toBe(200);
expect(res.body.results).toHaveLength(3);
const okResults = res.body.results.filter((r: { ok: boolean }) => r.ok);
const failedResults = res.body.results.filter((r: { ok: boolean }) => !r.ok);
expect(okResults).toHaveLength(2);
expect(failedResults).toHaveLength(1);
expect(failedResults[0].code).toBe('op_failed');
expect(failedResults[0].error).toMatch(/container crashed/);
});
it('reports stack_op_in_progress when a per-stack lock is already held', async () => {
const { StackOpLockService } = await import('../services/StackOpLockService');
StackOpLockService.getInstance().tryAcquire(1, 'busy-stack', 'deploy', 'someone-else');
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'restart', stackNames: ['busy-stack', 'free-stack'] });
expect(res.status).toBe(200);
const busy = res.body.results.find((r: { stackName: string }) => r.stackName === 'busy-stack');
const free = res.body.results.find((r: { stackName: string }) => r.stackName === 'free-stack');
expect(busy.ok).toBe(false);
expect(busy.code).toBe('stack_op_in_progress');
expect(busy.error).toMatch(/already deploying/i);
expect(free.ok).toBe(true);
});
it('rejects invalid stack names with code=invalid_name per-row', async () => {
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'restart', stackNames: ['..bad..', 'web'] });
expect(res.status).toBe(200);
const bad = res.body.results.find((r: { stackName: string }) => r.stackName === '..bad..');
expect(bad.ok).toBe(false);
expect(bad.code).toBe('invalid_name');
});
it('returns no_containers when a stack has nothing to restart', async () => {
mockGetContainersByStack.mockResolvedValueOnce([]);
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'restart', stackNames: ['empty-stack'] });
expect(res.status).toBe(200);
expect(res.body.results[0].ok).toBe(false);
expect(res.body.results[0].code).toBe('no_containers');
});
it('releases each stack lock after the op finishes', async () => {
await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'restart', stackNames: ['web', 'api'] });
const { StackOpLockService } = await import('../services/StackOpLockService');
expect(StackOpLockService.getInstance().size()).toBe(0);
});
it('runs at most BULK_PARALLELISM ops concurrently', async () => {
let inFlight = 0;
let maxInFlight = 0;
mockRestartContainer.mockImplementation(async () => {
inFlight += 1;
maxInFlight = Math.max(maxInFlight, inFlight);
await new Promise(r => setTimeout(r, 20));
inFlight -= 1;
});
const stackNames = Array.from({ length: 10 }, (_, i) => `stack-${i}`);
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'restart', stackNames });
expect(res.status).toBe(200);
expect(res.body.results).toHaveLength(10);
expect(maxInFlight).toBeLessThanOrEqual(4);
expect(maxInFlight).toBeGreaterThan(1);
});
it('handles stop action by dispatching stopContainer per stack', async () => {
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'stop', stackNames: ['web'] });
expect(res.status).toBe(200);
expect(res.body.results[0].ok).toBe(true);
expect(mockStopContainer).toHaveBeenCalledTimes(1);
expect(mockRestartContainer).not.toHaveBeenCalled();
});
it('handles update action (paid tier) by calling ComposeService.updateStack', async () => {
mockUpdateStack.mockResolvedValue(undefined);
const { LicenseService } = await import('../services/LicenseService');
const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
try {
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'update', stackNames: ['web'] });
expect(res.status).toBe(200);
expect(res.body.results[0].ok).toBe(true);
expect(mockUpdateStack).toHaveBeenCalledTimes(1);
} finally {
tierSpy.mockRestore();
}
});
it('returns policy_blocked per-row when the policy gate rejects an update', async () => {
const { LicenseService } = await import('../services/LicenseService');
const policyMod = await import('../services/PolicyEnforcement');
const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
const policySpy = vi.spyOn(policyMod, 'enforcePolicyPreDeploy').mockResolvedValue({
ok: false,
bypassed: false,
policy: { id: 1, name: 'block-criticals', node_id: null, node_identity: '', stack_pattern: null, max_severity: 'HIGH', block_on_deploy: 1, enabled: 1, replicated_from_control: 0, created_at: Date.now(), updated_at: Date.now() },
violations: [{ imageRef: 'nginx:latest', severity: 'CRITICAL', criticalCount: 3, highCount: 0, scanId: 1 }],
});
mockUpdateStack.mockResolvedValue(undefined);
try {
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'update', stackNames: ['web', 'api'] });
expect(res.status).toBe(200);
expect(res.body.results).toHaveLength(2);
expect(res.body.results.every((r: { ok: boolean }) => !r.ok)).toBe(true);
expect(res.body.results.every((r: { code: string }) => r.code === 'policy_blocked')).toBe(true);
expect(mockUpdateStack).not.toHaveBeenCalled();
} finally {
tierSpy.mockRestore();
policySpy.mockRestore();
}
});
it('dedupes repeated stackNames before scheduling work', async () => {
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'restart', stackNames: ['web', 'web', 'web'] });
expect(res.status).toBe(200);
expect(res.body.results).toHaveLength(1);
expect(res.body.results[0].ok).toBe(true);
expect(mockRestartContainer).toHaveBeenCalledTimes(1);
});
it('does not shadow POST /:stackName/restart for a stack literally named bulk', async () => {
// The bulk endpoint is mounted at /api/stacks/bulk (no trailing path).
// A stack named "bulk" must still be reachable at /api/stacks/bulk/restart
// because Express matches /:stackName/restart there, not the bulk handler.
const res = await request(app)
.post('/api/stacks/bulk/restart')
.set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(mockRestartContainer).toHaveBeenCalledTimes(1);
expect(res.body.success).toBe(true);
});
it('returns 403 on update action when caller is not on a paid tier', async () => {
const { LicenseService } = await import('../services/LicenseService');
const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
try {
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
.send({ action: 'update', stackNames: ['web'] });
expect(res.status).toBe(403);
expect(mockUpdateStack).not.toHaveBeenCalled();
} finally {
tierSpy.mockRestore();
}
});
});
+158 -1
View File
@@ -11,7 +11,7 @@ import { CacheService } from '../services/CacheService';
import { UpdatePreviewService } from '../services/UpdatePreviewService';
import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService';
import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement';
import { requirePermission } from '../middleware/permissions';
import { requirePermission, checkPermission } from '../middleware/permissions';
import { requirePaid, requireAdmin, effectiveTier } from '../middleware/tierGates';
import { NotificationService, type NotificationCategory } from '../services/NotificationService';
import { StackOpLockService, type StackOpAction } from '../services/StackOpLockService';
@@ -252,6 +252,163 @@ stacksRouter.get('/auto-update-settings', (req: Request, res: Response): void =>
}
});
type BulkLifecycleAction = 'start' | 'stop' | 'restart' | 'update';
const VALID_BULK_ACTIONS: ReadonlySet<BulkLifecycleAction> = new Set(['start', 'stop', 'restart', 'update']);
const BULK_PARALLELISM = 4;
const BULK_MAX_STACKS = 100;
interface BulkResultItem {
stackName: string;
ok: boolean;
error?: string;
code?: string;
}
async function runStackBulkOp(
req: Request,
stackName: string,
action: BulkLifecycleAction,
): Promise<BulkResultItem> {
if (!isValidStackName(stackName)) {
return { stackName, ok: false, error: 'Invalid stack name', code: 'invalid_name' };
}
if (!checkPermission(req, 'stack:deploy', 'stack', stackName)) {
return { stackName, ok: false, error: 'Permission denied', code: 'PERMISSION_DENIED' };
}
const fsSvc = FileSystemService.getInstance(req.nodeId);
const stackDir = path.join(fsSvc.getBaseDir(), stackName);
try {
if (!(await fsSvc.hasComposeFile(stackDir))) {
return { stackName, ok: false, error: 'Stack not found', code: 'not_found' };
}
} catch {
return { stackName, ok: false, error: 'Stack not found', code: 'not_found' };
}
const user = req.user?.username ?? 'system';
const lockAction: StackOpAction = action;
const lockResult = StackOpLockService.getInstance().tryAcquire(req.nodeId, stackName, lockAction, user);
if (!lockResult.acquired) {
return {
stackName,
ok: false,
error: `${stackName} is already ${STACK_OP_PRESENT_PARTICIPLE[lockResult.existing.action]}`,
code: 'stack_op_in_progress',
};
}
try {
if (action === 'update') {
const gate = await enforcePolicyPreDeploy(stackName, req.nodeId, buildPolicyGateOptions(req));
if (!gate.ok) {
return {
stackName,
ok: false,
error: `Policy "${gate.policy?.name}" blocked update`,
code: 'policy_blocked',
};
}
const atomic = effectiveTier(req) === 'paid';
await ComposeService.getInstance(req.nodeId).updateStack(stackName, getTerminalWs(), atomic);
DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
NotificationService.getInstance().broadcastEvent({
type: 'state-invalidate',
scope: 'image-updates',
nodeId: req.nodeId,
stackName,
action: 'stack-updated',
ts: Date.now(),
});
notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, user);
triggerPostDeployScan(stackName, req.nodeId).catch(err =>
console.error('[Security] Post-deploy scan failed for %s:', sanitizeForLog(stackName), err),
);
} else {
const outcome = await containerActionForStack(req.nodeId, stackName, action);
if (outcome.kind === 'no-containers') {
return { stackName, ok: false, error: 'No containers found for this stack', code: 'no_containers' };
}
if (outcome.kind === 'error') {
if (action !== 'start') notifyActionFailure(action, stackName, new Error(outcome.message));
return { stackName, ok: false, error: outcome.message, code: 'op_failed' };
}
const meta = CONTAINER_ACTION_META[action];
notifyActionSuccess(meta.category, `${stackName} ${meta.pastTense}`, stackName, user);
}
return { stackName, ok: true };
} catch (err) {
if (action !== 'start') notifyActionFailure(action, stackName, err);
return { stackName, ok: false, error: getErrorMessage(err, `${action} failed`), code: 'op_failed' };
} finally {
StackOpLockService.getInstance().release(req.nodeId, stackName);
}
}
async function runWithBoundedParallelism<T, R>(
items: T[],
limit: number,
task: (item: T) => Promise<R>,
): Promise<R[]> {
const results: R[] = new Array(items.length);
let next = 0;
const worker = async (): Promise<void> => {
while (next < items.length) {
const idx = next;
next += 1;
results[idx] = await task(items[idx]);
}
};
const workerCount = Math.min(limit, items.length);
await Promise.all(Array.from({ length: workerCount }, () => worker()));
return results;
}
stacksRouter.post('/bulk', async (req: Request, res: Response) => {
const body = req.body as { action?: unknown; stackNames?: unknown } | undefined;
const action = body?.action;
const stackNames = body?.stackNames;
if (typeof action !== 'string' || !VALID_BULK_ACTIONS.has(action as BulkLifecycleAction)) {
return res.status(400).json({
error: `Invalid action. Must be one of: ${[...VALID_BULK_ACTIONS].join(', ')}`,
});
}
if (!Array.isArray(stackNames) || stackNames.length === 0) {
return res.status(400).json({ error: 'stackNames must be a non-empty array' });
}
if (stackNames.length > BULK_MAX_STACKS) {
return res.status(400).json({ error: `Bulk operations are limited to ${BULK_MAX_STACKS} stacks per request` });
}
if (!stackNames.every(s => typeof s === 'string')) {
return res.status(400).json({ error: 'stackNames must be an array of strings' });
}
// Bulk update is paid-only by deliberate asymmetry with the single-stack
// POST /:stackName/update, which is open to all tiers (atomic backup is
// separately gated by effectiveTier inside the route). The bulk fan-out
// amplifies blast radius enough that we want a hard tier check here even
// though the per-stack route does not.
if (action === 'update' && !requirePaid(req, res)) return;
const typedAction = action as BulkLifecycleAction;
const typedNames = Array.from(new Set(stackNames as string[]));
const results = await runWithBoundedParallelism(
typedNames,
BULK_PARALLELISM,
name => runStackBulkOp(req, name, typedAction),
);
invalidateNodeCaches(req.nodeId);
const okCount = results.filter(r => r.ok).length;
console.log(
`[Stacks] Bulk ${sanitizeForLog(action)} completed: ${okCount}/${results.length} on node ${req.nodeId}`,
);
res.json({ action: typedAction, results });
});
stacksRouter.get('/:stackName/auto-update', (req: Request, res: Response): void => {
try {
const stackName = req.params.stackName as string;
+46 -23
View File
@@ -17,6 +17,18 @@ interface BulkCallbacks {
onAfter?: (files: string[]) => void;
}
interface BulkResultItem {
stackName: string;
ok: boolean;
error?: string;
code?: string;
}
interface BulkResponse {
action: BulkAction;
results: BulkResultItem[];
}
export function useBulkStackActions() {
const { isPaid } = useLicense();
@@ -33,32 +45,43 @@ export function useBulkStackActions() {
cbs?.onBefore?.(files);
const results = await Promise.allSettled(
files.map(file => {
const stackName = file.replace(/\.(yml|yaml)$/, '');
const headers: Record<string, string> = action === 'update' ? { 'x-bulk-mode': '1' } : {};
return apiFetch(`/stacks/${encodeURIComponent(stackName)}/${action}`, {
method: 'POST',
headers,
}).then(res => {
if (!res.ok) return Promise.reject(new Error(file));
return file;
});
})
);
const stackNames = files.map(file => file.replace(/\.(yml|yaml)$/, ''));
cbs?.onAfter?.(files);
try {
const response = await apiFetch('/stacks/bulk', {
method: 'POST',
body: JSON.stringify({ action, stackNames }),
});
const failed = results
.filter((r): r is PromiseRejectedResult => r.status === 'rejected')
.map(r => (r.reason as Error).message);
const okCount = results.length - failed.length;
cbs?.onAfter?.(files);
if (failed.length === 0) {
const noun = okCount === 1 ? 'stack' : 'stacks';
toast.success(`${okCount} ${noun} ${pastTense[action]}`);
} else {
toast.error(`${okCount} of ${files.length} ${pastTense[action]}; ${failed.length} failed: ${failed.join(', ')}`);
if (!response.ok) {
const errBody = await response.json().catch(() => ({}));
const errMsg = (errBody as { error?: string })?.error
?? `Bulk ${action} failed (HTTP ${response.status})`;
toast.error(errMsg);
return;
}
const payload = (await response.json()) as BulkResponse;
const results = Array.isArray(payload.results) ? payload.results : [];
const okCount = results.filter(r => r.ok).length;
const failed = results.filter(r => !r.ok);
if (failed.length === 0) {
const noun = okCount === 1 ? 'stack' : 'stacks';
toast.success(`${okCount} ${noun} ${pastTense[action]}`);
return;
}
const failedNames = failed.map(r => r.stackName).join(', ');
toast.error(
`${okCount} of ${results.length} ${pastTense[action]}; ${failed.length} failed: ${failedNames}`,
);
} catch (err) {
cbs?.onAfter?.(files);
console.error('Bulk action failed:', err);
toast.error(`Bulk ${action} failed: ${(err as Error).message}`);
}
}, [isPaid]);