From 5aedc52737c155be735b663809df05d213ef7d35 Mon Sep 17 00:00:00 2001 From: Anso Date: Sun, 24 May 2026 15:56:53 -0400 Subject: [PATCH] feat(stacks): server-side POST /api/stacks/bulk endpoint (#1185) * feat(stacks): server-side POST /api/stacks/bulk The frontend's bulk action UI fanned out N parallel POSTs to /api/stacks/:name/{start,stop,restart,update}. For 30 stacks on a remote node that was 30 round-trips through the proxy + auth + audit chain, with no shared mutex and partial-failure UX bolted on the client. The new endpoint accepts {action, stackNames} (max 100 names), runs ops under bounded parallelism (4 concurrent), reuses the per-(nodeId, stackName) lock from the lifecycle-mutex change so collisions report stack_op_in_progress as a per-row outcome, and returns {action, results: [{stackName, ok, error?, code?}, ...]} with a 200 envelope. Per-stack errors are rows, not response codes. Update action keeps the policy-enforcement check (per-stack, returns policy_blocked rows) and the post-deploy scan trigger so the single-stack security contract is preserved. State-invalidate and image-update notifications fire per successful row so the activity timeline and image-updates UI reflect bulk operations the same way as single-stack ones. Frontend useBulkStackActions swaps the Promise.allSettled fan-out for a single call; the per-stack toast aggregation moves to reading the results array. isPaid pre-flight stays in place to avoid a round trip for Community-tier users on update. * chore(stacks): dedupe bulk inputs; document tier asymmetry; regression test Three follow-ups from independent review: - Dedupe stackNames before scheduling so a payload like ['web','web'] produces one row, not one ok-row plus one stack_op_in_progress row whose presence depended on worker scheduling. - Add a route-ordering regression test verifying that a stack literally named 'bulk' is still reachable via /api/stacks/bulk/restart. Express matches the literal /bulk before /:stackName paths only at the no-suffix level; the :stackName/restart route still catches it. - Comment the deliberate tier asymmetry: bulk update is requirePaid; single-stack /:stackName/update is open to all tiers. The fan-out blast radius is the reason, and it matches the prior frontend gate. Existing 'policy_blocked per-row' test now uses the real ScanPolicy / PolicyViolation / PolicyEnforcementResult shapes (the first cut elided fields tsc strict-checked). --- .../src/__tests__/stack-bulk-routes.test.ts | 348 ++++++++++++++++++ backend/src/routes/stacks.ts | 159 +++++++- frontend/src/hooks/useBulkStackActions.ts | 69 ++-- 3 files changed, 552 insertions(+), 24 deletions(-) create mode 100644 backend/src/__tests__/stack-bulk-routes.test.ts diff --git a/backend/src/__tests__/stack-bulk-routes.test.ts b/backend/src/__tests__/stack-bulk-routes.test.ts new file mode 100644 index 00000000..19203014 --- /dev/null +++ b/backend/src/__tests__/stack-bulk-routes.test.ts @@ -0,0 +1,348 @@ +/** + * Integration tests for POST /api/stacks/bulk. + * + * The bulk endpoint replaces the frontend's per-stack fan-out for + * start/stop/restart/update with a single server-side request that runs the + * lifecycle ops under bounded parallelism and returns a per-stack outcome + * map. The endpoint reuses the per-(nodeId, stackName) lock from H-1 so a + * stack that is already busy reports `code: stack_op_in_progress` for that + * row instead of doubling the op. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; + +const { + mockUpdateStack, + mockGetContainersByStack, + mockRestartContainer, + mockStopContainer, + mockStartContainer, +} = vi.hoisted(() => ({ + mockUpdateStack: vi.fn(), + mockGetContainersByStack: vi.fn(), + mockRestartContainer: vi.fn(), + mockStopContainer: vi.fn(), + mockStartContainer: vi.fn(), +})); + +vi.mock('../services/ComposeService', async () => { + const actual = await vi.importActual( + '../services/ComposeService', + ); + return { + ...actual, + ComposeService: { + ...actual.ComposeService, + getInstance: () => ({ + updateStack: mockUpdateStack, + }), + }, + }; +}); + +vi.mock('../services/DockerController', async () => { + const actual = await vi.importActual( + '../services/DockerController', + ); + return { + ...actual, + default: { + ...actual.default, + getInstance: () => ({ + getContainersByStack: mockGetContainersByStack, + restartContainer: mockRestartContainer, + stopContainer: mockStopContainer, + startContainer: mockStartContainer, + }), + }, + }; +}); + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + getBaseDir: () => '/tmp/compose', + hasComposeFile: vi.fn().mockResolvedValue(true), + }), + }, +})); + +let tmpDir: string; +let app: import('express').Express; +let authCookie: string; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + authCookie = await loginAsTestAdmin(app); + + const { NotificationService } = await import('../services/NotificationService'); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); +}); + +afterAll(() => { + vi.restoreAllMocks(); + cleanupTestDb(tmpDir); +}); + +beforeEach(async () => { + mockUpdateStack.mockReset(); + mockGetContainersByStack.mockReset(); + mockRestartContainer.mockReset(); + mockStopContainer.mockReset(); + mockStartContainer.mockReset(); + mockGetContainersByStack.mockResolvedValue([{ Id: 'c1' }]); + const { StackOpLockService } = await import('../services/StackOpLockService'); + StackOpLockService.resetForTests(); +}); + +describe('POST /api/stacks/bulk request validation', () => { + it('rejects an unknown action with 400', async () => { + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'reboot', stackNames: ['web'] }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/Invalid action/); + }); + + it('rejects an empty stackNames array', async () => { + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'restart', stackNames: [] }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/non-empty array/); + }); + + it('rejects more than 100 stacks per request', async () => { + const stackNames = Array.from({ length: 101 }, (_, i) => `stack-${i}`); + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'restart', stackNames }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/limited to 100/); + }); + + it('rejects non-string stackNames entries', async () => { + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'restart', stackNames: ['web', 42, 'api'] }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/array of strings/); + }); +}); + +describe('POST /api/stacks/bulk execution', () => { + it('restarts three stacks and returns ok results for each', async () => { + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'restart', stackNames: ['web', 'api', 'db'] }); + + expect(res.status).toBe(200); + expect(res.body.action).toBe('restart'); + expect(res.body.results).toHaveLength(3); + expect(res.body.results.every((r: { ok: boolean }) => r.ok)).toBe(true); + expect(mockRestartContainer).toHaveBeenCalledTimes(3); + }); + + it('reports per-stack failures without short-circuiting other stacks', async () => { + let callCount = 0; + mockRestartContainer.mockImplementation(() => { + callCount += 1; + if (callCount === 2) return Promise.reject(new Error('container crashed')); + return Promise.resolve(); + }); + + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'restart', stackNames: ['web', 'api', 'db'] }); + + expect(res.status).toBe(200); + expect(res.body.results).toHaveLength(3); + const okResults = res.body.results.filter((r: { ok: boolean }) => r.ok); + const failedResults = res.body.results.filter((r: { ok: boolean }) => !r.ok); + expect(okResults).toHaveLength(2); + expect(failedResults).toHaveLength(1); + expect(failedResults[0].code).toBe('op_failed'); + expect(failedResults[0].error).toMatch(/container crashed/); + }); + + it('reports stack_op_in_progress when a per-stack lock is already held', async () => { + const { StackOpLockService } = await import('../services/StackOpLockService'); + StackOpLockService.getInstance().tryAcquire(1, 'busy-stack', 'deploy', 'someone-else'); + + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'restart', stackNames: ['busy-stack', 'free-stack'] }); + + expect(res.status).toBe(200); + const busy = res.body.results.find((r: { stackName: string }) => r.stackName === 'busy-stack'); + const free = res.body.results.find((r: { stackName: string }) => r.stackName === 'free-stack'); + expect(busy.ok).toBe(false); + expect(busy.code).toBe('stack_op_in_progress'); + expect(busy.error).toMatch(/already deploying/i); + expect(free.ok).toBe(true); + }); + + it('rejects invalid stack names with code=invalid_name per-row', async () => { + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'restart', stackNames: ['..bad..', 'web'] }); + + expect(res.status).toBe(200); + const bad = res.body.results.find((r: { stackName: string }) => r.stackName === '..bad..'); + expect(bad.ok).toBe(false); + expect(bad.code).toBe('invalid_name'); + }); + + it('returns no_containers when a stack has nothing to restart', async () => { + mockGetContainersByStack.mockResolvedValueOnce([]); + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'restart', stackNames: ['empty-stack'] }); + + expect(res.status).toBe(200); + expect(res.body.results[0].ok).toBe(false); + expect(res.body.results[0].code).toBe('no_containers'); + }); + + it('releases each stack lock after the op finishes', async () => { + await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'restart', stackNames: ['web', 'api'] }); + + const { StackOpLockService } = await import('../services/StackOpLockService'); + expect(StackOpLockService.getInstance().size()).toBe(0); + }); + + it('runs at most BULK_PARALLELISM ops concurrently', async () => { + let inFlight = 0; + let maxInFlight = 0; + mockRestartContainer.mockImplementation(async () => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise(r => setTimeout(r, 20)); + inFlight -= 1; + }); + + const stackNames = Array.from({ length: 10 }, (_, i) => `stack-${i}`); + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'restart', stackNames }); + + expect(res.status).toBe(200); + expect(res.body.results).toHaveLength(10); + expect(maxInFlight).toBeLessThanOrEqual(4); + expect(maxInFlight).toBeGreaterThan(1); + }); + + it('handles stop action by dispatching stopContainer per stack', async () => { + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'stop', stackNames: ['web'] }); + + expect(res.status).toBe(200); + expect(res.body.results[0].ok).toBe(true); + expect(mockStopContainer).toHaveBeenCalledTimes(1); + expect(mockRestartContainer).not.toHaveBeenCalled(); + }); + + it('handles update action (paid tier) by calling ComposeService.updateStack', async () => { + mockUpdateStack.mockResolvedValue(undefined); + const { LicenseService } = await import('../services/LicenseService'); + const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + try { + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'update', stackNames: ['web'] }); + + expect(res.status).toBe(200); + expect(res.body.results[0].ok).toBe(true); + expect(mockUpdateStack).toHaveBeenCalledTimes(1); + } finally { + tierSpy.mockRestore(); + } + }); + + it('returns policy_blocked per-row when the policy gate rejects an update', async () => { + const { LicenseService } = await import('../services/LicenseService'); + const policyMod = await import('../services/PolicyEnforcement'); + const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + const policySpy = vi.spyOn(policyMod, 'enforcePolicyPreDeploy').mockResolvedValue({ + ok: false, + bypassed: false, + policy: { id: 1, name: 'block-criticals', node_id: null, node_identity: '', stack_pattern: null, max_severity: 'HIGH', block_on_deploy: 1, enabled: 1, replicated_from_control: 0, created_at: Date.now(), updated_at: Date.now() }, + violations: [{ imageRef: 'nginx:latest', severity: 'CRITICAL', criticalCount: 3, highCount: 0, scanId: 1 }], + }); + mockUpdateStack.mockResolvedValue(undefined); + try { + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'update', stackNames: ['web', 'api'] }); + + expect(res.status).toBe(200); + expect(res.body.results).toHaveLength(2); + expect(res.body.results.every((r: { ok: boolean }) => !r.ok)).toBe(true); + expect(res.body.results.every((r: { code: string }) => r.code === 'policy_blocked')).toBe(true); + expect(mockUpdateStack).not.toHaveBeenCalled(); + } finally { + tierSpy.mockRestore(); + policySpy.mockRestore(); + } + }); + + it('dedupes repeated stackNames before scheduling work', async () => { + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'restart', stackNames: ['web', 'web', 'web'] }); + + expect(res.status).toBe(200); + expect(res.body.results).toHaveLength(1); + expect(res.body.results[0].ok).toBe(true); + expect(mockRestartContainer).toHaveBeenCalledTimes(1); + }); + + it('does not shadow POST /:stackName/restart for a stack literally named bulk', async () => { + // The bulk endpoint is mounted at /api/stacks/bulk (no trailing path). + // A stack named "bulk" must still be reachable at /api/stacks/bulk/restart + // because Express matches /:stackName/restart there, not the bulk handler. + const res = await request(app) + .post('/api/stacks/bulk/restart') + .set('Cookie', authCookie); + + expect(res.status).toBe(200); + expect(mockRestartContainer).toHaveBeenCalledTimes(1); + expect(res.body.success).toBe(true); + }); + + it('returns 403 on update action when caller is not on a paid tier', async () => { + const { LicenseService } = await import('../services/LicenseService'); + const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + try { + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'update', stackNames: ['web'] }); + + expect(res.status).toBe(403); + expect(mockUpdateStack).not.toHaveBeenCalled(); + } finally { + tierSpy.mockRestore(); + } + }); +}); diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index 865ab15f..7e666cdd 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -11,7 +11,7 @@ import { CacheService } from '../services/CacheService'; import { UpdatePreviewService } from '../services/UpdatePreviewService'; import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService'; import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement'; -import { requirePermission } from '../middleware/permissions'; +import { requirePermission, checkPermission } from '../middleware/permissions'; import { requirePaid, requireAdmin, effectiveTier } from '../middleware/tierGates'; import { NotificationService, type NotificationCategory } from '../services/NotificationService'; import { StackOpLockService, type StackOpAction } from '../services/StackOpLockService'; @@ -252,6 +252,163 @@ stacksRouter.get('/auto-update-settings', (req: Request, res: Response): void => } }); +type BulkLifecycleAction = 'start' | 'stop' | 'restart' | 'update'; +const VALID_BULK_ACTIONS: ReadonlySet = new Set(['start', 'stop', 'restart', 'update']); +const BULK_PARALLELISM = 4; +const BULK_MAX_STACKS = 100; + +interface BulkResultItem { + stackName: string; + ok: boolean; + error?: string; + code?: string; +} + +async function runStackBulkOp( + req: Request, + stackName: string, + action: BulkLifecycleAction, +): Promise { + if (!isValidStackName(stackName)) { + return { stackName, ok: false, error: 'Invalid stack name', code: 'invalid_name' }; + } + if (!checkPermission(req, 'stack:deploy', 'stack', stackName)) { + return { stackName, ok: false, error: 'Permission denied', code: 'PERMISSION_DENIED' }; + } + + const fsSvc = FileSystemService.getInstance(req.nodeId); + const stackDir = path.join(fsSvc.getBaseDir(), stackName); + try { + if (!(await fsSvc.hasComposeFile(stackDir))) { + return { stackName, ok: false, error: 'Stack not found', code: 'not_found' }; + } + } catch { + return { stackName, ok: false, error: 'Stack not found', code: 'not_found' }; + } + + const user = req.user?.username ?? 'system'; + const lockAction: StackOpAction = action; + const lockResult = StackOpLockService.getInstance().tryAcquire(req.nodeId, stackName, lockAction, user); + if (!lockResult.acquired) { + return { + stackName, + ok: false, + error: `${stackName} is already ${STACK_OP_PRESENT_PARTICIPLE[lockResult.existing.action]}`, + code: 'stack_op_in_progress', + }; + } + + try { + if (action === 'update') { + const gate = await enforcePolicyPreDeploy(stackName, req.nodeId, buildPolicyGateOptions(req)); + if (!gate.ok) { + return { + stackName, + ok: false, + error: `Policy "${gate.policy?.name}" blocked update`, + code: 'policy_blocked', + }; + } + const atomic = effectiveTier(req) === 'paid'; + await ComposeService.getInstance(req.nodeId).updateStack(stackName, getTerminalWs(), atomic); + DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName); + NotificationService.getInstance().broadcastEvent({ + type: 'state-invalidate', + scope: 'image-updates', + nodeId: req.nodeId, + stackName, + action: 'stack-updated', + ts: Date.now(), + }); + notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, user); + triggerPostDeployScan(stackName, req.nodeId).catch(err => + console.error('[Security] Post-deploy scan failed for %s:', sanitizeForLog(stackName), err), + ); + } else { + const outcome = await containerActionForStack(req.nodeId, stackName, action); + if (outcome.kind === 'no-containers') { + return { stackName, ok: false, error: 'No containers found for this stack', code: 'no_containers' }; + } + if (outcome.kind === 'error') { + if (action !== 'start') notifyActionFailure(action, stackName, new Error(outcome.message)); + return { stackName, ok: false, error: outcome.message, code: 'op_failed' }; + } + const meta = CONTAINER_ACTION_META[action]; + notifyActionSuccess(meta.category, `${stackName} ${meta.pastTense}`, stackName, user); + } + return { stackName, ok: true }; + } catch (err) { + if (action !== 'start') notifyActionFailure(action, stackName, err); + return { stackName, ok: false, error: getErrorMessage(err, `${action} failed`), code: 'op_failed' }; + } finally { + StackOpLockService.getInstance().release(req.nodeId, stackName); + } +} + +async function runWithBoundedParallelism( + items: T[], + limit: number, + task: (item: T) => Promise, +): Promise { + const results: R[] = new Array(items.length); + let next = 0; + const worker = async (): Promise => { + while (next < items.length) { + const idx = next; + next += 1; + results[idx] = await task(items[idx]); + } + }; + const workerCount = Math.min(limit, items.length); + await Promise.all(Array.from({ length: workerCount }, () => worker())); + return results; +} + +stacksRouter.post('/bulk', async (req: Request, res: Response) => { + const body = req.body as { action?: unknown; stackNames?: unknown } | undefined; + const action = body?.action; + const stackNames = body?.stackNames; + + if (typeof action !== 'string' || !VALID_BULK_ACTIONS.has(action as BulkLifecycleAction)) { + return res.status(400).json({ + error: `Invalid action. Must be one of: ${[...VALID_BULK_ACTIONS].join(', ')}`, + }); + } + if (!Array.isArray(stackNames) || stackNames.length === 0) { + return res.status(400).json({ error: 'stackNames must be a non-empty array' }); + } + if (stackNames.length > BULK_MAX_STACKS) { + return res.status(400).json({ error: `Bulk operations are limited to ${BULK_MAX_STACKS} stacks per request` }); + } + if (!stackNames.every(s => typeof s === 'string')) { + return res.status(400).json({ error: 'stackNames must be an array of strings' }); + } + + // Bulk update is paid-only by deliberate asymmetry with the single-stack + // POST /:stackName/update, which is open to all tiers (atomic backup is + // separately gated by effectiveTier inside the route). The bulk fan-out + // amplifies blast radius enough that we want a hard tier check here even + // though the per-stack route does not. + if (action === 'update' && !requirePaid(req, res)) return; + + const typedAction = action as BulkLifecycleAction; + const typedNames = Array.from(new Set(stackNames as string[])); + + const results = await runWithBoundedParallelism( + typedNames, + BULK_PARALLELISM, + name => runStackBulkOp(req, name, typedAction), + ); + + invalidateNodeCaches(req.nodeId); + const okCount = results.filter(r => r.ok).length; + console.log( + `[Stacks] Bulk ${sanitizeForLog(action)} completed: ${okCount}/${results.length} on node ${req.nodeId}`, + ); + + res.json({ action: typedAction, results }); +}); + stacksRouter.get('/:stackName/auto-update', (req: Request, res: Response): void => { try { const stackName = req.params.stackName as string; diff --git a/frontend/src/hooks/useBulkStackActions.ts b/frontend/src/hooks/useBulkStackActions.ts index e2b0845f..42d11f8e 100644 --- a/frontend/src/hooks/useBulkStackActions.ts +++ b/frontend/src/hooks/useBulkStackActions.ts @@ -17,6 +17,18 @@ interface BulkCallbacks { onAfter?: (files: string[]) => void; } +interface BulkResultItem { + stackName: string; + ok: boolean; + error?: string; + code?: string; +} + +interface BulkResponse { + action: BulkAction; + results: BulkResultItem[]; +} + export function useBulkStackActions() { const { isPaid } = useLicense(); @@ -33,32 +45,43 @@ export function useBulkStackActions() { cbs?.onBefore?.(files); - const results = await Promise.allSettled( - files.map(file => { - const stackName = file.replace(/\.(yml|yaml)$/, ''); - const headers: Record = action === 'update' ? { 'x-bulk-mode': '1' } : {}; - return apiFetch(`/stacks/${encodeURIComponent(stackName)}/${action}`, { - method: 'POST', - headers, - }).then(res => { - if (!res.ok) return Promise.reject(new Error(file)); - return file; - }); - }) - ); + const stackNames = files.map(file => file.replace(/\.(yml|yaml)$/, '')); - cbs?.onAfter?.(files); + try { + const response = await apiFetch('/stacks/bulk', { + method: 'POST', + body: JSON.stringify({ action, stackNames }), + }); - const failed = results - .filter((r): r is PromiseRejectedResult => r.status === 'rejected') - .map(r => (r.reason as Error).message); - const okCount = results.length - failed.length; + cbs?.onAfter?.(files); - if (failed.length === 0) { - const noun = okCount === 1 ? 'stack' : 'stacks'; - toast.success(`${okCount} ${noun} ${pastTense[action]}`); - } else { - toast.error(`${okCount} of ${files.length} ${pastTense[action]}; ${failed.length} failed: ${failed.join(', ')}`); + if (!response.ok) { + const errBody = await response.json().catch(() => ({})); + const errMsg = (errBody as { error?: string })?.error + ?? `Bulk ${action} failed (HTTP ${response.status})`; + toast.error(errMsg); + return; + } + + const payload = (await response.json()) as BulkResponse; + const results = Array.isArray(payload.results) ? payload.results : []; + const okCount = results.filter(r => r.ok).length; + const failed = results.filter(r => !r.ok); + + if (failed.length === 0) { + const noun = okCount === 1 ? 'stack' : 'stacks'; + toast.success(`${okCount} ${noun} ${pastTense[action]}`); + return; + } + + const failedNames = failed.map(r => r.stackName).join(', '); + toast.error( + `${okCount} of ${results.length} ${pastTense[action]}; ${failed.length} failed: ${failedNames}`, + ); + } catch (err) { + cbs?.onAfter?.(files); + console.error('Bulk action failed:', err); + toast.error(`Bulk ${action} failed: ${(err as Error).message}`); } }, [isPaid]);