mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-11 11:16:55 +00:00
a51547a158
* fix(monitor): decouple janitor disk-usage check from 30s cycle (F-6) `docker system df` (called by the MonitorService janitor check) can take 30+ seconds on Docker Desktop with many volumes. Running it on the 30s evaluate cycle compounded with the per-container stats fan-out and pushed the cycle to 140s+, blocking subsequent monitoring work. This change: - Moves the janitor disk-usage check into its own 15-minute cycle with a tight 8s timeout. A circuit breaker opens after 3 consecutive timeouts (60-minute cooldown) so a sick daemon stops pinning Dockerode sockets every tick. The first janitor tick is deferred 45 seconds past boot to avoid head-of-line collision with the initial monitor cycle's stats fan-out. - Adds a paired 8s `withTimeout` wrap to the admin prune-estimate routes (`/api/system/prune/estimate` and the dry-run path of `/api/system/prune/system`) so a slow df does not hang the admin tab. Both routes respond 503 with code `docker_df_slow` on timeout. - Factors `withTimeout` and `TimeoutError` into `utils/withTimeout.ts` so the route layer does not have to import from a service module. - Adds 10 unit tests covering the decoupling guardrail, breaker open/close, cooldown, threshold gate, the 100 MB reclaimable floor, re-entrancy, recovery logging, non-timeout error handling, and the full timer-cleanup contract of `stop()`. - Adds 4 integration tests for the prune routes covering the 503 timeout response, the success path, and the non-timeout 5xx path. * fix(fleet,monitor): extend F-6 timeout to fleet prune routes; close breaker-recovery log gap Codex audit findings on PR #1164: Major. The fleet routes that fan out prune-estimate work on local nodes (`POST /api/fleet/labels/fleet-prune` dry-run path and `POST /api/fleet/prune/estimate`) called `estimateSystemReclaim` without a timeout, so a slow local Docker daemon could still hang the fleet admin tab even though the system-maintenance routes were already bounded. Wrap both call sites with the shared `withTimeout(..., 8s)` and surface a "Docker daemon is busy" message via the per-target and per-node error channels the routes already used for other failures. The destructive (non-dry-run) prune path stays unwrapped because it calls `pruneSystem` / `pruneManagedOnly`, not `df`. Minor. The janitor circuit breaker zeroed `janitorConsecutiveTimeouts` when it opened, so a successful call after a full breaker-open cooldown slipped past the `if (counter > 0)` recovery-log branch and never emitted `[Monitor] Janitor disk-usage check recovered`. The operator observability signal was missing exactly when it mattered most. Extend the predicate to also trip on `janitorBreakerUntil > 0` (which stays set to its past timestamp after cooldown until the next success clears it), so recovery logs symmetrically for both partial-failure and post-breaker recovery paths. Added a dedicated test. Three new integration tests cover the fleet routes (timeout, success, and the estimate endpoint's per-node unreachable shape).
111 lines
4.2 KiB
TypeScript
111 lines
4.2 KiB
TypeScript
/**
|
|
* Route-level test for F-6: when `docker system df` is slow, the prune
|
|
* estimate endpoints must return 503 with code `docker_df_slow` instead
|
|
* of hanging the admin's tab. Mirrors the pattern from
|
|
* system-maintenance-self-protect.test.ts.
|
|
*
|
|
* Uses real timers because supertest dispatches lazily and vi.useFakeTimers
|
|
* does not compose cleanly with that pattern. Each timeout test waits the
|
|
* full 8s `withTimeout` budget, so two such tests add ~17s to the file.
|
|
*/
|
|
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
|
|
import request from 'supertest';
|
|
import jwt from 'jsonwebtoken';
|
|
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
|
|
|
|
let tmpDir: string;
|
|
let app: import('express').Express;
|
|
let authHeader: string;
|
|
let DockerController: typeof import('../services/DockerController').default;
|
|
let FileSystemService: typeof import('../services/FileSystemService').FileSystemService;
|
|
|
|
beforeAll(async () => {
|
|
tmpDir = await setupTestDb();
|
|
({ app } = await import('../index'));
|
|
({ default: DockerController } = await import('../services/DockerController'));
|
|
({ FileSystemService } = await import('../services/FileSystemService'));
|
|
// 10-minute expiry survives the full file even when two timeout tests
|
|
// burn ~8.5s each in real-timer mode.
|
|
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '10m' });
|
|
authHeader = `Bearer ${token}`;
|
|
});
|
|
|
|
afterAll(() => cleanupTestDb(tmpDir));
|
|
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
function stubFsStacks() {
|
|
vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({
|
|
getStacks: vi.fn().mockResolvedValue([]),
|
|
} as unknown as ReturnType<typeof FileSystemService.getInstance>);
|
|
}
|
|
|
|
function stubEstimate(impl: () => Promise<{ reclaimableBytes: number }>) {
|
|
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
|
estimateSystemReclaim: vi.fn().mockImplementation(impl),
|
|
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
|
}
|
|
|
|
describe('Prune estimate endpoints return 503 on slow docker df (F-6)', () => {
|
|
it('POST /api/system/prune/estimate returns 503 docker_df_slow when estimateSystemReclaim never settles', async () => {
|
|
stubFsStacks();
|
|
stubEstimate(() => new Promise(() => { /* never resolves */ }));
|
|
|
|
const t0 = Date.now();
|
|
const res = await request(app)
|
|
.post('/api/system/prune/estimate')
|
|
.set('Authorization', authHeader)
|
|
.send({ target: 'volumes', scope: 'all' });
|
|
const elapsed = Date.now() - t0;
|
|
|
|
expect(res.status).toBe(503);
|
|
expect(res.body.code).toBe('docker_df_slow');
|
|
expect(res.body.error).toMatch(/Docker daemon is busy/);
|
|
// Confirm the timeout actually fired (~8s), not an unrelated early
|
|
// 5xx that happened to look right.
|
|
expect(elapsed).toBeGreaterThanOrEqual(7_500);
|
|
expect(elapsed).toBeLessThan(15_000);
|
|
}, 20_000);
|
|
|
|
it('POST /api/system/prune/system dry-run returns 503 docker_df_slow when estimateSystemReclaim never settles', async () => {
|
|
stubFsStacks();
|
|
stubEstimate(() => new Promise(() => { /* never resolves */ }));
|
|
|
|
const res = await request(app)
|
|
.post('/api/system/prune/system')
|
|
.set('Authorization', authHeader)
|
|
.send({ target: 'volumes', scope: 'all', dryRun: true });
|
|
|
|
expect(res.status).toBe(503);
|
|
expect(res.body.code).toBe('docker_df_slow');
|
|
}, 20_000);
|
|
|
|
it('estimate route succeeds normally when estimateSystemReclaim resolves quickly', async () => {
|
|
stubFsStacks();
|
|
stubEstimate(() => Promise.resolve({ reclaimableBytes: 42 }));
|
|
|
|
const res = await request(app)
|
|
.post('/api/system/prune/estimate')
|
|
.set('Authorization', authHeader)
|
|
.send({ target: 'volumes', scope: 'all' });
|
|
|
|
expect(res.status).toBe(200);
|
|
expect(res.body.reclaimableBytes).toBe(42);
|
|
});
|
|
|
|
it('estimate route returns 5xx (not 503 docker_df_slow) on unrelated daemon error', async () => {
|
|
stubFsStacks();
|
|
stubEstimate(() => Promise.reject(new Error('daemon unreachable')));
|
|
|
|
const res = await request(app)
|
|
.post('/api/system/prune/estimate')
|
|
.set('Authorization', authHeader)
|
|
.send({ target: 'volumes', scope: 'all' });
|
|
|
|
expect(res.status).toBe(500);
|
|
expect(res.body.code).not.toBe('docker_df_slow');
|
|
});
|
|
});
|