mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-11 11:16:55 +00:00
perf(statuses): align stack-status cache TTL and invalidation with polling (#1814)
The 3s stack-statuses cache TTL never survived the 10s dashboard poll, so every ordinary poll recomputed. Raise the TTL to 15s and move the git-source label and self-identity enrichment inside the cached payload so cache hits serve fully decorated statuses with zero per-request work. Invalidation closes the gaps the longer TTL would otherwise widen: - DockerEventService drops stack-statuses:<nodeId> on container state events so the UI's state-invalidate refetch recomputes instead of hitting a stale entry. The narrow key only: container events do not reshape stack identity or file roots, and the stats key self-refreshes on its own 2s TTL. - git-source link and unlink invalidate node caches before responding, so the source label stays fresh without waiting for the TTL. - a payload whose enrichment degraded (identity probe failure or git-source scan failure) is never cached, so a mislabeled not-self or 'local' badge cannot persist for a full TTL window. Running outside Docker is not degradation, so host installs cache normally.
This commit is contained in:
@@ -14,12 +14,13 @@
|
||||
* layer rather than hitting the real Docker socket, so they run in CI
|
||||
* without requiring any external daemon.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, vi, beforeEach } from 'vitest';
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_PASSWORD } from './helpers/setupTestDb';
|
||||
import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsMock } from './helpers/arcstatsFsMock';
|
||||
import { GitSourceService } from '../services/GitSourceService';
|
||||
import type { PublicGitSource } from '../services/GitSourceService';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import * as selfStackGuard from '../helpers/selfStackGuard';
|
||||
|
||||
// ── Hoisted mocks (must come before importing the app) ─────────────────
|
||||
@@ -230,6 +231,25 @@ describe('GET /api/fleet/overview local-node memory', () => {
|
||||
// ── /api/stacks/statuses ───────────────────────────────────────────────
|
||||
|
||||
describe('GET /api/stacks/statuses caching', () => {
|
||||
// A healthy identity by default so computed results persist in the cache;
|
||||
// a degraded resolution would intentionally drop the entry after the call.
|
||||
// Tests that need a different identity override the mock on the shared spy.
|
||||
let identitySpy: ReturnType<typeof vi.spyOn>;
|
||||
|
||||
beforeEach(() => {
|
||||
identitySpy = vi
|
||||
.spyOn(selfStackGuard, 'resolveSelfStackIdentity')
|
||||
.mockResolvedValue({ projectName: 'sencho', labels: null, degraded: false });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
identitySpy.mockRestore();
|
||||
// The git-link test seeds a 'web' source row; drop it even when that
|
||||
// test fails mid-way so later tests never see it. Delete is a no-op when
|
||||
// the row is absent.
|
||||
GitSourceService.getInstance().delete('web');
|
||||
});
|
||||
|
||||
it('serves repeat calls from cache without re-invoking the filesystem', async () => {
|
||||
mockGetStacks.mockResolvedValue(['web', 'db']);
|
||||
mockGetBulkStackStatuses.mockResolvedValue({
|
||||
@@ -237,11 +257,12 @@ describe('GET /api/stacks/statuses caching', () => {
|
||||
db: { status: 'running' },
|
||||
});
|
||||
|
||||
await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
const first = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
|
||||
expect(mockGetStacks).toHaveBeenCalledTimes(1);
|
||||
expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(1);
|
||||
expect(second.body).toEqual(first.body);
|
||||
});
|
||||
|
||||
it('invalidates on POST /api/stacks', async () => {
|
||||
@@ -260,7 +281,7 @@ describe('GET /api/stacks/statuses caching', () => {
|
||||
expect(mockGetStacks).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('labels each stack with its git/local source, computed outside the cache', async () => {
|
||||
it('labels each stack with its git/local source, served from the cache', async () => {
|
||||
mockGetStacks.mockResolvedValue(['web.yml', 'db.yml']);
|
||||
mockGetBulkStackStatuses.mockResolvedValue({
|
||||
web: { status: 'running' },
|
||||
@@ -275,83 +296,143 @@ describe('GET /api/stacks/statuses caching', () => {
|
||||
expect(first.body['web.yml'].source).toBe('git');
|
||||
expect(first.body['db.yml'].source).toBe('local');
|
||||
|
||||
// Source is recomputed live even when the Docker-status payload is cached:
|
||||
// unlinking `web` flips it to local on the next request without a cache flush.
|
||||
listSpy.mockReturnValue([]);
|
||||
// The source label is part of the cached payload: the second request is a
|
||||
// cache hit and does not re-scan the git-source table. Link/unlink keep
|
||||
// the label fresh by invalidating the cache (git-source routes).
|
||||
const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
expect(second.body['web.yml'].source).toBe('local');
|
||||
expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(1); // status portion served from cache
|
||||
expect(second.body['web.yml'].source).toBe('git');
|
||||
expect(listSpy).toHaveBeenCalledTimes(1);
|
||||
expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(1);
|
||||
|
||||
listSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('resolves self-stack identity once per request, cache hits included, and labels each stack from it', async () => {
|
||||
it('resolves self-stack identity on cache miss only and serves hits from the cached payload', async () => {
|
||||
mockGetStacks.mockResolvedValue(['sencho.yml', 'web.yml']);
|
||||
mockGetBulkStackStatuses.mockResolvedValue({
|
||||
sencho: { status: 'running' },
|
||||
web: { status: 'running' },
|
||||
});
|
||||
const identitySpy = vi
|
||||
.spyOn(selfStackGuard, 'resolveSelfStackIdentity')
|
||||
.mockResolvedValue({ projectName: 'sencho', labels: null });
|
||||
|
||||
const first = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
|
||||
// Hoisted: one resolution per request (including the cache hit) serves
|
||||
// every stack in the response.
|
||||
expect(identitySpy).toHaveBeenCalledTimes(2);
|
||||
// Identity resolves once per computed fetch; the second request is a
|
||||
// cache hit and inherits the enriched isSelf flags.
|
||||
expect(identitySpy).toHaveBeenCalledTimes(1);
|
||||
for (const res of [first, second]) {
|
||||
expect(res.body['sencho.yml'].isSelf).toBe(true);
|
||||
expect(res.body['web.yml'].isSelf).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
identitySpy.mockRestore();
|
||||
it('reflects a Git link on the next statuses request without waiting for the TTL', async () => {
|
||||
mockGetStacks.mockResolvedValue(['web']);
|
||||
mockGetBulkStackStatuses.mockResolvedValue({ web: { status: 'running' } });
|
||||
// Stub upsert so the link does not clone the repository over the network.
|
||||
const upsertSpy = vi
|
||||
.spyOn(GitSourceService.getInstance(), 'upsert')
|
||||
.mockResolvedValue({ stack_name: 'web' } as Awaited<ReturnType<typeof GitSourceService.prototype.upsert>>);
|
||||
|
||||
const before = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
expect(before.body.web.source).toBe('local');
|
||||
|
||||
await request(app)
|
||||
.put('/api/stacks/web/git-source')
|
||||
.set('Cookie', authCookie)
|
||||
.send({
|
||||
repo_url: 'https://github.com/example/web.git',
|
||||
branch: 'main',
|
||||
compose_path: 'compose.yaml',
|
||||
auth_type: 'none',
|
||||
});
|
||||
// The stub skipped persistence; seed the row so the recompute sees it,
|
||||
// standing in for what a real upsert would have stored.
|
||||
DatabaseService.getInstance().upsertGitSource({
|
||||
stack_name: 'web',
|
||||
repo_url: 'https://github.com/example/web.git',
|
||||
branch: 'main',
|
||||
compose_path: 'compose.yaml',
|
||||
compose_paths: ['compose.yaml'],
|
||||
context_dir: null,
|
||||
sync_env: false,
|
||||
env_path: null,
|
||||
auth_type: 'none',
|
||||
encrypted_token: null,
|
||||
auto_apply_on_webhook: false,
|
||||
auto_deploy_on_apply: false,
|
||||
last_applied_commit_sha: null,
|
||||
last_applied_content_hash: null,
|
||||
pending_commit_sha: null,
|
||||
pending_compose_content: null,
|
||||
pending_env_content: null,
|
||||
pending_fetched_at: null,
|
||||
last_debounce_at: null,
|
||||
});
|
||||
|
||||
// Linking invalidated the statuses cache, so the next request recomputes
|
||||
// with the fresh label instead of serving the stale cached 'local'.
|
||||
const after = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
expect(after.body.web.source).toBe('git');
|
||||
expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(2);
|
||||
|
||||
upsertSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('skips identity resolution entirely when the node has no stacks', async () => {
|
||||
mockGetStacks.mockResolvedValue([]);
|
||||
mockGetBulkStackStatuses.mockResolvedValue({});
|
||||
const identitySpy = vi.spyOn(selfStackGuard, 'resolveSelfStackIdentity');
|
||||
|
||||
const res = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({});
|
||||
expect(identitySpy).not.toHaveBeenCalled();
|
||||
|
||||
identitySpy.mockRestore();
|
||||
});
|
||||
|
||||
it('serves 200 with isSelf false everywhere when identity resolution degrades', async () => {
|
||||
it('serves 200 with isSelf false everywhere when identity resolution degrades, and does not cache the degraded payload', async () => {
|
||||
mockGetStacks.mockResolvedValue(['web.yml', 'db.yml']);
|
||||
mockGetBulkStackStatuses.mockResolvedValue({
|
||||
web: { status: 'running' },
|
||||
db: { status: 'running' },
|
||||
});
|
||||
const identitySpy = vi
|
||||
.spyOn(selfStackGuard, 'resolveSelfStackIdentity')
|
||||
.mockResolvedValue({ projectName: null, labels: null });
|
||||
identitySpy.mockResolvedValue({ projectName: null, labels: null, degraded: true });
|
||||
|
||||
const res = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
const first = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body['web.yml'].isSelf).toBe(false);
|
||||
expect(res.body['db.yml'].isSelf).toBe(false);
|
||||
expect(first.status).toBe(200);
|
||||
expect(first.body['web.yml'].isSelf).toBe(false);
|
||||
expect(first.body['db.yml'].isSelf).toBe(false);
|
||||
|
||||
identitySpy.mockRestore();
|
||||
// A degraded resolution reports every stack as not-self, which would
|
||||
// un-gate destructive UI affordances on the Sencho stack itself, so it
|
||||
// must not persist: the next request recomputes and re-resolves instead
|
||||
// of serving a cached isSelf: false for a full TTL.
|
||||
const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
expect(second.status).toBe(200);
|
||||
expect(identitySpy).toHaveBeenCalledTimes(2);
|
||||
expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('falls back to local labels (200, not 500) when the git-source lookup throws', async () => {
|
||||
it('falls back to local labels (200, not 500) when the git-source lookup throws, and does not cache the fallback', async () => {
|
||||
mockGetStacks.mockResolvedValue(['web.yml']);
|
||||
mockGetBulkStackStatuses.mockResolvedValue({ web: { status: 'running' } });
|
||||
// The default healthy identity keeps the git-source scan as the only
|
||||
// degradation source.
|
||||
const listSpy = vi
|
||||
.spyOn(GitSourceService.getInstance(), 'list')
|
||||
.mockImplementation(() => { throw new Error('db locked'); });
|
||||
|
||||
const res = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body['web.yml'].source).toBe('local');
|
||||
const first = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
expect(first.status).toBe(200);
|
||||
expect(first.body['web.yml'].source).toBe('local');
|
||||
|
||||
// The 'local' fallback must not persist: the next request recomputes and
|
||||
// re-reads the git sources instead of serving the mislabel for a full TTL.
|
||||
const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
|
||||
expect(second.status).toBe(200);
|
||||
expect(listSpy).toHaveBeenCalledTimes(2);
|
||||
expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(2);
|
||||
|
||||
listSpy.mockRestore();
|
||||
});
|
||||
|
||||
@@ -90,6 +90,7 @@ function makeStream(): FakeStream {
|
||||
// ── Setup ──────────────────────────────────────────────────────────────
|
||||
|
||||
import { DockerEventService } from '../services/DockerEventService';
|
||||
import { CacheService } from '../services/CacheService';
|
||||
|
||||
let stream: FakeStream;
|
||||
let service: DockerEventService;
|
||||
@@ -1324,7 +1325,17 @@ describe('DockerEventService - hardening', () => {
|
||||
// ── State-invalidate broadcasts ────────────────────────────────────────
|
||||
|
||||
describe('DockerEventService - state-invalidate broadcasts', () => {
|
||||
it('broadcasts state-invalidate on container start', async () => {
|
||||
let invalidateSpy: ReturnType<typeof vi.spyOn>;
|
||||
|
||||
beforeEach(() => {
|
||||
invalidateSpy = vi.spyOn(CacheService.getInstance(), 'invalidate');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
invalidateSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('broadcasts state-invalidate and drops the statuses cache on container start', async () => {
|
||||
service = new DockerEventService(7, 'node-7');
|
||||
await service.start();
|
||||
|
||||
@@ -1344,6 +1355,11 @@ describe('DockerEventService - state-invalidate broadcasts', () => {
|
||||
containerId: 'aaa',
|
||||
action: 'start',
|
||||
}));
|
||||
// The UI refetch this broadcast triggers must recompute, not hit a
|
||||
// cache entry made stale by the event. Only the statuses key is
|
||||
// touched; the full invalidateNodeCaches helper is not used here.
|
||||
expect(invalidateSpy).toHaveBeenCalledTimes(1);
|
||||
expect(invalidateSpy).toHaveBeenCalledWith('stack-statuses:7');
|
||||
});
|
||||
|
||||
it('does not broadcast stack state-invalidate for Sencho self-container events', async () => {
|
||||
@@ -1368,6 +1384,7 @@ describe('DockerEventService - state-invalidate broadcasts', () => {
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(mockBroadcastEvent).not.toHaveBeenCalled();
|
||||
expect(invalidateSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('broadcasts state-invalidate on health_status:unhealthy', async () => {
|
||||
@@ -1386,6 +1403,7 @@ describe('DockerEventService - state-invalidate broadcasts', () => {
|
||||
(c[0] as { type?: string }).type === 'state-invalidate');
|
||||
expect(states.length).toBeGreaterThan(0);
|
||||
expect(states[0][0]).toMatchObject({ action: 'health_status', stackName: 'api' });
|
||||
expect(invalidateSpy).toHaveBeenCalledWith('stack-statuses:1');
|
||||
});
|
||||
|
||||
it('does not broadcast state-invalidate on non-state actions like exec_create', async () => {
|
||||
@@ -1401,6 +1419,7 @@ describe('DockerEventService - state-invalidate broadcasts', () => {
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(mockBroadcastEvent).not.toHaveBeenCalled();
|
||||
expect(invalidateSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
* Service-layer logic (encryption, error mapping, mutex, pending lifecycle)
|
||||
* is covered in git-source-service.test.ts.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import fs from 'fs';
|
||||
@@ -20,6 +20,17 @@ import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './he
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { GitSourceService } from '../services/GitSourceService';
|
||||
|
||||
// ── Hoisted mocks (must come before importing the app) ─────────────────
|
||||
|
||||
// The statuses cache carries the source label, so link/unlink must drop it.
|
||||
// Spy on the invalidation helper the routes call; the rest of the module
|
||||
// (remote-meta invalidation) stays real.
|
||||
const mockInvalidateNodeCaches = vi.hoisted(() => vi.fn());
|
||||
vi.mock('../helpers/cacheInvalidation', async () => {
|
||||
const actual = await vi.importActual<typeof import('../helpers/cacheInvalidation')>('../helpers/cacheInvalidation');
|
||||
return { ...actual, invalidateNodeCaches: mockInvalidateNodeCaches };
|
||||
});
|
||||
|
||||
function seedGitSource(stackName: string): void {
|
||||
DatabaseService.getInstance().upsertGitSource({
|
||||
stack_name: stackName,
|
||||
@@ -538,3 +549,66 @@ describe('GET /api/git-sources', () => {
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
});
|
||||
|
||||
describe('git-source routes: statuses-cache invalidation', () => {
|
||||
// The cached /stacks/statuses payload carries the source label, so link
|
||||
// and unlink must drop the cache; read-only routes must not.
|
||||
beforeEach(() => {
|
||||
mockInvalidateNodeCaches.mockClear();
|
||||
});
|
||||
|
||||
function seedStackDir(stackName: string): void {
|
||||
const composeDir = process.env.COMPOSE_DIR!;
|
||||
fs.mkdirSync(path.join(composeDir, stackName), { recursive: true });
|
||||
fs.writeFileSync(path.join(composeDir, stackName, 'compose.yaml'), 'services:\n x:\n image: nginx\n');
|
||||
}
|
||||
|
||||
it('invalidates node caches when linking a Git source', async () => {
|
||||
seedStackDir('inv-link');
|
||||
// Stub upsert so the assertion stays at the route layer without cloning a repo.
|
||||
const upsertSpy = vi.spyOn(GitSourceService.getInstance(), 'upsert')
|
||||
.mockResolvedValue({} as Awaited<ReturnType<typeof GitSourceService.prototype.upsert>>);
|
||||
const res = await request(app)
|
||||
.put('/api/stacks/inv-link/git-source')
|
||||
.set('Authorization', `Bearer ${adminToken()}`)
|
||||
.send({
|
||||
repo_url: 'https://github.com/example/inv-link.git',
|
||||
branch: 'main',
|
||||
compose_path: 'compose.yaml',
|
||||
auth_type: 'none',
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockInvalidateNodeCaches).toHaveBeenCalledTimes(1);
|
||||
expect(mockInvalidateNodeCaches).toHaveBeenCalledWith(expect.any(Number));
|
||||
upsertSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('invalidates node caches when unlinking a Git source', async () => {
|
||||
seedGitSource('inv-unlink');
|
||||
const res = await request(app)
|
||||
.delete('/api/stacks/inv-unlink/git-source')
|
||||
.set('Authorization', `Bearer ${adminToken()}`);
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockInvalidateNodeCaches).toHaveBeenCalledTimes(1);
|
||||
expect(mockInvalidateNodeCaches).toHaveBeenCalledWith(expect.any(Number));
|
||||
});
|
||||
|
||||
it('does not invalidate on GET of the Git source', async () => {
|
||||
seedStackDir('inv-get');
|
||||
seedGitSource('inv-get');
|
||||
const res = await request(app)
|
||||
.get('/api/stacks/inv-get/git-source')
|
||||
.set('Authorization', `Bearer ${adminToken()}`);
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockInvalidateNodeCaches).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not invalidate when dismissing a pending update', async () => {
|
||||
seedGitSource('inv-dismiss');
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/inv-dismiss/git-source/dismiss-pending')
|
||||
.set('Authorization', `Bearer ${adminToken()}`);
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockInvalidateNodeCaches).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -103,10 +103,10 @@ describe('[Stacks:debug] GET /api/stacks/statuses', () => {
|
||||
expect(second).toMatch(/cacheOutcome=hit/);
|
||||
// No docker call on a cache hit, so the subspan is null rather than 0.
|
||||
expect(second).toMatch(/dockerMs=null/);
|
||||
// Enrichment runs on every request, cache hits included, so its subspan
|
||||
// is a number on both legs.
|
||||
// Enrichment is part of the cached payload, so both subspans are
|
||||
// compute-only telemetry: numbers on the compute leg, null on a hit.
|
||||
expect(first).toMatch(/enrichmentMs=\d+/);
|
||||
expect(second).toMatch(/enrichmentMs=\d+/);
|
||||
expect(second).toMatch(/enrichmentMs=null/);
|
||||
// The compute ran the fetcher exactly once across both requests.
|
||||
expect(dockerCalls).toBe(1);
|
||||
});
|
||||
|
||||
@@ -161,10 +161,32 @@ describe('resolveSelfStackIdentity + isSelfStackByIdentity', () => {
|
||||
expect(isSelfStackByIdentity(identity, 'sencho', '/app/compose')).toBe(false);
|
||||
});
|
||||
|
||||
it('is false when no identity source is available', async () => {
|
||||
it('is false when no identity source is available, and not degraded', async () => {
|
||||
stubComposeProject(null);
|
||||
const identity = await resolveSelfStackIdentity();
|
||||
expect(isSelfStackByIdentity(identity, 'sencho')).toBe(false);
|
||||
// No source attempted a Docker call, so this is the healthy
|
||||
// "not running in Docker" state, not a degradation.
|
||||
expect(identity.degraded).toBe(false);
|
||||
});
|
||||
|
||||
it('marks the identity degraded when the container list read throws', async () => {
|
||||
const runtimeId = 'f'.repeat(64);
|
||||
process.env.HOSTNAME = runtimeId.slice(0, 12);
|
||||
stubComposeProject('sencho');
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDocker: () => ({
|
||||
listContainers: vi.fn().mockRejectedValue(new Error('socket unreachable')),
|
||||
}),
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
|
||||
const identity = await resolveSelfStackIdentity();
|
||||
// The surviving source still resolves, but the failed probe marks the
|
||||
// whole identity degraded so callers refuse to cache it.
|
||||
expect(identity.projectName).toBe('sencho');
|
||||
expect(identity.labels).toBeNull();
|
||||
expect(identity.degraded).toBe(true);
|
||||
expect(isSelfStackByIdentity(identity, 'sencho')).toBe(true);
|
||||
});
|
||||
|
||||
it('resolves identity with at most one container list read, shared by all stacks', async () => {
|
||||
|
||||
@@ -44,4 +44,8 @@ export const MFA_REPLAY_PURGE_INTERVAL_MS = 60 * 1000;
|
||||
// Keys are per-node: "stats:<nodeId>", "system-stats:<nodeId>", "stack-statuses:<nodeId>".
|
||||
export const STATS_CACHE_TTL_MS = 2_000;
|
||||
export const SYSTEM_STATS_CACHE_TTL_MS = 3_000;
|
||||
export const STACK_STATUSES_CACHE_TTL_MS = 3_000;
|
||||
// Stack statuses are cached past the frontend dashboard poll cadence (10s),
|
||||
// so ordinary polls hit instead of recomputing. Container events and
|
||||
// lifecycle mutations invalidate the key; 15s bounds worst-case staleness for
|
||||
// any missed invalidation path.
|
||||
export const STACK_STATUSES_CACHE_TTL_MS = 15_000;
|
||||
|
||||
@@ -50,19 +50,18 @@ function workingDirMatchesStack(workingDir: string | undefined, stackName: strin
|
||||
}
|
||||
|
||||
async function getRunningContainerLabels(): Promise<Record<string, string> | null> {
|
||||
try {
|
||||
const runtimeIds = await getRuntimeContainerIdCandidates();
|
||||
if (runtimeIds.length === 0) return null;
|
||||
const runtimeIds = await getRuntimeContainerIdCandidates();
|
||||
if (runtimeIds.length === 0) return null;
|
||||
|
||||
const containers = await DockerController.getInstance().getDocker().listContainers({ all: true }) as ListedContainer[];
|
||||
const selfContainer = containers.find((container) => {
|
||||
const containerId = container.Id;
|
||||
return typeof containerId === 'string' && runtimeIds.some(id => matchesContainerId(containerId, id));
|
||||
});
|
||||
return selfContainer?.Labels ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
// A listContainers failure (Docker socket unreachable) propagates so
|
||||
// resolveSelfStackIdentity can mark the resolution degraded. Callers that
|
||||
// need null-on-failure wrap this call themselves.
|
||||
const containers = await DockerController.getInstance().getDocker().listContainers({ all: true }) as ListedContainer[];
|
||||
const selfContainer = containers.find((container) => {
|
||||
const containerId = container.Id;
|
||||
return typeof containerId === 'string' && runtimeIds.some(id => matchesContainerId(containerId, id));
|
||||
});
|
||||
return selfContainer?.Labels ?? null;
|
||||
}
|
||||
|
||||
async function runningContainerMatchesStack(stackName: string, composeDir?: string): Promise<boolean> {
|
||||
@@ -89,10 +88,14 @@ export async function getSelfStackProjectName(): Promise<string | null> {
|
||||
|
||||
/** Directory name of the running Sencho compose project, when it is under COMPOSE_DIR. */
|
||||
export async function getSelfStackDirectoryName(composeDir?: string): Promise<string | null> {
|
||||
const labels = await getRunningContainerLabels();
|
||||
const workingDirStack = stackNameFromWorkingDir(labels?.['com.docker.compose.project.working_dir'], composeDir);
|
||||
if (workingDirStack) return workingDirStack;
|
||||
return getSelfStackProjectName();
|
||||
try {
|
||||
const labels = await getRunningContainerLabels();
|
||||
const workingDirStack = stackNameFromWorkingDir(labels?.['com.docker.compose.project.working_dir'], composeDir);
|
||||
if (workingDirStack) return workingDirStack;
|
||||
return getSelfStackProjectName();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** True when the stack appears to be the running Sencho compose project. */
|
||||
@@ -117,25 +120,46 @@ export async function isSelfStack(stackName: string, composeDir?: string): Promi
|
||||
export interface SelfStackIdentity {
|
||||
projectName: string | null;
|
||||
labels: Record<string, string> | null;
|
||||
/**
|
||||
* True when the container-labels probe failed (Docker socket unreachable).
|
||||
* A degraded identity cannot be trusted to classify every stack correctly,
|
||||
* so it must not be cached: the next request should re-resolve. Running
|
||||
* outside Docker is NOT degraded: both sources legitimately resolve to
|
||||
* null there and the identity is correct as-is.
|
||||
*/
|
||||
degraded: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves both identity sources. Each failure degrades only its own
|
||||
* source to null, matching the old per-stack behavior where a labels
|
||||
* failure never discarded the resolved project name.
|
||||
* Resolves both identity sources. A labels-probe failure degrades only that
|
||||
* source to null and marks the resolution degraded, matching the old
|
||||
* per-stack behavior where a labels failure never discarded the resolved
|
||||
* project name. getSelfStackProjectName swallows its own failures and
|
||||
* returns null, so it cannot mark the identity degraded.
|
||||
*/
|
||||
export async function resolveSelfStackIdentity(): Promise<SelfStackIdentity> {
|
||||
const projectName = await getSelfStackProjectName().catch((error) => {
|
||||
console.error('Failed to resolve self-stack project name; self-stack check degraded:', error);
|
||||
return null;
|
||||
});
|
||||
const labels = await getRunningContainerLabels().catch((error) => {
|
||||
const projectName = await getSelfStackProjectName();
|
||||
let labels: Record<string, string> | null = null;
|
||||
let degraded = false;
|
||||
try {
|
||||
labels = await getRunningContainerLabels();
|
||||
} catch (error) {
|
||||
console.error('Failed to resolve self-stack container labels; self-stack check degraded:', error);
|
||||
return null;
|
||||
});
|
||||
return { projectName, labels };
|
||||
degraded = true;
|
||||
}
|
||||
return { projectName, labels, degraded };
|
||||
}
|
||||
|
||||
/**
|
||||
* Identity used when no resolution is attempted (empty fleet). Not degraded:
|
||||
* with no stacks there is nothing to mislabel, so the payload caches as-is.
|
||||
*/
|
||||
export const UNRESOLVED_SELF_STACK_IDENTITY: SelfStackIdentity = {
|
||||
projectName: null,
|
||||
labels: null,
|
||||
degraded: false,
|
||||
};
|
||||
|
||||
/** isSelf semantics identical to isSelfStack() when the identity resolves successfully. */
|
||||
export function isSelfStackByIdentity(identity: SelfStackIdentity, stackName: string, composeDir?: string): boolean {
|
||||
if (identity.projectName === stackName) return true;
|
||||
|
||||
@@ -226,6 +226,13 @@ stackGitSourceRouter.put('/:stackName/git-source', async (req: Request, res: Res
|
||||
autoDeployOnApply,
|
||||
});
|
||||
|
||||
// The cached /stacks/statuses payload carries the source label; drop it
|
||||
// before responding so a client refetch on this response recomputes. The
|
||||
// full invalidateNodeCaches helper is deliberate here (matching every
|
||||
// other mutation route): link/unlink is a low-frequency user action, so
|
||||
// dropping the project-name map and file-root allowlists alongside is
|
||||
// harmless, unlike the high-frequency container-event path.
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
console.log(`[GitSource] Configured git source for ${stackName}`);
|
||||
res.json(source);
|
||||
} catch (error) {
|
||||
@@ -254,6 +261,13 @@ stackGitSourceRouter.delete('/:stackName/git-source', async (req: Request, res:
|
||||
return;
|
||||
}
|
||||
GitSourceService.getInstance().delete(stackName);
|
||||
// The cached /stacks/statuses payload carries the source label; drop it
|
||||
// before responding so a client refetch on this response recomputes. The
|
||||
// full invalidateNodeCaches helper is deliberate here (matching every
|
||||
// other mutation route): link/unlink is a low-frequency user action, so
|
||||
// dropping the project-name map and file-root allowlists alongside is
|
||||
// harmless, unlike the high-frequency container-event path.
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
console.log(`[GitSource] Removed git source for ${stackName}`);
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
|
||||
@@ -75,6 +75,7 @@ import {
|
||||
refuseIfSelfStack,
|
||||
resolveSelfStackIdentity,
|
||||
selfStackProtectedBulkResult,
|
||||
UNRESOLVED_SELF_STACK_IDENTITY,
|
||||
} from '../helpers/selfStackGuard';
|
||||
import { getActiveCapabilities, STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
@@ -344,6 +345,10 @@ stacksRouter.get('/statuses', async (req: Request, res: Response) => {
|
||||
let enrichmentMs: number | null = null;
|
||||
let count = 0;
|
||||
try {
|
||||
// Enrichment (git-source labels, self identity) is part of the cached
|
||||
// payload so cache hits serve fully decorated statuses with no per-request
|
||||
// work. The git label lookup failure still falls back to 'local' and must
|
||||
// not take down the primary status payload.
|
||||
const { value: result, outcome: fetchOutcome } = await CacheService.getInstance().getOrFetchWithMeta(
|
||||
`stack-statuses:${req.nodeId}`,
|
||||
STACK_STATUSES_CACHE_TTL_MS,
|
||||
@@ -359,38 +364,54 @@ stacksRouter.get('/statuses', async (req: Request, res: Response) => {
|
||||
const name = stack.replace(/\.(yml|yaml)$/, '');
|
||||
data[stack] = bulkInfo[name] ?? { status: 'unknown' };
|
||||
}
|
||||
return data;
|
||||
const enrichmentStartedAt = Date.now();
|
||||
let gitStackNames = new Set<string>();
|
||||
let gitSourcesDegraded = false;
|
||||
try {
|
||||
gitStackNames = new Set(GitSourceService.getInstance().list().map((s) => s.stack_name));
|
||||
} catch (sourceError) {
|
||||
console.error(`Failed to load git sources for status labels on node ${req.nodeId}; defaulting to local:`, sourceError);
|
||||
gitSourcesDegraded = true;
|
||||
}
|
||||
// Self-stack identity is resolved once per request instead of once per
|
||||
// stack, so cache misses pay a single container-list call, not N.
|
||||
const selfIdentity = stackNames.length > 0
|
||||
? await resolveSelfStackIdentity()
|
||||
: UNRESOLVED_SELF_STACK_IDENTITY;
|
||||
const withSource: Record<string, BulkStackInfo & { source: 'local' | 'git'; isSelf: boolean }> = {};
|
||||
const composeDir = FileSystemService.getInstance(req.nodeId).getBaseDir();
|
||||
for (const [stack, info] of Object.entries(data)) {
|
||||
const name = stack.replace(/\.(yml|yaml)$/, '');
|
||||
withSource[stack] = {
|
||||
...info,
|
||||
source: gitStackNames.has(name) ? 'git' : 'local',
|
||||
isSelf: isSelfStackByIdentity(selfIdentity, name, composeDir),
|
||||
};
|
||||
}
|
||||
enrichmentMs = Date.now() - enrichmentStartedAt;
|
||||
// The payload is flagged degraded when any enrichment source failed
|
||||
// (Docker socket unreachable, git-source scan failure) so the route
|
||||
// can refuse to let a mislabeled payload persist.
|
||||
return { data: withSource, degraded: selfIdentity.degraded || gitSourcesDegraded };
|
||||
},
|
||||
);
|
||||
cacheOutcome = fetchOutcome;
|
||||
count = Object.keys(result).length;
|
||||
const enrichmentStartedAt = Date.now();
|
||||
// Git-source labels are computed live, outside the cache, so linking or
|
||||
// unlinking a stack's Git source is reflected immediately. The Docker
|
||||
// status portion keeps its short TTL; only the cheap source label is fresh.
|
||||
// The label is cosmetic, so a lookup failure must not take down the primary
|
||||
// status payload: fall back to labeling everything 'local'.
|
||||
let gitStackNames = new Set<string>();
|
||||
try {
|
||||
gitStackNames = new Set(GitSourceService.getInstance().list().map((s) => s.stack_name));
|
||||
} catch (sourceError) {
|
||||
console.error('Failed to load git sources for status labels; defaulting to local:', sourceError);
|
||||
const { data, degraded } = result;
|
||||
count = Object.keys(data).length;
|
||||
// A degraded identity resolution (Docker socket unreachable) cannot be
|
||||
// trusted to classify every stack, which un-gates destructive UI
|
||||
// affordances on the Sencho stack itself, and a failed git-source scan
|
||||
// mislabels every source badge as 'local'. Never let either mislabel
|
||||
// persist for a full TTL: serve the live result, drop the cache entry,
|
||||
// and let the next request re-resolve. Everything between the fetch and
|
||||
// this invalidate is synchronous, so no concurrent reader can observe
|
||||
// the degraded entry. Running outside Docker is not degraded (both
|
||||
// identity sources legitimately resolve to null there), and an empty
|
||||
// fleet skips resolution entirely; both are cached as-is.
|
||||
if (fetchOutcome === 'computed' && count > 0 && degraded) {
|
||||
CacheService.getInstance().invalidate(`stack-statuses:${req.nodeId}`);
|
||||
}
|
||||
// Self-stack identity is resolved once per request instead of once per
|
||||
// stack, so cache hits no longer pay N container-list calls.
|
||||
const selfIdentity = count > 0 ? await resolveSelfStackIdentity() : { projectName: null, labels: null };
|
||||
const withSource: Record<string, BulkStackInfo & { source: 'local' | 'git' }> = {};
|
||||
const composeDir = FileSystemService.getInstance(req.nodeId).getBaseDir();
|
||||
for (const [stack, info] of Object.entries(result)) {
|
||||
const name = stack.replace(/\.(yml|yaml)$/, '');
|
||||
withSource[stack] = {
|
||||
...info,
|
||||
source: gitStackNames.has(name) ? 'git' : 'local',
|
||||
isSelf: isSelfStackByIdentity(selfIdentity, name, composeDir),
|
||||
};
|
||||
}
|
||||
enrichmentMs = Date.now() - enrichmentStartedAt;
|
||||
res.json(withSource);
|
||||
res.json(data);
|
||||
} catch (error) {
|
||||
outcome = 'error';
|
||||
console.error('Failed to fetch stack statuses:', error);
|
||||
|
||||
@@ -3,6 +3,7 @@ import { NodeRegistry } from './NodeRegistry';
|
||||
import { NotificationCategory, NotificationService } from './NotificationService';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import SelfIdentityService from './SelfIdentityService';
|
||||
import { CacheService } from './CacheService';
|
||||
import {
|
||||
classifyDie,
|
||||
classifyGapExit,
|
||||
@@ -449,8 +450,14 @@ export class DockerEventService {
|
||||
// Push a lightweight state-invalidate signal so connected UIs can
|
||||
// refetch stack statuses immediately on a real container event,
|
||||
// without waiting for the next polling tick. This is fire-and-forget
|
||||
// and is NOT persisted to the alerts history.
|
||||
// and is NOT persisted to the alerts history. Drop the statuses cache
|
||||
// key alongside the broadcast so the UI's refetch recomputes instead
|
||||
// of serving an entry the event just made stale. The full
|
||||
// invalidateNodeCaches helper is not used: container events do not
|
||||
// reshape the project-name map or file-root allowlists, and the stats
|
||||
// key self-refreshes on its own 2s TTL.
|
||||
if (STATE_INVALIDATE_ACTIONS.has(baseAction) && !isSelf) {
|
||||
CacheService.getInstance().invalidate(`stack-statuses:${this.nodeId}`);
|
||||
this.notifier.broadcastEvent({
|
||||
type: 'state-invalidate',
|
||||
scope: 'stack',
|
||||
|
||||
Reference in New Issue
Block a user