perf(statuses): align stack-status cache TTL and invalidation with polling (#1814)

The 3s stack-statuses cache TTL never survived the 10s dashboard poll, so
every ordinary poll recomputed. Raise the TTL to 15s and move the git-source
label and self-identity enrichment inside the cached payload so cache hits
serve fully decorated statuses with zero per-request work.

Invalidation closes the gaps the longer TTL would otherwise widen:

- DockerEventService drops stack-statuses:<nodeId> on container state events
  so the UI's state-invalidate refetch recomputes instead of hitting a stale
  entry. The narrow key only: container events do not reshape stack identity
  or file roots, and the stats key self-refreshes on its own 2s TTL.
- git-source link and unlink invalidate node caches before responding, so the
  source label stays fresh without waiting for the TTL.
- a payload whose enrichment degraded (identity probe failure or git-source
  scan failure) is never cached, so a mislabeled not-self or 'local' badge
  cannot persist for a full TTL window. Running outside Docker is not
  degradation, so host installs cache normally.
This commit is contained in:
Anso
2026-08-09 23:13:59 -04:00
committed by GitHub
parent 9798700401
commit 55ca82abb2
10 changed files with 362 additions and 96 deletions
+114 -33
View File
@@ -14,12 +14,13 @@
* layer rather than hitting the real Docker socket, so they run in CI
* without requiring any external daemon.
*/
import { describe, it, expect, beforeAll, afterAll, vi, beforeEach } from 'vitest';
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
import request from 'supertest';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_PASSWORD } from './helpers/setupTestDb';
import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsMock } from './helpers/arcstatsFsMock';
import { GitSourceService } from '../services/GitSourceService';
import type { PublicGitSource } from '../services/GitSourceService';
import { DatabaseService } from '../services/DatabaseService';
import * as selfStackGuard from '../helpers/selfStackGuard';
// ── Hoisted mocks (must come before importing the app) ─────────────────
@@ -230,6 +231,25 @@ describe('GET /api/fleet/overview local-node memory', () => {
// ── /api/stacks/statuses ───────────────────────────────────────────────
describe('GET /api/stacks/statuses caching', () => {
// A healthy identity by default so computed results persist in the cache;
// a degraded resolution would intentionally drop the entry after the call.
// Tests that need a different identity override the mock on the shared spy.
let identitySpy: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
identitySpy = vi
.spyOn(selfStackGuard, 'resolveSelfStackIdentity')
.mockResolvedValue({ projectName: 'sencho', labels: null, degraded: false });
});
afterEach(() => {
identitySpy.mockRestore();
// The git-link test seeds a 'web' source row; drop it even when that
// test fails mid-way so later tests never see it. Delete is a no-op when
// the row is absent.
GitSourceService.getInstance().delete('web');
});
it('serves repeat calls from cache without re-invoking the filesystem', async () => {
mockGetStacks.mockResolvedValue(['web', 'db']);
mockGetBulkStackStatuses.mockResolvedValue({
@@ -237,11 +257,12 @@ describe('GET /api/stacks/statuses caching', () => {
db: { status: 'running' },
});
await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
const first = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
expect(mockGetStacks).toHaveBeenCalledTimes(1);
expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(1);
expect(second.body).toEqual(first.body);
});
it('invalidates on POST /api/stacks', async () => {
@@ -260,7 +281,7 @@ describe('GET /api/stacks/statuses caching', () => {
expect(mockGetStacks).toHaveBeenCalledTimes(2);
});
it('labels each stack with its git/local source, computed outside the cache', async () => {
it('labels each stack with its git/local source, served from the cache', async () => {
mockGetStacks.mockResolvedValue(['web.yml', 'db.yml']);
mockGetBulkStackStatuses.mockResolvedValue({
web: { status: 'running' },
@@ -275,83 +296,143 @@ describe('GET /api/stacks/statuses caching', () => {
expect(first.body['web.yml'].source).toBe('git');
expect(first.body['db.yml'].source).toBe('local');
// Source is recomputed live even when the Docker-status payload is cached:
// unlinking `web` flips it to local on the next request without a cache flush.
listSpy.mockReturnValue([]);
// The source label is part of the cached payload: the second request is a
// cache hit and does not re-scan the git-source table. Link/unlink keep
// the label fresh by invalidating the cache (git-source routes).
const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
expect(second.body['web.yml'].source).toBe('local');
expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(1); // status portion served from cache
expect(second.body['web.yml'].source).toBe('git');
expect(listSpy).toHaveBeenCalledTimes(1);
expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(1);
listSpy.mockRestore();
});
it('resolves self-stack identity once per request, cache hits included, and labels each stack from it', async () => {
it('resolves self-stack identity on cache miss only and serves hits from the cached payload', async () => {
mockGetStacks.mockResolvedValue(['sencho.yml', 'web.yml']);
mockGetBulkStackStatuses.mockResolvedValue({
sencho: { status: 'running' },
web: { status: 'running' },
});
const identitySpy = vi
.spyOn(selfStackGuard, 'resolveSelfStackIdentity')
.mockResolvedValue({ projectName: 'sencho', labels: null });
const first = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
// Hoisted: one resolution per request (including the cache hit) serves
// every stack in the response.
expect(identitySpy).toHaveBeenCalledTimes(2);
// Identity resolves once per computed fetch; the second request is a
// cache hit and inherits the enriched isSelf flags.
expect(identitySpy).toHaveBeenCalledTimes(1);
for (const res of [first, second]) {
expect(res.body['sencho.yml'].isSelf).toBe(true);
expect(res.body['web.yml'].isSelf).toBe(false);
}
});
identitySpy.mockRestore();
it('reflects a Git link on the next statuses request without waiting for the TTL', async () => {
mockGetStacks.mockResolvedValue(['web']);
mockGetBulkStackStatuses.mockResolvedValue({ web: { status: 'running' } });
// Stub upsert so the link does not clone the repository over the network.
const upsertSpy = vi
.spyOn(GitSourceService.getInstance(), 'upsert')
.mockResolvedValue({ stack_name: 'web' } as Awaited<ReturnType<typeof GitSourceService.prototype.upsert>>);
const before = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
expect(before.body.web.source).toBe('local');
await request(app)
.put('/api/stacks/web/git-source')
.set('Cookie', authCookie)
.send({
repo_url: 'https://github.com/example/web.git',
branch: 'main',
compose_path: 'compose.yaml',
auth_type: 'none',
});
// The stub skipped persistence; seed the row so the recompute sees it,
// standing in for what a real upsert would have stored.
DatabaseService.getInstance().upsertGitSource({
stack_name: 'web',
repo_url: 'https://github.com/example/web.git',
branch: 'main',
compose_path: 'compose.yaml',
compose_paths: ['compose.yaml'],
context_dir: null,
sync_env: false,
env_path: null,
auth_type: 'none',
encrypted_token: null,
auto_apply_on_webhook: false,
auto_deploy_on_apply: false,
last_applied_commit_sha: null,
last_applied_content_hash: null,
pending_commit_sha: null,
pending_compose_content: null,
pending_env_content: null,
pending_fetched_at: null,
last_debounce_at: null,
});
// Linking invalidated the statuses cache, so the next request recomputes
// with the fresh label instead of serving the stale cached 'local'.
const after = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
expect(after.body.web.source).toBe('git');
expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(2);
upsertSpy.mockRestore();
});
it('skips identity resolution entirely when the node has no stacks', async () => {
mockGetStacks.mockResolvedValue([]);
mockGetBulkStackStatuses.mockResolvedValue({});
const identitySpy = vi.spyOn(selfStackGuard, 'resolveSelfStackIdentity');
const res = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(res.body).toEqual({});
expect(identitySpy).not.toHaveBeenCalled();
identitySpy.mockRestore();
});
it('serves 200 with isSelf false everywhere when identity resolution degrades', async () => {
it('serves 200 with isSelf false everywhere when identity resolution degrades, and does not cache the degraded payload', async () => {
mockGetStacks.mockResolvedValue(['web.yml', 'db.yml']);
mockGetBulkStackStatuses.mockResolvedValue({
web: { status: 'running' },
db: { status: 'running' },
});
const identitySpy = vi
.spyOn(selfStackGuard, 'resolveSelfStackIdentity')
.mockResolvedValue({ projectName: null, labels: null });
identitySpy.mockResolvedValue({ projectName: null, labels: null, degraded: true });
const res = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
const first = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(res.body['web.yml'].isSelf).toBe(false);
expect(res.body['db.yml'].isSelf).toBe(false);
expect(first.status).toBe(200);
expect(first.body['web.yml'].isSelf).toBe(false);
expect(first.body['db.yml'].isSelf).toBe(false);
identitySpy.mockRestore();
// A degraded resolution reports every stack as not-self, which would
// un-gate destructive UI affordances on the Sencho stack itself, so it
// must not persist: the next request recomputes and re-resolves instead
// of serving a cached isSelf: false for a full TTL.
const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
expect(second.status).toBe(200);
expect(identitySpy).toHaveBeenCalledTimes(2);
expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(2);
});
it('falls back to local labels (200, not 500) when the git-source lookup throws', async () => {
it('falls back to local labels (200, not 500) when the git-source lookup throws, and does not cache the fallback', async () => {
mockGetStacks.mockResolvedValue(['web.yml']);
mockGetBulkStackStatuses.mockResolvedValue({ web: { status: 'running' } });
// The default healthy identity keeps the git-source scan as the only
// degradation source.
const listSpy = vi
.spyOn(GitSourceService.getInstance(), 'list')
.mockImplementation(() => { throw new Error('db locked'); });
const res = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(res.body['web.yml'].source).toBe('local');
const first = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
expect(first.status).toBe(200);
expect(first.body['web.yml'].source).toBe('local');
// The 'local' fallback must not persist: the next request recomputes and
// re-reads the git sources instead of serving the mislabel for a full TTL.
const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
expect(second.status).toBe(200);
expect(listSpy).toHaveBeenCalledTimes(2);
expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(2);
listSpy.mockRestore();
});
@@ -90,6 +90,7 @@ function makeStream(): FakeStream {
// ── Setup ──────────────────────────────────────────────────────────────
import { DockerEventService } from '../services/DockerEventService';
import { CacheService } from '../services/CacheService';
let stream: FakeStream;
let service: DockerEventService;
@@ -1324,7 +1325,17 @@ describe('DockerEventService - hardening', () => {
// ── State-invalidate broadcasts ────────────────────────────────────────
describe('DockerEventService - state-invalidate broadcasts', () => {
it('broadcasts state-invalidate on container start', async () => {
let invalidateSpy: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
invalidateSpy = vi.spyOn(CacheService.getInstance(), 'invalidate');
});
afterEach(() => {
invalidateSpy.mockRestore();
});
it('broadcasts state-invalidate and drops the statuses cache on container start', async () => {
service = new DockerEventService(7, 'node-7');
await service.start();
@@ -1344,6 +1355,11 @@ describe('DockerEventService - state-invalidate broadcasts', () => {
containerId: 'aaa',
action: 'start',
}));
// The UI refetch this broadcast triggers must recompute, not hit a
// cache entry made stale by the event. Only the statuses key is
// touched; the full invalidateNodeCaches helper is not used here.
expect(invalidateSpy).toHaveBeenCalledTimes(1);
expect(invalidateSpy).toHaveBeenCalledWith('stack-statuses:7');
});
it('does not broadcast stack state-invalidate for Sencho self-container events', async () => {
@@ -1368,6 +1384,7 @@ describe('DockerEventService - state-invalidate broadcasts', () => {
await vi.advanceTimersByTimeAsync(1);
expect(mockBroadcastEvent).not.toHaveBeenCalled();
expect(invalidateSpy).not.toHaveBeenCalled();
});
it('broadcasts state-invalidate on health_status:unhealthy', async () => {
@@ -1386,6 +1403,7 @@ describe('DockerEventService - state-invalidate broadcasts', () => {
(c[0] as { type?: string }).type === 'state-invalidate');
expect(states.length).toBeGreaterThan(0);
expect(states[0][0]).toMatchObject({ action: 'health_status', stackName: 'api' });
expect(invalidateSpy).toHaveBeenCalledWith('stack-statuses:1');
});
it('does not broadcast state-invalidate on non-state actions like exec_create', async () => {
@@ -1401,6 +1419,7 @@ describe('DockerEventService - state-invalidate broadcasts', () => {
await vi.advanceTimersByTimeAsync(1);
expect(mockBroadcastEvent).not.toHaveBeenCalled();
expect(invalidateSpy).not.toHaveBeenCalled();
});
});
@@ -11,7 +11,7 @@
* Service-layer logic (encryption, error mapping, mutex, pending lifecycle)
* is covered in git-source-service.test.ts.
*/
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import fs from 'fs';
@@ -20,6 +20,17 @@ import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './he
import { DatabaseService } from '../services/DatabaseService';
import { GitSourceService } from '../services/GitSourceService';
// ── Hoisted mocks (must come before importing the app) ─────────────────
// The statuses cache carries the source label, so link/unlink must drop it.
// Spy on the invalidation helper the routes call; the rest of the module
// (remote-meta invalidation) stays real.
const mockInvalidateNodeCaches = vi.hoisted(() => vi.fn());
vi.mock('../helpers/cacheInvalidation', async () => {
const actual = await vi.importActual<typeof import('../helpers/cacheInvalidation')>('../helpers/cacheInvalidation');
return { ...actual, invalidateNodeCaches: mockInvalidateNodeCaches };
});
function seedGitSource(stackName: string): void {
DatabaseService.getInstance().upsertGitSource({
stack_name: stackName,
@@ -538,3 +549,66 @@ describe('GET /api/git-sources', () => {
expect(res.status).toBe(401);
});
});
describe('git-source routes: statuses-cache invalidation', () => {
// The cached /stacks/statuses payload carries the source label, so link
// and unlink must drop the cache; read-only routes must not.
beforeEach(() => {
mockInvalidateNodeCaches.mockClear();
});
function seedStackDir(stackName: string): void {
const composeDir = process.env.COMPOSE_DIR!;
fs.mkdirSync(path.join(composeDir, stackName), { recursive: true });
fs.writeFileSync(path.join(composeDir, stackName, 'compose.yaml'), 'services:\n x:\n image: nginx\n');
}
it('invalidates node caches when linking a Git source', async () => {
seedStackDir('inv-link');
// Stub upsert so the assertion stays at the route layer without cloning a repo.
const upsertSpy = vi.spyOn(GitSourceService.getInstance(), 'upsert')
.mockResolvedValue({} as Awaited<ReturnType<typeof GitSourceService.prototype.upsert>>);
const res = await request(app)
.put('/api/stacks/inv-link/git-source')
.set('Authorization', `Bearer ${adminToken()}`)
.send({
repo_url: 'https://github.com/example/inv-link.git',
branch: 'main',
compose_path: 'compose.yaml',
auth_type: 'none',
});
expect(res.status).toBe(200);
expect(mockInvalidateNodeCaches).toHaveBeenCalledTimes(1);
expect(mockInvalidateNodeCaches).toHaveBeenCalledWith(expect.any(Number));
upsertSpy.mockRestore();
});
it('invalidates node caches when unlinking a Git source', async () => {
seedGitSource('inv-unlink');
const res = await request(app)
.delete('/api/stacks/inv-unlink/git-source')
.set('Authorization', `Bearer ${adminToken()}`);
expect(res.status).toBe(200);
expect(mockInvalidateNodeCaches).toHaveBeenCalledTimes(1);
expect(mockInvalidateNodeCaches).toHaveBeenCalledWith(expect.any(Number));
});
it('does not invalidate on GET of the Git source', async () => {
seedStackDir('inv-get');
seedGitSource('inv-get');
const res = await request(app)
.get('/api/stacks/inv-get/git-source')
.set('Authorization', `Bearer ${adminToken()}`);
expect(res.status).toBe(200);
expect(mockInvalidateNodeCaches).not.toHaveBeenCalled();
});
it('does not invalidate when dismissing a pending update', async () => {
seedGitSource('inv-dismiss');
const res = await request(app)
.post('/api/stacks/inv-dismiss/git-source/dismiss-pending')
.set('Authorization', `Bearer ${adminToken()}`);
expect(res.status).toBe(200);
expect(mockInvalidateNodeCaches).not.toHaveBeenCalled();
});
});
@@ -103,10 +103,10 @@ describe('[Stacks:debug] GET /api/stacks/statuses', () => {
expect(second).toMatch(/cacheOutcome=hit/);
// No docker call on a cache hit, so the subspan is null rather than 0.
expect(second).toMatch(/dockerMs=null/);
// Enrichment runs on every request, cache hits included, so its subspan
// is a number on both legs.
// Enrichment is part of the cached payload, so both subspans are
// compute-only telemetry: numbers on the compute leg, null on a hit.
expect(first).toMatch(/enrichmentMs=\d+/);
expect(second).toMatch(/enrichmentMs=\d+/);
expect(second).toMatch(/enrichmentMs=null/);
// The compute ran the fetcher exactly once across both requests.
expect(dockerCalls).toBe(1);
});
+23 -1
View File
@@ -161,10 +161,32 @@ describe('resolveSelfStackIdentity + isSelfStackByIdentity', () => {
expect(isSelfStackByIdentity(identity, 'sencho', '/app/compose')).toBe(false);
});
it('is false when no identity source is available', async () => {
it('is false when no identity source is available, and not degraded', async () => {
stubComposeProject(null);
const identity = await resolveSelfStackIdentity();
expect(isSelfStackByIdentity(identity, 'sencho')).toBe(false);
// No source attempted a Docker call, so this is the healthy
// "not running in Docker" state, not a degradation.
expect(identity.degraded).toBe(false);
});
it('marks the identity degraded when the container list read throws', async () => {
const runtimeId = 'f'.repeat(64);
process.env.HOSTNAME = runtimeId.slice(0, 12);
stubComposeProject('sencho');
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDocker: () => ({
listContainers: vi.fn().mockRejectedValue(new Error('socket unreachable')),
}),
} as unknown as ReturnType<typeof DockerController.getInstance>);
const identity = await resolveSelfStackIdentity();
// The surviving source still resolves, but the failed probe marks the
// whole identity degraded so callers refuse to cache it.
expect(identity.projectName).toBe('sencho');
expect(identity.labels).toBeNull();
expect(identity.degraded).toBe(true);
expect(isSelfStackByIdentity(identity, 'sencho')).toBe(true);
});
it('resolves identity with at most one container list read, shared by all stacks', async () => {
+5 -1
View File
@@ -44,4 +44,8 @@ export const MFA_REPLAY_PURGE_INTERVAL_MS = 60 * 1000;
// Keys are per-node: "stats:<nodeId>", "system-stats:<nodeId>", "stack-statuses:<nodeId>".
export const STATS_CACHE_TTL_MS = 2_000;
export const SYSTEM_STATS_CACHE_TTL_MS = 3_000;
export const STACK_STATUSES_CACHE_TTL_MS = 3_000;
// Stack statuses are cached past the frontend dashboard poll cadence (10s),
// so ordinary polls hit instead of recomputing. Container events and
// lifecycle mutations invalidate the key; 15s bounds worst-case staleness for
// any missed invalidation path.
export const STACK_STATUSES_CACHE_TTL_MS = 15_000;
+51 -27
View File
@@ -50,19 +50,18 @@ function workingDirMatchesStack(workingDir: string | undefined, stackName: strin
}
async function getRunningContainerLabels(): Promise<Record<string, string> | null> {
try {
const runtimeIds = await getRuntimeContainerIdCandidates();
if (runtimeIds.length === 0) return null;
const runtimeIds = await getRuntimeContainerIdCandidates();
if (runtimeIds.length === 0) return null;
const containers = await DockerController.getInstance().getDocker().listContainers({ all: true }) as ListedContainer[];
const selfContainer = containers.find((container) => {
const containerId = container.Id;
return typeof containerId === 'string' && runtimeIds.some(id => matchesContainerId(containerId, id));
});
return selfContainer?.Labels ?? null;
} catch {
return null;
}
// A listContainers failure (Docker socket unreachable) propagates so
// resolveSelfStackIdentity can mark the resolution degraded. Callers that
// need null-on-failure wrap this call themselves.
const containers = await DockerController.getInstance().getDocker().listContainers({ all: true }) as ListedContainer[];
const selfContainer = containers.find((container) => {
const containerId = container.Id;
return typeof containerId === 'string' && runtimeIds.some(id => matchesContainerId(containerId, id));
});
return selfContainer?.Labels ?? null;
}
async function runningContainerMatchesStack(stackName: string, composeDir?: string): Promise<boolean> {
@@ -89,10 +88,14 @@ export async function getSelfStackProjectName(): Promise<string | null> {
/** Directory name of the running Sencho compose project, when it is under COMPOSE_DIR. */
export async function getSelfStackDirectoryName(composeDir?: string): Promise<string | null> {
const labels = await getRunningContainerLabels();
const workingDirStack = stackNameFromWorkingDir(labels?.['com.docker.compose.project.working_dir'], composeDir);
if (workingDirStack) return workingDirStack;
return getSelfStackProjectName();
try {
const labels = await getRunningContainerLabels();
const workingDirStack = stackNameFromWorkingDir(labels?.['com.docker.compose.project.working_dir'], composeDir);
if (workingDirStack) return workingDirStack;
return getSelfStackProjectName();
} catch {
return null;
}
}
/** True when the stack appears to be the running Sencho compose project. */
@@ -117,25 +120,46 @@ export async function isSelfStack(stackName: string, composeDir?: string): Promi
export interface SelfStackIdentity {
projectName: string | null;
labels: Record<string, string> | null;
/**
* True when the container-labels probe failed (Docker socket unreachable).
* A degraded identity cannot be trusted to classify every stack correctly,
* so it must not be cached: the next request should re-resolve. Running
* outside Docker is NOT degraded: both sources legitimately resolve to
* null there and the identity is correct as-is.
*/
degraded: boolean;
}
/**
* Resolves both identity sources. Each failure degrades only its own
* source to null, matching the old per-stack behavior where a labels
* failure never discarded the resolved project name.
* Resolves both identity sources. A labels-probe failure degrades only that
* source to null and marks the resolution degraded, matching the old
* per-stack behavior where a labels failure never discarded the resolved
* project name. getSelfStackProjectName swallows its own failures and
* returns null, so it cannot mark the identity degraded.
*/
export async function resolveSelfStackIdentity(): Promise<SelfStackIdentity> {
const projectName = await getSelfStackProjectName().catch((error) => {
console.error('Failed to resolve self-stack project name; self-stack check degraded:', error);
return null;
});
const labels = await getRunningContainerLabels().catch((error) => {
const projectName = await getSelfStackProjectName();
let labels: Record<string, string> | null = null;
let degraded = false;
try {
labels = await getRunningContainerLabels();
} catch (error) {
console.error('Failed to resolve self-stack container labels; self-stack check degraded:', error);
return null;
});
return { projectName, labels };
degraded = true;
}
return { projectName, labels, degraded };
}
/**
* Identity used when no resolution is attempted (empty fleet). Not degraded:
* with no stacks there is nothing to mislabel, so the payload caches as-is.
*/
export const UNRESOLVED_SELF_STACK_IDENTITY: SelfStackIdentity = {
projectName: null,
labels: null,
degraded: false,
};
/** isSelf semantics identical to isSelfStack() when the identity resolves successfully. */
export function isSelfStackByIdentity(identity: SelfStackIdentity, stackName: string, composeDir?: string): boolean {
if (identity.projectName === stackName) return true;
+14
View File
@@ -226,6 +226,13 @@ stackGitSourceRouter.put('/:stackName/git-source', async (req: Request, res: Res
autoDeployOnApply,
});
// The cached /stacks/statuses payload carries the source label; drop it
// before responding so a client refetch on this response recomputes. The
// full invalidateNodeCaches helper is deliberate here (matching every
// other mutation route): link/unlink is a low-frequency user action, so
// dropping the project-name map and file-root allowlists alongside is
// harmless, unlike the high-frequency container-event path.
invalidateNodeCaches(req.nodeId);
console.log(`[GitSource] Configured git source for ${stackName}`);
res.json(source);
} catch (error) {
@@ -254,6 +261,13 @@ stackGitSourceRouter.delete('/:stackName/git-source', async (req: Request, res:
return;
}
GitSourceService.getInstance().delete(stackName);
// The cached /stacks/statuses payload carries the source label; drop it
// before responding so a client refetch on this response recomputes. The
// full invalidateNodeCaches helper is deliberate here (matching every
// other mutation route): link/unlink is a low-frequency user action, so
// dropping the project-name map and file-root allowlists alongside is
// harmless, unlike the high-frequency container-event path.
invalidateNodeCaches(req.nodeId);
console.log(`[GitSource] Removed git source for ${stackName}`);
res.json({ success: true });
} catch (error) {
+49 -28
View File
@@ -75,6 +75,7 @@ import {
refuseIfSelfStack,
resolveSelfStackIdentity,
selfStackProtectedBulkResult,
UNRESOLVED_SELF_STACK_IDENTITY,
} from '../helpers/selfStackGuard';
import { getActiveCapabilities, STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry';
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
@@ -344,6 +345,10 @@ stacksRouter.get('/statuses', async (req: Request, res: Response) => {
let enrichmentMs: number | null = null;
let count = 0;
try {
// Enrichment (git-source labels, self identity) is part of the cached
// payload so cache hits serve fully decorated statuses with no per-request
// work. The git label lookup failure still falls back to 'local' and must
// not take down the primary status payload.
const { value: result, outcome: fetchOutcome } = await CacheService.getInstance().getOrFetchWithMeta(
`stack-statuses:${req.nodeId}`,
STACK_STATUSES_CACHE_TTL_MS,
@@ -359,38 +364,54 @@ stacksRouter.get('/statuses', async (req: Request, res: Response) => {
const name = stack.replace(/\.(yml|yaml)$/, '');
data[stack] = bulkInfo[name] ?? { status: 'unknown' };
}
return data;
const enrichmentStartedAt = Date.now();
let gitStackNames = new Set<string>();
let gitSourcesDegraded = false;
try {
gitStackNames = new Set(GitSourceService.getInstance().list().map((s) => s.stack_name));
} catch (sourceError) {
console.error(`Failed to load git sources for status labels on node ${req.nodeId}; defaulting to local:`, sourceError);
gitSourcesDegraded = true;
}
// Self-stack identity is resolved once per request instead of once per
// stack, so cache misses pay a single container-list call, not N.
const selfIdentity = stackNames.length > 0
? await resolveSelfStackIdentity()
: UNRESOLVED_SELF_STACK_IDENTITY;
const withSource: Record<string, BulkStackInfo & { source: 'local' | 'git'; isSelf: boolean }> = {};
const composeDir = FileSystemService.getInstance(req.nodeId).getBaseDir();
for (const [stack, info] of Object.entries(data)) {
const name = stack.replace(/\.(yml|yaml)$/, '');
withSource[stack] = {
...info,
source: gitStackNames.has(name) ? 'git' : 'local',
isSelf: isSelfStackByIdentity(selfIdentity, name, composeDir),
};
}
enrichmentMs = Date.now() - enrichmentStartedAt;
// The payload is flagged degraded when any enrichment source failed
// (Docker socket unreachable, git-source scan failure) so the route
// can refuse to let a mislabeled payload persist.
return { data: withSource, degraded: selfIdentity.degraded || gitSourcesDegraded };
},
);
cacheOutcome = fetchOutcome;
count = Object.keys(result).length;
const enrichmentStartedAt = Date.now();
// Git-source labels are computed live, outside the cache, so linking or
// unlinking a stack's Git source is reflected immediately. The Docker
// status portion keeps its short TTL; only the cheap source label is fresh.
// The label is cosmetic, so a lookup failure must not take down the primary
// status payload: fall back to labeling everything 'local'.
let gitStackNames = new Set<string>();
try {
gitStackNames = new Set(GitSourceService.getInstance().list().map((s) => s.stack_name));
} catch (sourceError) {
console.error('Failed to load git sources for status labels; defaulting to local:', sourceError);
const { data, degraded } = result;
count = Object.keys(data).length;
// A degraded identity resolution (Docker socket unreachable) cannot be
// trusted to classify every stack, which un-gates destructive UI
// affordances on the Sencho stack itself, and a failed git-source scan
// mislabels every source badge as 'local'. Never let either mislabel
// persist for a full TTL: serve the live result, drop the cache entry,
// and let the next request re-resolve. Everything between the fetch and
// this invalidate is synchronous, so no concurrent reader can observe
// the degraded entry. Running outside Docker is not degraded (both
// identity sources legitimately resolve to null there), and an empty
// fleet skips resolution entirely; both are cached as-is.
if (fetchOutcome === 'computed' && count > 0 && degraded) {
CacheService.getInstance().invalidate(`stack-statuses:${req.nodeId}`);
}
// Self-stack identity is resolved once per request instead of once per
// stack, so cache hits no longer pay N container-list calls.
const selfIdentity = count > 0 ? await resolveSelfStackIdentity() : { projectName: null, labels: null };
const withSource: Record<string, BulkStackInfo & { source: 'local' | 'git' }> = {};
const composeDir = FileSystemService.getInstance(req.nodeId).getBaseDir();
for (const [stack, info] of Object.entries(result)) {
const name = stack.replace(/\.(yml|yaml)$/, '');
withSource[stack] = {
...info,
source: gitStackNames.has(name) ? 'git' : 'local',
isSelf: isSelfStackByIdentity(selfIdentity, name, composeDir),
};
}
enrichmentMs = Date.now() - enrichmentStartedAt;
res.json(withSource);
res.json(data);
} catch (error) {
outcome = 'error';
console.error('Failed to fetch stack statuses:', error);
+8 -1
View File
@@ -3,6 +3,7 @@ import { NodeRegistry } from './NodeRegistry';
import { NotificationCategory, NotificationService } from './NotificationService';
import { DatabaseService } from './DatabaseService';
import SelfIdentityService from './SelfIdentityService';
import { CacheService } from './CacheService';
import {
classifyDie,
classifyGapExit,
@@ -449,8 +450,14 @@ export class DockerEventService {
// Push a lightweight state-invalidate signal so connected UIs can
// refetch stack statuses immediately on a real container event,
// without waiting for the next polling tick. This is fire-and-forget
// and is NOT persisted to the alerts history.
// and is NOT persisted to the alerts history. Drop the statuses cache
// key alongside the broadcast so the UI's refetch recomputes instead
// of serving an entry the event just made stale. The full
// invalidateNodeCaches helper is not used: container events do not
// reshape the project-name map or file-root allowlists, and the stats
// key self-refreshes on its own 2s TTL.
if (STATE_INVALIDATE_ACTIONS.has(baseAction) && !isSelf) {
CacheService.getInstance().invalidate(`stack-statuses:${this.nodeId}`);
this.notifier.broadcastEvent({
type: 'state-invalidate',
scope: 'stack',