perf(statuses): align stack-status cache TTL and invalidation with polling (#1814)

The 3s stack-statuses cache TTL never survived the 10s dashboard poll, so
every ordinary poll recomputed. Raise the TTL to 15s and move the git-source
label and self-identity enrichment inside the cached payload so cache hits
serve fully decorated statuses with zero per-request work.

Invalidation closes the gaps the longer TTL would otherwise widen:

- DockerEventService drops stack-statuses:<nodeId> on container state events
  so the UI's state-invalidate refetch recomputes instead of hitting a stale
  entry. The narrow key only: container events do not reshape stack identity
  or file roots, and the stats key self-refreshes on its own 2s TTL.
- git-source link and unlink invalidate node caches before responding, so the
  source label stays fresh without waiting for the TTL.
- a payload whose enrichment degraded (identity probe failure or git-source
  scan failure) is never cached, so a mislabeled not-self or 'local' badge
  cannot persist for a full TTL window. Running outside Docker is not
  degradation, so host installs cache normally.

backend/src/__tests__/cache-endpoints.test.ts

@@ -14,12 +14,13 @@
  * layer rather than hitting the real Docker socket, so they run in CI
  * without requiring any external daemon.
  */
-import { describe, it, expect, beforeAll, afterAll, vi, beforeEach } from 'vitest';
+import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
 import request from 'supertest';
 import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_PASSWORD } from './helpers/setupTestDb';
 import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsMock } from './helpers/arcstatsFsMock';
 import { GitSourceService } from '../services/GitSourceService';
 import type { PublicGitSource } from '../services/GitSourceService';
+import { DatabaseService } from '../services/DatabaseService';
 import * as selfStackGuard from '../helpers/selfStackGuard';
 
 // ── Hoisted mocks (must come before importing the app) ─────────────────
@@ -230,6 +231,25 @@ describe('GET /api/fleet/overview local-node memory', () => {
 // ── /api/stacks/statuses ───────────────────────────────────────────────
 
 describe('GET /api/stacks/statuses caching', () => {
+  // A healthy identity by default so computed results persist in the cache;
+  // a degraded resolution would intentionally drop the entry after the call.
+  // Tests that need a different identity override the mock on the shared spy.
+  let identitySpy: ReturnType<typeof vi.spyOn>;
+
+  beforeEach(() => {
+    identitySpy = vi
+      .spyOn(selfStackGuard, 'resolveSelfStackIdentity')
+      .mockResolvedValue({ projectName: 'sencho', labels: null, degraded: false });
+  });
+
+  afterEach(() => {
+    identitySpy.mockRestore();
+    // The git-link test seeds a 'web' source row; drop it even when that
+    // test fails mid-way so later tests never see it. Delete is a no-op when
+    // the row is absent.
+    GitSourceService.getInstance().delete('web');
+  });
+
   it('serves repeat calls from cache without re-invoking the filesystem', async () => {
     mockGetStacks.mockResolvedValue(['web', 'db']);
     mockGetBulkStackStatuses.mockResolvedValue({
@@ -237,11 +257,12 @@ describe('GET /api/stacks/statuses caching', () => {
       db: { status: 'running' },
     });
 
-    await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
-    await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
+    const first = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
+    const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
 
     expect(mockGetStacks).toHaveBeenCalledTimes(1);
     expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(1);
+    expect(second.body).toEqual(first.body);
   });
 
   it('invalidates on POST /api/stacks', async () => {
@@ -260,7 +281,7 @@ describe('GET /api/stacks/statuses caching', () => {
     expect(mockGetStacks).toHaveBeenCalledTimes(2);
   });
 
-  it('labels each stack with its git/local source, computed outside the cache', async () => {
+  it('labels each stack with its git/local source, served from the cache', async () => {
     mockGetStacks.mockResolvedValue(['web.yml', 'db.yml']);
     mockGetBulkStackStatuses.mockResolvedValue({
       web: { status: 'running' },
@@ -275,83 +296,143 @@ describe('GET /api/stacks/statuses caching', () => {
     expect(first.body['web.yml'].source).toBe('git');
     expect(first.body['db.yml'].source).toBe('local');
 
-    // Source is recomputed live even when the Docker-status payload is cached:
-    // unlinking `web` flips it to local on the next request without a cache flush.
-    listSpy.mockReturnValue([]);
+    // The source label is part of the cached payload: the second request is a
+    // cache hit and does not re-scan the git-source table. Link/unlink keep
+    // the label fresh by invalidating the cache (git-source routes).
     const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
-    expect(second.body['web.yml'].source).toBe('local');
-    expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(1); // status portion served from cache
+    expect(second.body['web.yml'].source).toBe('git');
+    expect(listSpy).toHaveBeenCalledTimes(1);
+    expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(1);
 
     listSpy.mockRestore();
   });
 
-  it('resolves self-stack identity once per request, cache hits included, and labels each stack from it', async () => {
+  it('resolves self-stack identity on cache miss only and serves hits from the cached payload', async () => {
     mockGetStacks.mockResolvedValue(['sencho.yml', 'web.yml']);
     mockGetBulkStackStatuses.mockResolvedValue({
       sencho: { status: 'running' },
       web: { status: 'running' },
     });
-    const identitySpy = vi
-      .spyOn(selfStackGuard, 'resolveSelfStackIdentity')
-      .mockResolvedValue({ projectName: 'sencho', labels: null });
 
     const first = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
     const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
 
-    // Hoisted: one resolution per request (including the cache hit) serves
-    // every stack in the response.
-    expect(identitySpy).toHaveBeenCalledTimes(2);
+    // Identity resolves once per computed fetch; the second request is a
+    // cache hit and inherits the enriched isSelf flags.
+    expect(identitySpy).toHaveBeenCalledTimes(1);
     for (const res of [first, second]) {
       expect(res.body['sencho.yml'].isSelf).toBe(true);
       expect(res.body['web.yml'].isSelf).toBe(false);
     }
+  });
 
-    identitySpy.mockRestore();
+  it('reflects a Git link on the next statuses request without waiting for the TTL', async () => {
+    mockGetStacks.mockResolvedValue(['web']);
+    mockGetBulkStackStatuses.mockResolvedValue({ web: { status: 'running' } });
+    // Stub upsert so the link does not clone the repository over the network.
+    const upsertSpy = vi
+      .spyOn(GitSourceService.getInstance(), 'upsert')
+      .mockResolvedValue({ stack_name: 'web' } as Awaited<ReturnType<typeof GitSourceService.prototype.upsert>>);
+
+    const before = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
+    expect(before.body.web.source).toBe('local');
+
+    await request(app)
+      .put('/api/stacks/web/git-source')
+      .set('Cookie', authCookie)
+      .send({
+        repo_url: 'https://github.com/example/web.git',
+        branch: 'main',
+        compose_path: 'compose.yaml',
+        auth_type: 'none',
+      });
+    // The stub skipped persistence; seed the row so the recompute sees it,
+    // standing in for what a real upsert would have stored.
+    DatabaseService.getInstance().upsertGitSource({
+      stack_name: 'web',
+      repo_url: 'https://github.com/example/web.git',
+      branch: 'main',
+      compose_path: 'compose.yaml',
+      compose_paths: ['compose.yaml'],
+      context_dir: null,
+      sync_env: false,
+      env_path: null,
+      auth_type: 'none',
+      encrypted_token: null,
+      auto_apply_on_webhook: false,
+      auto_deploy_on_apply: false,
+      last_applied_commit_sha: null,
+      last_applied_content_hash: null,
+      pending_commit_sha: null,
+      pending_compose_content: null,
+      pending_env_content: null,
+      pending_fetched_at: null,
+      last_debounce_at: null,
+    });
+
+    // Linking invalidated the statuses cache, so the next request recomputes
+    // with the fresh label instead of serving the stale cached 'local'.
+    const after = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
+    expect(after.body.web.source).toBe('git');
+    expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(2);
+
+    upsertSpy.mockRestore();
   });
 
   it('skips identity resolution entirely when the node has no stacks', async () => {
     mockGetStacks.mockResolvedValue([]);
     mockGetBulkStackStatuses.mockResolvedValue({});
-    const identitySpy = vi.spyOn(selfStackGuard, 'resolveSelfStackIdentity');
 
     const res = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
 
     expect(res.status).toBe(200);
     expect(res.body).toEqual({});
     expect(identitySpy).not.toHaveBeenCalled();
-
-    identitySpy.mockRestore();
   });
 
-  it('serves 200 with isSelf false everywhere when identity resolution degrades', async () => {
+  it('serves 200 with isSelf false everywhere when identity resolution degrades, and does not cache the degraded payload', async () => {
     mockGetStacks.mockResolvedValue(['web.yml', 'db.yml']);
     mockGetBulkStackStatuses.mockResolvedValue({
       web: { status: 'running' },
       db: { status: 'running' },
     });
-    const identitySpy = vi
-      .spyOn(selfStackGuard, 'resolveSelfStackIdentity')
-      .mockResolvedValue({ projectName: null, labels: null });
+    identitySpy.mockResolvedValue({ projectName: null, labels: null, degraded: true });
 
-    const res = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
+    const first = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
 
-    expect(res.status).toBe(200);
-    expect(res.body['web.yml'].isSelf).toBe(false);
-    expect(res.body['db.yml'].isSelf).toBe(false);
+    expect(first.status).toBe(200);
+    expect(first.body['web.yml'].isSelf).toBe(false);
+    expect(first.body['db.yml'].isSelf).toBe(false);
 
-    identitySpy.mockRestore();
+    // A degraded resolution reports every stack as not-self, which would
+    // un-gate destructive UI affordances on the Sencho stack itself, so it
+    // must not persist: the next request recomputes and re-resolves instead
+    // of serving a cached isSelf: false for a full TTL.
+    const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
+    expect(second.status).toBe(200);
+    expect(identitySpy).toHaveBeenCalledTimes(2);
+    expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(2);
   });
 
-  it('falls back to local labels (200, not 500) when the git-source lookup throws', async () => {
+  it('falls back to local labels (200, not 500) when the git-source lookup throws, and does not cache the fallback', async () => {
     mockGetStacks.mockResolvedValue(['web.yml']);
     mockGetBulkStackStatuses.mockResolvedValue({ web: { status: 'running' } });
+    // The default healthy identity keeps the git-source scan as the only
+    // degradation source.
     const listSpy = vi
       .spyOn(GitSourceService.getInstance(), 'list')
       .mockImplementation(() => { throw new Error('db locked'); });
 
-    const res = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
-    expect(res.status).toBe(200);
-    expect(res.body['web.yml'].source).toBe('local');
+    const first = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
+    expect(first.status).toBe(200);
+    expect(first.body['web.yml'].source).toBe('local');
+
+    // The 'local' fallback must not persist: the next request recomputes and
+    // re-reads the git sources instead of serving the mislabel for a full TTL.
+    const second = await request(app).get('/api/stacks/statuses').set('Cookie', authCookie);
+    expect(second.status).toBe(200);
+    expect(listSpy).toHaveBeenCalledTimes(2);
+    expect(mockGetBulkStackStatuses).toHaveBeenCalledTimes(2);
 
     listSpy.mockRestore();
   });

backend/src/__tests__/docker-event-service.test.ts

@@ -90,6 +90,7 @@ function makeStream(): FakeStream {
 // ── Setup ──────────────────────────────────────────────────────────────
 
 import { DockerEventService } from '../services/DockerEventService';
+import { CacheService } from '../services/CacheService';
 
 let stream: FakeStream;
 let service: DockerEventService;
@@ -1324,7 +1325,17 @@ describe('DockerEventService - hardening', () => {
 // ── State-invalidate broadcasts ────────────────────────────────────────
 
 describe('DockerEventService - state-invalidate broadcasts', () => {
-    it('broadcasts state-invalidate on container start', async () => {
+    let invalidateSpy: ReturnType<typeof vi.spyOn>;
+
+    beforeEach(() => {
+        invalidateSpy = vi.spyOn(CacheService.getInstance(), 'invalidate');
+    });
+
+    afterEach(() => {
+        invalidateSpy.mockRestore();
+    });
+
+    it('broadcasts state-invalidate and drops the statuses cache on container start', async () => {
         service = new DockerEventService(7, 'node-7');
         await service.start();
 
@@ -1344,6 +1355,11 @@ describe('DockerEventService - state-invalidate broadcasts', () => {
             containerId: 'aaa',
             action: 'start',
         }));
+        // The UI refetch this broadcast triggers must recompute, not hit a
+        // cache entry made stale by the event. Only the statuses key is
+        // touched; the full invalidateNodeCaches helper is not used here.
+        expect(invalidateSpy).toHaveBeenCalledTimes(1);
+        expect(invalidateSpy).toHaveBeenCalledWith('stack-statuses:7');
     });
 
     it('does not broadcast stack state-invalidate for Sencho self-container events', async () => {
@@ -1368,6 +1384,7 @@ describe('DockerEventService - state-invalidate broadcasts', () => {
         await vi.advanceTimersByTimeAsync(1);
 
         expect(mockBroadcastEvent).not.toHaveBeenCalled();
+        expect(invalidateSpy).not.toHaveBeenCalled();
     });
 
     it('broadcasts state-invalidate on health_status:unhealthy', async () => {
@@ -1386,6 +1403,7 @@ describe('DockerEventService - state-invalidate broadcasts', () => {
             (c[0] as { type?: string }).type === 'state-invalidate');
         expect(states.length).toBeGreaterThan(0);
         expect(states[0][0]).toMatchObject({ action: 'health_status', stackName: 'api' });
+        expect(invalidateSpy).toHaveBeenCalledWith('stack-statuses:1');
     });
 
     it('does not broadcast state-invalidate on non-state actions like exec_create', async () => {
@@ -1401,6 +1419,7 @@ describe('DockerEventService - state-invalidate broadcasts', () => {
         await vi.advanceTimersByTimeAsync(1);
 
         expect(mockBroadcastEvent).not.toHaveBeenCalled();
+        expect(invalidateSpy).not.toHaveBeenCalled();
     });
 });
 

backend/src/__tests__/git-source-routes.test.ts

@@ -11,7 +11,7 @@
  * Service-layer logic (encryption, error mapping, mutex, pending lifecycle)
  * is covered in git-source-service.test.ts.
  */
-import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
+import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
 import request from 'supertest';
 import jwt from 'jsonwebtoken';
 import fs from 'fs';
@@ -20,6 +20,17 @@ import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './he
 import { DatabaseService } from '../services/DatabaseService';
 import { GitSourceService } from '../services/GitSourceService';
 
+// ── Hoisted mocks (must come before importing the app) ─────────────────
+
+// The statuses cache carries the source label, so link/unlink must drop it.
+// Spy on the invalidation helper the routes call; the rest of the module
+// (remote-meta invalidation) stays real.
+const mockInvalidateNodeCaches = vi.hoisted(() => vi.fn());
+vi.mock('../helpers/cacheInvalidation', async () => {
+    const actual = await vi.importActual<typeof import('../helpers/cacheInvalidation')>('../helpers/cacheInvalidation');
+    return { ...actual, invalidateNodeCaches: mockInvalidateNodeCaches };
+});
+
 function seedGitSource(stackName: string): void {
     DatabaseService.getInstance().upsertGitSource({
         stack_name: stackName,
@@ -538,3 +549,66 @@ describe('GET /api/git-sources', () => {
         expect(res.status).toBe(401);
     });
 });
+
+describe('git-source routes: statuses-cache invalidation', () => {
+    // The cached /stacks/statuses payload carries the source label, so link
+    // and unlink must drop the cache; read-only routes must not.
+    beforeEach(() => {
+        mockInvalidateNodeCaches.mockClear();
+    });
+
+    function seedStackDir(stackName: string): void {
+        const composeDir = process.env.COMPOSE_DIR!;
+        fs.mkdirSync(path.join(composeDir, stackName), { recursive: true });
+        fs.writeFileSync(path.join(composeDir, stackName, 'compose.yaml'), 'services:\n  x:\n    image: nginx\n');
+    }
+
+    it('invalidates node caches when linking a Git source', async () => {
+        seedStackDir('inv-link');
+        // Stub upsert so the assertion stays at the route layer without cloning a repo.
+        const upsertSpy = vi.spyOn(GitSourceService.getInstance(), 'upsert')
+            .mockResolvedValue({} as Awaited<ReturnType<typeof GitSourceService.prototype.upsert>>);
+        const res = await request(app)
+            .put('/api/stacks/inv-link/git-source')
+            .set('Authorization', `Bearer ${adminToken()}`)
+            .send({
+                repo_url: 'https://github.com/example/inv-link.git',
+                branch: 'main',
+                compose_path: 'compose.yaml',
+                auth_type: 'none',
+            });
+        expect(res.status).toBe(200);
+        expect(mockInvalidateNodeCaches).toHaveBeenCalledTimes(1);
+        expect(mockInvalidateNodeCaches).toHaveBeenCalledWith(expect.any(Number));
+        upsertSpy.mockRestore();
+    });
+
+    it('invalidates node caches when unlinking a Git source', async () => {
+        seedGitSource('inv-unlink');
+        const res = await request(app)
+            .delete('/api/stacks/inv-unlink/git-source')
+            .set('Authorization', `Bearer ${adminToken()}`);
+        expect(res.status).toBe(200);
+        expect(mockInvalidateNodeCaches).toHaveBeenCalledTimes(1);
+        expect(mockInvalidateNodeCaches).toHaveBeenCalledWith(expect.any(Number));
+    });
+
+    it('does not invalidate on GET of the Git source', async () => {
+        seedStackDir('inv-get');
+        seedGitSource('inv-get');
+        const res = await request(app)
+            .get('/api/stacks/inv-get/git-source')
+            .set('Authorization', `Bearer ${adminToken()}`);
+        expect(res.status).toBe(200);
+        expect(mockInvalidateNodeCaches).not.toHaveBeenCalled();
+    });
+
+    it('does not invalidate when dismissing a pending update', async () => {
+        seedGitSource('inv-dismiss');
+        const res = await request(app)
+            .post('/api/stacks/inv-dismiss/git-source/dismiss-pending')
+            .set('Authorization', `Bearer ${adminToken()}`);
+        expect(res.status).toBe(200);
+        expect(mockInvalidateNodeCaches).not.toHaveBeenCalled();
+    });
+});

backend/src/__tests__/hydration-timing-routes.test.ts

@@ -103,10 +103,10 @@ describe('[Stacks:debug] GET /api/stacks/statuses', () => {
     expect(second).toMatch(/cacheOutcome=hit/);
     // No docker call on a cache hit, so the subspan is null rather than 0.
     expect(second).toMatch(/dockerMs=null/);
-    // Enrichment runs on every request, cache hits included, so its subspan
-    // is a number on both legs.
+    // Enrichment is part of the cached payload, so both subspans are
+    // compute-only telemetry: numbers on the compute leg, null on a hit.
     expect(first).toMatch(/enrichmentMs=\d+/);
-    expect(second).toMatch(/enrichmentMs=\d+/);
+    expect(second).toMatch(/enrichmentMs=null/);
     // The compute ran the fetcher exactly once across both requests.
     expect(dockerCalls).toBe(1);
   });

backend/src/__tests__/self-stack-guard.test.ts

@@ -161,10 +161,32 @@ describe('resolveSelfStackIdentity + isSelfStackByIdentity', () => {
     expect(isSelfStackByIdentity(identity, 'sencho', '/app/compose')).toBe(false);
   });
 
-  it('is false when no identity source is available', async () => {
+  it('is false when no identity source is available, and not degraded', async () => {
     stubComposeProject(null);
     const identity = await resolveSelfStackIdentity();
     expect(isSelfStackByIdentity(identity, 'sencho')).toBe(false);
+    // No source attempted a Docker call, so this is the healthy
+    // "not running in Docker" state, not a degradation.
+    expect(identity.degraded).toBe(false);
+  });
+
+  it('marks the identity degraded when the container list read throws', async () => {
+    const runtimeId = 'f'.repeat(64);
+    process.env.HOSTNAME = runtimeId.slice(0, 12);
+    stubComposeProject('sencho');
+    vi.spyOn(DockerController, 'getInstance').mockReturnValue({
+      getDocker: () => ({
+        listContainers: vi.fn().mockRejectedValue(new Error('socket unreachable')),
+      }),
+    } as unknown as ReturnType<typeof DockerController.getInstance>);
+
+    const identity = await resolveSelfStackIdentity();
+    // The surviving source still resolves, but the failed probe marks the
+    // whole identity degraded so callers refuse to cache it.
+    expect(identity.projectName).toBe('sencho');
+    expect(identity.labels).toBeNull();
+    expect(identity.degraded).toBe(true);
+    expect(isSelfStackByIdentity(identity, 'sencho')).toBe(true);
   });
 
   it('resolves identity with at most one container list read, shared by all stacks', async () => {