diff --git a/backend/src/__tests__/mesh-data-plane-revalidate.test.ts b/backend/src/__tests__/mesh-data-plane-revalidate.test.ts new file mode 100644 index 00000000..b63563f9 --- /dev/null +++ b/backend/src/__tests__/mesh-data-plane-revalidate.test.ts @@ -0,0 +1,649 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import type { MeshActivityEvent, MeshDataPlaneStatus } from '../services/MeshService'; + +let tmpDir: string; +let MeshService: typeof import('../services/MeshService').MeshService; +let DockerController: typeof import('../services/DockerController').default; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let MESH_RECREATE_THROTTLE_MS: number; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ MeshService, MESH_RECREATE_THROTTLE_MS } = await import('../services/MeshService')); + ({ default: DockerController } = await import('../services/DockerController')); + ({ DatabaseService } = await import('../services/DatabaseService')); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +type MutableSvc = { + activity: MeshActivityEvent[]; + senchoIp: string | null; + meshSubnet: string; + networkSetupError: string | null; + dataPlaneStatus: MeshDataPlaneStatus; + dataPlaneRevalidateInFlight: boolean; + dataPlaneRevalidateTimer?: NodeJS.Timeout; + lastRecreateAttemptAt: number; + started: boolean; +}; + +let prevHostnameEnv: string | undefined; + +beforeEach(() => { + prevHostnameEnv = process.env.HOSTNAME; + // Mirror the production recipe (`docker run --hostname sencho ...`) so + // the revalidator's attachment check resolves via container-name match + // against `Containers..Name`. The 12-char container-ID prefix path + // is exercised by tests that drop the `name` field. + process.env.HOSTNAME = 'sencho'; + const svc = MeshService.getInstance() as unknown as MutableSvc; + svc.activity = []; + svc.senchoIp = '172.30.0.2'; + svc.meshSubnet = '172.30.0.0/24'; + svc.networkSetupError = null; + svc.dataPlaneStatus = { + ok: true, + reason: 'ok', + message: null, + subnet: '172.30.0.0/24', + }; + svc.dataPlaneRevalidateInFlight = false; + svc.lastRecreateAttemptAt = 0; + DatabaseService.getInstance().updateGlobalSetting('mesh_auto_recreate', '0'); +}); + +afterEach(() => { + if (prevHostnameEnv === undefined) delete process.env.HOSTNAME; + else process.env.HOSTNAME = prevHostnameEnv; + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +type FakeController = { + createNetwork: ReturnType; + inspectNetwork: ReturnType; + connectContainerToNetwork: ReturnType; +}; + +function mockDocker(overrides: Partial = {}): FakeController { + const fake: FakeController = { + createNetwork: vi.fn().mockResolvedValue(undefined), + inspectNetwork: vi.fn(), + connectContainerToNetwork: vi.fn().mockResolvedValue(undefined), + ...overrides, + }; + vi.spyOn(DockerController, 'getInstance').mockReturnValue( + fake as unknown as ReturnType, + ); + return fake; +} + +function networkPresent(subnet: string, attached: Array): { + IPAM: { Config: Array<{ Subnet: string; IPRange?: string }> }; + Containers: Record; +} { + const containers: Record = {}; + for (const entry of attached) { + if (typeof entry === 'string') { + containers[entry] = { Name: '/workload' }; + } else { + containers[entry.id] = { Name: `/${entry.name}` }; + } + } + return { + IPAM: { Config: [{ Subnet: subnet, IPRange: '172.30.0.128/25' }] }, + Containers: containers, + }; +} + +function lastTransition(svc: import('../services/MeshService').MeshService): MeshActivityEvent | undefined { + const all = (svc as unknown as MutableSvc).activity; + return all + .filter((e) => e.type === 'mesh.enable' || e.type === 'mesh.disable') + .filter((e) => typeof e.details?.prevReason === 'string') + .slice(-1)[0]; +} + +describe('MeshService.revalidateDataPlane — short-circuits', () => { + it('skips when boot has not finished (not_started)', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + svc.dataPlaneStatus = { ok: false, reason: 'not_started', message: 'init', subnet: '' }; + const fake = mockDocker(); + await MeshService.getInstance().revalidateDataPlane(); + expect(fake.inspectNetwork).not.toHaveBeenCalled(); + expect(svc.dataPlaneStatus.reason).toBe('not_started'); + }); + + it('skips in dev mode (not_in_docker)', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + svc.dataPlaneStatus = { ok: false, reason: 'not_in_docker', message: 'dev', subnet: '172.30.0.0/24' }; + const fake = mockDocker(); + await MeshService.getInstance().revalidateDataPlane(); + expect(fake.inspectNetwork).not.toHaveBeenCalled(); + expect(svc.dataPlaneStatus.reason).toBe('not_in_docker'); + }); + + it('skips when env config error (subnet_invalid)', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + svc.dataPlaneStatus = { ok: false, reason: 'subnet_invalid', message: 'bad cidr', subnet: 'nope' }; + const fake = mockDocker(); + await MeshService.getInstance().revalidateDataPlane(); + expect(fake.inspectNetwork).not.toHaveBeenCalled(); + expect(svc.dataPlaneStatus.reason).toBe('subnet_invalid'); + }); + + it('skips when a revalidate is already in flight (re-entry guard)', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + svc.dataPlaneRevalidateInFlight = true; + const fake = mockDocker(); + await MeshService.getInstance().revalidateDataPlane(); + expect(fake.inspectNetwork).not.toHaveBeenCalled(); + }); +}); + +describe('MeshService.revalidateDataPlane — happy paths', () => { + it('is a silent no-op when network and attachment are both healthy', async () => { + const fake = mockDocker({ + inspectNetwork: vi.fn().mockResolvedValue( + networkPresent('172.30.0.0/24', [{ id: 'aaaa11112222333344445555', name: 'sencho' }]), + ), + }); + const svc = MeshService.getInstance(); + const before = (svc as unknown as MutableSvc).activity.length; + await svc.revalidateDataPlane(); + expect(fake.inspectNetwork).toHaveBeenCalledTimes(1); + expect(svc.getDataPlaneStatus().reason).toBe('ok'); + // No transition entry appended — idempotent on stable state. + expect((svc as unknown as MutableSvc).activity.length).toBe(before); + }); + + it('matches the self-attachment check by full container-ID prefix when no Name is set', async () => { + const fake = mockDocker({ + inspectNetwork: vi.fn().mockResolvedValue({ + IPAM: { Config: [{ Subnet: '172.30.0.0/24', IPRange: '172.30.0.128/25' }] }, + // Mimic the Docker default HOSTNAME=<12-char short id> setup + // by writing a containers entry with NO Name but an ID that + // begins with our HOSTNAME-equivalent 12-char prefix. + Containers: { '5dd0ce69ef50abcdefabcdef12345678abcdef1234567890abcd12345678': { Name: '' } }, + }), + }); + process.env.HOSTNAME = '5dd0ce69ef50'; + const svc = MeshService.getInstance(); + await svc.revalidateDataPlane(); + expect(fake.inspectNetwork).toHaveBeenCalledTimes(1); + expect(svc.getDataPlaneStatus().reason).toBe('ok'); + }); + + it('preserves status as `unknown` when HOSTNAME is a short hex prefix and no Name match', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + // HOSTNAME='ab' is short AND hex: it could plausibly be a + // container-ID prefix, but length < 12 makes the prefix path + // unsafe. Name does not match. We genuinely cannot identify + // ourselves; preserve prior status. + process.env.HOSTNAME = 'ab'; + mockDocker({ + inspectNetwork: vi.fn().mockResolvedValue({ + IPAM: { Config: [{ Subnet: '172.30.0.0/24', IPRange: '172.30.0.128/25' }] }, + Containers: { 'ff00112233445566ffffffffff': { Name: '/something-else' } }, + }), + }); + await MeshService.getInstance().revalidateDataPlane(); + // Prior status preserved (ok), no transition to attach_failed. + expect(svc.dataPlaneStatus.reason).toBe('ok'); + }); + + it('classifies as `detached` when HOSTNAME is non-hex (operator-set) and no Name match', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + // HOSTNAME='sencho' is non-hex. It cannot collide with any + // container ID regardless of length, so Name is the only path + // and a Name miss means we are definitely not attached. + process.env.HOSTNAME = 'sencho'; + mockDocker({ + inspectNetwork: vi.fn().mockResolvedValue( + networkPresent('172.30.0.0/24', [{ id: 'ffffeeeeddddccccaaa', name: 'someoneelse' }]), + ), + }); + await MeshService.getInstance().revalidateDataPlane(); + expect(svc.dataPlaneStatus.reason).toBe('attach_failed'); + }); + + it('recovers from subnet_mismatch when operator removed the conflicting network', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + svc.dataPlaneStatus = { + ok: false, + reason: 'subnet_mismatch', + message: 'old conflict', + subnet: '172.30.0.0/24', + }; + svc.networkSetupError = 'old conflict'; + mockDocker({ + inspectNetwork: vi.fn().mockResolvedValue( + networkPresent('172.30.0.0/24', [{ id: 'aaaabbbbccccdddd', name: 'sencho' }]), + ), + }); + await MeshService.getInstance().revalidateDataPlane(); + expect(svc.dataPlaneStatus.reason).toBe('ok'); + expect(svc.dataPlaneStatus.ok).toBe(true); + // Transition into ok clears the legacy raw-error string so the two + // status surfaces (typed discriminator + raw string) stay coherent. + expect(svc.networkSetupError).toBeNull(); + const t = lastTransition(MeshService.getInstance()); + expect(t?.type).toBe('mesh.enable'); + expect(t?.details?.prevReason).toBe('subnet_mismatch'); + expect(t?.details?.nextReason).toBe('ok'); + }); + + it('recovers from subnet_overlap when operator removed the conflicting network', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + svc.dataPlaneStatus = { + ok: false, + reason: 'subnet_overlap', + message: 'overlap', + subnet: '172.30.0.0/24', + }; + mockDocker({ + inspectNetwork: vi.fn().mockResolvedValue( + networkPresent('172.30.0.0/24', [{ id: 'abcdef012345678901', name: 'sencho' }]), + ), + }); + await MeshService.getInstance().revalidateDataPlane(); + expect(svc.dataPlaneStatus.reason).toBe('ok'); + }); +}); + +describe('MeshService.revalidateDataPlane — failure transitions', () => { + it('flips to not_found when sencho_mesh has been removed', async () => { + mockDocker({ + // inspectNetwork resolves null via the 404 path inside the snapshot helper. + inspectNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('no such network'), { statusCode: 404 }), + ), + }); + const svc = MeshService.getInstance(); + const consoleWarn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + await svc.revalidateDataPlane(); + const status = svc.getDataPlaneStatus(); + expect(status.ok).toBe(false); + expect(status.reason).toBe('not_found'); + expect(status.subnet).toBe('172.30.0.0/24'); + expect(status.message).toMatch(/is not present/i); + // Mirrored to console.warn so docker logs surfaces the transition. + expect(consoleWarn).toHaveBeenCalled(); + const t = lastTransition(svc); + expect(t?.type).toBe('mesh.disable'); + expect(t?.details?.prevReason).toBe('ok'); + expect(t?.details?.nextReason).toBe('not_found'); + }); + + it('flips to subnet_mismatch when the network was externally recreated at a different subnet', async () => { + mockDocker({ + inspectNetwork: vi.fn().mockResolvedValue( + networkPresent('10.42.0.0/24', [{ id: 'someotherid12345', name: 'sencho' }]), + ), + }); + const svc = MeshService.getInstance(); + await svc.revalidateDataPlane(); + const status = svc.getDataPlaneStatus(); + expect(status.reason).toBe('subnet_mismatch'); + expect(status.message).toMatch(/172\.30\.0\.0\/24/); + expect(status.message).toMatch(/10\.42\.0\.0\/24/); + }); + + it('flips to attach_failed when network is present but Sencho is no longer attached', async () => { + mockDocker({ + inspectNetwork: vi.fn().mockResolvedValue( + // Attached containers: someone else, not Sencho. + networkPresent('172.30.0.0/24', [{ id: 'other-id', name: 'other-container' }]), + ), + }); + const svc = MeshService.getInstance(); + await svc.revalidateDataPlane(); + const status = svc.getDataPlaneStatus(); + expect(status.reason).toBe('attach_failed'); + expect(status.message).toMatch(/no longer attached/i); + }); + + it('flips from attach_failed to not_found when the network is then removed', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + svc.dataPlaneStatus = { + ok: false, + reason: 'attach_failed', + message: 'Sencho is no longer attached', + subnet: '172.30.0.0/24', + }; + mockDocker({ + inspectNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('no such network'), { statusCode: 404 }), + ), + }); + await MeshService.getInstance().revalidateDataPlane(); + expect(svc.dataPlaneStatus.reason).toBe('not_found'); + }); + + it('flips from subnet_mismatch to not_found when the conflicting network is then removed', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + svc.dataPlaneStatus = { + ok: false, + reason: 'subnet_mismatch', + message: 'mismatch', + subnet: '172.30.0.0/24', + }; + mockDocker({ + inspectNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('no such network'), { statusCode: 404 }), + ), + }); + await MeshService.getInstance().revalidateDataPlane(); + expect(svc.dataPlaneStatus.reason).toBe('not_found'); + }); + + it('flips from not_found to subnet_mismatch when an external recreate lands at a different subnet', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + svc.dataPlaneStatus = { + ok: false, + reason: 'not_found', + message: 'removed', + subnet: '172.30.0.0/24', + }; + mockDocker({ + inspectNetwork: vi.fn().mockResolvedValue( + networkPresent('10.43.0.0/24', [{ id: 'newone', name: 'someoneelse' }]), + ), + }); + await MeshService.getInstance().revalidateDataPlane(); + expect(svc.dataPlaneStatus.reason).toBe('subnet_mismatch'); + expect(svc.dataPlaneStatus.subnet).toBe('172.30.0.0/24'); // our configured subnet, unchanged + expect(svc.dataPlaneStatus.message).toMatch(/10\.43\.0\.0\/24/); + }); + + it('refreshes the message + observed subnet on a SECOND subnet_mismatch observation against a different external subnet', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + // First mismatch lands at 10.42.0.0/24. + mockDocker({ + inspectNetwork: vi.fn().mockResolvedValue( + networkPresent('10.42.0.0/24', [{ id: 'newone', name: 'unrelated' }]), + ), + }); + await MeshService.getInstance().revalidateDataPlane(); + expect(svc.dataPlaneStatus.reason).toBe('subnet_mismatch'); + expect(svc.dataPlaneStatus.message).toMatch(/10\.42\.0\.0\/24/); + const firstMessage = svc.dataPlaneStatus.message; + const transitionsAfterFirst = svc.activity + .filter((e) => e.details && typeof e.details.prevReason === 'string').length; + + // Second mismatch lands at 10.43.0.0/24. Reason stays + // `subnet_mismatch`, but the message must be refreshed so + // /api/health reports the current subnet, not the stale one. + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + createNetwork: vi.fn(), + inspectNetwork: vi.fn().mockResolvedValue( + networkPresent('10.43.0.0/24', [{ id: 'newone2', name: 'unrelated' }]), + ), + connectContainerToNetwork: vi.fn(), + } as unknown as ReturnType); + await MeshService.getInstance().revalidateDataPlane(); + expect(svc.dataPlaneStatus.reason).toBe('subnet_mismatch'); + expect(svc.dataPlaneStatus.message).not.toBe(firstMessage); + expect(svc.dataPlaneStatus.message).toMatch(/10\.43\.0\.0\/24/); + + // Log surface stays quiet: reason did not change, so no second + // activity entry beyond the first transition. + const transitionsAfterSecond = svc.activity + .filter((e) => e.details && typeof e.details.prevReason === 'string').length; + expect(transitionsAfterSecond).toBe(transitionsAfterFirst); + }); + + it('preserves the prior status on a transient Docker error (no flap)', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + mockDocker({ + inspectNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('connect ECONNREFUSED'), { statusCode: 500 }), + ), + }); + await MeshService.getInstance().revalidateDataPlane(); + // Daemon transient: status stays at ok, no transition appended. + expect(svc.dataPlaneStatus.reason).toBe('ok'); + expect(lastTransition(MeshService.getInstance())).toBeUndefined(); + }); +}); + +describe('MeshService.transitionDataPlane — idempotency', () => { + it('records a single transition entry across two ticks observing the same not_found state', async () => { + const svc = MeshService.getInstance(); + mockDocker({ + inspectNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('no such network'), { statusCode: 404 }), + ), + }); + const before = (svc as unknown as MutableSvc).activity.length; + await svc.revalidateDataPlane(); + await svc.revalidateDataPlane(); + const after = (svc as unknown as MutableSvc).activity.length; + // Two ticks, but only the first one triggers the ok -> not_found + // transition. The second tick sees prev.reason === next.reason and + // no-ops in transitionDataPlane. + // (Auto-recreate is OFF in this test so the only growth source is + // the transition itself.) + expect(after - before).toBe(1); + }); +}); + +describe('MeshService.attemptInPlaceRecreate — auto-recreate off (default)', () => { + it('does NOT call createNetwork when the setting is off', async () => { + const fake = mockDocker({ + inspectNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('no such network'), { statusCode: 404 }), + ), + }); + const svc = MeshService.getInstance(); + await svc.revalidateDataPlane(); + expect(svc.getDataPlaneStatus().reason).toBe('not_found'); + expect(fake.createNetwork).not.toHaveBeenCalled(); + }); +}); + +describe('MeshService.attemptInPlaceRecreate — auto-recreate on', () => { + beforeEach(() => { + DatabaseService.getInstance().updateGlobalSetting('mesh_auto_recreate', '1'); + }); + + it('recreates the network at the same subnet and re-attaches Sencho on success', async () => { + const fake = mockDocker({ + inspectNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('no such network'), { statusCode: 404 }), + ), + }); + const svc = MeshService.getInstance(); + await svc.revalidateDataPlane(); + // First the revalidator surfaced the bad state... + expect(fake.createNetwork).toHaveBeenCalledTimes(1); + expect(fake.createNetwork).toHaveBeenCalledWith(expect.objectContaining({ + Name: 'sencho_mesh', + IPAM: { Config: [expect.objectContaining({ Subnet: '172.30.0.0/24' })] }, + })); + expect(fake.connectContainerToNetwork).toHaveBeenCalledTimes(1); + // Final status flips back to ok. + expect(svc.getDataPlaneStatus().reason).toBe('ok'); + // And the previously chosen senchoIp is preserved end-to-end: + // ensureSelfAttached MUST NOT clear it on the recovery branch. + expect((svc as unknown as MutableSvc).senchoIp).toBe('172.30.0.2'); + }); + + it('records subnet_overlap (and does NOT drift) when createNetwork rejects with overlap', async () => { + const fake = mockDocker({ + inspectNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('no such network'), { statusCode: 404 }), + ), + createNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('Pool overlaps with other one on this address space'), { statusCode: 500 }), + ), + }); + const svc = MeshService.getInstance(); + await svc.revalidateDataPlane(); + expect(fake.createNetwork).toHaveBeenCalledTimes(1); + const status = svc.getDataPlaneStatus(); + expect(status.reason).toBe('subnet_overlap'); + // The chosen subnet has NOT changed; auto-recreate never iterates + // candidates because that would invalidate existing override files. + expect(status.subnet).toBe('172.30.0.0/24'); + }); + + it('preserves senchoIp across a create failure, then re-attaches on the next successful retry', async () => { + // Tick 1: createNetwork throws overlap. recordRecreateFailure + // must NOT clear `senchoIp`, otherwise the next tick's + // attachment check (gated on `this.senchoIp`) is skipped and + // the revalidator could silently flip back to `ok` against a + // newly created but un-attached network. + const inspectNetwork = vi.fn().mockRejectedValueOnce( + Object.assign(new Error('no such network'), { statusCode: 404 }), + ); + const createNetwork = vi.fn() + .mockRejectedValueOnce(Object.assign( + new Error('Pool overlaps with other one on this address space'), + { statusCode: 500 }, + )) + .mockResolvedValueOnce(undefined); + const connectContainerToNetwork = vi.fn().mockResolvedValue(undefined); + const fake = mockDocker({ inspectNetwork, createNetwork, connectContainerToNetwork }); + const svc = MeshService.getInstance() as unknown as MutableSvc; + const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(5_000_000); + await MeshService.getInstance().revalidateDataPlane(); + expect(svc.dataPlaneStatus.reason).toBe('subnet_overlap'); + expect(svc.senchoIp).toBe('172.30.0.2'); + + // Advance past throttle and let the next tick succeed. + // From this tick the inspect returns the freshly created network. + nowSpy.mockReturnValue(5_000_000 + MESH_RECREATE_THROTTLE_MS + 1); + inspectNetwork.mockRejectedValueOnce( + Object.assign(new Error('no such network'), { statusCode: 404 }), + ); + await MeshService.getInstance().revalidateDataPlane(); + // createNetwork now succeeds, ensureSelfAttached runs against the + // preserved senchoIp and binds Sencho back to the new network. + expect(createNetwork).toHaveBeenCalledTimes(2); + expect(connectContainerToNetwork).toHaveBeenCalledWith( + 'sencho_mesh', + 'sencho', + { ipv4Address: '172.30.0.2' }, + ); + expect(svc.dataPlaneStatus.reason).toBe('ok'); + expect(svc.senchoIp).toBe('172.30.0.2'); + expect(fake.inspectNetwork).toHaveBeenCalled(); + }); + + it('surfaces attach_failed honestly when createNetwork succeeds but ensureSelfAttached throws', async () => { + // Tick 1: createNetwork succeeds, connectContainerToNetwork + // throws (e.g. another workload is squatting `.2`). + // recordRecreateFailure preserves `senchoIp` so the next tick + // can still classify the attachment state correctly via the + // snapshot path. + const createNetwork = vi.fn().mockResolvedValue(undefined); + const connectContainerToNetwork = vi.fn() + .mockRejectedValueOnce(new Error('Address already in use')); + const inspectNetwork = vi.fn() + .mockRejectedValueOnce(Object.assign(new Error('no such network'), { statusCode: 404 })) + // Tick 2: network now exists from the tick-1 createNetwork + // success, but Sencho's container is NOT in the Containers + // map (attach threw). + .mockResolvedValueOnce( + networkPresent('172.30.0.0/24', [{ id: 'squatter-id', name: 'squatter' }]), + ); + mockDocker({ inspectNetwork, createNetwork, connectContainerToNetwork }); + const svc = MeshService.getInstance() as unknown as MutableSvc; + await MeshService.getInstance().revalidateDataPlane(); + expect(svc.dataPlaneStatus.reason).toBe('ip_in_use'); + expect(svc.senchoIp).toBe('172.30.0.2'); // preserved + // Next revalidator tick sees the new network but no self- + // attachment. The reason flips to `attach_failed` honestly + // instead of silently reporting `ok` against an un-attached + // network (the BLOCKER scenario this fix exists for). + await MeshService.getInstance().revalidateDataPlane(); + expect(svc.dataPlaneStatus.reason).toBe('attach_failed'); + expect(svc.senchoIp).toBe('172.30.0.2'); + }); + + it('does NOT flap subnet_overlap back to not_found during the recreate throttle window', async () => { + const fake = mockDocker({ + inspectNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('no such network'), { statusCode: 404 }), + ), + createNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('Pool overlaps with other one on this address space'), { statusCode: 500 }), + ), + }); + const svc = MeshService.getInstance(); + const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(2_000_000); + // Tick 1: failure-classifies as subnet_overlap. + await svc.revalidateDataPlane(); + expect(svc.getDataPlaneStatus().reason).toBe('subnet_overlap'); + // Tick 2 (10s later): network still missing, but the recreate + // throttle is still active. The operator-actionable reason + // (subnet_overlap) must NOT be downgraded to the generic + // not_found. The createNetwork call must NOT be retried either. + nowSpy.mockReturnValue(2_000_000 + 10_000); + await svc.revalidateDataPlane(); + expect(svc.getDataPlaneStatus().reason).toBe('subnet_overlap'); + expect(fake.createNetwork).toHaveBeenCalledTimes(1); + // Throttle elapses; tick retries the recreate and re-classifies. + nowSpy.mockReturnValue(2_000_000 + MESH_RECREATE_THROTTLE_MS + 1); + await svc.revalidateDataPlane(); + expect(fake.createNetwork).toHaveBeenCalledTimes(2); + }); + + it('throttles repeat recreate attempts within MESH_RECREATE_THROTTLE_MS', async () => { + const fake = mockDocker({ + inspectNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('no such network'), { statusCode: 404 }), + ), + createNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('Pool overlaps with other one on this address space'), { statusCode: 500 }), + ), + }); + const svc = MeshService.getInstance(); + const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + await svc.revalidateDataPlane(); + await svc.revalidateDataPlane(); + await svc.revalidateDataPlane(); + // Three ticks, but only ONE createNetwork because the throttle + // window blocks the second and third attempts. + expect(fake.createNetwork).toHaveBeenCalledTimes(1); + // Advance past the throttle window and confirm the next tick is + // willing to try again. + nowSpy.mockReturnValue(1_000_000 + MESH_RECREATE_THROTTLE_MS + 1); + await svc.revalidateDataPlane(); + expect(fake.createNetwork).toHaveBeenCalledTimes(2); + }); +}); + +describe('MeshService.dataPlaneRevalidateTimer — lifecycle', () => { + it('start() schedules the timer, stop() clears it', async () => { + const svc = MeshService.getInstance() as unknown as MutableSvc; + // The shared singleton may have been start()-ed already by another + // suite; clear any previous timer and reset 'started' so we can + // exercise start/stop cleanly without crossing real I/O paths. + if (svc.dataPlaneRevalidateTimer) { + clearInterval(svc.dataPlaneRevalidateTimer); + svc.dataPlaneRevalidateTimer = undefined; + } + // Drive the lifecycle by stubbing the singleton's slow boot work so + // start() returns quickly. We only care about the timer wiring. + const inst = MeshService.getInstance(); + const ms = inst as unknown as Record; + ms.setupMeshNetwork = vi.fn().mockResolvedValue(undefined); + ms.refreshAliasCache = vi.fn().mockResolvedValue(undefined); + ms.syncForwarderListeners = vi.fn().mockResolvedValue(undefined); + ms.regenerateAllOverrides = vi.fn().mockResolvedValue(undefined); + ms.proactiveBridgeFanout = vi.fn().mockReturnValue(undefined); + ms.startBridgeReconcileLoop = vi.fn().mockReturnValue(undefined); + ms.stopBridgeReconcileLoop = vi.fn().mockReturnValue(undefined); + svc.started = false; + await inst.start(); + expect(svc.dataPlaneRevalidateTimer).toBeDefined(); + await inst.stop(); + expect(svc.dataPlaneRevalidateTimer).toBeUndefined(); + }); +}); diff --git a/backend/src/__tests__/settings-routes.test.ts b/backend/src/__tests__/settings-routes.test.ts index 9b882180..d0a1da28 100644 --- a/backend/src/__tests__/settings-routes.test.ts +++ b/backend/src/__tests__/settings-routes.test.ts @@ -105,6 +105,47 @@ describe('POST /api/settings (single-key write)', () => { const settings = DatabaseService.getInstance().getGlobalSettings(); expect(settings.host_cpu_limit).toBe('75'); }); + + it('rejects an enum-shaped key whose value is not one of the allowed literals', async () => { + // Regression: the single-key POST previously wrote `String(value)` + // without re-validating, so an allowlisted enum-shaped key like + // `mesh_auto_recreate` could store arbitrary strings (`'banana'`) + // that the bulk PATCH would later refuse. The single-key path now + // routes through the same SettingsPatchSchema as PATCH. + const res = await request(app) + .post('/api/settings') + .set('Cookie', adminCookie) + .send({ key: 'mesh_auto_recreate', value: 'banana' }); + expect(res.status).toBe(400); + expect(res.body.error).toBe('Validation failed'); + expect(res.body.details).toBeInstanceOf(Object); + const settings = DatabaseService.getInstance().getGlobalSettings(); + // Confirm the bad write did NOT leak through. + expect(settings.mesh_auto_recreate).not.toBe('banana'); + }); + + it('accepts a well-formed mesh_auto_recreate write', async () => { + const res = await request(app) + .post('/api/settings') + .set('Cookie', adminCookie) + .send({ key: 'mesh_auto_recreate', value: '1' }); + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(DatabaseService.getInstance().getGlobalSettings().mesh_auto_recreate).toBe('1'); + // Reset for any later tests that read the value. + DatabaseService.getInstance().updateGlobalSetting('mesh_auto_recreate', '0'); + }); + + it('rejects an out-of-range numeric value on the single-key path (now schema-validated)', async () => { + // Same regression class as mesh_auto_recreate=banana, but exercised + // on a numeric setting to lock down the schema routing. + const res = await request(app) + .post('/api/settings') + .set('Cookie', adminCookie) + .send({ key: 'host_cpu_limit', value: '9999' }); + expect(res.status).toBe(400); + expect(res.body.error).toBe('Validation failed'); + }); }); describe('PATCH /api/settings (bulk update)', () => { diff --git a/backend/src/routes/settings.ts b/backend/src/routes/settings.ts index dc3d41c8..1cf77767 100644 --- a/backend/src/routes/settings.ts +++ b/backend/src/routes/settings.ts @@ -22,6 +22,7 @@ const ALLOWED_SETTING_KEYS = new Set([ 'metrics_retention_hours', 'log_retention_days', 'audit_retention_days', + 'mesh_auto_recreate', ]); // Bulk PATCH schema. All keys optional; present keys are fully validated. @@ -37,6 +38,7 @@ const SettingsPatchSchema = z.object({ metrics_retention_hours: z.coerce.number().int().min(1).max(8760).transform(String), log_retention_days: z.coerce.number().int().min(1).max(365).transform(String), audit_retention_days: z.coerce.number().int().min(1).max(365).transform(String), + mesh_auto_recreate: z.enum(['0', '1']), }).partial(); export const settingsRouter = Router(); @@ -66,7 +68,30 @@ settingsRouter.post('/', authMiddleware, async (req: Request, res: Response): Pr res.status(400).json({ error: 'Setting value is required' }); return; } - DatabaseService.getInstance().updateGlobalSetting(key, String(value)); + // Route the single-key write through the same per-key schema used by + // the bulk PATCH so allowlisted-but-malformed values (e.g. `true`, + // `banana`, out-of-range integers) cannot bypass validation just + // because they came in via the single-key path. The schema coerces + // numeric settings to strings and rejects enum-shaped settings that + // are not one of the allowed literals. + const parsed = SettingsPatchSchema.safeParse({ [key]: value }); + if (!parsed.success) { + res.status(400).json({ + error: 'Validation failed', + details: parsed.error.flatten().fieldErrors, + }); + return; + } + const validated = (parsed.data as Record)[key]; + if (validated === undefined) { + // Defensive: the schema is `.partial()`, so an unknown key would + // pass through silently. We already gated on ALLOWED_SETTING_KEYS, + // but reject explicitly if the key is somehow missing from the + // schema's shape (drift between the allowlist and the schema). + res.status(400).json({ error: `Setting key has no validator: ${key}` }); + return; + } + DatabaseService.getInstance().updateGlobalSetting(key, validated); res.json({ success: true }); } catch (error) { console.error('Failed to update setting:', error); diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 220442f2..2e11350c 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -1237,6 +1237,7 @@ export class DatabaseService { stmt.run('log_retention_days', '30'); stmt.run('trivy_auto_update', '0'); stmt.run('trivy_last_notified_version', ''); + stmt.run('mesh_auto_recreate', '0'); // Seed the default local node if none exists const nodeCount = (this.db.prepare('SELECT COUNT(*) as count FROM nodes').get() as any)?.count || 0; diff --git a/backend/src/services/MeshService.ts b/backend/src/services/MeshService.ts index 77827d5f..03e62e25 100644 --- a/backend/src/services/MeshService.ts +++ b/backend/src/services/MeshService.ts @@ -26,6 +26,16 @@ const ALIAS_REFRESH_INTERVAL_MS = 60_000; const PROBE_TIMEOUT_MS = 5_000; const SLOW_PROBE_THRESHOLD_MS = 500; const DEFAULT_MESH_SUBNET = '172.30.0.0/24'; +// Cadence at which `revalidateDataPlane` re-evaluates the Docker network and +// Sencho's attachment. Bounds the staleness of `getDataPlaneStatus()` between +// runtime mesh changes (operator `docker network rm`, external recreate at a +// different subnet, container detach) and the next /api/health response. +export const DATA_PLANE_REVALIDATE_INTERVAL_MS = 10_000; +// After a failed in-place recreate attempt, refuse to retry within this +// window so a persistent CIDR overlap does not spam the Docker daemon on +// every 10s tick. Matches the cadence pattern used by other notification +// dedup paths (e.g. PolicyEnforcement.notifyTrivyMissingOnce). +export const MESH_RECREATE_THROTTLE_MS = 60_000; /** * Subnets attempted in order when SENCHO_MESH_SUBNET is unset and no @@ -129,7 +139,8 @@ export type MeshDataPlaneReason = | 'subnet_mismatch' // sencho_mesh already exists with a different subnet | 'ip_in_use' // another container squats +2 | 'attach_failed' // self-attach failed for any other reason - | 'not_in_docker'; // HOSTNAME unset or self-container lookup returned 404 + | 'not_in_docker' // HOSTNAME unset or self-container lookup returned 404 + | 'not_found'; // sencho_mesh was removed after boot (revalidator-only) export interface MeshDataPlaneStatus { ok: boolean; @@ -362,6 +373,19 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { // upstream tunnel is the most authoritative source. Cleared on tunnel // close. private proxyTunnelSelfCentralNodeId: number | null = null; + // 10s revalidator timer that re-evaluates the mesh data plane against + // Docker's current state so `/api/health` and the dashboard banner do + // not serve stale values after the operator removes or recreates + // sencho_mesh while Sencho is running. + private dataPlaneRevalidateTimer?: NodeJS.Timeout; + // Re-entrancy guard: two ticks of the revalidator can in principle + // overlap if a Docker inspect takes longer than the cadence. Skip the + // overlapping tick rather than fire concurrent network inspects. + private dataPlaneRevalidateInFlight = false; + // Wall-clock of the last auto-recreate attempt (success or failure). + // Used by `attemptInPlaceRecreate` to enforce MESH_RECREATE_THROTTLE_MS + // so a persistent overlap does not spam createNetwork on every tick. + private lastRecreateAttemptAt = 0; private constructor() { super(); @@ -454,6 +478,17 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } })(); }, ALIAS_REFRESH_INTERVAL_MS); + // Data-plane revalidator: bounds /api/health staleness at + // DATA_PLANE_REVALIDATE_INTERVAL_MS when the operator removes, + // recreates, or disconnects Sencho from sencho_mesh at runtime. + this.dataPlaneRevalidateTimer = setInterval(() => { + void this.revalidateDataPlane().catch((err) => { + console.warn( + '[MeshService] data plane revalidate failed:', + sanitizeForLog((err as Error).message), + ); + }); + }, DATA_PLANE_REVALIDATE_INTERVAL_MS); const dpReason = this.dataPlaneStatus.reason; const dataPlane = this.senchoIp ? 'ok' : `unavailable (${dpReason}: ${this.networkSetupError ?? 'unknown'})`; @@ -483,6 +518,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { clearInterval(this.aliasRefreshTimer); this.aliasRefreshTimer = undefined; } + if (this.dataPlaneRevalidateTimer) { + clearInterval(this.dataPlaneRevalidateTimer); + this.dataPlaneRevalidateTimer = undefined; + } this.stopBridgeReconcileLoop(); await this.forwarder.shutdown(); } @@ -586,7 +625,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { * real failures. */ private recordSetupFailure( - reason: Exclude, + reason: Exclude, err: unknown, level: MeshActivityLevel, // The subnet_invalid path fires before `this.meshSubnet` is assigned, @@ -995,6 +1034,370 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } } + /** + * One-shot snapshot of `sencho_mesh`: subnet, IPRange, and the set of + * attached containers (full ID + Name). Returns `null` when the + * network does not exist (Docker inspect 404). Used by the + * revalidator to keep `/api/health` fresh after the operator + * removes, recreates, or disconnects Sencho from the mesh at + * runtime. Real daemon errors propagate so the revalidator can + * preserve the prior status rather than flap it on a single + * transient. + */ + private async inspectMeshNetworkSnapshot(): Promise< + { + subnet: string; + ipRange: string | null; + containers: Array<{ id: string; name: string | null }>; + } | null + > { + const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId()); + try { + const info = await dc.inspectNetwork(SENCHO_MESH_NETWORK) as { + IPAM?: { Config?: Array<{ Subnet?: string; IPRange?: string }> }; + Containers?: Record; + } | undefined; + const cfg = info?.IPAM?.Config?.[0]; + if (!cfg?.Subnet) return null; + const containers = Object.entries(info?.Containers ?? {}).map(([id, c]) => ({ + id, + name: c?.Name ?? null, + })); + return { + subnet: cfg.Subnet, + ipRange: cfg.IPRange ?? null, + containers, + }; + } catch (err) { + const e = err as { statusCode?: number }; + if (e?.statusCode === 404) return null; + throw err; + } + } + + /** + * Result of the self-attachment check during revalidation: + * - `attached`: Sencho's container is in the network's Containers map. + * - `detached`: Sencho's identity was determinable AND not present. + * - `unknown`: cannot determine (HOSTNAME unset / too short to be a + * safe container-ID prefix). Caller preserves prior status. + */ + private isSelfStillAttached( + containers: Array<{ id: string; name: string | null }>, + ): 'attached' | 'detached' | 'unknown' { + const hostname = process.env.HOSTNAME; + if (!hostname) return 'unknown'; + // Match either by container Name (operator set `--hostname`, the + // container's `Name` in the network's Containers map equals that + // value with a leading `/`) or by full container-ID prefix + // (Docker's default HOSTNAME is the 12-char short ID; the + // Containers map keys are the full 64-char IDs). + // + // The ID-prefix path is gated by two checks to avoid false + // positives: HOSTNAME must be all hex (`0-9a-f`) AND at least 12 + // chars. Container IDs are pure hex, so a non-hex HOSTNAME (e.g. + // operator-set `--hostname sencho`) cannot collide with any + // container ID regardless of length, and a short hex HOSTNAME + // (e.g. `--hostname ab`) could match unrelated containers. + // + // Network-inspect's `Name` field carries a leading `/`; normalize + // before comparing against HOSTNAME which never has it. + const matchByName = (name: string | null): boolean => { + if (!name) return false; + const normalized = name.startsWith('/') ? name.slice(1) : name; + return normalized === hostname; + }; + const isHexOnly = /^[0-9a-f]+$/.test(hostname); + const idPrefixUsable = isHexOnly && hostname.length >= 12; + for (const c of containers) { + if (matchByName(c.name)) return 'attached'; + if (idPrefixUsable && c.id.startsWith(hostname)) return 'attached'; + } + // No match. + // - Non-hex HOSTNAME (e.g. `sencho`, `mynode`): the operator set + // it explicitly. It cannot be a container-ID prefix. The Name + // path was the only reliable signal and it failed, so we are + // detached. + // - Hex HOSTNAME >= 12 chars: ID-prefix path was run and missed. + // Detached. + // - Hex HOSTNAME < 12 chars (rare, only when an operator passes + // `--hostname` with a hex string shorter than 12 chars): we + // could not safely ID-prefix-match and Name did not match. We + // genuinely do not know; preserve the prior status. + if (!isHexOnly) return 'detached'; + if (hostname.length >= 12) return 'detached'; + return 'unknown'; + } + + /** + * Reads the persisted `mesh_auto_recreate` global setting. Defaults to + * off (`'0'`); only `'1'` enables the auto-recreate behaviour in + * `attemptInPlaceRecreate`. Wrapped in try/catch so a transient DB + * failure (e.g. SQLite briefly locked during a backup) cannot crash + * the revalidator timer; on failure we default to off so a missing + * read never accidentally triggers a Docker mutation. + */ + private isMeshAutoRecreateEnabled(): boolean { + try { + const settings = DatabaseService.getInstance().getGlobalSettings(); + return settings['mesh_auto_recreate'] === '1'; + } catch (err) { + console.warn( + '[MeshService] could not read mesh_auto_recreate setting:', + sanitizeForLog((err as Error).message), + ); + return false; + } + } + + /** + * Single transition path for `dataPlaneStatus`. Two-tier semantics: + * - Fields (`message`, `subnet`) are always refreshed when they + * drift, so `/api/health` never carries stale numbers (e.g. two + * consecutive `subnet_mismatch` observations against different + * external subnets both show their own remote subnet). + * - Activity ring + `console.{log,warn}` mirrors fire only when + * the `reason` discriminator actually changes, so the 10s timer + * can tick indefinitely on a stable mesh without spamming the + * log surface. + * `networkSetupError` follows the same coherence rule it had before + * the revalidator landed: cleared on recovery to `ok`, otherwise + * tracks the typed status's `message`. + */ + private transitionDataPlane(next: MeshDataPlaneStatus): void { + const prev = this.dataPlaneStatus; + const reasonChanged = prev.reason !== next.reason; + const fieldsChanged = + reasonChanged || + prev.message !== next.message || + prev.subnet !== next.subnet || + prev.ok !== next.ok; + if (!fieldsChanged) return; + this.dataPlaneStatus = next; + if (next.ok) { + this.networkSetupError = null; + } else if (next.message) { + // Keep the legacy raw-error string in step with the typed + // discriminator so callers that still read `networkSetupError` + // (optInStack, applyLocalOverride, regenerateAllOverrides) see + // the same message the typed status surfaces. + this.networkSetupError = next.message; + } + if (!reasonChanged) return; + const line = next.ok + ? `[Mesh] data plane recovered (reason ${prev.reason} -> ${next.reason}, subnet ${next.subnet})` + : `[Mesh] data plane changed (reason ${prev.reason} -> ${next.reason}, subnet ${next.subnet}): ${next.message ?? 'no detail'}`; + if (next.ok) console.log(line); + else console.warn(line); + this.logActivity({ + source: 'mesh', + level: next.ok ? 'info' : 'warn', + type: next.ok ? 'mesh.enable' : 'mesh.disable', + message: line, + details: { prevReason: prev.reason, nextReason: next.reason, subnet: next.subnet }, + }); + } + + /** + * 10s tick body. Reports current Docker reality into + * `dataPlaneStatus`; never mutates `senchoIp` or `meshSubnet` + * (those are pinned at boot to keep override files stable). The + * `mesh_auto_recreate` setting opts into a single bounded + * recreate-on-the-same-subnet attempt via `attemptInPlaceRecreate` + * when the network has been removed. Public so unit tests can call + * it directly without manipulating timers. + * + * Short-circuits in states where the truth cannot have changed + * within this process: `not_started` (boot still in flight), + * `not_in_docker` (dev mode; HOSTNAME unset for the whole process), + * and `subnet_invalid` (env config error; resolved only by restart + * with new env). + */ + public async revalidateDataPlane(): Promise { + if (this.dataPlaneRevalidateInFlight) return; + const reason = this.dataPlaneStatus.reason; + if (reason === 'not_started' || reason === 'not_in_docker' || reason === 'subnet_invalid') { + return; + } + this.dataPlaneRevalidateInFlight = true; + try { + let snapshot: Awaited>; + try { + snapshot = await this.inspectMeshNetworkSnapshot(); + } catch { + // Transient Docker daemon failure. Preserve current status; + // next tick retries. Anti-flap by design. + return; + } + + if (snapshot === null) { + // Anti-flap during the auto-recreate throttle window: + // `attemptInPlaceRecreate` may have just classified the + // failure (`subnet_overlap`, `ip_in_use`, `attach_failed`). + // Those reasons are more actionable than the generic + // `not_found`, and the network is still missing because the + // recreate could not finish, not because the operator just + // removed it. Preserve the recreate-failure status until + // the throttle elapses and the next attempt re-classifies. + const prev = this.dataPlaneStatus; + const inThrottleWindow = + this.lastRecreateAttemptAt > 0 && + Date.now() - this.lastRecreateAttemptAt < MESH_RECREATE_THROTTLE_MS; + const isRecreateFailureReason = + prev.reason === 'subnet_overlap' || + prev.reason === 'subnet_mismatch' || + prev.reason === 'ip_in_use' || + prev.reason === 'attach_failed'; + if (inThrottleWindow && isRecreateFailureReason) { + return; + } + this.transitionDataPlane({ + ok: false, + reason: 'not_found', + message: `${SENCHO_MESH_NETWORK} is not present on this host. Mesh routing is offline; restart Sencho to recreate the network.`, + subnet: this.meshSubnet, + }); + if (this.isMeshAutoRecreateEnabled()) { + await this.attemptInPlaceRecreate(); + } + return; + } + + if (this.meshSubnet && snapshot.subnet !== this.meshSubnet) { + this.transitionDataPlane({ + ok: false, + reason: 'subnet_mismatch', + message: `${SENCHO_MESH_NETWORK} now uses ${snapshot.subnet}, Sencho is configured for ${this.meshSubnet}. Restart Sencho to adopt the new subnet.`, + subnet: this.meshSubnet, + }); + return; + } + + if (this.senchoIp) { + const attachment = this.isSelfStillAttached(snapshot.containers); + if (attachment === 'unknown') { + // Cannot determine our own identity (HOSTNAME unset or + // too short to be a safe ID prefix). Preserve the prior + // status; the boot-time `ensureSelfAttached` already + // classified the not-in-Docker case. + return; + } + if (attachment === 'detached') { + this.transitionDataPlane({ + ok: false, + reason: 'attach_failed', + message: `Sencho is no longer attached to ${SENCHO_MESH_NETWORK}; restart Sencho to re-attach.`, + subnet: this.meshSubnet, + }); + return; + } + } + + this.transitionDataPlane({ + ok: true, + reason: 'ok', + message: null, + subnet: this.meshSubnet, + }); + } finally { + this.dataPlaneRevalidateInFlight = false; + } + } + + /** + * Records a runtime recreate failure WITHOUT clearing `senchoIp`. + * Distinct from `recordSetupFailure`, which is boot-only and clears + * the IP as part of the failure-disables-mesh contract. At runtime + * the boot-resolved IP must survive a transient failure so a later + * successful attempt can drive `ensureSelfAttached` and re-bind + * Sencho to the freshly created network. Otherwise the next + * revalidator tick that observes the new network would skip the + * attachment check (gated on `this.senchoIp`) and silently report + * `ok` while Sencho is in fact detached. + */ + private recordRecreateFailure( + reason: Exclude, + err: unknown, + subnet: string, + ): void { + const message = err instanceof Error ? err.message : String(err); + this.transitionDataPlane({ + ok: false, + reason, + message, + subnet, + }); + } + + /** + * Opt-in auto-recreate path. Only invoked from the revalidator when + * the network was observed missing AND the operator has flipped + * `mesh_auto_recreate` to `'1'` in Settings -> System. Hard-prefers + * `this.meshSubnet` (the subnet chosen at boot) and never iterates + * the candidate list, because changing the chosen subnet here would + * invalidate every existing `extra_hosts` override on disk and + * silently break cross-node routing for opted-in stacks. A real + * overlap on the prior subnet is reported via `subnet_overlap` so + * the operator can take action. + * + * Throttled by MESH_RECREATE_THROTTLE_MS to keep a persistent + * conflict from spamming the daemon on every 10s tick. + */ + private async attemptInPlaceRecreate(): Promise { + const now = Date.now(); + if (now - this.lastRecreateAttemptAt < MESH_RECREATE_THROTTLE_MS) { + return; + } + this.lastRecreateAttemptAt = now; + // A revalidator-driven recreate is only meaningful when the + // boot-resolved Sencho IP is known. If `senchoIp` is null, + // setupMeshNetwork either never ran (impossible: revalidator + // short-circuits on `not_started`) or recorded a hard failure + // (`subnet_invalid` / `not_in_docker`, both of which also + // short-circuit). Defensive bail. + if (!this.senchoIp) return; + this.logActivity({ + source: 'mesh', + level: 'info', + type: 'mesh.enable', + message: `attempting auto-recreate of ${SENCHO_MESH_NETWORK} at ${this.meshSubnet}`, + details: { subnet: this.meshSubnet }, + }); + try { + await this.createMeshNetwork(this.meshSubnet); + } catch (err) { + // Preserve `senchoIp` so a later successful retry (after the + // throttle elapses + the overlap clears) can re-attach. + this.recordRecreateFailure( + this.classifyMeshNetworkError(err), + err, + this.meshSubnet, + ); + return; + } + try { + await this.ensureSelfAttached(); + } catch (err) { + // Same preservation rationale; the attach can succeed on a + // later attempt (e.g. when whatever was squatting the IP + // gets out of the way). The network now exists, so the next + // revalidator tick will surface the attachment failure + // honestly through the snapshot path. + this.recordRecreateFailure( + this.classifySelfAttachError(err), + err, + this.meshSubnet, + ); + return; + } + this.transitionDataPlane({ + ok: true, + reason: 'ok', + message: null, + subnet: this.meshSubnet, + }); + } + /** * Bind the forwarder's listeners to every alias port across the fleet * and release any listeners no longer in the alias set. Called from diff --git a/docs/features/sencho-mesh.mdx b/docs/features/sencho-mesh.mdx index b5d697d1..f6f29b4b 100644 --- a/docs/features/sencho-mesh.mdx +++ b/docs/features/sencho-mesh.mdx @@ -165,6 +165,8 @@ services: Sencho pins itself to ` + 2` (so `10.42.0.2` for the example above; the bridge's `+1` gateway sits between). The `sencho_mesh` network is created with an IPRange covering the upper half of the subnet, so Docker's auto-allocation hands meshed workloads ` + 128` and up by default and never grabs Sencho's static IP while Sencho is offline. Each node is configured independently; the alias registry pushes the correct local IP to each node's override file. +Sencho re-checks the mesh data plane every 10 seconds and updates `/api/health` and the dashboard banner with the current state, so removing or recreating `sencho_mesh` while Sencho is running is reflected within one tick instead of waiting for a restart. The reconciler is report-only by default: if the network disappears, Sencho surfaces `not_found` and waits for the operator to restart. To have Sencho rebuild the network at the same subnet on the next tick, flip **Settings → System → Mesh data plane → Auto-recreate mesh network** on. Auto-recreate never iterates the subnet candidate list (it would invalidate every existing `extra_hosts` override on disk); if the original subnet is no longer free, Sencho reports `subnet_overlap` and stops. Repeat failures are throttled to one attempt per minute. + ## Security and trust boundaries **Authentication.** Cross-node traffic rides an authenticated WebSocket between Sencho instances. Pilot Agent nodes authenticate with the long-lived JWT issued at enrollment (stored at `/app/data/pilot.jwt` on the agent). Distributed API Proxy nodes authenticate with the [Node Token](/features/multi-node#add-a-remote-node-distributed-api-proxy) generated on the remote. Restricted API token scopes (read-only, deploy-only) cannot carry mesh traffic; only a full-admin Node Token can. diff --git a/frontend/src/components/fleet/MeshDataPlaneBanner.tsx b/frontend/src/components/fleet/MeshDataPlaneBanner.tsx index 5969190f..cbd785d2 100644 --- a/frontend/src/components/fleet/MeshDataPlaneBanner.tsx +++ b/frontend/src/components/fleet/MeshDataPlaneBanner.tsx @@ -10,6 +10,7 @@ const HEADLINES: Record strin subnet_mismatch: (s) => `sencho_mesh already exists with a different subnet than ${s.subnet}.`, ip_in_use: (s) => `Another container is using Sencho's address on ${s.subnet}.`, attach_failed: () => 'Sencho could not attach to its own mesh network.', + not_found: (s) => `sencho_mesh is not present on this host (last known subnet ${s.subnet}).`, }; function isActionable(reason: Reason): reason is ActionableReason { diff --git a/frontend/src/components/settings/SystemSection.tsx b/frontend/src/components/settings/SystemSection.tsx index c5608861..43f9f30c 100644 --- a/frontend/src/components/settings/SystemSection.tsx +++ b/frontend/src/components/settings/SystemSection.tsx @@ -5,6 +5,7 @@ import { cn } from '@/lib/utils'; import { apiFetch } from '@/lib/api'; import { toast } from '@/components/ui/toast-store'; import { useNodes } from '@/context/NodeContext'; +import { useAuth } from '@/context/AuthContext'; import { DEFAULT_SETTINGS } from './types'; import type { PatchableSettings } from './types'; import { SettingsSection } from './SettingsSection'; @@ -132,7 +133,7 @@ function SettingsSkeleton() { ); } -type SystemFields = Pick; +type SystemFields = Pick; const DEFAULT_SYSTEM: SystemFields = { host_cpu_limit: DEFAULT_SETTINGS.host_cpu_limit, @@ -141,10 +142,12 @@ const DEFAULT_SYSTEM: SystemFields = { host_alert_suppression_mins: DEFAULT_SETTINGS.host_alert_suppression_mins, docker_janitor_gb: DEFAULT_SETTINGS.docker_janitor_gb, global_crash: DEFAULT_SETTINGS.global_crash, + mesh_auto_recreate: DEFAULT_SETTINGS.mesh_auto_recreate, }; export function SystemSection({ onDirtyChange }: SystemSectionProps) { const { activeNode } = useNodes(); + const { isAdmin } = useAuth(); const [settings, setSettings] = useState({ ...DEFAULT_SYSTEM }); const serverSettingsRef = useRef({ ...DEFAULT_SYSTEM }); const [isLoading, setIsLoading] = useState(false); @@ -159,6 +162,7 @@ export function SystemSection({ onDirtyChange }: SystemSectionProps) { if (settings.host_alert_suppression_mins !== baseline.host_alert_suppression_mins) n++; if (settings.docker_janitor_gb !== baseline.docker_janitor_gb) n++; if (settings.global_crash !== baseline.global_crash) n++; + if (settings.mesh_auto_recreate !== baseline.mesh_auto_recreate) n++; return n; }, [settings]); @@ -193,6 +197,7 @@ export function SystemSection({ onDirtyChange }: SystemSectionProps) { host_alert_suppression_mins: nodeData.host_alert_suppression_mins ?? DEFAULT_SETTINGS.host_alert_suppression_mins, docker_janitor_gb: nodeData.docker_janitor_gb ?? DEFAULT_SETTINGS.docker_janitor_gb, global_crash: (nodeData.global_crash as '0' | '1') ?? DEFAULT_SETTINGS.global_crash, + mesh_auto_recreate: (nodeData.mesh_auto_recreate as '0' | '1') ?? DEFAULT_SETTINGS.mesh_auto_recreate, }; setSettings(safe); serverSettingsRef.current = { ...safe }; @@ -314,6 +319,29 @@ export function SystemSection({ onDirtyChange }: SystemSectionProps) { + {/* + Mesh-data-plane recreate touches Docker (createNetwork + + connectContainerToNetwork) on the backend, which is admin-gated + by `requireAdmin` on the settings route. Hide the affordance + for non-admins so the toggle does not surface a 403 on save. + The rest of the System section stays visible because it + matches the existing pattern (host thresholds, janitor, alert + suppression all visible read-only to non-admins). + */} + {isAdmin && ( + + + onSettingChange('mesh_auto_recreate', next ? '1' : '0')} + /> + + + )} + {isSaving ? ( diff --git a/frontend/src/components/settings/types.ts b/frontend/src/components/settings/types.ts index 2526d8a9..56f984a1 100644 --- a/frontend/src/components/settings/types.ts +++ b/frontend/src/components/settings/types.ts @@ -10,6 +10,7 @@ export interface PatchableSettings { metrics_retention_hours?: string; log_retention_days?: string; audit_retention_days?: string; + mesh_auto_recreate?: '0' | '1'; } export const DEFAULT_SETTINGS: PatchableSettings = { @@ -24,6 +25,7 @@ export const DEFAULT_SETTINGS: PatchableSettings = { metrics_retention_hours: '24', log_retention_days: '30', audit_retention_days: '90', + mesh_auto_recreate: '0', }; export type SectionId = diff --git a/frontend/src/types/mesh.ts b/frontend/src/types/mesh.ts index 0afd3be6..e81e6fb0 100644 --- a/frontend/src/types/mesh.ts +++ b/frontend/src/types/mesh.ts @@ -8,7 +8,8 @@ export type MeshDataPlaneReason = | 'subnet_mismatch' | 'ip_in_use' | 'attach_failed' - | 'not_in_docker'; + | 'not_in_docker' + | 'not_found'; export interface MeshDataPlaneStatus { ok: boolean;