From 3c934066f3d49d5850a42475821a2d47dbf52b2e Mon Sep 17 00:00:00 2001 From: Anso Date: Wed, 29 Jul 2026 12:55:30 -0400 Subject: [PATCH] fix(networking): ignore verified Mesh attachments in drift (#1729) --- .../compose-network-inspector.test.ts | 192 ++++++++- backend/src/__tests__/drift-detection.test.ts | 69 ++++ .../__tests__/managed-mesh-attachment.test.ts | 124 ++++++ .../mesh-remove-override-remote.test.ts | 34 +- backend/src/__tests__/mesh-service.test.ts | 382 ++++++++++++++++++ .../src/__tests__/networking-summary.test.ts | 36 ++ backend/src/services/DriftDetectionService.ts | 34 +- backend/src/services/MeshService.ts | 292 +++++++++++-- .../network/composeNetworkInspector.ts | 12 +- .../services/network/managedMeshAttachment.ts | 55 +++ .../src/services/network/networkingSummary.ts | 10 +- backend/src/services/network/normalize.ts | 10 +- docs/features/compose-networking.mdx | 2 +- docs/features/stack-drift.mdx | 4 +- 14 files changed, 1206 insertions(+), 50 deletions(-) create mode 100644 backend/src/__tests__/managed-mesh-attachment.test.ts create mode 100644 backend/src/services/network/managedMeshAttachment.ts diff --git a/backend/src/__tests__/compose-network-inspector.test.ts b/backend/src/__tests__/compose-network-inspector.test.ts index 702a9328..2ca505b8 100644 --- a/backend/src/__tests__/compose-network-inspector.test.ts +++ b/backend/src/__tests__/compose-network-inspector.test.ts @@ -4,14 +4,21 @@ * runtime-vs-Compose drift comparison (system/default/external networks and * stopped containers are not flagged). */ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import type { EffectiveModel, EffService } from '../services/preflight/effectiveModel'; import type { DeclaredCompose } from '../helpers/composeDependencyParse'; import type { DependencySnapshot, DependencyContainer, DependencyNetwork } from '../services/DockerController'; import { fromEffectiveModel, fromDeclaredCompose, compareStackNetworks, runtimeResourceName, parseAccessUrlPorts, + type ManagedNetworkAttachmentPredicate, } from '../services/network/normalize'; -import { assembleStackNetworkFacts } from '../services/network/composeNetworkInspector'; +import { assembleStackNetworkFacts, buildStackNetworkFacts } from '../services/network/composeNetworkInspector'; +import { buildNodeNetworkingFindings } from '../services/network/networkingFindings'; +import type { NetworkingNetworkBase } from '../services/network/networkingTypes'; +import DockerController from '../services/DockerController'; +import { ComposeService } from '../services/ComposeService'; +import { DatabaseService } from '../services/DatabaseService'; +import { FileSystemService } from '../services/FileSystemService'; function effSvc(over: Partial = {}): EffService { const hasHealthcheck = over.hasHealthcheck ?? true; @@ -264,6 +271,187 @@ describe('compareStackNetworks', () => { expect(drift.foreignNetworkAttachments).toEqual([]); }); + it('removes managed Mesh drift before Networking findings while preserving advanced-driver info', () => { + const meshOnlyModel: EffectiveModel = { + projectName: 'myapp', + services: [], + networks: {}, + volumes: {}, + }; + const snap = snapshot( + [container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })], + [depNet({ id: 'm', name: 'sencho_mesh', driver: 'macvlan', composeProject: null, stack: null })], + ); + const facts = assembleStackNetworkFacts( + 'myapp', + meshOnlyModel, + null, + snap, + (_runtimeContainer, networkName) => networkName === 'sencho_mesh', + ); + const baseNetworks: NetworkingNetworkBase[] = [{ + id: 'm', + name: 'sencho_mesh', + driver: 'macvlan', + scope: 'local', + isSystem: false, + ingress: false, + composeProject: null, + stack: null, + connectedCount: 1, + isSencho: true, + ownership: 'sencho-managed', + declaredByStacks: [], + declaredExternalByStacks: [], + isExternalDependency: false, + }]; + + const findings = buildNodeNetworkingFindings(1, snap, [facts], baseNetworks); + + expect(facts.drift.runtimeOnlyAttachments).toEqual([]); + expect(facts.drift.foreignNetworkAttachments).toEqual([]); + expect(findings.some(f => f.kind === 'network-undeclared' || f.kind === 'foreign-network-attachment')).toBe(false); + expect(findings).toContainEqual(expect.objectContaining({ + kind: 'advanced-driver-caveat', + severity: 'info', + network: 'sencho_mesh', + })); + }); + + it('keeps Networking facts available and Mesh drift actionable when opt-in authority fails', async () => { + const snap = snapshot( + [container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })], + [depNet({ name: 'sencho_mesh', composeProject: null, stack: null })], + ); + vi.spyOn(ComposeService, 'getInstance').mockReturnValue({ + renderConfig: vi.fn().mockResolvedValue({ + rendered: JSON.stringify({ name: 'myapp', services: { web: { image: 'nginx:1.27' } } }), + stderr: '', + code: 0, + timedOut: false, + }), + } as unknown as ComposeService); + vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({ + getStacks: vi.fn().mockResolvedValue(['myapp']), + } as unknown as FileSystemService); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getDependencySnapshot: vi.fn().mockResolvedValue(snap), + } as unknown as DockerController); + vi.spyOn(DatabaseService, 'getInstance').mockImplementation(() => { + throw new Error('database unavailable'); + }); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + try { + const facts = await buildStackNetworkFacts(1, 'myapp'); + + expect(facts.runtime).toBe('available'); + expect(facts.drift.foreignNetworkAttachments).toEqual([{ container: 'web1', network: 'sencho_mesh' }]); + } finally { + vi.restoreAllMocks(); + } + }); + + it('threads opted-in authority through Networking facts and preserves advanced-driver info', async () => { + const snap = snapshot( + [container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })], + [depNet({ id: 'm', name: 'sencho_mesh', driver: 'macvlan', composeProject: null, stack: null })], + ); + vi.spyOn(ComposeService, 'getInstance').mockReturnValue({ + renderConfig: vi.fn().mockResolvedValue({ + rendered: JSON.stringify({ name: 'myapp', services: { web: { image: 'nginx:1.27' } } }), + stderr: '', + code: 0, + timedOut: false, + }), + } as unknown as ComposeService); + vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({ + getStacks: vi.fn().mockResolvedValue(['myapp']), + } as unknown as FileSystemService); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getDependencySnapshot: vi.fn().mockResolvedValue(snap), + } as unknown as DockerController); + vi.spyOn(DatabaseService, 'getInstance').mockReturnValue({ + isMeshStackEnabled: vi.fn().mockReturnValue(true), + getStackExposureIntents: vi.fn().mockReturnValue([]), + getStackDossier: vi.fn().mockReturnValue(null), + } as unknown as DatabaseService); + + try { + const facts = await buildStackNetworkFacts(1, 'myapp'); + const baseNetworks: NetworkingNetworkBase[] = [{ + id: 'm', name: 'sencho_mesh', driver: 'macvlan', scope: 'local', isSystem: false, + ingress: false, composeProject: null, stack: null, connectedCount: 1, isSencho: true, + ownership: 'sencho-managed', declaredByStacks: [], declaredExternalByStacks: [], + isExternalDependency: false, + }]; + const findings = buildNodeNetworkingFindings(1, snap, [facts], baseNetworks); + + expect(facts.drift.runtimeOnlyAttachments).toEqual([]); + expect(facts.drift.foreignNetworkAttachments).toEqual([]); + expect(findings.filter(f => f.kind === 'network-undeclared' || f.kind === 'foreign-network-attachment')).toEqual([]); + expect(findings).toContainEqual(expect.objectContaining({ + kind: 'advanced-driver-caveat', + severity: 'info', + network: 'sencho_mesh', + })); + } finally { + vi.restoreAllMocks(); + } + }); + + it('ignores a verified Sencho Mesh attachment for the Sencho container', () => { + const snap = snapshot( + [container({ id: 'sencho-id', name: 'sencho', service: 'sencho', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })], + [depNet({ name: 'sencho_mesh', composeProject: null, stack: null })], + ); + const managed: ManagedNetworkAttachmentPredicate = (runtimeContainer, networkName) => + runtimeContainer.id === 'sencho-id' && networkName === 'sencho_mesh'; + + const drift = compareStackNetworks(declared, snap, 'myapp', managed); + + expect(drift.runtimeOnlyAttachments).toEqual([]); + expect(drift.foreignNetworkAttachments).toEqual([]); + }); + + it('ignores a verified Mesh attachment for an opted-in application stack', () => { + const snap = snapshot( + [container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })], + [depNet({ name: 'sencho_mesh', composeProject: null, stack: null })], + ); + const managed: ManagedNetworkAttachmentPredicate = (_runtimeContainer, networkName) => + networkName === 'sencho_mesh'; + + const drift = compareStackNetworks(declared, snap, 'myapp', managed); + + expect(drift.runtimeOnlyAttachments).toEqual([]); + expect(drift.foreignNetworkAttachments).toEqual([]); + }); + + it('keeps an unverified manual Mesh attachment actionable', () => { + const snap = snapshot( + [container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })], + [depNet({ name: 'sencho_mesh', composeProject: null, stack: null })], + ); + + const drift = compareStackNetworks(declared, snap, 'myapp', () => false); + + expect(drift.runtimeOnlyAttachments).toEqual([]); + expect(drift.foreignNetworkAttachments).toEqual([{ container: 'web1', network: 'sencho_mesh' }]); + }); + + it('keeps sencho_extra actionable even when the stack is Mesh-managed', () => { + const snap = snapshot( + [container({ networks: [{ name: 'sencho_extra', id: 'e', ip: '' }] })], + [depNet({ name: 'sencho_extra', composeProject: null, stack: null })], + ); + + const drift = compareStackNetworks(declared, snap, 'myapp', () => true); + + expect(drift.runtimeOnlyAttachments).toEqual([]); + expect(drift.foreignNetworkAttachments).toEqual([{ container: 'web1', network: 'sencho_extra' }]); + }); + it('does not flag attachments from stopped containers', () => { const snap = snapshot( [container({ state: 'exited', networks: [{ name: 'myapp_extra', id: 'b', ip: '' }] })], diff --git a/backend/src/__tests__/drift-detection.test.ts b/backend/src/__tests__/drift-detection.test.ts index c5198e68..fa810e2d 100644 --- a/backend/src/__tests__/drift-detection.test.ts +++ b/backend/src/__tests__/drift-detection.test.ts @@ -18,6 +18,7 @@ import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompos import type { DeclaredCompose, DeclaredService, DeclaredPort } from '../helpers/composeDependencyParse'; import type { EffectiveModel, EffService } from '../services/preflight/effectiveModel'; import { fromDeclaredCompose, fromEffectiveModel } from '../services/network/normalize'; +import { DatabaseService } from '../services/DatabaseService'; // ── builders ──────────────────────────────────────────────────────────── @@ -580,6 +581,32 @@ describe('assembleStackDrift - network drift', () => { expect(report.findings.filter(f => f.kind.startsWith('network-'))).toEqual([]); expect(report.status).toBe('in-sync'); }); + + it('reports in-sync when a verified Mesh attachment is the only runtime difference', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([service({ name: 'web' })]), + containers: [container({ id: 'c1', service: 'web', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })], + networks: [depNet('sencho_mesh', { composeProject: null, stack: null })], + managedNetworkAttachment: (_runtimeContainer, networkName) => networkName === 'sencho_mesh', + }); + + expect(report.status).toBe('in-sync'); + expect(report.findings.filter(f => f.kind === 'network-undeclared')).toEqual([]); + }); + + it('keeps an unverified manual Mesh attachment drifted', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([service({ name: 'web' })]), + containers: [container({ id: 'c1', service: 'web', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })], + networks: [depNet('sencho_mesh', { composeProject: null, stack: null })], + managedNetworkAttachment: () => false, + }); + + expect(report.status).toBe('drifted'); + expect(report.findings.filter(f => f.kind === 'network-undeclared')).toHaveLength(1); + }); }); // ── declaredFromEffectiveModel ───────────────────────────────────────────── @@ -815,4 +842,46 @@ describe('buildStackDriftReport - boundaries', () => { expect(findingKinds(report)).toContain('network-undeclared'); vi.restoreAllMocks(); }); + + it('keeps the report available and Mesh drift actionable when opt-in authority fails', async () => { + const snapshot: DependencySnapshot = { + containers: [container({ id: 'c1', service: 'web', stack: 'app', image: 'nginx:1.25', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })], + networks: [depNet('sencho_mesh', { composeProject: null, stack: null })], + volumes: [], + }; + stubDockerRender({ name: 'app', services: { web: { image: 'nginx:1.25' } } }); + stubFsAndSnapshot(snapshot); + vi.spyOn(DatabaseService, 'getInstance').mockImplementation(() => { + throw new Error('database unavailable'); + }); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + const report = await buildStackDriftReport(0, 'app'); + + expect(report.status).toBe('drifted'); + expect(report.findings).toContainEqual(expect.objectContaining({ + kind: 'network-undeclared', + actual: 'sencho_mesh', + })); + vi.restoreAllMocks(); + }); + + it('reports in-sync through the public builder when DB authority opts the stack into Mesh', async () => { + const snapshot: DependencySnapshot = { + containers: [container({ id: 'c1', service: 'web', stack: 'app', image: 'nginx:1.25', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })], + networks: [depNet('sencho_mesh', { composeProject: null, stack: null })], + volumes: [], + }; + stubDockerRender({ name: 'app', services: { web: { image: 'nginx:1.25' } } }); + stubFsAndSnapshot(snapshot); + vi.spyOn(DatabaseService, 'getInstance').mockReturnValue({ + isMeshStackEnabled: vi.fn().mockReturnValue(true), + } as unknown as DatabaseService); + + const report = await buildStackDriftReport(0, 'app'); + + expect(report.status).toBe('in-sync'); + expect(report.findings.filter(f => f.kind === 'network-undeclared')).toEqual([]); + vi.restoreAllMocks(); + }); }); diff --git a/backend/src/__tests__/managed-mesh-attachment.test.ts b/backend/src/__tests__/managed-mesh-attachment.test.ts new file mode 100644 index 00000000..c80fdf11 --- /dev/null +++ b/backend/src/__tests__/managed-mesh-attachment.test.ts @@ -0,0 +1,124 @@ +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { DatabaseService } from '../services/DatabaseService'; +import SelfIdentityService from '../services/SelfIdentityService'; +import type { DependencyContainer } from '../services/DockerController'; +import { resolveManagedMeshAttachment } from '../services/network/managedMeshAttachment'; + +const originalDataDir = process.env.DATA_DIR; +const originalMode = process.env.SENCHO_MODE; +let tempDir: string | null = null; + +function runtimeContainer(overrides: Partial = {}): DependencyContainer { + return { + id: 'app-id', + name: 'app-web-1', + service: 'web', + composeProject: 'app', + stack: 'app', + state: 'running', + exitCode: null, + image: 'nginx:latest', + networks: [], + volumes: [], + ports: [], + ...overrides, + }; +} + +function stubAuthorities(meshStackEnabled: boolean, ownContainers: string[] = []): void { + vi.spyOn(DatabaseService, 'getInstance').mockReturnValue({ + isMeshStackEnabled: vi.fn().mockReturnValue(meshStackEnabled), + } as unknown as DatabaseService); + vi.spyOn(SelfIdentityService, 'getInstance').mockReturnValue({ + isOwnContainer: vi.fn((idOrName: string) => ownContainers.includes(idOrName)), + } as unknown as SelfIdentityService); +} + +afterEach(async () => { + vi.restoreAllMocks(); + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; + if (originalMode === undefined) delete process.env.SENCHO_MODE; + else process.env.SENCHO_MODE = originalMode; + if (tempDir) await fs.rm(tempDir, { recursive: true, force: true }); + tempDir = null; +}); + +describe('resolveManagedMeshAttachment', () => { + it('authorizes the canonical Mesh attachment for a centrally opted-in stack', async () => { + process.env.SENCHO_MODE = 'server'; + stubAuthorities(true); + const isManaged = await resolveManagedMeshAttachment(1, 'app'); + + expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(true); + expect(isManaged(runtimeContainer(), 'sencho_extra')).toBe(false); + }); + + it('authorizes the canonical Mesh attachment for the actual Sencho container', async () => { + process.env.SENCHO_MODE = 'server'; + stubAuthorities(false, ['sencho-id']); + const isManaged = await resolveManagedMeshAttachment(1, 'sencho'); + + expect(isManaged(runtimeContainer({ id: 'sencho-id', name: 'sencho' }), 'sencho_mesh')).toBe(true); + expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(false); + }); + + it('keeps a manual Mesh attachment actionable for an opted-out stack', async () => { + process.env.SENCHO_MODE = 'server'; + stubAuthorities(false); + const isManaged = await resolveManagedMeshAttachment(1, 'app'); + + expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(false); + }); + + it('uses Pilot override presence as the authoritative opt-in representation', async () => { + process.env.SENCHO_MODE = 'pilot'; + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'sencho-mesh-drift-')); + process.env.DATA_DIR = tempDir; + const overrideDir = path.join(tempDir, 'mesh', 'overrides', '7'); + await fs.mkdir(overrideDir, { recursive: true }); + await fs.writeFile(path.join(overrideDir, 'app.override.yml'), 'services: {}\n'); + stubAuthorities(false); + + const isManaged = await resolveManagedMeshAttachment(7, 'app'); + + expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(true); + }); + + it('does not let stale server override presence supersede opted-out DB state', async () => { + process.env.SENCHO_MODE = 'server'; + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'sencho-mesh-drift-')); + process.env.DATA_DIR = tempDir; + const overrideDir = path.join(tempDir, 'mesh', 'overrides', '1'); + await fs.mkdir(overrideDir, { recursive: true }); + await fs.writeFile(path.join(overrideDir, 'app.override.yml'), 'services: {}\n'); + stubAuthorities(false); + + const isManaged = await resolveManagedMeshAttachment(1, 'app'); + + expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(false); + }); + + it('fails closed when Mesh opt-in state cannot be read', async () => { + process.env.SENCHO_MODE = 'server'; + vi.spyOn(DatabaseService, 'getInstance').mockImplementation(() => { + throw new Error('database unavailable'); + }); + vi.spyOn(SelfIdentityService, 'getInstance').mockReturnValue({ + isOwnContainer: vi.fn().mockReturnValue(false), + } as unknown as SelfIdentityService); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + const isManaged = await resolveManagedMeshAttachment(1, 'app'); + + expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(false); + expect(warn).toHaveBeenCalledWith( + '[NetworkDrift] Could not verify Mesh opt-in state for %s:', + 'app', + 'database unavailable', + ); + }); +}); diff --git a/backend/src/__tests__/mesh-remove-override-remote.test.ts b/backend/src/__tests__/mesh-remove-override-remote.test.ts index 5fc15682..a027ea9a 100644 --- a/backend/src/__tests__/mesh-remove-override-remote.test.ts +++ b/backend/src/__tests__/mesh-remove-override-remote.test.ts @@ -96,7 +96,7 @@ describe('MeshService.removeOverrideFromNode (remote dispatch)', () => { db.deleteNode(remoteNodeId); }); - it('swallows network errors from the remote so the disable cascade can continue', async () => { + it('rejects network errors so callers can preserve authoritative opt-in state', async () => { const svc = MeshService.getInstance(); const db = DatabaseService.getInstance(); const remoteNodeId = db.addNode({ @@ -117,11 +117,37 @@ describe('MeshService.removeOverrideFromNode (remote dispatch)', () => { vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('ECONNREFUSED')); const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ }); - // Must not throw: the remote being offline is a tolerable condition; - // the cascade upstream uses Promise.allSettled and continues. - await expect(svc.removeOverrideFromNode(remoteNodeId, 'sample-stack')).resolves.toBeUndefined(); + await expect(svc.removeOverrideFromNode(remoteNodeId, 'sample-stack')).rejects.toThrow('ECONNREFUSED'); expect(warnSpy).toHaveBeenCalled(); db.deleteNode(remoteNodeId); }); + + it('rejects a non-success response from a busy remote target', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const remoteNodeId = db.addNode({ + name: 'remove-override-busy-test', + type: 'remote', + mode: 'proxy', + compose_dir: '/tmp', + is_default: false, + api_url: 'https://remote.example.com:1852', + api_token: 'remote-tok', + }); + + vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ + apiUrl: 'https://remote.example.com:1852', + apiToken: 'remote-tok', + }); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response('another operation is already in progress', { status: 500 }), + ); + vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ }); + + await expect(svc.removeOverrideFromNode(remoteNodeId, 'sample-stack')) + .rejects.toThrow('HTTP 500'); + + db.deleteNode(remoteNodeId); + }); }); diff --git a/backend/src/__tests__/mesh-service.test.ts b/backend/src/__tests__/mesh-service.test.ts index ea28d274..b4cdd0b7 100644 --- a/backend/src/__tests__/mesh-service.test.ts +++ b/backend/src/__tests__/mesh-service.test.ts @@ -1,6 +1,7 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { EventEmitter } from 'events'; import fsSync from 'fs'; +import fs from 'fs/promises'; import path from 'path'; import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; import { getSenchoIpFromSubnet, MeshError, type MeshTarget, type MeshTcpStreamLike } from '../services/MeshService'; @@ -20,6 +21,7 @@ afterAll(() => { }); beforeEach(() => { + process.env.SENCHO_MODE = 'server'; const db = DatabaseService.getInstance().getDb(); db.prepare('DELETE FROM mesh_stacks').run(); db.prepare('DELETE FROM nodes WHERE is_default = 0').run(); @@ -106,6 +108,84 @@ describe('MeshService.optInStack', () => { expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false); }); + it('restores opt-in authority when target override removal is rejected', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + db.insertMeshStack(localNodeId, 'busy-stack', 'setup'); + vi.spyOn(svc, 'removeOverrideFromNode').mockRejectedValue( + new Error('HTTP 500: another operation is already in progress'), + ); + + await expect(svc.optOutStack(localNodeId, 'busy-stack', 'tester')) + .rejects.toThrow('another operation is already in progress'); + + expect(db.isMeshStackEnabled(localNodeId, 'busy-stack')).toBe(true); + }); + + it('serializes concurrent opt-out requests for the same node', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + db.insertMeshStack(localNodeId, 'queued-stack', 'setup'); + let releaseRemoval!: () => void; + const removalPending = new Promise((resolve) => { + releaseRemoval = resolve; + }); + const removeSpy = vi.spyOn(svc, 'removeOverrideFromNode').mockReturnValue(removalPending); + vi.spyOn(svc as unknown as { regenerateOverridesAcrossFleet: () => Promise }, 'regenerateOverridesAcrossFleet') + .mockResolvedValue(undefined); + vi.spyOn(svc as unknown as { cascadeRecomposeAcrossFleet: () => void }, 'cascadeRecomposeAcrossFleet') + .mockImplementation(() => { /* noop */ }); + vi.spyOn(svc, 'triggerRedeploy').mockImplementation(() => { /* noop */ }); + + const first = svc.optOutStack(localNodeId, 'queued-stack', 'tester'); + await vi.waitFor(() => expect(removeSpy).toHaveBeenCalledTimes(1)); + const second = svc.optOutStack(localNodeId, 'queued-stack', 'tester'); + await Promise.resolve(); + expect(removeSpy).toHaveBeenCalledTimes(1); + + releaseRemoval(); + await Promise.all([first, second]); + + expect(removeSpy).toHaveBeenCalledTimes(1); + expect(db.isMeshStackEnabled(localNodeId, 'queued-stack')).toBe(false); + }); + + it('serializes different Mesh mutations per node without blocking another node', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + const remoteNodeId = db.addNode({ + name: 'independent-node', type: 'remote', mode: 'pilot_agent', + compose_dir: '/tmp', is_default: false, api_url: '', api_token: '', + }); + db.setNodeMeshEnabled(localNodeId, true); + db.insertMeshStack(localNodeId, 'held-stack', 'setup'); + let releaseRemoval!: () => void; + const removalPending = new Promise((resolve) => { + releaseRemoval = resolve; + }); + vi.spyOn(svc, 'removeOverrideFromNode').mockReturnValue(removalPending); + vi.spyOn(svc as unknown as { regenerateOverridesAcrossFleet: () => Promise }, 'regenerateOverridesAcrossFleet') + .mockResolvedValue(undefined); + vi.spyOn(svc as unknown as { cascadeRecomposeAcrossFleet: () => void }, 'cascadeRecomposeAcrossFleet') + .mockImplementation(() => { /* noop */ }); + vi.spyOn(svc, 'triggerRedeploy').mockImplementation(() => { /* noop */ }); + + const optOut = svc.optOutStack(localNodeId, 'held-stack', 'tester'); + await vi.waitFor(() => expect(svc.removeOverrideFromNode).toHaveBeenCalledTimes(1)); + const disable = svc.disableForNode(localNodeId, 'tester'); + await svc.enableForNode(remoteNodeId); + + expect(db.getNodeMeshEnabled(localNodeId)).toBe(true); + expect(db.getNodeMeshEnabled(remoteNodeId)).toBe(true); + releaseRemoval(); + await Promise.all([optOut, disable]); + expect(db.getNodeMeshEnabled(localNodeId)).toBe(false); + db.deleteNode(remoteNodeId); + }); + it('rejects an invalid stack name (path traversal attempt)', async () => { const svc = MeshService.getInstance(); const db = DatabaseService.getInstance(); @@ -427,6 +507,44 @@ describe('MeshService.disableForNode', () => { db.deleteNode(remoteNodeId); }); + it('keeps failed remote stacks authoritative when node disable is incomplete', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const remoteNodeId = db.addNode({ + name: 'remote-partial-disable', type: 'remote', mode: 'pilot_agent', + compose_dir: '/tmp', is_default: false, api_url: '', api_token: '', + }); + db.setNodeMeshEnabled(remoteNodeId, true); + db.insertMeshStack(remoteNodeId, 'removed-stack', 'setup'); + db.insertMeshStack(remoteNodeId, 'busy-stack', 'setup'); + vi.spyOn(svc, 'removeOverrideFromNode').mockImplementation(async (_nodeId, stackName) => { + if (stackName === 'busy-stack') throw new Error('target busy'); + }); + vi.spyOn( + svc as unknown as { regenerateOverridesAcrossFleet: () => Promise }, + 'regenerateOverridesAcrossFleet', + ).mockResolvedValue(undefined); + vi.spyOn( + svc as unknown as { cascadeRecomposeAcrossFleet: () => void }, + 'cascadeRecomposeAcrossFleet', + ).mockImplementation(() => { /* noop */ }); + const redeployed: string[] = []; + vi.spyOn(svc, 'triggerRedeploy').mockImplementation((_nodeId, stackName) => { + redeployed.push(stackName); + }); + vi.spyOn(svc as unknown as { refreshAliasCache: () => Promise }, 'refreshAliasCache') + .mockRejectedValue(new Error('refresh failed')); + vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ }); + + await expect(svc.disableForNode(remoteNodeId, 'tester')).rejects.toThrow('busy-stack'); + + expect(db.getNodeMeshEnabled(remoteNodeId)).toBe(true); + expect(db.isMeshStackEnabled(remoteNodeId, 'removed-stack')).toBe(false); + expect(db.isMeshStackEnabled(remoteNodeId, 'busy-stack')).toBe(true); + expect(redeployed).toContain('removed-stack'); + db.deleteNode(remoteNodeId); + }); + it('defaults the actor when none is supplied so legacy callers still log a non-empty actor', async () => { const svc = MeshService.getInstance(); const db = DatabaseService.getInstance(); @@ -673,6 +791,62 @@ describe('MeshService.optInStack rollback', () => { .rejects.toThrow(/simulated remote pilot offline/); expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false); }); + + it('retains remote authority when the push outcome is unknown', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const remoteNodeId = db.addNode({ + name: 'ambiguous-push', type: 'remote', mode: 'pilot_agent', + compose_dir: '/tmp', is_default: false, api_url: '', api_token: '', + }); + vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise }, 'inspectStackServices') + .mockResolvedValue([{ service: 'web', ports: [8080] }]); + vi.spyOn(svc, 'pushOverrideToNode').mockRejectedValue(new Error('connection reset')); + + await expect(svc.optInStack(remoteNodeId, 'ambiguous-stack', 'tester')) + .rejects.toThrow('connection reset'); + + expect(db.isMeshStackEnabled(remoteNodeId, 'ambiguous-stack')).toBe(true); + db.deleteNode(remoteNodeId); + }); + + it('treats a remote gateway error as an ambiguous push outcome', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const remoteNodeId = db.addNode({ + name: 'gateway-error-push', type: 'remote', mode: 'pilot_agent', + compose_dir: '/tmp', is_default: false, api_url: '', api_token: '', + }); + vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise }, 'inspectStackServices') + .mockResolvedValue([{ service: 'web', ports: [8080] }]); + vi.spyOn(svc as unknown as { proxyFetch: () => Promise }, 'proxyFetch') + .mockResolvedValue(new Response('gateway timeout', { status: 502 })); + + await expect(svc.optInStack(remoteNodeId, 'gateway-error-stack', 'tester')) + .rejects.toThrow('HTTP 502'); + + expect(db.isMeshStackEnabled(remoteNodeId, 'gateway-error-stack')).toBe(true); + db.deleteNode(remoteNodeId); + }); + + it('rolls back remote authority when the target explicitly rejects the push', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const remoteNodeId = db.addNode({ + name: 'rejected-push', type: 'remote', mode: 'pilot_agent', + compose_dir: '/tmp', is_default: false, api_url: '', api_token: '', + }); + vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise }, 'inspectStackServices') + .mockResolvedValue([{ service: 'web', ports: [8080] }]); + vi.spyOn(svc, 'pushOverrideToNode') + .mockRejectedValue(new MeshError('push_failed', 'target rejected override')); + + await expect(svc.optInStack(remoteNodeId, 'rejected-stack', 'tester')) + .rejects.toThrow('target rejected override'); + + expect(db.isMeshStackEnabled(remoteNodeId, 'rejected-stack')).toBe(false); + db.deleteNode(remoteNodeId); + }); }); describe('MeshService.optInStack guard rails (network setup)', () => { @@ -984,6 +1158,7 @@ describe('MeshService.ensureStackOverride (BUG-1 fix)', () => { const db = DatabaseService.getInstance(); const localNodeId = db.getNodes()[0].id; + process.env.SENCHO_MODE = 'pilot'; // Simulate the pilot scenario: no mesh_stacks row (isMeshStackEnabled → false), // but the override file already exists on disk, pushed by central via D-1. const dataDir = process.env.DATA_DIR as string; @@ -1008,6 +1183,184 @@ describe('MeshService.ensureStackOverride (BUG-1 fix)', () => { // Cleanup. fsSync.unlinkSync(overrideFile); + process.env.SENCHO_MODE = 'server'; + }); + + it('persists and removes proxy-target opt-in state with a pushed local override', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']); + + const file = await svc.applyLocalOverride('proxy-stack', []); + + expect(file).not.toBeNull(); + const yaml = fsSync.readFileSync(file as string, 'utf8'); + expect(yaml).toContain('web:'); + expect(yaml).toContain('sencho_mesh'); + expect(fsSync.readdirSync(path.dirname(file as string)).some((name) => name.includes('proxy-stack') && name.endsWith('.tmp'))).toBe(false); + expect(db.isMeshStackEnabled(localNodeId, 'proxy-stack')).toBe(true); + + await svc.removeLocalOverride('proxy-stack'); + + expect(db.isMeshStackEnabled(localNodeId, 'proxy-stack')).toBe(false); + }); + + it('does not write a proxy override when DB authority cannot be recorded', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']); + vi.spyOn(db, 'insertMeshStack').mockImplementation(() => { + throw new Error('database unavailable'); + }); + + await expect(svc.applyLocalOverride('db-failure', [])).rejects.toThrow('database unavailable'); + + const overrideFile = path.join( + process.env.DATA_DIR as string, + 'mesh', + 'overrides', + String(localNodeId), + 'db-failure.override.yml', + ); + expect(fsSync.existsSync(overrideFile)).toBe(false); + }); + + it('restores an existing override when DB authority cannot be recorded', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId)); + fsSync.mkdirSync(overrideDir, { recursive: true }); + const overrideFile = path.join(overrideDir, 'db-replacement-failure.override.yml'); + const originalYaml = 'services:\n prior:\n networks:\n - sencho_mesh\n'; + fsSync.writeFileSync(overrideFile, originalYaml, 'utf8'); + vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']); + vi.spyOn(db, 'insertMeshStack').mockImplementation(() => { + throw new Error('database unavailable'); + }); + + await expect(svc.applyLocalOverride('db-replacement-failure', [])).rejects.toThrow('database unavailable'); + + expect(fsSync.readFileSync(overrideFile, 'utf8')).toBe(originalYaml); + expect(db.isMeshStackEnabled(localNodeId, 'db-replacement-failure')).toBe(false); + expect(fsSync.readdirSync(overrideDir).some((name) => name.includes('db-replacement-failure') && name.endsWith('.tmp'))).toBe(false); + }); + + it('does not publish DB authority or a final file when atomic override publication fails', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']); + vi.spyOn(fs, 'rename').mockRejectedValue(new Error('rename failed')); + + await expect(svc.applyLocalOverride('write-failure', [])).rejects.toThrow('rename failed'); + + const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId)); + expect(db.isMeshStackEnabled(localNodeId, 'write-failure')).toBe(false); + expect(fsSync.existsSync(path.join(overrideDir, 'write-failure.override.yml'))).toBe(false); + expect(fsSync.readdirSync(overrideDir).some((name) => name.includes('write-failure'))).toBe(false); + }); + + it('preserves the prior override and authority when atomic replacement fails', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId)); + fsSync.mkdirSync(overrideDir, { recursive: true }); + const overrideFile = path.join(overrideDir, 'replacement-failure.override.yml'); + const originalYaml = 'services:\n prior:\n networks:\n - sencho_mesh\n'; + fsSync.writeFileSync(overrideFile, originalYaml, 'utf8'); + db.insertMeshStack(localNodeId, 'replacement-failure', 'tester'); + vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']); + vi.spyOn(fs, 'rename').mockRejectedValue(new Error('rename failed')); + + await expect(svc.applyLocalOverride('replacement-failure', [])).rejects.toThrow('rename failed'); + + expect(fsSync.readFileSync(overrideFile, 'utf8')).toBe(originalYaml); + expect(db.isMeshStackEnabled(localNodeId, 'replacement-failure')).toBe(true); + }); + + it('prevents overlapping override mutations for the same stack', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + let releaseServices!: (services: string[]) => void; + const servicesPending = new Promise((resolve) => { + releaseServices = resolve; + }); + vi.spyOn(svc, 'getDeclaredStackServiceNames').mockReturnValue(servicesPending); + + const first = svc.applyLocalOverride('concurrent-stack', []); + await vi.waitFor(() => expect(svc.getDeclaredStackServiceNames).toHaveBeenCalledTimes(1)); + + await expect(svc.applyLocalOverride('concurrent-stack', [])).rejects.toThrow('another operation'); + releaseServices(['web']); + await first; + + expect(db.isMeshStackEnabled(localNodeId, 'concurrent-stack')).toBe(true); + }); + + it('prevents removal from overlapping an in-flight override apply', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + let releaseServices!: (services: string[]) => void; + const servicesPending = new Promise((resolve) => { + releaseServices = resolve; + }); + vi.spyOn(svc, 'getDeclaredStackServiceNames').mockReturnValue(servicesPending); + + const apply = svc.applyLocalOverride('apply-remove-stack', []); + await vi.waitFor(() => expect(svc.getDeclaredStackServiceNames).toHaveBeenCalledTimes(1)); + + await expect(svc.removeLocalOverride('apply-remove-stack')).rejects.toThrow('another operation'); + releaseServices(['web']); + const file = await apply; + + expect(file).not.toBeNull(); + expect(fsSync.existsSync(file as string)).toBe(true); + expect(db.isMeshStackEnabled(localNodeId, 'apply-remove-stack')).toBe(true); + }); + + it('does not report committed removal as failed when alias refresh fails', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId)); + fsSync.mkdirSync(overrideDir, { recursive: true }); + const overrideFile = path.join(overrideDir, 'refresh-failure.override.yml'); + fsSync.writeFileSync(overrideFile, 'services: {}\n', 'utf8'); + db.insertMeshStack(localNodeId, 'refresh-failure', 'setup'); + (svc as unknown as { pilotAliasOverlay: Map }).pilotAliasOverlay.set('refresh-failure', []); + vi.spyOn(svc as unknown as { refreshAliasCache: () => Promise }, 'refreshAliasCache') + .mockRejectedValue(new Error('refresh failed')); + vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ }); + + await expect(svc.removeLocalOverride('refresh-failure')).resolves.toBeUndefined(); + + expect(fsSync.existsSync(overrideFile)).toBe(false); + expect(db.isMeshStackEnabled(localNodeId, 'refresh-failure')).toBe(false); + }); + + it('does not report committed apply as failed when alias refresh fails', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']); + vi.spyOn(svc as unknown as { refreshAliasCache: () => Promise }, 'refreshAliasCache') + .mockRejectedValue(new Error('refresh failed')); + vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ }); + + const file = await svc.applyLocalOverride('apply-refresh-failure', [], [{ + host: 'web.example', nodeId: localNodeId, nodeName: 'local', stackName: 'apply-refresh-failure', + serviceName: 'web', port: 8080, + }]); + + expect(file).not.toBeNull(); + expect(fsSync.existsSync(file as string)).toBe(true); + expect(db.isMeshStackEnabled(localNodeId, 'apply-refresh-failure')).toBe(true); }); it('returns null for pilot nodes when no pushed override file exists', async () => { @@ -1015,9 +1368,38 @@ describe('MeshService.ensureStackOverride (BUG-1 fix)', () => { const db = DatabaseService.getInstance(); const localNodeId = db.getNodes()[0].id; + process.env.SENCHO_MODE = 'pilot'; // No mesh_stacks row, no file on disk. const result = await svc.ensureStackOverride(localNodeId, 'no-such-stack'); expect(result).toBeNull(); + process.env.SENCHO_MODE = 'server'; + }); + + it('does not use stale override presence as authority on a server', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId)); + fsSync.mkdirSync(overrideDir, { recursive: true }); + const overrideFile = path.join(overrideDir, 'stale-server.override.yml'); + fsSync.writeFileSync(overrideFile, 'services: {}\n', 'utf8'); + + const result = await svc.ensureStackOverride(localNodeId, 'stale-server'); + + expect(result).toBeNull(); + fsSync.unlinkSync(overrideFile); + }); + + it('restores proxy-target DB authority when override removal fails', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + db.insertMeshStack(localNodeId, 'unlink-failure', 'tester'); + vi.spyOn(fs, 'unlink').mockRejectedValue(Object.assign(new Error('permission denied'), { code: 'EACCES' })); + + await expect(svc.removeLocalOverride('unlink-failure')).rejects.toThrow('permission denied'); + + expect(db.isMeshStackEnabled(localNodeId, 'unlink-failure')).toBe(true); }); }); diff --git a/backend/src/__tests__/networking-summary.test.ts b/backend/src/__tests__/networking-summary.test.ts index caa3d011..b774b16b 100644 --- a/backend/src/__tests__/networking-summary.test.ts +++ b/backend/src/__tests__/networking-summary.test.ts @@ -45,6 +45,7 @@ describe('networking summary', () => { afterEach(() => { vi.restoreAllMocks(); DatabaseService.getInstance().deleteStackExposureIntents(1, STACK); + DatabaseService.getInstance().deleteMeshStack(1, STACK); fs.rmSync(stackDir, { recursive: true, force: true }); }); @@ -72,6 +73,41 @@ describe('networking summary', () => { expect(res.body.networkDrift.stacks).toContain(STACK); }); + it('does not count an opted-in Mesh attachment as network drift', async () => { + DatabaseService.getInstance().insertMeshStack(1, STACK, 'tester'); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getDependencySnapshot: vi.fn().mockResolvedValue({ + containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }], volumes: [], ports: [] }], + networks: [ + { id: 'm', name: 'sencho_mesh', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null }, + ], + volumes: [], + }), + } as unknown as DockerController); + + const res = await request(app).get('/api/networking/summary').set('Authorization', authHeader); + + expect(res.status).toBe(200); + expect(res.body.networkDrift).toEqual({ count: 0, stacks: [] }); + }); + + it('counts an opted-out manual Mesh attachment as network drift', async () => { + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getDependencySnapshot: vi.fn().mockResolvedValue({ + containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }], volumes: [], ports: [] }], + networks: [ + { id: 'm', name: 'sencho_mesh', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null }, + ], + volumes: [], + }), + } as unknown as DockerController); + + const res = await request(app).get('/api/networking/summary').set('Authorization', authHeader); + + expect(res.status).toBe(200); + expect(res.body.networkDrift).toEqual({ count: 1, stacks: [STACK] }); + }); + it('still reports declared signals when the snapshot is unavailable (drift skipped)', async () => { vi.spyOn(DockerController, 'getInstance').mockReturnValue({ getDependencySnapshot: vi.fn().mockRejectedValue(new Error('docker down')), diff --git a/backend/src/services/DriftDetectionService.ts b/backend/src/services/DriftDetectionService.ts index 3151b4d2..69bd9da4 100644 --- a/backend/src/services/DriftDetectionService.ts +++ b/backend/src/services/DriftDetectionService.ts @@ -5,7 +5,12 @@ import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompos import type { DeclaredCompose, DeclaredService } from '../helpers/composeDependencyParse'; import { parseMissingRequiredVars } from '../helpers/envVarParse'; import { parseEffectiveModel } from './preflight/effectiveModel'; -import { compareStackNetworks, fromDeclaredCompose } from './network/normalize'; +import { + compareStackNetworks, + fromDeclaredCompose, + type ManagedNetworkAttachmentPredicate, +} from './network/normalize'; +import { resolveManagedMeshAttachment } from './network/managedMeshAttachment'; import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog'; import { getErrorMessage } from '../utils/errors'; import { isCleanOneShotCompletion } from '../utils/oneShotCompletion'; @@ -111,6 +116,8 @@ export interface AssembleStackDriftInput { containers: DependencyContainer[]; /** Every network on the node (for resolving foreign vs stack-owned attachments). */ networks?: DependencyNetwork[]; + /** Authoritative runtime attachments that are intentionally absent from authored Compose. */ + managedNetworkAttachment?: ManagedNetworkAttachmentPredicate; /** Set when the compose file could not be parsed. */ parseError?: string; } @@ -137,12 +144,18 @@ function networkDriftFindings( declared: DeclaredCompose, containers: DependencyContainer[], networks: DependencyNetwork[], + managedNetworkAttachment?: ManagedNetworkAttachmentPredicate, ): StackDriftFinding[] { // Runtime resource names use the Compose project (top-level `name:` when set), // not the stack directory, so a stack with `name:` resolves its networks the // same way Docker does. Containers are still attributed to the stack directory. const normalized = fromDeclaredCompose(declared, declared.projectName ?? stack); - const facts = compareStackNetworks(normalized, { containers, networks, volumes: [] }, stack); + const facts = compareStackNetworks( + normalized, + { containers, networks, volumes: [] }, + stack, + managedNetworkAttachment, + ); const findings: StackDriftFinding[] = []; const serviceByContainer = new Map(containers.map(c => [c.name, c.service ?? c.name])); @@ -293,7 +306,13 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe } } - findings.push(...networkDriftFindings(stack, declared, containers, networks)); + findings.push(...networkDriftFindings( + stack, + declared, + containers, + networks, + input.managedNetworkAttachment, + )); const status: StackDriftStatus = findings.length > 0 ? 'drifted' : 'in-sync'; return { stack, status, hasComposeFile: true, hasContainers, findings }; @@ -384,5 +403,12 @@ export async function buildStackDriftReport(nodeId: number, stackName: string): }; } - return assembleStackDrift({ stack: stackName, declared: render.declared, containers, networks }); + const managedNetworkAttachment = await resolveManagedMeshAttachment(nodeId, stackName); + return assembleStackDrift({ + stack: stackName, + declared: render.declared, + containers, + networks, + managedNetworkAttachment, + }); } diff --git a/backend/src/services/MeshService.ts b/backend/src/services/MeshService.ts index b7b0ff85..ca87971d 100644 --- a/backend/src/services/MeshService.ts +++ b/backend/src/services/MeshService.ts @@ -1,6 +1,7 @@ import net from 'net'; import path from 'path'; import fs from 'fs/promises'; +import { randomUUID } from 'crypto'; import { EventEmitter } from 'events'; import * as YAML from 'yaml'; import { ComposeService } from './ComposeService'; @@ -20,6 +21,7 @@ import { STREAM_PENDING_DATA_MAX_BYTES } from '../pilot/protocol'; import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; import { isDebugEnabled } from '../utils/debug'; import { isPathWithinBase, isValidStackName, isValidRelativeStackPath } from '../utils/validation'; +import { getErrorMessage } from '../utils/errors'; import { PORT as SENCHO_LISTEN_PORT } from '../helpers/constants'; import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate'; @@ -1497,6 +1499,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { // --- Opt-in / opt-out --- public async optInStack(nodeId: number, stackName: string, actor: string): Promise { + return this.runMeshNodeMutation(nodeId, () => this.optInStackExclusive(nodeId, stackName, actor)); + } + + private async optInStackExclusive(nodeId: number, stackName: string, actor: string): Promise { this.logDiag('opt-in start', { nodeId, stackName, actor }); const t0 = Date.now(); if (!isValidStackName(stackName)) { @@ -1541,19 +1547,41 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } db.insertMeshStack(nodeId, stackName, actor); - await this.refreshAliasCache(); - await this.syncForwarderListeners(); + try { + await this.refreshAliasCache(); + await this.syncForwarderListeners(); + } catch (error) { + db.deleteMeshStack(nodeId, stackName); + try { + await this.refreshAliasCache(); + await this.syncForwarderListeners(); + } catch (rollbackError) { + console.warn('[MeshService] Failed to refresh aliases after opt-in rollback:', sanitizeForLog(getErrorMessage(rollbackError, 'unknown'))); + } + throw error; + } - // Push the just-opted-in stack's override loudly. If this fails the - // DB state is invalid (alias claimed but remote pilot has no - // override file) so roll back rather than leave a half-state that - // future opt-in calls would short-circuit on `isMeshStackEnabled`. + // Push the just-opted-in stack's override loudly. Explicit target + // rejection rolls back the row. A remote transport failure is + // ambiguous because the target may already have committed, so retain + // authority and let normal regeneration reconcile it. try { await this.pushOverrideToNode(nodeId, stackName); } catch (err) { - db.deleteMeshStack(nodeId, stackName); - await this.refreshAliasCache(); - await this.syncForwarderListeners(); + const node = db.getNode(nodeId); + const explicitlyRejected = node?.type !== 'remote' || err instanceof MeshError; + if (explicitlyRejected) { + db.deleteMeshStack(nodeId, stackName); + await this.refreshAliasCache(); + await this.syncForwarderListeners(); + } else { + this.logActivity({ + source: 'mesh', level: 'warn', type: 'forwarder.error', + nodeId, + message: `mesh override push outcome unknown for ${stackName}; retaining opt-in authority for reconciliation`, + details: { stackName }, + }); + } throw err; } // Regenerate every other meshed stack's override across the fleet @@ -1584,6 +1612,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } public async optOutStack(nodeId: number, stackName: string, actor: string): Promise { + return this.runMeshNodeMutation(nodeId, () => this.optOutStackExclusive(nodeId, stackName, actor)); + } + + private async optOutStackExclusive(nodeId: number, stackName: string, actor: string): Promise { this.logDiag('opt-out start', { nodeId, stackName, actor }); if (!isValidStackName(stackName)) { throw new MeshError('denied', `invalid stack name: ${stackName}`); @@ -1591,9 +1623,18 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { const db = DatabaseService.getInstance(); if (!db.isMeshStackEnabled(nodeId, stackName)) return; db.deleteMeshStack(nodeId, stackName); - await this.removeOverrideFromNode(nodeId, stackName); - await this.refreshAliasCache(); - await this.syncForwarderListeners(); + try { + await this.removeOverrideFromNode(nodeId, stackName); + } catch (error) { + db.insertMeshStack(nodeId, stackName, actor); + throw error; + } + try { + await this.refreshAliasCache(); + await this.syncForwarderListeners(); + } catch (error) { + console.warn('[MeshService] Failed to refresh aliases after committed opt-out:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + } // The opted-out row is already deleted, so listMeshStacks() will not // include it. Walk the remaining fleet-wide rows so every other // meshed stack regenerates its override without the dropped alias. @@ -1620,6 +1661,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } public async enableForNode(nodeId: number): Promise { + return this.runMeshNodeMutation(nodeId, () => this.enableForNodeExclusive(nodeId)); + } + + private async enableForNodeExclusive(nodeId: number): Promise { this.logDiag('enable-for-node', { nodeId }); DatabaseService.getInstance().setNodeMeshEnabled(nodeId, true); this.logActivity({ @@ -1642,26 +1687,42 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { nodeId: number, actor: string = 'system:mesh.disable', ): Promise { + return this.runMeshNodeMutation(nodeId, () => this.disableForNodeExclusive(nodeId, actor)); + } + + private async disableForNodeExclusive(nodeId: number, actor: string): Promise { this.logDiag('disable-for-node start', { nodeId, actor }); const t0 = Date.now(); - DatabaseService.getInstance().setNodeMeshEnabled(nodeId, false); - const stacks = DatabaseService.getInstance().listMeshStacks(nodeId); - for (const s of stacks) { - DatabaseService.getInstance().deleteMeshStack(nodeId, s.stack_name); - } + const db = DatabaseService.getInstance(); + const stacks = db.listMeshStacks(nodeId); // Dispatch DELETE /api/mesh/local-override/:stack for remote nodes // (pilot or proxy) so the override file pushed earlier via // applyLocalOverride is removed; falls back to local deletion for // local nodes. Parallelize per the regenerateOverridesForNode // rationale: each remote call is its own HTTP round-trip, so // awaiting sequentially turns N stacks into N serialised DELETEs. - // `allSettled` so a single failure does not abort the others - // (removeOverrideFromNode already swallows errors internally). - await Promise.allSettled( + // `allSettled` so a single failure does not abort the others. + const removals = await Promise.allSettled( stacks.map((s) => this.removeOverrideFromNode(nodeId, s.stack_name)), ); - await this.refreshAliasCache(); - await this.syncForwarderListeners(); + const failed: string[] = []; + const removed: typeof stacks = []; + removals.forEach((result, index) => { + const stack = stacks[index]; + if (result.status === 'fulfilled') { + db.deleteMeshStack(nodeId, stack.stack_name); + removed.push(stack); + } else { + failed.push(stack.stack_name); + } + }); + if (failed.length === 0) db.setNodeMeshEnabled(nodeId, false); + try { + await this.refreshAliasCache(); + await this.syncForwarderListeners(); + } catch (error) { + console.warn('[MeshService] Failed to refresh aliases after committed node disable changes:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + } // Mirror optOutStack: regenerate every remaining node's override // without the dropped aliases, recompose the rest of the fleet so // their containers shed the stale extra_hosts, and redeploy the @@ -1669,9 +1730,15 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { // sencho_mesh network and lose the alias entries they owned. await this.regenerateOverridesAcrossFleet(); this.cascadeRecomposeAcrossFleet(undefined, undefined, actor); - for (const s of stacks) { + for (const s of removed) { this.triggerRedeploy(nodeId, s.stack_name, actor); } + if (failed.length > 0) { + throw new MeshError( + 'push_failed', + `Could not disable Mesh on node ${nodeId}: override removal failed for ${failed.join(', ')}.`, + ); + } this.logDiag('disable-for-node complete', { nodeId, stacks: stacks.length, ms: Date.now() - t0 }); this.logActivity({ source: 'mesh', level: 'info', type: 'mesh.disable', @@ -1679,6 +1746,24 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { }); } + private readonly meshNodeMutations = new Map>(); + + private async runMeshNodeMutation(nodeId: number, operation: () => Promise): Promise { + const previous = this.meshNodeMutations.get(nodeId) ?? Promise.resolve(); + let release!: () => void; + const pending = new Promise((resolve) => { + release = resolve; + }); + this.meshNodeMutations.set(nodeId, pending); + await previous; + try { + return await operation(); + } finally { + release(); + if (this.meshNodeMutations.get(nodeId) === pending) this.meshNodeMutations.delete(nodeId); + } + } + // --- Override file management --- public async ensureStackOverride(nodeId: number, stackName: string): Promise { @@ -1690,6 +1775,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { // lives on central per the C-3 design). Use file-presence as the // fallback: if central pushed an override via applyLocalOverride, return // that path so ComposeService picks it up on the next deploy. + if (process.env.SENCHO_MODE !== 'pilot') return null; const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`); if (!isPathWithinBase(file, dir)) return null; try { @@ -1754,13 +1840,37 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { portAliases?: MeshGlobalAlias[], ): Promise { if (!isValidStackName(stackName)) return null; - if (!this.senchoIp) { + const senchoIp = this.senchoIp; + if (!senchoIp) { throw new MeshError( 'push_failed', this.networkSetupError || 'mesh data plane unavailable on this node', ); } const localNodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const lock = await StackOpLockService.getInstance().runExclusive( + localNodeId, + stackName, + 'deploy', + 'system:mesh.override', + () => this.applyLocalOverrideExclusive(localNodeId, stackName, aliases, senchoIp, portAliases), + ); + if (!lock.ran) { + throw new MeshError( + 'push_failed', + `Cannot apply Mesh override for "${stackName}": another operation (${lock.existing.action}) is already in progress.`, + ); + } + return lock.result; + } + + private async applyLocalOverrideExclusive( + localNodeId: number, + stackName: string, + aliases: MeshAlias[], + senchoIp: string, + portAliases?: MeshGlobalAlias[], + ): Promise { const serviceNames = await this.getDeclaredStackServiceNames(stackName, localNodeId); const dir = this.overrideDirFor(localNodeId); @@ -1785,6 +1895,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { message: `mesh override preserved for ${stackName}: declared services unreadable, keeping ${existing.length} existing entries`, details: { stackName, preservedServices: existing }, }); + this.recordLocalOverrideIntent(localNodeId, stackName); return file; } } @@ -1792,13 +1903,24 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { const yaml = generateOverrideYaml({ services: serviceNames, aliases, - senchoIp: this.senchoIp, + senchoIp, }); - await fs.writeFile(file, yaml, 'utf8'); + const previousYaml = await this.readOverrideContent(file); + await this.writeOverrideAtomically(file, yaml); + try { + this.recordLocalOverrideIntent(localNodeId, stackName); + } catch (error) { + await this.restoreOverrideAfterAuthorityFailure(file, previousYaml); + throw error; + } if (portAliases && portAliases.length > 0) { this.pilotAliasOverlay.set(stackName, portAliases); - await this.refreshAliasCache(); - await this.syncForwarderListeners(); + try { + await this.refreshAliasCache(); + await this.syncForwarderListeners(); + } catch (error) { + console.warn('[MeshService] Failed to refresh aliases after committed override apply:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + } } return file; } @@ -1810,13 +1932,95 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { public async removeLocalOverride(stackName: string): Promise { if (!isValidStackName(stackName)) return; const localNodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const lock = await StackOpLockService.getInstance().runExclusive( + localNodeId, + stackName, + 'deploy', + 'system:mesh.override', + () => this.removeLocalOverrideExclusive(localNodeId, stackName), + ); + if (!lock.ran) { + throw new MeshError( + 'push_failed', + `Cannot remove Mesh override for "${stackName}": another operation (${lock.existing.action}) is already in progress.`, + ); + } + } + + private async removeLocalOverrideExclusive(localNodeId: number, stackName: string): Promise { const dir = this.overrideDirFor(localNodeId); const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`); if (!isPathWithinBase(file, dir)) return; - try { await fs.unlink(file); } catch { /* ignore not-exist */ } + const db = DatabaseService.getInstance(); + const hadIntent = db.isMeshStackEnabled(localNodeId, stackName); + if (hadIntent) db.deleteMeshStack(localNodeId, stackName); + try { + await fs.unlink(file); + } catch (error) { + if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) { + if (hadIntent) db.insertMeshStack(localNodeId, stackName, null); + throw error; + } + } if (this.pilotAliasOverlay.delete(stackName)) { - await this.refreshAliasCache(); - await this.syncForwarderListeners(); + try { + await this.refreshAliasCache(); + await this.syncForwarderListeners(); + } catch (error) { + console.warn('[MeshService] Failed to refresh aliases after committed override removal:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + } + } + } + + private async writeOverrideAtomically(file: string, yaml: string): Promise { + const dir = path.dirname(file); + const tempFile = path.resolve(dir, `.${path.basename(file)}.${randomUUID()}.tmp`); + if (!isPathWithinBase(tempFile, dir)) throw new Error('Invalid Mesh override temporary path'); + + let handle: Awaited> | null = null; + try { + handle = await fs.open(tempFile, 'wx'); + await handle.writeFile(yaml, 'utf8'); + await handle.sync(); + await handle.close(); + handle = null; + await fs.rename(tempFile, file); + } catch (error) { + if (handle) { + try { await handle.close(); } catch (closeError) { + console.warn('[MeshService] Failed to close temporary override:', sanitizeForLog(getErrorMessage(closeError, 'unknown'))); + } + } + try { + await fs.unlink(tempFile); + } catch (cleanupError) { + if (!(cleanupError instanceof Error && 'code' in cleanupError && cleanupError.code === 'ENOENT')) { + console.warn('[MeshService] Failed to clean up temporary override:', sanitizeForLog(getErrorMessage(cleanupError, 'unknown'))); + } + } + throw error; + } + } + + private async readOverrideContent(file: string): Promise { + try { + return await fs.readFile(file, 'utf8'); + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null; + throw error; + } + } + + private async restoreOverrideAfterAuthorityFailure(file: string, previousYaml: string | null): Promise { + try { + if (previousYaml === null) { + await fs.unlink(file); + } else { + await this.writeOverrideAtomically(file, previousYaml); + } + } catch (error) { + if (previousYaml === null && error instanceof Error && 'code' in error && error.code === 'ENOENT') return; + console.warn('[MeshService] Failed to restore override after authority error:', sanitizeForLog(getErrorMessage(error, 'unknown'))); } } @@ -1825,7 +2029,11 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { const dir = this.overrideDirFor(nodeId); const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`); if (!isPathWithinBase(file, dir)) return; - try { await fs.unlink(file); } catch { /* ignore not-exist */ } + try { + await fs.unlink(file); + } catch (error) { + if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error; + } } private overrideDirFor(nodeId: number): string { @@ -1833,6 +2041,13 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { return path.join(dataDir, 'mesh', 'overrides', String(nodeId)); } + private recordLocalOverrideIntent(nodeId: number, stackName: string): void { + if (process.env.SENCHO_MODE === 'pilot') return; + const db = DatabaseService.getInstance(); + if (db.isMeshStackEnabled(nodeId, stackName)) return; + db.insertMeshStack(nodeId, stackName, null); + } + private async regenerateOverridesForNode(nodeId: number, skipStack?: string): Promise { const db = DatabaseService.getInstance(); const stacks = db.listMeshStacks(nodeId); @@ -2331,7 +2546,11 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { ); } if (!res.ok) { - throw new MeshError('push_failed', `HTTP ${res.status} from node ${node.name}`); + const message = `HTTP ${res.status} from node ${node.name}`; + if (res.status >= 400 && res.status < 500) { + throw new MeshError('push_failed', message); + } + throw new Error(message); } } @@ -2436,15 +2655,20 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } try { - await this.proxyFetch( + const response = await this.proxyFetch( nodeId, 'DELETE', `/api/mesh/local-override/${encodeURIComponent(stackName)}`, undefined, 5_000, ); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`HTTP ${response.status} from node ${node.name}: ${body.slice(0, 256)}`); + } } catch (err) { console.warn('[MeshService] removeOverrideFromNode failed:', sanitizeForLog((err as Error).message)); + throw err; } } diff --git a/backend/src/services/network/composeNetworkInspector.ts b/backend/src/services/network/composeNetworkInspector.ts index 1cd05f1b..9e667404 100644 --- a/backend/src/services/network/composeNetworkInspector.ts +++ b/backend/src/services/network/composeNetworkInspector.ts @@ -23,6 +23,8 @@ import type { NetworkDriftFacts, NetworkFactNetwork, NetworkFactService, NetworkRuntimeState, StackNetworkFacts, } from './types'; import { classifyMissingExternalNetworks, type MissingExternalNetwork } from './missingExternalNetworks'; +import { resolveManagedMeshAttachment } from './managedMeshAttachment'; +import type { ManagedNetworkAttachmentPredicate } from './normalize'; import { getErrorMessage } from '../../utils/errors'; import { redactSensitiveText, sanitizeForLog } from '../../utils/safeLog'; @@ -43,6 +45,7 @@ export function assembleStackNetworkFacts( model: EffectiveModel | null, renderError: string | null, snapshot: DependencySnapshot | null, + managedNetworkAttachment?: ManagedNetworkAttachmentPredicate, ): StackNetworkFacts { const runtime: NetworkRuntimeState = snapshot ? 'available' : 'unavailable'; @@ -82,7 +85,9 @@ export function assembleStackNetworkFacts( extraHosts: s.extraHosts, })); - const drift = snapshot ? compareStackNetworks(fromEffectiveModel(model), snapshot, stackName) : EMPTY_DRIFT; + const drift = snapshot + ? compareStackNetworks(fromEffectiveModel(model), snapshot, stackName, managedNetworkAttachment) + : EMPTY_DRIFT; const missingExternalNetworks: MissingExternalNetwork[] = snapshot ? classifyMissingExternalNetworks( model, @@ -147,5 +152,8 @@ export async function buildStackNetworkFacts( } } - return assembleStackNetworkFacts(stackName, model, renderError, snapshot); + const managedNetworkAttachment = snapshot && model + ? await resolveManagedMeshAttachment(nodeId, stackName) + : undefined; + return assembleStackNetworkFacts(stackName, model, renderError, snapshot, managedNetworkAttachment); } diff --git a/backend/src/services/network/managedMeshAttachment.ts b/backend/src/services/network/managedMeshAttachment.ts new file mode 100644 index 00000000..93c8ce4c --- /dev/null +++ b/backend/src/services/network/managedMeshAttachment.ts @@ -0,0 +1,55 @@ +import fs from 'fs/promises'; +import path from 'path'; +import { DatabaseService } from '../DatabaseService'; +import { SENCHO_MESH_NETWORK } from '../MeshComposeOverride'; +import SelfIdentityService from '../SelfIdentityService'; +import { getErrorMessage } from '../../utils/errors'; +import { sanitizeForLog } from '../../utils/safeLog'; +import { isPathWithinBase, isValidStackName } from '../../utils/validation'; +import type { ManagedNetworkAttachmentPredicate } from './normalize'; + +async function hasPilotMeshOverride(nodeId: number, stackName: string): Promise { + if (process.env.SENCHO_MODE !== 'pilot' || !isValidStackName(stackName)) return false; + + const dataDir = process.env.DATA_DIR || '/app/data'; + const overrideDir = path.resolve(dataDir, 'mesh', 'overrides', String(nodeId)); + const overridePath = path.resolve(overrideDir, `${path.basename(stackName)}.override.yml`); + if (!isPathWithinBase(overridePath, overrideDir)) return false; + + try { + await fs.access(overridePath); + return true; + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return false; + console.warn( + '[NetworkDrift] Could not verify Pilot Mesh override for %s:', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + return false; + } +} + +export async function resolveManagedMeshAttachment( + nodeId: number, + stackName: string, +): Promise { + let stackManaged = false; + try { + stackManaged = DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName); + } catch (error) { + console.warn( + '[NetworkDrift] Could not verify Mesh opt-in state for %s:', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } + if (!stackManaged) stackManaged = await hasPilotMeshOverride(nodeId, stackName); + const selfIdentity = SelfIdentityService.getInstance(); + + return (container, networkName) => networkName === SENCHO_MESH_NETWORK && ( + stackManaged + || selfIdentity.isOwnContainer(container.id) + || selfIdentity.isOwnContainer(container.name) + ); +} diff --git a/backend/src/services/network/networkingSummary.ts b/backend/src/services/network/networkingSummary.ts index aeb24696..80874865 100644 --- a/backend/src/services/network/networkingSummary.ts +++ b/backend/src/services/network/networkingSummary.ts @@ -10,6 +10,7 @@ import { FileSystemService } from '../FileSystemService'; import { DatabaseService } from '../DatabaseService'; import { parseComposeDependencies } from '../../helpers/composeDependencyParse'; import { assembleStackDrift } from '../DriftDetectionService'; +import { resolveManagedMeshAttachment } from './managedMeshAttachment'; import { isHostNetwork, isLoopback } from './normalize'; import { getErrorMessage } from '../../utils/errors'; import { sanitizeForLog } from '../../utils/safeLog'; @@ -86,7 +87,14 @@ export async function computeNodeNetworkingSummary(nodeId: number): Promise c.stack === stack); - const report = assembleStackDrift({ stack, declared, containers, networks: snapshot.networks }); + const managedNetworkAttachment = await resolveManagedMeshAttachment(nodeId, stack); + const report = assembleStackDrift({ + stack, + declared, + containers, + networks: snapshot.networks, + managedNetworkAttachment, + }); if (report.findings.some(f => f.kind === 'network-undeclared' || f.kind === 'network-missing')) networkDrift.push(stack); } } diff --git a/backend/src/services/network/normalize.ts b/backend/src/services/network/normalize.ts index c2694cb1..523c4f18 100644 --- a/backend/src/services/network/normalize.ts +++ b/backend/src/services/network/normalize.ts @@ -8,8 +8,9 @@ */ import type { EffectiveModel } from '../preflight/effectiveModel'; import type { DeclaredCompose } from '../../helpers/composeDependencyParse'; -import type { DependencySnapshot } from '../DockerController'; +import type { DependencyContainer, DependencySnapshot } from '../DockerController'; import type { NetworkDriftFacts } from './types'; +import { SENCHO_MESH_NETWORK } from '../MeshComposeOverride'; /** Container states that count as "deployed" for drift, matching DriftDetectionService. */ const RUNNING_STATES = new Set(['running', 'restarting']); @@ -62,6 +63,11 @@ export interface NormalizedNetworkModel { services: { name: string; networkKeys: string[]; networkMode?: string }[]; } +export type ManagedNetworkAttachmentPredicate = ( + container: DependencyContainer, + networkName: string, +) => boolean; + /** Rendered model: resource names are already resolved by `docker compose config`. */ export function fromEffectiveModel(m: EffectiveModel): NormalizedNetworkModel { const networks: NormalizedNetworkModel['networks'] = {}; @@ -97,6 +103,7 @@ export function compareStackNetworks( declared: NormalizedNetworkModel, snapshot: DependencySnapshot, stackName: string, + isManagedAttachment: ManagedNetworkAttachmentPredicate = () => false, ): NetworkDriftFacts { const runtimeOnlyAttachments: NetworkDriftFacts['runtimeOnlyAttachments'] = []; const foreignNetworkAttachments: NetworkDriftFacts['foreignNetworkAttachments'] = []; @@ -118,6 +125,7 @@ export function compareStackNetworks( const net = networkByName.get(attached.name); if (SYSTEM_NETWORK_NAMES.has(attached.name) || net?.isSystem) continue; if (declaredRuntimeNames.has(attached.name)) { usedRuntimeNames.add(attached.name); continue; } + if (attached.name === SENCHO_MESH_NETWORK && isManagedAttachment(c, attached.name)) continue; if (net?.stack === stackName || attached.name.startsWith(`${declared.projectName}_`)) { runtimeOnlyAttachments.push({ container: c.name, service: c.service, network: attached.name }); } else { diff --git a/docs/features/compose-networking.mdx b/docs/features/compose-networking.mdx index f1cad0bb..8da7bdcf 100644 --- a/docs/features/compose-networking.mdx +++ b/docs/features/compose-networking.mdx @@ -153,7 +153,7 @@ When the node is reachable, the tab compares the declared effective model agains | **Declared but unused** | A network is declared in the Compose file but no currently running service is connected to it. Often seen when a service is stopped or removed without `docker compose down`. | | **Missing from runtime** | A network is declared but does not exist in Docker. The stack may not have been deployed, or the network was deleted externally. | -System-managed networks (`bridge`, `host`, `none`) and Docker's implicit default bridge are excluded from all drift findings. +System-managed networks (`bridge`, `host`, `none`) and Docker's implicit default bridge are excluded from all drift findings. Attachments to `sencho_mesh` are also excluded when Sencho verifies that the container is its own instance or that the stack is opted into Sencho Mesh. A manual attachment from an opted-out stack remains visible as drift. When the runtime matches the Compose file, the section shows a green **runtime matches compose** card. diff --git a/docs/features/stack-drift.mdx b/docs/features/stack-drift.mdx index 81cf7404..246e90e1 100644 --- a/docs/features/stack-drift.mdx +++ b/docs/features/stack-drift.mdx @@ -83,7 +83,7 @@ A Compose port range such as `8000-8002:8000-8002` is compared conservatively. B ### Network findings and the Networking tab -The **network-undeclared** and **network-missing** findings reuse the same comparison the stack's [Networking](/features/compose-networking) tab uses, so the two surfaces never disagree about a network attachment. The difference is history: this tab persists findings in the drift ledger over time, while the Networking tab always shows the current live state with no history. A stack with an open network finding also counts toward the **Drift** chip in the Fleet Overview's Networking filter group, so a network-attachment mismatch is visible both per-stack here and across the fleet there. +The **network-undeclared** and **network-missing** findings reuse the same comparison the stack's [Networking](/features/compose-networking) tab uses, so the two surfaces never disagree about a network attachment. Sencho-verified `sencho_mesh` attachments for its own container and Mesh-opted-in stacks are excluded, while a manual attachment from an opted-out stack remains actionable. The difference is history: this tab persists findings in the drift ledger over time, while the Networking tab always shows the current live state with no history. A stack with an open network finding also counts toward the **Drift** chip in the Fleet Overview's Networking filter group, so a network-attachment mismatch is visible both per-stack here and across the fleet there. ## When drift is recorded @@ -93,6 +93,8 @@ The **network-undeclared** and **network-missing** findings reuse the same compa **After every deploy or update**, Sencho automatically records a new baseline hash and runs a full reconciliation. You do not need to click re-check after deploying; the ledger is updated as part of the deploy pipeline. +An open ledger entry for an attachment that is no longer reported clears during the next re-check or post-deploy reconciliation. Opening the tab refreshes the live report but does not change persisted history. +