fix(networking): ignore verified Mesh attachments in drift (#1729)

This commit is contained in:
Anso
2026-07-29 12:55:30 -04:00
committed by GitHub
parent 55644cf87d
commit 3c934066f3
14 changed files with 1206 additions and 50 deletions
@@ -4,14 +4,21 @@
* runtime-vs-Compose drift comparison (system/default/external networks and
* stopped containers are not flagged).
*/
import { describe, it, expect } from 'vitest';
import { describe, it, expect, vi } from 'vitest';
import type { EffectiveModel, EffService } from '../services/preflight/effectiveModel';
import type { DeclaredCompose } from '../helpers/composeDependencyParse';
import type { DependencySnapshot, DependencyContainer, DependencyNetwork } from '../services/DockerController';
import {
fromEffectiveModel, fromDeclaredCompose, compareStackNetworks, runtimeResourceName, parseAccessUrlPorts,
type ManagedNetworkAttachmentPredicate,
} from '../services/network/normalize';
import { assembleStackNetworkFacts } from '../services/network/composeNetworkInspector';
import { assembleStackNetworkFacts, buildStackNetworkFacts } from '../services/network/composeNetworkInspector';
import { buildNodeNetworkingFindings } from '../services/network/networkingFindings';
import type { NetworkingNetworkBase } from '../services/network/networkingTypes';
import DockerController from '../services/DockerController';
import { ComposeService } from '../services/ComposeService';
import { DatabaseService } from '../services/DatabaseService';
import { FileSystemService } from '../services/FileSystemService';
function effSvc(over: Partial<EffService> = {}): EffService {
const hasHealthcheck = over.hasHealthcheck ?? true;
@@ -264,6 +271,187 @@ describe('compareStackNetworks', () => {
expect(drift.foreignNetworkAttachments).toEqual([]);
});
it('removes managed Mesh drift before Networking findings while preserving advanced-driver info', () => {
const meshOnlyModel: EffectiveModel = {
projectName: 'myapp',
services: [],
networks: {},
volumes: {},
};
const snap = snapshot(
[container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
[depNet({ id: 'm', name: 'sencho_mesh', driver: 'macvlan', composeProject: null, stack: null })],
);
const facts = assembleStackNetworkFacts(
'myapp',
meshOnlyModel,
null,
snap,
(_runtimeContainer, networkName) => networkName === 'sencho_mesh',
);
const baseNetworks: NetworkingNetworkBase[] = [{
id: 'm',
name: 'sencho_mesh',
driver: 'macvlan',
scope: 'local',
isSystem: false,
ingress: false,
composeProject: null,
stack: null,
connectedCount: 1,
isSencho: true,
ownership: 'sencho-managed',
declaredByStacks: [],
declaredExternalByStacks: [],
isExternalDependency: false,
}];
const findings = buildNodeNetworkingFindings(1, snap, [facts], baseNetworks);
expect(facts.drift.runtimeOnlyAttachments).toEqual([]);
expect(facts.drift.foreignNetworkAttachments).toEqual([]);
expect(findings.some(f => f.kind === 'network-undeclared' || f.kind === 'foreign-network-attachment')).toBe(false);
expect(findings).toContainEqual(expect.objectContaining({
kind: 'advanced-driver-caveat',
severity: 'info',
network: 'sencho_mesh',
}));
});
it('keeps Networking facts available and Mesh drift actionable when opt-in authority fails', async () => {
const snap = snapshot(
[container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
[depNet({ name: 'sencho_mesh', composeProject: null, stack: null })],
);
vi.spyOn(ComposeService, 'getInstance').mockReturnValue({
renderConfig: vi.fn().mockResolvedValue({
rendered: JSON.stringify({ name: 'myapp', services: { web: { image: 'nginx:1.27' } } }),
stderr: '',
code: 0,
timedOut: false,
}),
} as unknown as ComposeService);
vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({
getStacks: vi.fn().mockResolvedValue(['myapp']),
} as unknown as FileSystemService);
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDependencySnapshot: vi.fn().mockResolvedValue(snap),
} as unknown as DockerController);
vi.spyOn(DatabaseService, 'getInstance').mockImplementation(() => {
throw new Error('database unavailable');
});
vi.spyOn(console, 'warn').mockImplementation(() => undefined);
try {
const facts = await buildStackNetworkFacts(1, 'myapp');
expect(facts.runtime).toBe('available');
expect(facts.drift.foreignNetworkAttachments).toEqual([{ container: 'web1', network: 'sencho_mesh' }]);
} finally {
vi.restoreAllMocks();
}
});
it('threads opted-in authority through Networking facts and preserves advanced-driver info', async () => {
const snap = snapshot(
[container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
[depNet({ id: 'm', name: 'sencho_mesh', driver: 'macvlan', composeProject: null, stack: null })],
);
vi.spyOn(ComposeService, 'getInstance').mockReturnValue({
renderConfig: vi.fn().mockResolvedValue({
rendered: JSON.stringify({ name: 'myapp', services: { web: { image: 'nginx:1.27' } } }),
stderr: '',
code: 0,
timedOut: false,
}),
} as unknown as ComposeService);
vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({
getStacks: vi.fn().mockResolvedValue(['myapp']),
} as unknown as FileSystemService);
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDependencySnapshot: vi.fn().mockResolvedValue(snap),
} as unknown as DockerController);
vi.spyOn(DatabaseService, 'getInstance').mockReturnValue({
isMeshStackEnabled: vi.fn().mockReturnValue(true),
getStackExposureIntents: vi.fn().mockReturnValue([]),
getStackDossier: vi.fn().mockReturnValue(null),
} as unknown as DatabaseService);
try {
const facts = await buildStackNetworkFacts(1, 'myapp');
const baseNetworks: NetworkingNetworkBase[] = [{
id: 'm', name: 'sencho_mesh', driver: 'macvlan', scope: 'local', isSystem: false,
ingress: false, composeProject: null, stack: null, connectedCount: 1, isSencho: true,
ownership: 'sencho-managed', declaredByStacks: [], declaredExternalByStacks: [],
isExternalDependency: false,
}];
const findings = buildNodeNetworkingFindings(1, snap, [facts], baseNetworks);
expect(facts.drift.runtimeOnlyAttachments).toEqual([]);
expect(facts.drift.foreignNetworkAttachments).toEqual([]);
expect(findings.filter(f => f.kind === 'network-undeclared' || f.kind === 'foreign-network-attachment')).toEqual([]);
expect(findings).toContainEqual(expect.objectContaining({
kind: 'advanced-driver-caveat',
severity: 'info',
network: 'sencho_mesh',
}));
} finally {
vi.restoreAllMocks();
}
});
it('ignores a verified Sencho Mesh attachment for the Sencho container', () => {
const snap = snapshot(
[container({ id: 'sencho-id', name: 'sencho', service: 'sencho', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
[depNet({ name: 'sencho_mesh', composeProject: null, stack: null })],
);
const managed: ManagedNetworkAttachmentPredicate = (runtimeContainer, networkName) =>
runtimeContainer.id === 'sencho-id' && networkName === 'sencho_mesh';
const drift = compareStackNetworks(declared, snap, 'myapp', managed);
expect(drift.runtimeOnlyAttachments).toEqual([]);
expect(drift.foreignNetworkAttachments).toEqual([]);
});
it('ignores a verified Mesh attachment for an opted-in application stack', () => {
const snap = snapshot(
[container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
[depNet({ name: 'sencho_mesh', composeProject: null, stack: null })],
);
const managed: ManagedNetworkAttachmentPredicate = (_runtimeContainer, networkName) =>
networkName === 'sencho_mesh';
const drift = compareStackNetworks(declared, snap, 'myapp', managed);
expect(drift.runtimeOnlyAttachments).toEqual([]);
expect(drift.foreignNetworkAttachments).toEqual([]);
});
it('keeps an unverified manual Mesh attachment actionable', () => {
const snap = snapshot(
[container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
[depNet({ name: 'sencho_mesh', composeProject: null, stack: null })],
);
const drift = compareStackNetworks(declared, snap, 'myapp', () => false);
expect(drift.runtimeOnlyAttachments).toEqual([]);
expect(drift.foreignNetworkAttachments).toEqual([{ container: 'web1', network: 'sencho_mesh' }]);
});
it('keeps sencho_extra actionable even when the stack is Mesh-managed', () => {
const snap = snapshot(
[container({ networks: [{ name: 'sencho_extra', id: 'e', ip: '' }] })],
[depNet({ name: 'sencho_extra', composeProject: null, stack: null })],
);
const drift = compareStackNetworks(declared, snap, 'myapp', () => true);
expect(drift.runtimeOnlyAttachments).toEqual([]);
expect(drift.foreignNetworkAttachments).toEqual([{ container: 'web1', network: 'sencho_extra' }]);
});
it('does not flag attachments from stopped containers', () => {
const snap = snapshot(
[container({ state: 'exited', networks: [{ name: 'myapp_extra', id: 'b', ip: '' }] })],
@@ -18,6 +18,7 @@ import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompos
import type { DeclaredCompose, DeclaredService, DeclaredPort } from '../helpers/composeDependencyParse';
import type { EffectiveModel, EffService } from '../services/preflight/effectiveModel';
import { fromDeclaredCompose, fromEffectiveModel } from '../services/network/normalize';
import { DatabaseService } from '../services/DatabaseService';
// ── builders ────────────────────────────────────────────────────────────
@@ -580,6 +581,32 @@ describe('assembleStackDrift - network drift', () => {
expect(report.findings.filter(f => f.kind.startsWith('network-'))).toEqual([]);
expect(report.status).toBe('in-sync');
});
it('reports in-sync when a verified Mesh attachment is the only runtime difference', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([service({ name: 'web' })]),
containers: [container({ id: 'c1', service: 'web', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
networks: [depNet('sencho_mesh', { composeProject: null, stack: null })],
managedNetworkAttachment: (_runtimeContainer, networkName) => networkName === 'sencho_mesh',
});
expect(report.status).toBe('in-sync');
expect(report.findings.filter(f => f.kind === 'network-undeclared')).toEqual([]);
});
it('keeps an unverified manual Mesh attachment drifted', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([service({ name: 'web' })]),
containers: [container({ id: 'c1', service: 'web', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
networks: [depNet('sencho_mesh', { composeProject: null, stack: null })],
managedNetworkAttachment: () => false,
});
expect(report.status).toBe('drifted');
expect(report.findings.filter(f => f.kind === 'network-undeclared')).toHaveLength(1);
});
});
// ── declaredFromEffectiveModel ─────────────────────────────────────────────
@@ -815,4 +842,46 @@ describe('buildStackDriftReport - boundaries', () => {
expect(findingKinds(report)).toContain('network-undeclared');
vi.restoreAllMocks();
});
it('keeps the report available and Mesh drift actionable when opt-in authority fails', async () => {
const snapshot: DependencySnapshot = {
containers: [container({ id: 'c1', service: 'web', stack: 'app', image: 'nginx:1.25', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
networks: [depNet('sencho_mesh', { composeProject: null, stack: null })],
volumes: [],
};
stubDockerRender({ name: 'app', services: { web: { image: 'nginx:1.25' } } });
stubFsAndSnapshot(snapshot);
vi.spyOn(DatabaseService, 'getInstance').mockImplementation(() => {
throw new Error('database unavailable');
});
vi.spyOn(console, 'warn').mockImplementation(() => undefined);
const report = await buildStackDriftReport(0, 'app');
expect(report.status).toBe('drifted');
expect(report.findings).toContainEqual(expect.objectContaining({
kind: 'network-undeclared',
actual: 'sencho_mesh',
}));
vi.restoreAllMocks();
});
it('reports in-sync through the public builder when DB authority opts the stack into Mesh', async () => {
const snapshot: DependencySnapshot = {
containers: [container({ id: 'c1', service: 'web', stack: 'app', image: 'nginx:1.25', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
networks: [depNet('sencho_mesh', { composeProject: null, stack: null })],
volumes: [],
};
stubDockerRender({ name: 'app', services: { web: { image: 'nginx:1.25' } } });
stubFsAndSnapshot(snapshot);
vi.spyOn(DatabaseService, 'getInstance').mockReturnValue({
isMeshStackEnabled: vi.fn().mockReturnValue(true),
} as unknown as DatabaseService);
const report = await buildStackDriftReport(0, 'app');
expect(report.status).toBe('in-sync');
expect(report.findings.filter(f => f.kind === 'network-undeclared')).toEqual([]);
vi.restoreAllMocks();
});
});
@@ -0,0 +1,124 @@
import fs from 'fs/promises';
import os from 'os';
import path from 'path';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { DatabaseService } from '../services/DatabaseService';
import SelfIdentityService from '../services/SelfIdentityService';
import type { DependencyContainer } from '../services/DockerController';
import { resolveManagedMeshAttachment } from '../services/network/managedMeshAttachment';
const originalDataDir = process.env.DATA_DIR;
const originalMode = process.env.SENCHO_MODE;
let tempDir: string | null = null;
function runtimeContainer(overrides: Partial<DependencyContainer> = {}): DependencyContainer {
return {
id: 'app-id',
name: 'app-web-1',
service: 'web',
composeProject: 'app',
stack: 'app',
state: 'running',
exitCode: null,
image: 'nginx:latest',
networks: [],
volumes: [],
ports: [],
...overrides,
};
}
function stubAuthorities(meshStackEnabled: boolean, ownContainers: string[] = []): void {
vi.spyOn(DatabaseService, 'getInstance').mockReturnValue({
isMeshStackEnabled: vi.fn().mockReturnValue(meshStackEnabled),
} as unknown as DatabaseService);
vi.spyOn(SelfIdentityService, 'getInstance').mockReturnValue({
isOwnContainer: vi.fn((idOrName: string) => ownContainers.includes(idOrName)),
} as unknown as SelfIdentityService);
}
afterEach(async () => {
vi.restoreAllMocks();
if (originalDataDir === undefined) delete process.env.DATA_DIR;
else process.env.DATA_DIR = originalDataDir;
if (originalMode === undefined) delete process.env.SENCHO_MODE;
else process.env.SENCHO_MODE = originalMode;
if (tempDir) await fs.rm(tempDir, { recursive: true, force: true });
tempDir = null;
});
describe('resolveManagedMeshAttachment', () => {
it('authorizes the canonical Mesh attachment for a centrally opted-in stack', async () => {
process.env.SENCHO_MODE = 'server';
stubAuthorities(true);
const isManaged = await resolveManagedMeshAttachment(1, 'app');
expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(true);
expect(isManaged(runtimeContainer(), 'sencho_extra')).toBe(false);
});
it('authorizes the canonical Mesh attachment for the actual Sencho container', async () => {
process.env.SENCHO_MODE = 'server';
stubAuthorities(false, ['sencho-id']);
const isManaged = await resolveManagedMeshAttachment(1, 'sencho');
expect(isManaged(runtimeContainer({ id: 'sencho-id', name: 'sencho' }), 'sencho_mesh')).toBe(true);
expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(false);
});
it('keeps a manual Mesh attachment actionable for an opted-out stack', async () => {
process.env.SENCHO_MODE = 'server';
stubAuthorities(false);
const isManaged = await resolveManagedMeshAttachment(1, 'app');
expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(false);
});
it('uses Pilot override presence as the authoritative opt-in representation', async () => {
process.env.SENCHO_MODE = 'pilot';
tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'sencho-mesh-drift-'));
process.env.DATA_DIR = tempDir;
const overrideDir = path.join(tempDir, 'mesh', 'overrides', '7');
await fs.mkdir(overrideDir, { recursive: true });
await fs.writeFile(path.join(overrideDir, 'app.override.yml'), 'services: {}\n');
stubAuthorities(false);
const isManaged = await resolveManagedMeshAttachment(7, 'app');
expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(true);
});
it('does not let stale server override presence supersede opted-out DB state', async () => {
process.env.SENCHO_MODE = 'server';
tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'sencho-mesh-drift-'));
process.env.DATA_DIR = tempDir;
const overrideDir = path.join(tempDir, 'mesh', 'overrides', '1');
await fs.mkdir(overrideDir, { recursive: true });
await fs.writeFile(path.join(overrideDir, 'app.override.yml'), 'services: {}\n');
stubAuthorities(false);
const isManaged = await resolveManagedMeshAttachment(1, 'app');
expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(false);
});
it('fails closed when Mesh opt-in state cannot be read', async () => {
process.env.SENCHO_MODE = 'server';
vi.spyOn(DatabaseService, 'getInstance').mockImplementation(() => {
throw new Error('database unavailable');
});
vi.spyOn(SelfIdentityService, 'getInstance').mockReturnValue({
isOwnContainer: vi.fn().mockReturnValue(false),
} as unknown as SelfIdentityService);
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
const isManaged = await resolveManagedMeshAttachment(1, 'app');
expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(false);
expect(warn).toHaveBeenCalledWith(
'[NetworkDrift] Could not verify Mesh opt-in state for %s:',
'app',
'database unavailable',
);
});
});
@@ -96,7 +96,7 @@ describe('MeshService.removeOverrideFromNode (remote dispatch)', () => {
db.deleteNode(remoteNodeId);
});
it('swallows network errors from the remote so the disable cascade can continue', async () => {
it('rejects network errors so callers can preserve authoritative opt-in state', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const remoteNodeId = db.addNode({
@@ -117,11 +117,37 @@ describe('MeshService.removeOverrideFromNode (remote dispatch)', () => {
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('ECONNREFUSED'));
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ });
// Must not throw: the remote being offline is a tolerable condition;
// the cascade upstream uses Promise.allSettled and continues.
await expect(svc.removeOverrideFromNode(remoteNodeId, 'sample-stack')).resolves.toBeUndefined();
await expect(svc.removeOverrideFromNode(remoteNodeId, 'sample-stack')).rejects.toThrow('ECONNREFUSED');
expect(warnSpy).toHaveBeenCalled();
db.deleteNode(remoteNodeId);
});
it('rejects a non-success response from a busy remote target', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const remoteNodeId = db.addNode({
name: 'remove-override-busy-test',
type: 'remote',
mode: 'proxy',
compose_dir: '/tmp',
is_default: false,
api_url: 'https://remote.example.com:1852',
api_token: 'remote-tok',
});
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'remote-tok',
});
vi.spyOn(globalThis, 'fetch').mockResolvedValue(
new Response('another operation is already in progress', { status: 500 }),
);
vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ });
await expect(svc.removeOverrideFromNode(remoteNodeId, 'sample-stack'))
.rejects.toThrow('HTTP 500');
db.deleteNode(remoteNodeId);
});
});
+382
View File
@@ -1,6 +1,7 @@
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { EventEmitter } from 'events';
import fsSync from 'fs';
import fs from 'fs/promises';
import path from 'path';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import { getSenchoIpFromSubnet, MeshError, type MeshTarget, type MeshTcpStreamLike } from '../services/MeshService';
@@ -20,6 +21,7 @@ afterAll(() => {
});
beforeEach(() => {
process.env.SENCHO_MODE = 'server';
const db = DatabaseService.getInstance().getDb();
db.prepare('DELETE FROM mesh_stacks').run();
db.prepare('DELETE FROM nodes WHERE is_default = 0').run();
@@ -106,6 +108,84 @@ describe('MeshService.optInStack', () => {
expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false);
});
it('restores opt-in authority when target override removal is rejected', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
db.insertMeshStack(localNodeId, 'busy-stack', 'setup');
vi.spyOn(svc, 'removeOverrideFromNode').mockRejectedValue(
new Error('HTTP 500: another operation is already in progress'),
);
await expect(svc.optOutStack(localNodeId, 'busy-stack', 'tester'))
.rejects.toThrow('another operation is already in progress');
expect(db.isMeshStackEnabled(localNodeId, 'busy-stack')).toBe(true);
});
it('serializes concurrent opt-out requests for the same node', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
db.insertMeshStack(localNodeId, 'queued-stack', 'setup');
let releaseRemoval!: () => void;
const removalPending = new Promise<void>((resolve) => {
releaseRemoval = resolve;
});
const removeSpy = vi.spyOn(svc, 'removeOverrideFromNode').mockReturnValue(removalPending);
vi.spyOn(svc as unknown as { regenerateOverridesAcrossFleet: () => Promise<void> }, 'regenerateOverridesAcrossFleet')
.mockResolvedValue(undefined);
vi.spyOn(svc as unknown as { cascadeRecomposeAcrossFleet: () => void }, 'cascadeRecomposeAcrossFleet')
.mockImplementation(() => { /* noop */ });
vi.spyOn(svc, 'triggerRedeploy').mockImplementation(() => { /* noop */ });
const first = svc.optOutStack(localNodeId, 'queued-stack', 'tester');
await vi.waitFor(() => expect(removeSpy).toHaveBeenCalledTimes(1));
const second = svc.optOutStack(localNodeId, 'queued-stack', 'tester');
await Promise.resolve();
expect(removeSpy).toHaveBeenCalledTimes(1);
releaseRemoval();
await Promise.all([first, second]);
expect(removeSpy).toHaveBeenCalledTimes(1);
expect(db.isMeshStackEnabled(localNodeId, 'queued-stack')).toBe(false);
});
it('serializes different Mesh mutations per node without blocking another node', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
const remoteNodeId = db.addNode({
name: 'independent-node', type: 'remote', mode: 'pilot_agent',
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
});
db.setNodeMeshEnabled(localNodeId, true);
db.insertMeshStack(localNodeId, 'held-stack', 'setup');
let releaseRemoval!: () => void;
const removalPending = new Promise<void>((resolve) => {
releaseRemoval = resolve;
});
vi.spyOn(svc, 'removeOverrideFromNode').mockReturnValue(removalPending);
vi.spyOn(svc as unknown as { regenerateOverridesAcrossFleet: () => Promise<void> }, 'regenerateOverridesAcrossFleet')
.mockResolvedValue(undefined);
vi.spyOn(svc as unknown as { cascadeRecomposeAcrossFleet: () => void }, 'cascadeRecomposeAcrossFleet')
.mockImplementation(() => { /* noop */ });
vi.spyOn(svc, 'triggerRedeploy').mockImplementation(() => { /* noop */ });
const optOut = svc.optOutStack(localNodeId, 'held-stack', 'tester');
await vi.waitFor(() => expect(svc.removeOverrideFromNode).toHaveBeenCalledTimes(1));
const disable = svc.disableForNode(localNodeId, 'tester');
await svc.enableForNode(remoteNodeId);
expect(db.getNodeMeshEnabled(localNodeId)).toBe(true);
expect(db.getNodeMeshEnabled(remoteNodeId)).toBe(true);
releaseRemoval();
await Promise.all([optOut, disable]);
expect(db.getNodeMeshEnabled(localNodeId)).toBe(false);
db.deleteNode(remoteNodeId);
});
it('rejects an invalid stack name (path traversal attempt)', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
@@ -427,6 +507,44 @@ describe('MeshService.disableForNode', () => {
db.deleteNode(remoteNodeId);
});
it('keeps failed remote stacks authoritative when node disable is incomplete', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const remoteNodeId = db.addNode({
name: 'remote-partial-disable', type: 'remote', mode: 'pilot_agent',
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
});
db.setNodeMeshEnabled(remoteNodeId, true);
db.insertMeshStack(remoteNodeId, 'removed-stack', 'setup');
db.insertMeshStack(remoteNodeId, 'busy-stack', 'setup');
vi.spyOn(svc, 'removeOverrideFromNode').mockImplementation(async (_nodeId, stackName) => {
if (stackName === 'busy-stack') throw new Error('target busy');
});
vi.spyOn(
svc as unknown as { regenerateOverridesAcrossFleet: () => Promise<void> },
'regenerateOverridesAcrossFleet',
).mockResolvedValue(undefined);
vi.spyOn(
svc as unknown as { cascadeRecomposeAcrossFleet: () => void },
'cascadeRecomposeAcrossFleet',
).mockImplementation(() => { /* noop */ });
const redeployed: string[] = [];
vi.spyOn(svc, 'triggerRedeploy').mockImplementation((_nodeId, stackName) => {
redeployed.push(stackName);
});
vi.spyOn(svc as unknown as { refreshAliasCache: () => Promise<void> }, 'refreshAliasCache')
.mockRejectedValue(new Error('refresh failed'));
vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ });
await expect(svc.disableForNode(remoteNodeId, 'tester')).rejects.toThrow('busy-stack');
expect(db.getNodeMeshEnabled(remoteNodeId)).toBe(true);
expect(db.isMeshStackEnabled(remoteNodeId, 'removed-stack')).toBe(false);
expect(db.isMeshStackEnabled(remoteNodeId, 'busy-stack')).toBe(true);
expect(redeployed).toContain('removed-stack');
db.deleteNode(remoteNodeId);
});
it('defaults the actor when none is supplied so legacy callers still log a non-empty actor', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
@@ -673,6 +791,62 @@ describe('MeshService.optInStack rollback', () => {
.rejects.toThrow(/simulated remote pilot offline/);
expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false);
});
it('retains remote authority when the push outcome is unknown', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const remoteNodeId = db.addNode({
name: 'ambiguous-push', type: 'remote', mode: 'pilot_agent',
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
});
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
.mockResolvedValue([{ service: 'web', ports: [8080] }]);
vi.spyOn(svc, 'pushOverrideToNode').mockRejectedValue(new Error('connection reset'));
await expect(svc.optInStack(remoteNodeId, 'ambiguous-stack', 'tester'))
.rejects.toThrow('connection reset');
expect(db.isMeshStackEnabled(remoteNodeId, 'ambiguous-stack')).toBe(true);
db.deleteNode(remoteNodeId);
});
it('treats a remote gateway error as an ambiguous push outcome', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const remoteNodeId = db.addNode({
name: 'gateway-error-push', type: 'remote', mode: 'pilot_agent',
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
});
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
.mockResolvedValue([{ service: 'web', ports: [8080] }]);
vi.spyOn(svc as unknown as { proxyFetch: () => Promise<Response> }, 'proxyFetch')
.mockResolvedValue(new Response('gateway timeout', { status: 502 }));
await expect(svc.optInStack(remoteNodeId, 'gateway-error-stack', 'tester'))
.rejects.toThrow('HTTP 502');
expect(db.isMeshStackEnabled(remoteNodeId, 'gateway-error-stack')).toBe(true);
db.deleteNode(remoteNodeId);
});
it('rolls back remote authority when the target explicitly rejects the push', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const remoteNodeId = db.addNode({
name: 'rejected-push', type: 'remote', mode: 'pilot_agent',
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
});
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
.mockResolvedValue([{ service: 'web', ports: [8080] }]);
vi.spyOn(svc, 'pushOverrideToNode')
.mockRejectedValue(new MeshError('push_failed', 'target rejected override'));
await expect(svc.optInStack(remoteNodeId, 'rejected-stack', 'tester'))
.rejects.toThrow('target rejected override');
expect(db.isMeshStackEnabled(remoteNodeId, 'rejected-stack')).toBe(false);
db.deleteNode(remoteNodeId);
});
});
describe('MeshService.optInStack guard rails (network setup)', () => {
@@ -984,6 +1158,7 @@ describe('MeshService.ensureStackOverride (BUG-1 fix)', () => {
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
process.env.SENCHO_MODE = 'pilot';
// Simulate the pilot scenario: no mesh_stacks row (isMeshStackEnabled → false),
// but the override file already exists on disk, pushed by central via D-1.
const dataDir = process.env.DATA_DIR as string;
@@ -1008,6 +1183,184 @@ describe('MeshService.ensureStackOverride (BUG-1 fix)', () => {
// Cleanup.
fsSync.unlinkSync(overrideFile);
process.env.SENCHO_MODE = 'server';
});
it('persists and removes proxy-target opt-in state with a pushed local override', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']);
const file = await svc.applyLocalOverride('proxy-stack', []);
expect(file).not.toBeNull();
const yaml = fsSync.readFileSync(file as string, 'utf8');
expect(yaml).toContain('web:');
expect(yaml).toContain('sencho_mesh');
expect(fsSync.readdirSync(path.dirname(file as string)).some((name) => name.includes('proxy-stack') && name.endsWith('.tmp'))).toBe(false);
expect(db.isMeshStackEnabled(localNodeId, 'proxy-stack')).toBe(true);
await svc.removeLocalOverride('proxy-stack');
expect(db.isMeshStackEnabled(localNodeId, 'proxy-stack')).toBe(false);
});
it('does not write a proxy override when DB authority cannot be recorded', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']);
vi.spyOn(db, 'insertMeshStack').mockImplementation(() => {
throw new Error('database unavailable');
});
await expect(svc.applyLocalOverride('db-failure', [])).rejects.toThrow('database unavailable');
const overrideFile = path.join(
process.env.DATA_DIR as string,
'mesh',
'overrides',
String(localNodeId),
'db-failure.override.yml',
);
expect(fsSync.existsSync(overrideFile)).toBe(false);
});
it('restores an existing override when DB authority cannot be recorded', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId));
fsSync.mkdirSync(overrideDir, { recursive: true });
const overrideFile = path.join(overrideDir, 'db-replacement-failure.override.yml');
const originalYaml = 'services:\n prior:\n networks:\n - sencho_mesh\n';
fsSync.writeFileSync(overrideFile, originalYaml, 'utf8');
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']);
vi.spyOn(db, 'insertMeshStack').mockImplementation(() => {
throw new Error('database unavailable');
});
await expect(svc.applyLocalOverride('db-replacement-failure', [])).rejects.toThrow('database unavailable');
expect(fsSync.readFileSync(overrideFile, 'utf8')).toBe(originalYaml);
expect(db.isMeshStackEnabled(localNodeId, 'db-replacement-failure')).toBe(false);
expect(fsSync.readdirSync(overrideDir).some((name) => name.includes('db-replacement-failure') && name.endsWith('.tmp'))).toBe(false);
});
it('does not publish DB authority or a final file when atomic override publication fails', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']);
vi.spyOn(fs, 'rename').mockRejectedValue(new Error('rename failed'));
await expect(svc.applyLocalOverride('write-failure', [])).rejects.toThrow('rename failed');
const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId));
expect(db.isMeshStackEnabled(localNodeId, 'write-failure')).toBe(false);
expect(fsSync.existsSync(path.join(overrideDir, 'write-failure.override.yml'))).toBe(false);
expect(fsSync.readdirSync(overrideDir).some((name) => name.includes('write-failure'))).toBe(false);
});
it('preserves the prior override and authority when atomic replacement fails', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId));
fsSync.mkdirSync(overrideDir, { recursive: true });
const overrideFile = path.join(overrideDir, 'replacement-failure.override.yml');
const originalYaml = 'services:\n prior:\n networks:\n - sencho_mesh\n';
fsSync.writeFileSync(overrideFile, originalYaml, 'utf8');
db.insertMeshStack(localNodeId, 'replacement-failure', 'tester');
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']);
vi.spyOn(fs, 'rename').mockRejectedValue(new Error('rename failed'));
await expect(svc.applyLocalOverride('replacement-failure', [])).rejects.toThrow('rename failed');
expect(fsSync.readFileSync(overrideFile, 'utf8')).toBe(originalYaml);
expect(db.isMeshStackEnabled(localNodeId, 'replacement-failure')).toBe(true);
});
it('prevents overlapping override mutations for the same stack', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
let releaseServices!: (services: string[]) => void;
const servicesPending = new Promise<string[]>((resolve) => {
releaseServices = resolve;
});
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockReturnValue(servicesPending);
const first = svc.applyLocalOverride('concurrent-stack', []);
await vi.waitFor(() => expect(svc.getDeclaredStackServiceNames).toHaveBeenCalledTimes(1));
await expect(svc.applyLocalOverride('concurrent-stack', [])).rejects.toThrow('another operation');
releaseServices(['web']);
await first;
expect(db.isMeshStackEnabled(localNodeId, 'concurrent-stack')).toBe(true);
});
it('prevents removal from overlapping an in-flight override apply', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
let releaseServices!: (services: string[]) => void;
const servicesPending = new Promise<string[]>((resolve) => {
releaseServices = resolve;
});
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockReturnValue(servicesPending);
const apply = svc.applyLocalOverride('apply-remove-stack', []);
await vi.waitFor(() => expect(svc.getDeclaredStackServiceNames).toHaveBeenCalledTimes(1));
await expect(svc.removeLocalOverride('apply-remove-stack')).rejects.toThrow('another operation');
releaseServices(['web']);
const file = await apply;
expect(file).not.toBeNull();
expect(fsSync.existsSync(file as string)).toBe(true);
expect(db.isMeshStackEnabled(localNodeId, 'apply-remove-stack')).toBe(true);
});
it('does not report committed removal as failed when alias refresh fails', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId));
fsSync.mkdirSync(overrideDir, { recursive: true });
const overrideFile = path.join(overrideDir, 'refresh-failure.override.yml');
fsSync.writeFileSync(overrideFile, 'services: {}\n', 'utf8');
db.insertMeshStack(localNodeId, 'refresh-failure', 'setup');
(svc as unknown as { pilotAliasOverlay: Map<string, unknown> }).pilotAliasOverlay.set('refresh-failure', []);
vi.spyOn(svc as unknown as { refreshAliasCache: () => Promise<void> }, 'refreshAliasCache')
.mockRejectedValue(new Error('refresh failed'));
vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ });
await expect(svc.removeLocalOverride('refresh-failure')).resolves.toBeUndefined();
expect(fsSync.existsSync(overrideFile)).toBe(false);
expect(db.isMeshStackEnabled(localNodeId, 'refresh-failure')).toBe(false);
});
it('does not report committed apply as failed when alias refresh fails', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']);
vi.spyOn(svc as unknown as { refreshAliasCache: () => Promise<void> }, 'refreshAliasCache')
.mockRejectedValue(new Error('refresh failed'));
vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ });
const file = await svc.applyLocalOverride('apply-refresh-failure', [], [{
host: 'web.example', nodeId: localNodeId, nodeName: 'local', stackName: 'apply-refresh-failure',
serviceName: 'web', port: 8080,
}]);
expect(file).not.toBeNull();
expect(fsSync.existsSync(file as string)).toBe(true);
expect(db.isMeshStackEnabled(localNodeId, 'apply-refresh-failure')).toBe(true);
});
it('returns null for pilot nodes when no pushed override file exists', async () => {
@@ -1015,9 +1368,38 @@ describe('MeshService.ensureStackOverride (BUG-1 fix)', () => {
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
process.env.SENCHO_MODE = 'pilot';
// No mesh_stacks row, no file on disk.
const result = await svc.ensureStackOverride(localNodeId, 'no-such-stack');
expect(result).toBeNull();
process.env.SENCHO_MODE = 'server';
});
it('does not use stale override presence as authority on a server', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId));
fsSync.mkdirSync(overrideDir, { recursive: true });
const overrideFile = path.join(overrideDir, 'stale-server.override.yml');
fsSync.writeFileSync(overrideFile, 'services: {}\n', 'utf8');
const result = await svc.ensureStackOverride(localNodeId, 'stale-server');
expect(result).toBeNull();
fsSync.unlinkSync(overrideFile);
});
it('restores proxy-target DB authority when override removal fails', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
db.insertMeshStack(localNodeId, 'unlink-failure', 'tester');
vi.spyOn(fs, 'unlink').mockRejectedValue(Object.assign(new Error('permission denied'), { code: 'EACCES' }));
await expect(svc.removeLocalOverride('unlink-failure')).rejects.toThrow('permission denied');
expect(db.isMeshStackEnabled(localNodeId, 'unlink-failure')).toBe(true);
});
});
@@ -45,6 +45,7 @@ describe('networking summary', () => {
afterEach(() => {
vi.restoreAllMocks();
DatabaseService.getInstance().deleteStackExposureIntents(1, STACK);
DatabaseService.getInstance().deleteMeshStack(1, STACK);
fs.rmSync(stackDir, { recursive: true, force: true });
});
@@ -72,6 +73,41 @@ describe('networking summary', () => {
expect(res.body.networkDrift.stacks).toContain(STACK);
});
it('does not count an opted-in Mesh attachment as network drift', async () => {
DatabaseService.getInstance().insertMeshStack(1, STACK, 'tester');
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDependencySnapshot: vi.fn().mockResolvedValue({
containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }], volumes: [], ports: [] }],
networks: [
{ id: 'm', name: 'sencho_mesh', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null },
],
volumes: [],
}),
} as unknown as DockerController);
const res = await request(app).get('/api/networking/summary').set('Authorization', authHeader);
expect(res.status).toBe(200);
expect(res.body.networkDrift).toEqual({ count: 0, stacks: [] });
});
it('counts an opted-out manual Mesh attachment as network drift', async () => {
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDependencySnapshot: vi.fn().mockResolvedValue({
containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }], volumes: [], ports: [] }],
networks: [
{ id: 'm', name: 'sencho_mesh', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null },
],
volumes: [],
}),
} as unknown as DockerController);
const res = await request(app).get('/api/networking/summary').set('Authorization', authHeader);
expect(res.status).toBe(200);
expect(res.body.networkDrift).toEqual({ count: 1, stacks: [STACK] });
});
it('still reports declared signals when the snapshot is unavailable (drift skipped)', async () => {
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDependencySnapshot: vi.fn().mockRejectedValue(new Error('docker down')),
+30 -4
View File
@@ -5,7 +5,12 @@ import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompos
import type { DeclaredCompose, DeclaredService } from '../helpers/composeDependencyParse';
import { parseMissingRequiredVars } from '../helpers/envVarParse';
import { parseEffectiveModel } from './preflight/effectiveModel';
import { compareStackNetworks, fromDeclaredCompose } from './network/normalize';
import {
compareStackNetworks,
fromDeclaredCompose,
type ManagedNetworkAttachmentPredicate,
} from './network/normalize';
import { resolveManagedMeshAttachment } from './network/managedMeshAttachment';
import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog';
import { getErrorMessage } from '../utils/errors';
import { isCleanOneShotCompletion } from '../utils/oneShotCompletion';
@@ -111,6 +116,8 @@ export interface AssembleStackDriftInput {
containers: DependencyContainer[];
/** Every network on the node (for resolving foreign vs stack-owned attachments). */
networks?: DependencyNetwork[];
/** Authoritative runtime attachments that are intentionally absent from authored Compose. */
managedNetworkAttachment?: ManagedNetworkAttachmentPredicate;
/** Set when the compose file could not be parsed. */
parseError?: string;
}
@@ -137,12 +144,18 @@ function networkDriftFindings(
declared: DeclaredCompose,
containers: DependencyContainer[],
networks: DependencyNetwork[],
managedNetworkAttachment?: ManagedNetworkAttachmentPredicate,
): StackDriftFinding[] {
// Runtime resource names use the Compose project (top-level `name:` when set),
// not the stack directory, so a stack with `name:` resolves its networks the
// same way Docker does. Containers are still attributed to the stack directory.
const normalized = fromDeclaredCompose(declared, declared.projectName ?? stack);
const facts = compareStackNetworks(normalized, { containers, networks, volumes: [] }, stack);
const facts = compareStackNetworks(
normalized,
{ containers, networks, volumes: [] },
stack,
managedNetworkAttachment,
);
const findings: StackDriftFinding[] = [];
const serviceByContainer = new Map(containers.map(c => [c.name, c.service ?? c.name]));
@@ -293,7 +306,13 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe
}
}
findings.push(...networkDriftFindings(stack, declared, containers, networks));
findings.push(...networkDriftFindings(
stack,
declared,
containers,
networks,
input.managedNetworkAttachment,
));
const status: StackDriftStatus = findings.length > 0 ? 'drifted' : 'in-sync';
return { stack, status, hasComposeFile: true, hasContainers, findings };
@@ -384,5 +403,12 @@ export async function buildStackDriftReport(nodeId: number, stackName: string):
};
}
return assembleStackDrift({ stack: stackName, declared: render.declared, containers, networks });
const managedNetworkAttachment = await resolveManagedMeshAttachment(nodeId, stackName);
return assembleStackDrift({
stack: stackName,
declared: render.declared,
containers,
networks,
managedNetworkAttachment,
});
}
+258 -34
View File
@@ -1,6 +1,7 @@
import net from 'net';
import path from 'path';
import fs from 'fs/promises';
import { randomUUID } from 'crypto';
import { EventEmitter } from 'events';
import * as YAML from 'yaml';
import { ComposeService } from './ComposeService';
@@ -20,6 +21,7 @@ import { STREAM_PENDING_DATA_MAX_BYTES } from '../pilot/protocol';
import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
import { isDebugEnabled } from '../utils/debug';
import { isPathWithinBase, isValidStackName, isValidRelativeStackPath } from '../utils/validation';
import { getErrorMessage } from '../utils/errors';
import { PORT as SENCHO_LISTEN_PORT } from '../helpers/constants';
import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate';
@@ -1497,6 +1499,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
// --- Opt-in / opt-out ---
public async optInStack(nodeId: number, stackName: string, actor: string): Promise<void> {
return this.runMeshNodeMutation(nodeId, () => this.optInStackExclusive(nodeId, stackName, actor));
}
private async optInStackExclusive(nodeId: number, stackName: string, actor: string): Promise<void> {
this.logDiag('opt-in start', { nodeId, stackName, actor });
const t0 = Date.now();
if (!isValidStackName(stackName)) {
@@ -1541,19 +1547,41 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
}
db.insertMeshStack(nodeId, stackName, actor);
await this.refreshAliasCache();
await this.syncForwarderListeners();
try {
await this.refreshAliasCache();
await this.syncForwarderListeners();
} catch (error) {
db.deleteMeshStack(nodeId, stackName);
try {
await this.refreshAliasCache();
await this.syncForwarderListeners();
} catch (rollbackError) {
console.warn('[MeshService] Failed to refresh aliases after opt-in rollback:', sanitizeForLog(getErrorMessage(rollbackError, 'unknown')));
}
throw error;
}
// Push the just-opted-in stack's override loudly. If this fails the
// DB state is invalid (alias claimed but remote pilot has no
// override file) so roll back rather than leave a half-state that
// future opt-in calls would short-circuit on `isMeshStackEnabled`.
// Push the just-opted-in stack's override loudly. Explicit target
// rejection rolls back the row. A remote transport failure is
// ambiguous because the target may already have committed, so retain
// authority and let normal regeneration reconcile it.
try {
await this.pushOverrideToNode(nodeId, stackName);
} catch (err) {
db.deleteMeshStack(nodeId, stackName);
await this.refreshAliasCache();
await this.syncForwarderListeners();
const node = db.getNode(nodeId);
const explicitlyRejected = node?.type !== 'remote' || err instanceof MeshError;
if (explicitlyRejected) {
db.deleteMeshStack(nodeId, stackName);
await this.refreshAliasCache();
await this.syncForwarderListeners();
} else {
this.logActivity({
source: 'mesh', level: 'warn', type: 'forwarder.error',
nodeId,
message: `mesh override push outcome unknown for ${stackName}; retaining opt-in authority for reconciliation`,
details: { stackName },
});
}
throw err;
}
// Regenerate every other meshed stack's override across the fleet
@@ -1584,6 +1612,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
}
public async optOutStack(nodeId: number, stackName: string, actor: string): Promise<void> {
return this.runMeshNodeMutation(nodeId, () => this.optOutStackExclusive(nodeId, stackName, actor));
}
private async optOutStackExclusive(nodeId: number, stackName: string, actor: string): Promise<void> {
this.logDiag('opt-out start', { nodeId, stackName, actor });
if (!isValidStackName(stackName)) {
throw new MeshError('denied', `invalid stack name: ${stackName}`);
@@ -1591,9 +1623,18 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
const db = DatabaseService.getInstance();
if (!db.isMeshStackEnabled(nodeId, stackName)) return;
db.deleteMeshStack(nodeId, stackName);
await this.removeOverrideFromNode(nodeId, stackName);
await this.refreshAliasCache();
await this.syncForwarderListeners();
try {
await this.removeOverrideFromNode(nodeId, stackName);
} catch (error) {
db.insertMeshStack(nodeId, stackName, actor);
throw error;
}
try {
await this.refreshAliasCache();
await this.syncForwarderListeners();
} catch (error) {
console.warn('[MeshService] Failed to refresh aliases after committed opt-out:', sanitizeForLog(getErrorMessage(error, 'unknown')));
}
// The opted-out row is already deleted, so listMeshStacks() will not
// include it. Walk the remaining fleet-wide rows so every other
// meshed stack regenerates its override without the dropped alias.
@@ -1620,6 +1661,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
}
public async enableForNode(nodeId: number): Promise<void> {
return this.runMeshNodeMutation(nodeId, () => this.enableForNodeExclusive(nodeId));
}
private async enableForNodeExclusive(nodeId: number): Promise<void> {
this.logDiag('enable-for-node', { nodeId });
DatabaseService.getInstance().setNodeMeshEnabled(nodeId, true);
this.logActivity({
@@ -1642,26 +1687,42 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
nodeId: number,
actor: string = 'system:mesh.disable',
): Promise<void> {
return this.runMeshNodeMutation(nodeId, () => this.disableForNodeExclusive(nodeId, actor));
}
private async disableForNodeExclusive(nodeId: number, actor: string): Promise<void> {
this.logDiag('disable-for-node start', { nodeId, actor });
const t0 = Date.now();
DatabaseService.getInstance().setNodeMeshEnabled(nodeId, false);
const stacks = DatabaseService.getInstance().listMeshStacks(nodeId);
for (const s of stacks) {
DatabaseService.getInstance().deleteMeshStack(nodeId, s.stack_name);
}
const db = DatabaseService.getInstance();
const stacks = db.listMeshStacks(nodeId);
// Dispatch DELETE /api/mesh/local-override/:stack for remote nodes
// (pilot or proxy) so the override file pushed earlier via
// applyLocalOverride is removed; falls back to local deletion for
// local nodes. Parallelize per the regenerateOverridesForNode
// rationale: each remote call is its own HTTP round-trip, so
// awaiting sequentially turns N stacks into N serialised DELETEs.
// `allSettled` so a single failure does not abort the others
// (removeOverrideFromNode already swallows errors internally).
await Promise.allSettled(
// `allSettled` so a single failure does not abort the others.
const removals = await Promise.allSettled(
stacks.map((s) => this.removeOverrideFromNode(nodeId, s.stack_name)),
);
await this.refreshAliasCache();
await this.syncForwarderListeners();
const failed: string[] = [];
const removed: typeof stacks = [];
removals.forEach((result, index) => {
const stack = stacks[index];
if (result.status === 'fulfilled') {
db.deleteMeshStack(nodeId, stack.stack_name);
removed.push(stack);
} else {
failed.push(stack.stack_name);
}
});
if (failed.length === 0) db.setNodeMeshEnabled(nodeId, false);
try {
await this.refreshAliasCache();
await this.syncForwarderListeners();
} catch (error) {
console.warn('[MeshService] Failed to refresh aliases after committed node disable changes:', sanitizeForLog(getErrorMessage(error, 'unknown')));
}
// Mirror optOutStack: regenerate every remaining node's override
// without the dropped aliases, recompose the rest of the fleet so
// their containers shed the stale extra_hosts, and redeploy the
@@ -1669,9 +1730,15 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
// sencho_mesh network and lose the alias entries they owned.
await this.regenerateOverridesAcrossFleet();
this.cascadeRecomposeAcrossFleet(undefined, undefined, actor);
for (const s of stacks) {
for (const s of removed) {
this.triggerRedeploy(nodeId, s.stack_name, actor);
}
if (failed.length > 0) {
throw new MeshError(
'push_failed',
`Could not disable Mesh on node ${nodeId}: override removal failed for ${failed.join(', ')}.`,
);
}
this.logDiag('disable-for-node complete', { nodeId, stacks: stacks.length, ms: Date.now() - t0 });
this.logActivity({
source: 'mesh', level: 'info', type: 'mesh.disable',
@@ -1679,6 +1746,24 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
});
}
private readonly meshNodeMutations = new Map<number, Promise<void>>();
private async runMeshNodeMutation<T>(nodeId: number, operation: () => Promise<T>): Promise<T> {
const previous = this.meshNodeMutations.get(nodeId) ?? Promise.resolve();
let release!: () => void;
const pending = new Promise<void>((resolve) => {
release = resolve;
});
this.meshNodeMutations.set(nodeId, pending);
await previous;
try {
return await operation();
} finally {
release();
if (this.meshNodeMutations.get(nodeId) === pending) this.meshNodeMutations.delete(nodeId);
}
}
// --- Override file management ---
public async ensureStackOverride(nodeId: number, stackName: string): Promise<string | null> {
@@ -1690,6 +1775,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
// lives on central per the C-3 design). Use file-presence as the
// fallback: if central pushed an override via applyLocalOverride, return
// that path so ComposeService picks it up on the next deploy.
if (process.env.SENCHO_MODE !== 'pilot') return null;
const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`);
if (!isPathWithinBase(file, dir)) return null;
try {
@@ -1754,13 +1840,37 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
portAliases?: MeshGlobalAlias[],
): Promise<string | null> {
if (!isValidStackName(stackName)) return null;
if (!this.senchoIp) {
const senchoIp = this.senchoIp;
if (!senchoIp) {
throw new MeshError(
'push_failed',
this.networkSetupError || 'mesh data plane unavailable on this node',
);
}
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
const lock = await StackOpLockService.getInstance().runExclusive(
localNodeId,
stackName,
'deploy',
'system:mesh.override',
() => this.applyLocalOverrideExclusive(localNodeId, stackName, aliases, senchoIp, portAliases),
);
if (!lock.ran) {
throw new MeshError(
'push_failed',
`Cannot apply Mesh override for "${stackName}": another operation (${lock.existing.action}) is already in progress.`,
);
}
return lock.result;
}
private async applyLocalOverrideExclusive(
localNodeId: number,
stackName: string,
aliases: MeshAlias[],
senchoIp: string,
portAliases?: MeshGlobalAlias[],
): Promise<string | null> {
const serviceNames = await this.getDeclaredStackServiceNames(stackName, localNodeId);
const dir = this.overrideDirFor(localNodeId);
@@ -1785,6 +1895,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
message: `mesh override preserved for ${stackName}: declared services unreadable, keeping ${existing.length} existing entries`,
details: { stackName, preservedServices: existing },
});
this.recordLocalOverrideIntent(localNodeId, stackName);
return file;
}
}
@@ -1792,13 +1903,24 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
const yaml = generateOverrideYaml({
services: serviceNames,
aliases,
senchoIp: this.senchoIp,
senchoIp,
});
await fs.writeFile(file, yaml, 'utf8');
const previousYaml = await this.readOverrideContent(file);
await this.writeOverrideAtomically(file, yaml);
try {
this.recordLocalOverrideIntent(localNodeId, stackName);
} catch (error) {
await this.restoreOverrideAfterAuthorityFailure(file, previousYaml);
throw error;
}
if (portAliases && portAliases.length > 0) {
this.pilotAliasOverlay.set(stackName, portAliases);
await this.refreshAliasCache();
await this.syncForwarderListeners();
try {
await this.refreshAliasCache();
await this.syncForwarderListeners();
} catch (error) {
console.warn('[MeshService] Failed to refresh aliases after committed override apply:', sanitizeForLog(getErrorMessage(error, 'unknown')));
}
}
return file;
}
@@ -1810,13 +1932,95 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
public async removeLocalOverride(stackName: string): Promise<void> {
if (!isValidStackName(stackName)) return;
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
const lock = await StackOpLockService.getInstance().runExclusive(
localNodeId,
stackName,
'deploy',
'system:mesh.override',
() => this.removeLocalOverrideExclusive(localNodeId, stackName),
);
if (!lock.ran) {
throw new MeshError(
'push_failed',
`Cannot remove Mesh override for "${stackName}": another operation (${lock.existing.action}) is already in progress.`,
);
}
}
private async removeLocalOverrideExclusive(localNodeId: number, stackName: string): Promise<void> {
const dir = this.overrideDirFor(localNodeId);
const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`);
if (!isPathWithinBase(file, dir)) return;
try { await fs.unlink(file); } catch { /* ignore not-exist */ }
const db = DatabaseService.getInstance();
const hadIntent = db.isMeshStackEnabled(localNodeId, stackName);
if (hadIntent) db.deleteMeshStack(localNodeId, stackName);
try {
await fs.unlink(file);
} catch (error) {
if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) {
if (hadIntent) db.insertMeshStack(localNodeId, stackName, null);
throw error;
}
}
if (this.pilotAliasOverlay.delete(stackName)) {
await this.refreshAliasCache();
await this.syncForwarderListeners();
try {
await this.refreshAliasCache();
await this.syncForwarderListeners();
} catch (error) {
console.warn('[MeshService] Failed to refresh aliases after committed override removal:', sanitizeForLog(getErrorMessage(error, 'unknown')));
}
}
}
private async writeOverrideAtomically(file: string, yaml: string): Promise<void> {
const dir = path.dirname(file);
const tempFile = path.resolve(dir, `.${path.basename(file)}.${randomUUID()}.tmp`);
if (!isPathWithinBase(tempFile, dir)) throw new Error('Invalid Mesh override temporary path');
let handle: Awaited<ReturnType<typeof fs.open>> | null = null;
try {
handle = await fs.open(tempFile, 'wx');
await handle.writeFile(yaml, 'utf8');
await handle.sync();
await handle.close();
handle = null;
await fs.rename(tempFile, file);
} catch (error) {
if (handle) {
try { await handle.close(); } catch (closeError) {
console.warn('[MeshService] Failed to close temporary override:', sanitizeForLog(getErrorMessage(closeError, 'unknown')));
}
}
try {
await fs.unlink(tempFile);
} catch (cleanupError) {
if (!(cleanupError instanceof Error && 'code' in cleanupError && cleanupError.code === 'ENOENT')) {
console.warn('[MeshService] Failed to clean up temporary override:', sanitizeForLog(getErrorMessage(cleanupError, 'unknown')));
}
}
throw error;
}
}
private async readOverrideContent(file: string): Promise<string | null> {
try {
return await fs.readFile(file, 'utf8');
} catch (error) {
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null;
throw error;
}
}
private async restoreOverrideAfterAuthorityFailure(file: string, previousYaml: string | null): Promise<void> {
try {
if (previousYaml === null) {
await fs.unlink(file);
} else {
await this.writeOverrideAtomically(file, previousYaml);
}
} catch (error) {
if (previousYaml === null && error instanceof Error && 'code' in error && error.code === 'ENOENT') return;
console.warn('[MeshService] Failed to restore override after authority error:', sanitizeForLog(getErrorMessage(error, 'unknown')));
}
}
@@ -1825,7 +2029,11 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
const dir = this.overrideDirFor(nodeId);
const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`);
if (!isPathWithinBase(file, dir)) return;
try { await fs.unlink(file); } catch { /* ignore not-exist */ }
try {
await fs.unlink(file);
} catch (error) {
if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error;
}
}
private overrideDirFor(nodeId: number): string {
@@ -1833,6 +2041,13 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
return path.join(dataDir, 'mesh', 'overrides', String(nodeId));
}
private recordLocalOverrideIntent(nodeId: number, stackName: string): void {
if (process.env.SENCHO_MODE === 'pilot') return;
const db = DatabaseService.getInstance();
if (db.isMeshStackEnabled(nodeId, stackName)) return;
db.insertMeshStack(nodeId, stackName, null);
}
private async regenerateOverridesForNode(nodeId: number, skipStack?: string): Promise<void> {
const db = DatabaseService.getInstance();
const stacks = db.listMeshStacks(nodeId);
@@ -2331,7 +2546,11 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
);
}
if (!res.ok) {
throw new MeshError('push_failed', `HTTP ${res.status} from node ${node.name}`);
const message = `HTTP ${res.status} from node ${node.name}`;
if (res.status >= 400 && res.status < 500) {
throw new MeshError('push_failed', message);
}
throw new Error(message);
}
}
@@ -2436,15 +2655,20 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
}
try {
await this.proxyFetch(
const response = await this.proxyFetch(
nodeId,
'DELETE',
`/api/mesh/local-override/${encodeURIComponent(stackName)}`,
undefined,
5_000,
);
if (!response.ok) {
const body = await response.text().catch(() => '');
throw new Error(`HTTP ${response.status} from node ${node.name}: ${body.slice(0, 256)}`);
}
} catch (err) {
console.warn('[MeshService] removeOverrideFromNode failed:', sanitizeForLog((err as Error).message));
throw err;
}
}
@@ -23,6 +23,8 @@ import type {
NetworkDriftFacts, NetworkFactNetwork, NetworkFactService, NetworkRuntimeState, StackNetworkFacts,
} from './types';
import { classifyMissingExternalNetworks, type MissingExternalNetwork } from './missingExternalNetworks';
import { resolveManagedMeshAttachment } from './managedMeshAttachment';
import type { ManagedNetworkAttachmentPredicate } from './normalize';
import { getErrorMessage } from '../../utils/errors';
import { redactSensitiveText, sanitizeForLog } from '../../utils/safeLog';
@@ -43,6 +45,7 @@ export function assembleStackNetworkFacts(
model: EffectiveModel | null,
renderError: string | null,
snapshot: DependencySnapshot | null,
managedNetworkAttachment?: ManagedNetworkAttachmentPredicate,
): StackNetworkFacts {
const runtime: NetworkRuntimeState = snapshot ? 'available' : 'unavailable';
@@ -82,7 +85,9 @@ export function assembleStackNetworkFacts(
extraHosts: s.extraHosts,
}));
const drift = snapshot ? compareStackNetworks(fromEffectiveModel(model), snapshot, stackName) : EMPTY_DRIFT;
const drift = snapshot
? compareStackNetworks(fromEffectiveModel(model), snapshot, stackName, managedNetworkAttachment)
: EMPTY_DRIFT;
const missingExternalNetworks: MissingExternalNetwork[] = snapshot
? classifyMissingExternalNetworks(
model,
@@ -147,5 +152,8 @@ export async function buildStackNetworkFacts(
}
}
return assembleStackNetworkFacts(stackName, model, renderError, snapshot);
const managedNetworkAttachment = snapshot && model
? await resolveManagedMeshAttachment(nodeId, stackName)
: undefined;
return assembleStackNetworkFacts(stackName, model, renderError, snapshot, managedNetworkAttachment);
}
@@ -0,0 +1,55 @@
import fs from 'fs/promises';
import path from 'path';
import { DatabaseService } from '../DatabaseService';
import { SENCHO_MESH_NETWORK } from '../MeshComposeOverride';
import SelfIdentityService from '../SelfIdentityService';
import { getErrorMessage } from '../../utils/errors';
import { sanitizeForLog } from '../../utils/safeLog';
import { isPathWithinBase, isValidStackName } from '../../utils/validation';
import type { ManagedNetworkAttachmentPredicate } from './normalize';
async function hasPilotMeshOverride(nodeId: number, stackName: string): Promise<boolean> {
if (process.env.SENCHO_MODE !== 'pilot' || !isValidStackName(stackName)) return false;
const dataDir = process.env.DATA_DIR || '/app/data';
const overrideDir = path.resolve(dataDir, 'mesh', 'overrides', String(nodeId));
const overridePath = path.resolve(overrideDir, `${path.basename(stackName)}.override.yml`);
if (!isPathWithinBase(overridePath, overrideDir)) return false;
try {
await fs.access(overridePath);
return true;
} catch (error) {
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return false;
console.warn(
'[NetworkDrift] Could not verify Pilot Mesh override for %s:',
sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
return false;
}
}
export async function resolveManagedMeshAttachment(
nodeId: number,
stackName: string,
): Promise<ManagedNetworkAttachmentPredicate> {
let stackManaged = false;
try {
stackManaged = DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName);
} catch (error) {
console.warn(
'[NetworkDrift] Could not verify Mesh opt-in state for %s:',
sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
}
if (!stackManaged) stackManaged = await hasPilotMeshOverride(nodeId, stackName);
const selfIdentity = SelfIdentityService.getInstance();
return (container, networkName) => networkName === SENCHO_MESH_NETWORK && (
stackManaged
|| selfIdentity.isOwnContainer(container.id)
|| selfIdentity.isOwnContainer(container.name)
);
}
@@ -10,6 +10,7 @@ import { FileSystemService } from '../FileSystemService';
import { DatabaseService } from '../DatabaseService';
import { parseComposeDependencies } from '../../helpers/composeDependencyParse';
import { assembleStackDrift } from '../DriftDetectionService';
import { resolveManagedMeshAttachment } from './managedMeshAttachment';
import { isHostNetwork, isLoopback } from './normalize';
import { getErrorMessage } from '../../utils/errors';
import { sanitizeForLog } from '../../utils/safeLog';
@@ -86,7 +87,14 @@ export async function computeNodeNetworkingSummary(nodeId: number): Promise<Node
if (snapshot) {
// declared.parseError is already excluded above, so the drift report is authoritative.
const containers = snapshot.containers.filter(c => c.stack === stack);
const report = assembleStackDrift({ stack, declared, containers, networks: snapshot.networks });
const managedNetworkAttachment = await resolveManagedMeshAttachment(nodeId, stack);
const report = assembleStackDrift({
stack,
declared,
containers,
networks: snapshot.networks,
managedNetworkAttachment,
});
if (report.findings.some(f => f.kind === 'network-undeclared' || f.kind === 'network-missing')) networkDrift.push(stack);
}
}
+9 -1
View File
@@ -8,8 +8,9 @@
*/
import type { EffectiveModel } from '../preflight/effectiveModel';
import type { DeclaredCompose } from '../../helpers/composeDependencyParse';
import type { DependencySnapshot } from '../DockerController';
import type { DependencyContainer, DependencySnapshot } from '../DockerController';
import type { NetworkDriftFacts } from './types';
import { SENCHO_MESH_NETWORK } from '../MeshComposeOverride';
/** Container states that count as "deployed" for drift, matching DriftDetectionService. */
const RUNNING_STATES = new Set(['running', 'restarting']);
@@ -62,6 +63,11 @@ export interface NormalizedNetworkModel {
services: { name: string; networkKeys: string[]; networkMode?: string }[];
}
export type ManagedNetworkAttachmentPredicate = (
container: DependencyContainer,
networkName: string,
) => boolean;
/** Rendered model: resource names are already resolved by `docker compose config`. */
export function fromEffectiveModel(m: EffectiveModel): NormalizedNetworkModel {
const networks: NormalizedNetworkModel['networks'] = {};
@@ -97,6 +103,7 @@ export function compareStackNetworks(
declared: NormalizedNetworkModel,
snapshot: DependencySnapshot,
stackName: string,
isManagedAttachment: ManagedNetworkAttachmentPredicate = () => false,
): NetworkDriftFacts {
const runtimeOnlyAttachments: NetworkDriftFacts['runtimeOnlyAttachments'] = [];
const foreignNetworkAttachments: NetworkDriftFacts['foreignNetworkAttachments'] = [];
@@ -118,6 +125,7 @@ export function compareStackNetworks(
const net = networkByName.get(attached.name);
if (SYSTEM_NETWORK_NAMES.has(attached.name) || net?.isSystem) continue;
if (declaredRuntimeNames.has(attached.name)) { usedRuntimeNames.add(attached.name); continue; }
if (attached.name === SENCHO_MESH_NETWORK && isManagedAttachment(c, attached.name)) continue;
if (net?.stack === stackName || attached.name.startsWith(`${declared.projectName}_`)) {
runtimeOnlyAttachments.push({ container: c.name, service: c.service, network: attached.name });
} else {
+1 -1
View File
@@ -153,7 +153,7 @@ When the node is reachable, the tab compares the declared effective model agains
| **Declared but unused** | A network is declared in the Compose file but no currently running service is connected to it. Often seen when a service is stopped or removed without `docker compose down`. |
| **Missing from runtime** | A network is declared but does not exist in Docker. The stack may not have been deployed, or the network was deleted externally. |
System-managed networks (`bridge`, `host`, `none`) and Docker's implicit default bridge are excluded from all drift findings.
System-managed networks (`bridge`, `host`, `none`) and Docker's implicit default bridge are excluded from all drift findings. Attachments to `sencho_mesh` are also excluded when Sencho verifies that the container is its own instance or that the stack is opted into Sencho Mesh. A manual attachment from an opted-out stack remains visible as drift.
When the runtime matches the Compose file, the section shows a green **runtime matches compose** card.
+3 -1
View File
@@ -83,7 +83,7 @@ A Compose port range such as `8000-8002:8000-8002` is compared conservatively. B
### Network findings and the Networking tab
The **network-undeclared** and **network-missing** findings reuse the same comparison the stack's [Networking](/features/compose-networking) tab uses, so the two surfaces never disagree about a network attachment. The difference is history: this tab persists findings in the drift ledger over time, while the Networking tab always shows the current live state with no history. A stack with an open network finding also counts toward the **Drift** chip in the Fleet Overview's Networking filter group, so a network-attachment mismatch is visible both per-stack here and across the fleet there.
The **network-undeclared** and **network-missing** findings reuse the same comparison the stack's [Networking](/features/compose-networking) tab uses, so the two surfaces never disagree about a network attachment. Sencho-verified `sencho_mesh` attachments for its own container and Mesh-opted-in stacks are excluded, while a manual attachment from an opted-out stack remains actionable. The difference is history: this tab persists findings in the drift ledger over time, while the Networking tab always shows the current live state with no history. A stack with an open network finding also counts toward the **Drift** chip in the Fleet Overview's Networking filter group, so a network-attachment mismatch is visible both per-stack here and across the fleet there.
## When drift is recorded
@@ -93,6 +93,8 @@ The **network-undeclared** and **network-missing** findings reuse the same compa
**After every deploy or update**, Sencho automatically records a new baseline hash and runs a full reconciliation. You do not need to click re-check after deploying; the ledger is updated as part of the deploy pipeline.
An open ledger entry for an attachment that is no longer reported clears during the next re-check or post-deploy reconciliation. Opening the tab refreshes the live report but does not change persisted history.
<img
src="/images/stack-drift/drift-history.png"
alt="The Drift tab showing both the Findings section and the Drift history section with an open finding marked just now"