mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-05 16:37:46 +00:00
fix(networking): ignore verified Mesh attachments in drift (#1729)
This commit is contained in:
@@ -4,14 +4,21 @@
|
||||
* runtime-vs-Compose drift comparison (system/default/external networks and
|
||||
* stopped containers are not flagged).
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import type { EffectiveModel, EffService } from '../services/preflight/effectiveModel';
|
||||
import type { DeclaredCompose } from '../helpers/composeDependencyParse';
|
||||
import type { DependencySnapshot, DependencyContainer, DependencyNetwork } from '../services/DockerController';
|
||||
import {
|
||||
fromEffectiveModel, fromDeclaredCompose, compareStackNetworks, runtimeResourceName, parseAccessUrlPorts,
|
||||
type ManagedNetworkAttachmentPredicate,
|
||||
} from '../services/network/normalize';
|
||||
import { assembleStackNetworkFacts } from '../services/network/composeNetworkInspector';
|
||||
import { assembleStackNetworkFacts, buildStackNetworkFacts } from '../services/network/composeNetworkInspector';
|
||||
import { buildNodeNetworkingFindings } from '../services/network/networkingFindings';
|
||||
import type { NetworkingNetworkBase } from '../services/network/networkingTypes';
|
||||
import DockerController from '../services/DockerController';
|
||||
import { ComposeService } from '../services/ComposeService';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
|
||||
function effSvc(over: Partial<EffService> = {}): EffService {
|
||||
const hasHealthcheck = over.hasHealthcheck ?? true;
|
||||
@@ -264,6 +271,187 @@ describe('compareStackNetworks', () => {
|
||||
expect(drift.foreignNetworkAttachments).toEqual([]);
|
||||
});
|
||||
|
||||
it('removes managed Mesh drift before Networking findings while preserving advanced-driver info', () => {
|
||||
const meshOnlyModel: EffectiveModel = {
|
||||
projectName: 'myapp',
|
||||
services: [],
|
||||
networks: {},
|
||||
volumes: {},
|
||||
};
|
||||
const snap = snapshot(
|
||||
[container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
|
||||
[depNet({ id: 'm', name: 'sencho_mesh', driver: 'macvlan', composeProject: null, stack: null })],
|
||||
);
|
||||
const facts = assembleStackNetworkFacts(
|
||||
'myapp',
|
||||
meshOnlyModel,
|
||||
null,
|
||||
snap,
|
||||
(_runtimeContainer, networkName) => networkName === 'sencho_mesh',
|
||||
);
|
||||
const baseNetworks: NetworkingNetworkBase[] = [{
|
||||
id: 'm',
|
||||
name: 'sencho_mesh',
|
||||
driver: 'macvlan',
|
||||
scope: 'local',
|
||||
isSystem: false,
|
||||
ingress: false,
|
||||
composeProject: null,
|
||||
stack: null,
|
||||
connectedCount: 1,
|
||||
isSencho: true,
|
||||
ownership: 'sencho-managed',
|
||||
declaredByStacks: [],
|
||||
declaredExternalByStacks: [],
|
||||
isExternalDependency: false,
|
||||
}];
|
||||
|
||||
const findings = buildNodeNetworkingFindings(1, snap, [facts], baseNetworks);
|
||||
|
||||
expect(facts.drift.runtimeOnlyAttachments).toEqual([]);
|
||||
expect(facts.drift.foreignNetworkAttachments).toEqual([]);
|
||||
expect(findings.some(f => f.kind === 'network-undeclared' || f.kind === 'foreign-network-attachment')).toBe(false);
|
||||
expect(findings).toContainEqual(expect.objectContaining({
|
||||
kind: 'advanced-driver-caveat',
|
||||
severity: 'info',
|
||||
network: 'sencho_mesh',
|
||||
}));
|
||||
});
|
||||
|
||||
it('keeps Networking facts available and Mesh drift actionable when opt-in authority fails', async () => {
|
||||
const snap = snapshot(
|
||||
[container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
|
||||
[depNet({ name: 'sencho_mesh', composeProject: null, stack: null })],
|
||||
);
|
||||
vi.spyOn(ComposeService, 'getInstance').mockReturnValue({
|
||||
renderConfig: vi.fn().mockResolvedValue({
|
||||
rendered: JSON.stringify({ name: 'myapp', services: { web: { image: 'nginx:1.27' } } }),
|
||||
stderr: '',
|
||||
code: 0,
|
||||
timedOut: false,
|
||||
}),
|
||||
} as unknown as ComposeService);
|
||||
vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({
|
||||
getStacks: vi.fn().mockResolvedValue(['myapp']),
|
||||
} as unknown as FileSystemService);
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDependencySnapshot: vi.fn().mockResolvedValue(snap),
|
||||
} as unknown as DockerController);
|
||||
vi.spyOn(DatabaseService, 'getInstance').mockImplementation(() => {
|
||||
throw new Error('database unavailable');
|
||||
});
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
|
||||
try {
|
||||
const facts = await buildStackNetworkFacts(1, 'myapp');
|
||||
|
||||
expect(facts.runtime).toBe('available');
|
||||
expect(facts.drift.foreignNetworkAttachments).toEqual([{ container: 'web1', network: 'sencho_mesh' }]);
|
||||
} finally {
|
||||
vi.restoreAllMocks();
|
||||
}
|
||||
});
|
||||
|
||||
it('threads opted-in authority through Networking facts and preserves advanced-driver info', async () => {
|
||||
const snap = snapshot(
|
||||
[container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
|
||||
[depNet({ id: 'm', name: 'sencho_mesh', driver: 'macvlan', composeProject: null, stack: null })],
|
||||
);
|
||||
vi.spyOn(ComposeService, 'getInstance').mockReturnValue({
|
||||
renderConfig: vi.fn().mockResolvedValue({
|
||||
rendered: JSON.stringify({ name: 'myapp', services: { web: { image: 'nginx:1.27' } } }),
|
||||
stderr: '',
|
||||
code: 0,
|
||||
timedOut: false,
|
||||
}),
|
||||
} as unknown as ComposeService);
|
||||
vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({
|
||||
getStacks: vi.fn().mockResolvedValue(['myapp']),
|
||||
} as unknown as FileSystemService);
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDependencySnapshot: vi.fn().mockResolvedValue(snap),
|
||||
} as unknown as DockerController);
|
||||
vi.spyOn(DatabaseService, 'getInstance').mockReturnValue({
|
||||
isMeshStackEnabled: vi.fn().mockReturnValue(true),
|
||||
getStackExposureIntents: vi.fn().mockReturnValue([]),
|
||||
getStackDossier: vi.fn().mockReturnValue(null),
|
||||
} as unknown as DatabaseService);
|
||||
|
||||
try {
|
||||
const facts = await buildStackNetworkFacts(1, 'myapp');
|
||||
const baseNetworks: NetworkingNetworkBase[] = [{
|
||||
id: 'm', name: 'sencho_mesh', driver: 'macvlan', scope: 'local', isSystem: false,
|
||||
ingress: false, composeProject: null, stack: null, connectedCount: 1, isSencho: true,
|
||||
ownership: 'sencho-managed', declaredByStacks: [], declaredExternalByStacks: [],
|
||||
isExternalDependency: false,
|
||||
}];
|
||||
const findings = buildNodeNetworkingFindings(1, snap, [facts], baseNetworks);
|
||||
|
||||
expect(facts.drift.runtimeOnlyAttachments).toEqual([]);
|
||||
expect(facts.drift.foreignNetworkAttachments).toEqual([]);
|
||||
expect(findings.filter(f => f.kind === 'network-undeclared' || f.kind === 'foreign-network-attachment')).toEqual([]);
|
||||
expect(findings).toContainEqual(expect.objectContaining({
|
||||
kind: 'advanced-driver-caveat',
|
||||
severity: 'info',
|
||||
network: 'sencho_mesh',
|
||||
}));
|
||||
} finally {
|
||||
vi.restoreAllMocks();
|
||||
}
|
||||
});
|
||||
|
||||
it('ignores a verified Sencho Mesh attachment for the Sencho container', () => {
|
||||
const snap = snapshot(
|
||||
[container({ id: 'sencho-id', name: 'sencho', service: 'sencho', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
|
||||
[depNet({ name: 'sencho_mesh', composeProject: null, stack: null })],
|
||||
);
|
||||
const managed: ManagedNetworkAttachmentPredicate = (runtimeContainer, networkName) =>
|
||||
runtimeContainer.id === 'sencho-id' && networkName === 'sencho_mesh';
|
||||
|
||||
const drift = compareStackNetworks(declared, snap, 'myapp', managed);
|
||||
|
||||
expect(drift.runtimeOnlyAttachments).toEqual([]);
|
||||
expect(drift.foreignNetworkAttachments).toEqual([]);
|
||||
});
|
||||
|
||||
it('ignores a verified Mesh attachment for an opted-in application stack', () => {
|
||||
const snap = snapshot(
|
||||
[container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
|
||||
[depNet({ name: 'sencho_mesh', composeProject: null, stack: null })],
|
||||
);
|
||||
const managed: ManagedNetworkAttachmentPredicate = (_runtimeContainer, networkName) =>
|
||||
networkName === 'sencho_mesh';
|
||||
|
||||
const drift = compareStackNetworks(declared, snap, 'myapp', managed);
|
||||
|
||||
expect(drift.runtimeOnlyAttachments).toEqual([]);
|
||||
expect(drift.foreignNetworkAttachments).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps an unverified manual Mesh attachment actionable', () => {
|
||||
const snap = snapshot(
|
||||
[container({ networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
|
||||
[depNet({ name: 'sencho_mesh', composeProject: null, stack: null })],
|
||||
);
|
||||
|
||||
const drift = compareStackNetworks(declared, snap, 'myapp', () => false);
|
||||
|
||||
expect(drift.runtimeOnlyAttachments).toEqual([]);
|
||||
expect(drift.foreignNetworkAttachments).toEqual([{ container: 'web1', network: 'sencho_mesh' }]);
|
||||
});
|
||||
|
||||
it('keeps sencho_extra actionable even when the stack is Mesh-managed', () => {
|
||||
const snap = snapshot(
|
||||
[container({ networks: [{ name: 'sencho_extra', id: 'e', ip: '' }] })],
|
||||
[depNet({ name: 'sencho_extra', composeProject: null, stack: null })],
|
||||
);
|
||||
|
||||
const drift = compareStackNetworks(declared, snap, 'myapp', () => true);
|
||||
|
||||
expect(drift.runtimeOnlyAttachments).toEqual([]);
|
||||
expect(drift.foreignNetworkAttachments).toEqual([{ container: 'web1', network: 'sencho_extra' }]);
|
||||
});
|
||||
|
||||
it('does not flag attachments from stopped containers', () => {
|
||||
const snap = snapshot(
|
||||
[container({ state: 'exited', networks: [{ name: 'myapp_extra', id: 'b', ip: '' }] })],
|
||||
|
||||
@@ -18,6 +18,7 @@ import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompos
|
||||
import type { DeclaredCompose, DeclaredService, DeclaredPort } from '../helpers/composeDependencyParse';
|
||||
import type { EffectiveModel, EffService } from '../services/preflight/effectiveModel';
|
||||
import { fromDeclaredCompose, fromEffectiveModel } from '../services/network/normalize';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
|
||||
// ── builders ────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -580,6 +581,32 @@ describe('assembleStackDrift - network drift', () => {
|
||||
expect(report.findings.filter(f => f.kind.startsWith('network-'))).toEqual([]);
|
||||
expect(report.status).toBe('in-sync');
|
||||
});
|
||||
|
||||
it('reports in-sync when a verified Mesh attachment is the only runtime difference', () => {
|
||||
const report = assembleStackDrift({
|
||||
stack: 'app',
|
||||
declared: declared([service({ name: 'web' })]),
|
||||
containers: [container({ id: 'c1', service: 'web', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
|
||||
networks: [depNet('sencho_mesh', { composeProject: null, stack: null })],
|
||||
managedNetworkAttachment: (_runtimeContainer, networkName) => networkName === 'sencho_mesh',
|
||||
});
|
||||
|
||||
expect(report.status).toBe('in-sync');
|
||||
expect(report.findings.filter(f => f.kind === 'network-undeclared')).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps an unverified manual Mesh attachment drifted', () => {
|
||||
const report = assembleStackDrift({
|
||||
stack: 'app',
|
||||
declared: declared([service({ name: 'web' })]),
|
||||
containers: [container({ id: 'c1', service: 'web', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
|
||||
networks: [depNet('sencho_mesh', { composeProject: null, stack: null })],
|
||||
managedNetworkAttachment: () => false,
|
||||
});
|
||||
|
||||
expect(report.status).toBe('drifted');
|
||||
expect(report.findings.filter(f => f.kind === 'network-undeclared')).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── declaredFromEffectiveModel ─────────────────────────────────────────────
|
||||
@@ -815,4 +842,46 @@ describe('buildStackDriftReport - boundaries', () => {
|
||||
expect(findingKinds(report)).toContain('network-undeclared');
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('keeps the report available and Mesh drift actionable when opt-in authority fails', async () => {
|
||||
const snapshot: DependencySnapshot = {
|
||||
containers: [container({ id: 'c1', service: 'web', stack: 'app', image: 'nginx:1.25', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
|
||||
networks: [depNet('sencho_mesh', { composeProject: null, stack: null })],
|
||||
volumes: [],
|
||||
};
|
||||
stubDockerRender({ name: 'app', services: { web: { image: 'nginx:1.25' } } });
|
||||
stubFsAndSnapshot(snapshot);
|
||||
vi.spyOn(DatabaseService, 'getInstance').mockImplementation(() => {
|
||||
throw new Error('database unavailable');
|
||||
});
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
|
||||
const report = await buildStackDriftReport(0, 'app');
|
||||
|
||||
expect(report.status).toBe('drifted');
|
||||
expect(report.findings).toContainEqual(expect.objectContaining({
|
||||
kind: 'network-undeclared',
|
||||
actual: 'sencho_mesh',
|
||||
}));
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('reports in-sync through the public builder when DB authority opts the stack into Mesh', async () => {
|
||||
const snapshot: DependencySnapshot = {
|
||||
containers: [container({ id: 'c1', service: 'web', stack: 'app', image: 'nginx:1.25', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }] })],
|
||||
networks: [depNet('sencho_mesh', { composeProject: null, stack: null })],
|
||||
volumes: [],
|
||||
};
|
||||
stubDockerRender({ name: 'app', services: { web: { image: 'nginx:1.25' } } });
|
||||
stubFsAndSnapshot(snapshot);
|
||||
vi.spyOn(DatabaseService, 'getInstance').mockReturnValue({
|
||||
isMeshStackEnabled: vi.fn().mockReturnValue(true),
|
||||
} as unknown as DatabaseService);
|
||||
|
||||
const report = await buildStackDriftReport(0, 'app');
|
||||
|
||||
expect(report.status).toBe('in-sync');
|
||||
expect(report.findings.filter(f => f.kind === 'network-undeclared')).toEqual([]);
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
import fs from 'fs/promises';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import SelfIdentityService from '../services/SelfIdentityService';
|
||||
import type { DependencyContainer } from '../services/DockerController';
|
||||
import { resolveManagedMeshAttachment } from '../services/network/managedMeshAttachment';
|
||||
|
||||
const originalDataDir = process.env.DATA_DIR;
|
||||
const originalMode = process.env.SENCHO_MODE;
|
||||
let tempDir: string | null = null;
|
||||
|
||||
function runtimeContainer(overrides: Partial<DependencyContainer> = {}): DependencyContainer {
|
||||
return {
|
||||
id: 'app-id',
|
||||
name: 'app-web-1',
|
||||
service: 'web',
|
||||
composeProject: 'app',
|
||||
stack: 'app',
|
||||
state: 'running',
|
||||
exitCode: null,
|
||||
image: 'nginx:latest',
|
||||
networks: [],
|
||||
volumes: [],
|
||||
ports: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function stubAuthorities(meshStackEnabled: boolean, ownContainers: string[] = []): void {
|
||||
vi.spyOn(DatabaseService, 'getInstance').mockReturnValue({
|
||||
isMeshStackEnabled: vi.fn().mockReturnValue(meshStackEnabled),
|
||||
} as unknown as DatabaseService);
|
||||
vi.spyOn(SelfIdentityService, 'getInstance').mockReturnValue({
|
||||
isOwnContainer: vi.fn((idOrName: string) => ownContainers.includes(idOrName)),
|
||||
} as unknown as SelfIdentityService);
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
if (originalDataDir === undefined) delete process.env.DATA_DIR;
|
||||
else process.env.DATA_DIR = originalDataDir;
|
||||
if (originalMode === undefined) delete process.env.SENCHO_MODE;
|
||||
else process.env.SENCHO_MODE = originalMode;
|
||||
if (tempDir) await fs.rm(tempDir, { recursive: true, force: true });
|
||||
tempDir = null;
|
||||
});
|
||||
|
||||
describe('resolveManagedMeshAttachment', () => {
|
||||
it('authorizes the canonical Mesh attachment for a centrally opted-in stack', async () => {
|
||||
process.env.SENCHO_MODE = 'server';
|
||||
stubAuthorities(true);
|
||||
const isManaged = await resolveManagedMeshAttachment(1, 'app');
|
||||
|
||||
expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(true);
|
||||
expect(isManaged(runtimeContainer(), 'sencho_extra')).toBe(false);
|
||||
});
|
||||
|
||||
it('authorizes the canonical Mesh attachment for the actual Sencho container', async () => {
|
||||
process.env.SENCHO_MODE = 'server';
|
||||
stubAuthorities(false, ['sencho-id']);
|
||||
const isManaged = await resolveManagedMeshAttachment(1, 'sencho');
|
||||
|
||||
expect(isManaged(runtimeContainer({ id: 'sencho-id', name: 'sencho' }), 'sencho_mesh')).toBe(true);
|
||||
expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(false);
|
||||
});
|
||||
|
||||
it('keeps a manual Mesh attachment actionable for an opted-out stack', async () => {
|
||||
process.env.SENCHO_MODE = 'server';
|
||||
stubAuthorities(false);
|
||||
const isManaged = await resolveManagedMeshAttachment(1, 'app');
|
||||
|
||||
expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(false);
|
||||
});
|
||||
|
||||
it('uses Pilot override presence as the authoritative opt-in representation', async () => {
|
||||
process.env.SENCHO_MODE = 'pilot';
|
||||
tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'sencho-mesh-drift-'));
|
||||
process.env.DATA_DIR = tempDir;
|
||||
const overrideDir = path.join(tempDir, 'mesh', 'overrides', '7');
|
||||
await fs.mkdir(overrideDir, { recursive: true });
|
||||
await fs.writeFile(path.join(overrideDir, 'app.override.yml'), 'services: {}\n');
|
||||
stubAuthorities(false);
|
||||
|
||||
const isManaged = await resolveManagedMeshAttachment(7, 'app');
|
||||
|
||||
expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(true);
|
||||
});
|
||||
|
||||
it('does not let stale server override presence supersede opted-out DB state', async () => {
|
||||
process.env.SENCHO_MODE = 'server';
|
||||
tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'sencho-mesh-drift-'));
|
||||
process.env.DATA_DIR = tempDir;
|
||||
const overrideDir = path.join(tempDir, 'mesh', 'overrides', '1');
|
||||
await fs.mkdir(overrideDir, { recursive: true });
|
||||
await fs.writeFile(path.join(overrideDir, 'app.override.yml'), 'services: {}\n');
|
||||
stubAuthorities(false);
|
||||
|
||||
const isManaged = await resolveManagedMeshAttachment(1, 'app');
|
||||
|
||||
expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(false);
|
||||
});
|
||||
|
||||
it('fails closed when Mesh opt-in state cannot be read', async () => {
|
||||
process.env.SENCHO_MODE = 'server';
|
||||
vi.spyOn(DatabaseService, 'getInstance').mockImplementation(() => {
|
||||
throw new Error('database unavailable');
|
||||
});
|
||||
vi.spyOn(SelfIdentityService, 'getInstance').mockReturnValue({
|
||||
isOwnContainer: vi.fn().mockReturnValue(false),
|
||||
} as unknown as SelfIdentityService);
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
|
||||
const isManaged = await resolveManagedMeshAttachment(1, 'app');
|
||||
|
||||
expect(isManaged(runtimeContainer(), 'sencho_mesh')).toBe(false);
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
'[NetworkDrift] Could not verify Mesh opt-in state for %s:',
|
||||
'app',
|
||||
'database unavailable',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -96,7 +96,7 @@ describe('MeshService.removeOverrideFromNode (remote dispatch)', () => {
|
||||
db.deleteNode(remoteNodeId);
|
||||
});
|
||||
|
||||
it('swallows network errors from the remote so the disable cascade can continue', async () => {
|
||||
it('rejects network errors so callers can preserve authoritative opt-in state', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const remoteNodeId = db.addNode({
|
||||
@@ -117,11 +117,37 @@ describe('MeshService.removeOverrideFromNode (remote dispatch)', () => {
|
||||
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('ECONNREFUSED'));
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ });
|
||||
|
||||
// Must not throw: the remote being offline is a tolerable condition;
|
||||
// the cascade upstream uses Promise.allSettled and continues.
|
||||
await expect(svc.removeOverrideFromNode(remoteNodeId, 'sample-stack')).resolves.toBeUndefined();
|
||||
await expect(svc.removeOverrideFromNode(remoteNodeId, 'sample-stack')).rejects.toThrow('ECONNREFUSED');
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
|
||||
db.deleteNode(remoteNodeId);
|
||||
});
|
||||
|
||||
it('rejects a non-success response from a busy remote target', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const remoteNodeId = db.addNode({
|
||||
name: 'remove-override-busy-test',
|
||||
type: 'remote',
|
||||
mode: 'proxy',
|
||||
compose_dir: '/tmp',
|
||||
is_default: false,
|
||||
api_url: 'https://remote.example.com:1852',
|
||||
api_token: 'remote-tok',
|
||||
});
|
||||
|
||||
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
|
||||
apiUrl: 'https://remote.example.com:1852',
|
||||
apiToken: 'remote-tok',
|
||||
});
|
||||
vi.spyOn(globalThis, 'fetch').mockResolvedValue(
|
||||
new Response('another operation is already in progress', { status: 500 }),
|
||||
);
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ });
|
||||
|
||||
await expect(svc.removeOverrideFromNode(remoteNodeId, 'sample-stack'))
|
||||
.rejects.toThrow('HTTP 500');
|
||||
|
||||
db.deleteNode(remoteNodeId);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { EventEmitter } from 'events';
|
||||
import fsSync from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
import { getSenchoIpFromSubnet, MeshError, type MeshTarget, type MeshTcpStreamLike } from '../services/MeshService';
|
||||
@@ -20,6 +21,7 @@ afterAll(() => {
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.SENCHO_MODE = 'server';
|
||||
const db = DatabaseService.getInstance().getDb();
|
||||
db.prepare('DELETE FROM mesh_stacks').run();
|
||||
db.prepare('DELETE FROM nodes WHERE is_default = 0').run();
|
||||
@@ -106,6 +108,84 @@ describe('MeshService.optInStack', () => {
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false);
|
||||
});
|
||||
|
||||
it('restores opt-in authority when target override removal is rejected', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
db.insertMeshStack(localNodeId, 'busy-stack', 'setup');
|
||||
vi.spyOn(svc, 'removeOverrideFromNode').mockRejectedValue(
|
||||
new Error('HTTP 500: another operation is already in progress'),
|
||||
);
|
||||
|
||||
await expect(svc.optOutStack(localNodeId, 'busy-stack', 'tester'))
|
||||
.rejects.toThrow('another operation is already in progress');
|
||||
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'busy-stack')).toBe(true);
|
||||
});
|
||||
|
||||
it('serializes concurrent opt-out requests for the same node', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
db.insertMeshStack(localNodeId, 'queued-stack', 'setup');
|
||||
let releaseRemoval!: () => void;
|
||||
const removalPending = new Promise<void>((resolve) => {
|
||||
releaseRemoval = resolve;
|
||||
});
|
||||
const removeSpy = vi.spyOn(svc, 'removeOverrideFromNode').mockReturnValue(removalPending);
|
||||
vi.spyOn(svc as unknown as { regenerateOverridesAcrossFleet: () => Promise<void> }, 'regenerateOverridesAcrossFleet')
|
||||
.mockResolvedValue(undefined);
|
||||
vi.spyOn(svc as unknown as { cascadeRecomposeAcrossFleet: () => void }, 'cascadeRecomposeAcrossFleet')
|
||||
.mockImplementation(() => { /* noop */ });
|
||||
vi.spyOn(svc, 'triggerRedeploy').mockImplementation(() => { /* noop */ });
|
||||
|
||||
const first = svc.optOutStack(localNodeId, 'queued-stack', 'tester');
|
||||
await vi.waitFor(() => expect(removeSpy).toHaveBeenCalledTimes(1));
|
||||
const second = svc.optOutStack(localNodeId, 'queued-stack', 'tester');
|
||||
await Promise.resolve();
|
||||
expect(removeSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
releaseRemoval();
|
||||
await Promise.all([first, second]);
|
||||
|
||||
expect(removeSpy).toHaveBeenCalledTimes(1);
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'queued-stack')).toBe(false);
|
||||
});
|
||||
|
||||
it('serializes different Mesh mutations per node without blocking another node', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
const remoteNodeId = db.addNode({
|
||||
name: 'independent-node', type: 'remote', mode: 'pilot_agent',
|
||||
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
|
||||
});
|
||||
db.setNodeMeshEnabled(localNodeId, true);
|
||||
db.insertMeshStack(localNodeId, 'held-stack', 'setup');
|
||||
let releaseRemoval!: () => void;
|
||||
const removalPending = new Promise<void>((resolve) => {
|
||||
releaseRemoval = resolve;
|
||||
});
|
||||
vi.spyOn(svc, 'removeOverrideFromNode').mockReturnValue(removalPending);
|
||||
vi.spyOn(svc as unknown as { regenerateOverridesAcrossFleet: () => Promise<void> }, 'regenerateOverridesAcrossFleet')
|
||||
.mockResolvedValue(undefined);
|
||||
vi.spyOn(svc as unknown as { cascadeRecomposeAcrossFleet: () => void }, 'cascadeRecomposeAcrossFleet')
|
||||
.mockImplementation(() => { /* noop */ });
|
||||
vi.spyOn(svc, 'triggerRedeploy').mockImplementation(() => { /* noop */ });
|
||||
|
||||
const optOut = svc.optOutStack(localNodeId, 'held-stack', 'tester');
|
||||
await vi.waitFor(() => expect(svc.removeOverrideFromNode).toHaveBeenCalledTimes(1));
|
||||
const disable = svc.disableForNode(localNodeId, 'tester');
|
||||
await svc.enableForNode(remoteNodeId);
|
||||
|
||||
expect(db.getNodeMeshEnabled(localNodeId)).toBe(true);
|
||||
expect(db.getNodeMeshEnabled(remoteNodeId)).toBe(true);
|
||||
releaseRemoval();
|
||||
await Promise.all([optOut, disable]);
|
||||
expect(db.getNodeMeshEnabled(localNodeId)).toBe(false);
|
||||
db.deleteNode(remoteNodeId);
|
||||
});
|
||||
|
||||
it('rejects an invalid stack name (path traversal attempt)', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
@@ -427,6 +507,44 @@ describe('MeshService.disableForNode', () => {
|
||||
db.deleteNode(remoteNodeId);
|
||||
});
|
||||
|
||||
it('keeps failed remote stacks authoritative when node disable is incomplete', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const remoteNodeId = db.addNode({
|
||||
name: 'remote-partial-disable', type: 'remote', mode: 'pilot_agent',
|
||||
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
|
||||
});
|
||||
db.setNodeMeshEnabled(remoteNodeId, true);
|
||||
db.insertMeshStack(remoteNodeId, 'removed-stack', 'setup');
|
||||
db.insertMeshStack(remoteNodeId, 'busy-stack', 'setup');
|
||||
vi.spyOn(svc, 'removeOverrideFromNode').mockImplementation(async (_nodeId, stackName) => {
|
||||
if (stackName === 'busy-stack') throw new Error('target busy');
|
||||
});
|
||||
vi.spyOn(
|
||||
svc as unknown as { regenerateOverridesAcrossFleet: () => Promise<void> },
|
||||
'regenerateOverridesAcrossFleet',
|
||||
).mockResolvedValue(undefined);
|
||||
vi.spyOn(
|
||||
svc as unknown as { cascadeRecomposeAcrossFleet: () => void },
|
||||
'cascadeRecomposeAcrossFleet',
|
||||
).mockImplementation(() => { /* noop */ });
|
||||
const redeployed: string[] = [];
|
||||
vi.spyOn(svc, 'triggerRedeploy').mockImplementation((_nodeId, stackName) => {
|
||||
redeployed.push(stackName);
|
||||
});
|
||||
vi.spyOn(svc as unknown as { refreshAliasCache: () => Promise<void> }, 'refreshAliasCache')
|
||||
.mockRejectedValue(new Error('refresh failed'));
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ });
|
||||
|
||||
await expect(svc.disableForNode(remoteNodeId, 'tester')).rejects.toThrow('busy-stack');
|
||||
|
||||
expect(db.getNodeMeshEnabled(remoteNodeId)).toBe(true);
|
||||
expect(db.isMeshStackEnabled(remoteNodeId, 'removed-stack')).toBe(false);
|
||||
expect(db.isMeshStackEnabled(remoteNodeId, 'busy-stack')).toBe(true);
|
||||
expect(redeployed).toContain('removed-stack');
|
||||
db.deleteNode(remoteNodeId);
|
||||
});
|
||||
|
||||
it('defaults the actor when none is supplied so legacy callers still log a non-empty actor', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
@@ -673,6 +791,62 @@ describe('MeshService.optInStack rollback', () => {
|
||||
.rejects.toThrow(/simulated remote pilot offline/);
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false);
|
||||
});
|
||||
|
||||
it('retains remote authority when the push outcome is unknown', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const remoteNodeId = db.addNode({
|
||||
name: 'ambiguous-push', type: 'remote', mode: 'pilot_agent',
|
||||
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
|
||||
});
|
||||
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
|
||||
.mockResolvedValue([{ service: 'web', ports: [8080] }]);
|
||||
vi.spyOn(svc, 'pushOverrideToNode').mockRejectedValue(new Error('connection reset'));
|
||||
|
||||
await expect(svc.optInStack(remoteNodeId, 'ambiguous-stack', 'tester'))
|
||||
.rejects.toThrow('connection reset');
|
||||
|
||||
expect(db.isMeshStackEnabled(remoteNodeId, 'ambiguous-stack')).toBe(true);
|
||||
db.deleteNode(remoteNodeId);
|
||||
});
|
||||
|
||||
it('treats a remote gateway error as an ambiguous push outcome', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const remoteNodeId = db.addNode({
|
||||
name: 'gateway-error-push', type: 'remote', mode: 'pilot_agent',
|
||||
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
|
||||
});
|
||||
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
|
||||
.mockResolvedValue([{ service: 'web', ports: [8080] }]);
|
||||
vi.spyOn(svc as unknown as { proxyFetch: () => Promise<Response> }, 'proxyFetch')
|
||||
.mockResolvedValue(new Response('gateway timeout', { status: 502 }));
|
||||
|
||||
await expect(svc.optInStack(remoteNodeId, 'gateway-error-stack', 'tester'))
|
||||
.rejects.toThrow('HTTP 502');
|
||||
|
||||
expect(db.isMeshStackEnabled(remoteNodeId, 'gateway-error-stack')).toBe(true);
|
||||
db.deleteNode(remoteNodeId);
|
||||
});
|
||||
|
||||
it('rolls back remote authority when the target explicitly rejects the push', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const remoteNodeId = db.addNode({
|
||||
name: 'rejected-push', type: 'remote', mode: 'pilot_agent',
|
||||
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
|
||||
});
|
||||
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
|
||||
.mockResolvedValue([{ service: 'web', ports: [8080] }]);
|
||||
vi.spyOn(svc, 'pushOverrideToNode')
|
||||
.mockRejectedValue(new MeshError('push_failed', 'target rejected override'));
|
||||
|
||||
await expect(svc.optInStack(remoteNodeId, 'rejected-stack', 'tester'))
|
||||
.rejects.toThrow('target rejected override');
|
||||
|
||||
expect(db.isMeshStackEnabled(remoteNodeId, 'rejected-stack')).toBe(false);
|
||||
db.deleteNode(remoteNodeId);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MeshService.optInStack guard rails (network setup)', () => {
|
||||
@@ -984,6 +1158,7 @@ describe('MeshService.ensureStackOverride (BUG-1 fix)', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
|
||||
process.env.SENCHO_MODE = 'pilot';
|
||||
// Simulate the pilot scenario: no mesh_stacks row (isMeshStackEnabled → false),
|
||||
// but the override file already exists on disk, pushed by central via D-1.
|
||||
const dataDir = process.env.DATA_DIR as string;
|
||||
@@ -1008,6 +1183,184 @@ describe('MeshService.ensureStackOverride (BUG-1 fix)', () => {
|
||||
|
||||
// Cleanup.
|
||||
fsSync.unlinkSync(overrideFile);
|
||||
process.env.SENCHO_MODE = 'server';
|
||||
});
|
||||
|
||||
it('persists and removes proxy-target opt-in state with a pushed local override', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']);
|
||||
|
||||
const file = await svc.applyLocalOverride('proxy-stack', []);
|
||||
|
||||
expect(file).not.toBeNull();
|
||||
const yaml = fsSync.readFileSync(file as string, 'utf8');
|
||||
expect(yaml).toContain('web:');
|
||||
expect(yaml).toContain('sencho_mesh');
|
||||
expect(fsSync.readdirSync(path.dirname(file as string)).some((name) => name.includes('proxy-stack') && name.endsWith('.tmp'))).toBe(false);
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'proxy-stack')).toBe(true);
|
||||
|
||||
await svc.removeLocalOverride('proxy-stack');
|
||||
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'proxy-stack')).toBe(false);
|
||||
});
|
||||
|
||||
it('does not write a proxy override when DB authority cannot be recorded', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']);
|
||||
vi.spyOn(db, 'insertMeshStack').mockImplementation(() => {
|
||||
throw new Error('database unavailable');
|
||||
});
|
||||
|
||||
await expect(svc.applyLocalOverride('db-failure', [])).rejects.toThrow('database unavailable');
|
||||
|
||||
const overrideFile = path.join(
|
||||
process.env.DATA_DIR as string,
|
||||
'mesh',
|
||||
'overrides',
|
||||
String(localNodeId),
|
||||
'db-failure.override.yml',
|
||||
);
|
||||
expect(fsSync.existsSync(overrideFile)).toBe(false);
|
||||
});
|
||||
|
||||
it('restores an existing override when DB authority cannot be recorded', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId));
|
||||
fsSync.mkdirSync(overrideDir, { recursive: true });
|
||||
const overrideFile = path.join(overrideDir, 'db-replacement-failure.override.yml');
|
||||
const originalYaml = 'services:\n prior:\n networks:\n - sencho_mesh\n';
|
||||
fsSync.writeFileSync(overrideFile, originalYaml, 'utf8');
|
||||
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']);
|
||||
vi.spyOn(db, 'insertMeshStack').mockImplementation(() => {
|
||||
throw new Error('database unavailable');
|
||||
});
|
||||
|
||||
await expect(svc.applyLocalOverride('db-replacement-failure', [])).rejects.toThrow('database unavailable');
|
||||
|
||||
expect(fsSync.readFileSync(overrideFile, 'utf8')).toBe(originalYaml);
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'db-replacement-failure')).toBe(false);
|
||||
expect(fsSync.readdirSync(overrideDir).some((name) => name.includes('db-replacement-failure') && name.endsWith('.tmp'))).toBe(false);
|
||||
});
|
||||
|
||||
it('does not publish DB authority or a final file when atomic override publication fails', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']);
|
||||
vi.spyOn(fs, 'rename').mockRejectedValue(new Error('rename failed'));
|
||||
|
||||
await expect(svc.applyLocalOverride('write-failure', [])).rejects.toThrow('rename failed');
|
||||
|
||||
const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId));
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'write-failure')).toBe(false);
|
||||
expect(fsSync.existsSync(path.join(overrideDir, 'write-failure.override.yml'))).toBe(false);
|
||||
expect(fsSync.readdirSync(overrideDir).some((name) => name.includes('write-failure'))).toBe(false);
|
||||
});
|
||||
|
||||
it('preserves the prior override and authority when atomic replacement fails', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId));
|
||||
fsSync.mkdirSync(overrideDir, { recursive: true });
|
||||
const overrideFile = path.join(overrideDir, 'replacement-failure.override.yml');
|
||||
const originalYaml = 'services:\n prior:\n networks:\n - sencho_mesh\n';
|
||||
fsSync.writeFileSync(overrideFile, originalYaml, 'utf8');
|
||||
db.insertMeshStack(localNodeId, 'replacement-failure', 'tester');
|
||||
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']);
|
||||
vi.spyOn(fs, 'rename').mockRejectedValue(new Error('rename failed'));
|
||||
|
||||
await expect(svc.applyLocalOverride('replacement-failure', [])).rejects.toThrow('rename failed');
|
||||
|
||||
expect(fsSync.readFileSync(overrideFile, 'utf8')).toBe(originalYaml);
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'replacement-failure')).toBe(true);
|
||||
});
|
||||
|
||||
it('prevents overlapping override mutations for the same stack', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
let releaseServices!: (services: string[]) => void;
|
||||
const servicesPending = new Promise<string[]>((resolve) => {
|
||||
releaseServices = resolve;
|
||||
});
|
||||
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockReturnValue(servicesPending);
|
||||
|
||||
const first = svc.applyLocalOverride('concurrent-stack', []);
|
||||
await vi.waitFor(() => expect(svc.getDeclaredStackServiceNames).toHaveBeenCalledTimes(1));
|
||||
|
||||
await expect(svc.applyLocalOverride('concurrent-stack', [])).rejects.toThrow('another operation');
|
||||
releaseServices(['web']);
|
||||
await first;
|
||||
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'concurrent-stack')).toBe(true);
|
||||
});
|
||||
|
||||
it('prevents removal from overlapping an in-flight override apply', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
let releaseServices!: (services: string[]) => void;
|
||||
const servicesPending = new Promise<string[]>((resolve) => {
|
||||
releaseServices = resolve;
|
||||
});
|
||||
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockReturnValue(servicesPending);
|
||||
|
||||
const apply = svc.applyLocalOverride('apply-remove-stack', []);
|
||||
await vi.waitFor(() => expect(svc.getDeclaredStackServiceNames).toHaveBeenCalledTimes(1));
|
||||
|
||||
await expect(svc.removeLocalOverride('apply-remove-stack')).rejects.toThrow('another operation');
|
||||
releaseServices(['web']);
|
||||
const file = await apply;
|
||||
|
||||
expect(file).not.toBeNull();
|
||||
expect(fsSync.existsSync(file as string)).toBe(true);
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'apply-remove-stack')).toBe(true);
|
||||
});
|
||||
|
||||
it('does not report committed removal as failed when alias refresh fails', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId));
|
||||
fsSync.mkdirSync(overrideDir, { recursive: true });
|
||||
const overrideFile = path.join(overrideDir, 'refresh-failure.override.yml');
|
||||
fsSync.writeFileSync(overrideFile, 'services: {}\n', 'utf8');
|
||||
db.insertMeshStack(localNodeId, 'refresh-failure', 'setup');
|
||||
(svc as unknown as { pilotAliasOverlay: Map<string, unknown> }).pilotAliasOverlay.set('refresh-failure', []);
|
||||
vi.spyOn(svc as unknown as { refreshAliasCache: () => Promise<void> }, 'refreshAliasCache')
|
||||
.mockRejectedValue(new Error('refresh failed'));
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ });
|
||||
|
||||
await expect(svc.removeLocalOverride('refresh-failure')).resolves.toBeUndefined();
|
||||
|
||||
expect(fsSync.existsSync(overrideFile)).toBe(false);
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'refresh-failure')).toBe(false);
|
||||
});
|
||||
|
||||
it('does not report committed apply as failed when alias refresh fails', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
vi.spyOn(svc, 'getDeclaredStackServiceNames').mockResolvedValue(['web']);
|
||||
vi.spyOn(svc as unknown as { refreshAliasCache: () => Promise<void> }, 'refreshAliasCache')
|
||||
.mockRejectedValue(new Error('refresh failed'));
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => { /* silence */ });
|
||||
|
||||
const file = await svc.applyLocalOverride('apply-refresh-failure', [], [{
|
||||
host: 'web.example', nodeId: localNodeId, nodeName: 'local', stackName: 'apply-refresh-failure',
|
||||
serviceName: 'web', port: 8080,
|
||||
}]);
|
||||
|
||||
expect(file).not.toBeNull();
|
||||
expect(fsSync.existsSync(file as string)).toBe(true);
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'apply-refresh-failure')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns null for pilot nodes when no pushed override file exists', async () => {
|
||||
@@ -1015,9 +1368,38 @@ describe('MeshService.ensureStackOverride (BUG-1 fix)', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
|
||||
process.env.SENCHO_MODE = 'pilot';
|
||||
// No mesh_stacks row, no file on disk.
|
||||
const result = await svc.ensureStackOverride(localNodeId, 'no-such-stack');
|
||||
expect(result).toBeNull();
|
||||
process.env.SENCHO_MODE = 'server';
|
||||
});
|
||||
|
||||
it('does not use stale override presence as authority on a server', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
const overrideDir = path.join(process.env.DATA_DIR as string, 'mesh', 'overrides', String(localNodeId));
|
||||
fsSync.mkdirSync(overrideDir, { recursive: true });
|
||||
const overrideFile = path.join(overrideDir, 'stale-server.override.yml');
|
||||
fsSync.writeFileSync(overrideFile, 'services: {}\n', 'utf8');
|
||||
|
||||
const result = await svc.ensureStackOverride(localNodeId, 'stale-server');
|
||||
|
||||
expect(result).toBeNull();
|
||||
fsSync.unlinkSync(overrideFile);
|
||||
});
|
||||
|
||||
it('restores proxy-target DB authority when override removal fails', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
db.insertMeshStack(localNodeId, 'unlink-failure', 'tester');
|
||||
vi.spyOn(fs, 'unlink').mockRejectedValue(Object.assign(new Error('permission denied'), { code: 'EACCES' }));
|
||||
|
||||
await expect(svc.removeLocalOverride('unlink-failure')).rejects.toThrow('permission denied');
|
||||
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'unlink-failure')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ describe('networking summary', () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
DatabaseService.getInstance().deleteStackExposureIntents(1, STACK);
|
||||
DatabaseService.getInstance().deleteMeshStack(1, STACK);
|
||||
fs.rmSync(stackDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
@@ -72,6 +73,41 @@ describe('networking summary', () => {
|
||||
expect(res.body.networkDrift.stacks).toContain(STACK);
|
||||
});
|
||||
|
||||
it('does not count an opted-in Mesh attachment as network drift', async () => {
|
||||
DatabaseService.getInstance().insertMeshStack(1, STACK, 'tester');
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDependencySnapshot: vi.fn().mockResolvedValue({
|
||||
containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }], volumes: [], ports: [] }],
|
||||
networks: [
|
||||
{ id: 'm', name: 'sencho_mesh', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null },
|
||||
],
|
||||
volumes: [],
|
||||
}),
|
||||
} as unknown as DockerController);
|
||||
|
||||
const res = await request(app).get('/api/networking/summary').set('Authorization', authHeader);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.networkDrift).toEqual({ count: 0, stacks: [] });
|
||||
});
|
||||
|
||||
it('counts an opted-out manual Mesh attachment as network drift', async () => {
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDependencySnapshot: vi.fn().mockResolvedValue({
|
||||
containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'sencho_mesh', id: 'm', ip: '' }], volumes: [], ports: [] }],
|
||||
networks: [
|
||||
{ id: 'm', name: 'sencho_mesh', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null },
|
||||
],
|
||||
volumes: [],
|
||||
}),
|
||||
} as unknown as DockerController);
|
||||
|
||||
const res = await request(app).get('/api/networking/summary').set('Authorization', authHeader);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.networkDrift).toEqual({ count: 1, stacks: [STACK] });
|
||||
});
|
||||
|
||||
it('still reports declared signals when the snapshot is unavailable (drift skipped)', async () => {
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDependencySnapshot: vi.fn().mockRejectedValue(new Error('docker down')),
|
||||
|
||||
@@ -5,7 +5,12 @@ import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompos
|
||||
import type { DeclaredCompose, DeclaredService } from '../helpers/composeDependencyParse';
|
||||
import { parseMissingRequiredVars } from '../helpers/envVarParse';
|
||||
import { parseEffectiveModel } from './preflight/effectiveModel';
|
||||
import { compareStackNetworks, fromDeclaredCompose } from './network/normalize';
|
||||
import {
|
||||
compareStackNetworks,
|
||||
fromDeclaredCompose,
|
||||
type ManagedNetworkAttachmentPredicate,
|
||||
} from './network/normalize';
|
||||
import { resolveManagedMeshAttachment } from './network/managedMeshAttachment';
|
||||
import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { isCleanOneShotCompletion } from '../utils/oneShotCompletion';
|
||||
@@ -111,6 +116,8 @@ export interface AssembleStackDriftInput {
|
||||
containers: DependencyContainer[];
|
||||
/** Every network on the node (for resolving foreign vs stack-owned attachments). */
|
||||
networks?: DependencyNetwork[];
|
||||
/** Authoritative runtime attachments that are intentionally absent from authored Compose. */
|
||||
managedNetworkAttachment?: ManagedNetworkAttachmentPredicate;
|
||||
/** Set when the compose file could not be parsed. */
|
||||
parseError?: string;
|
||||
}
|
||||
@@ -137,12 +144,18 @@ function networkDriftFindings(
|
||||
declared: DeclaredCompose,
|
||||
containers: DependencyContainer[],
|
||||
networks: DependencyNetwork[],
|
||||
managedNetworkAttachment?: ManagedNetworkAttachmentPredicate,
|
||||
): StackDriftFinding[] {
|
||||
// Runtime resource names use the Compose project (top-level `name:` when set),
|
||||
// not the stack directory, so a stack with `name:` resolves its networks the
|
||||
// same way Docker does. Containers are still attributed to the stack directory.
|
||||
const normalized = fromDeclaredCompose(declared, declared.projectName ?? stack);
|
||||
const facts = compareStackNetworks(normalized, { containers, networks, volumes: [] }, stack);
|
||||
const facts = compareStackNetworks(
|
||||
normalized,
|
||||
{ containers, networks, volumes: [] },
|
||||
stack,
|
||||
managedNetworkAttachment,
|
||||
);
|
||||
const findings: StackDriftFinding[] = [];
|
||||
|
||||
const serviceByContainer = new Map(containers.map(c => [c.name, c.service ?? c.name]));
|
||||
@@ -293,7 +306,13 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe
|
||||
}
|
||||
}
|
||||
|
||||
findings.push(...networkDriftFindings(stack, declared, containers, networks));
|
||||
findings.push(...networkDriftFindings(
|
||||
stack,
|
||||
declared,
|
||||
containers,
|
||||
networks,
|
||||
input.managedNetworkAttachment,
|
||||
));
|
||||
|
||||
const status: StackDriftStatus = findings.length > 0 ? 'drifted' : 'in-sync';
|
||||
return { stack, status, hasComposeFile: true, hasContainers, findings };
|
||||
@@ -384,5 +403,12 @@ export async function buildStackDriftReport(nodeId: number, stackName: string):
|
||||
};
|
||||
}
|
||||
|
||||
return assembleStackDrift({ stack: stackName, declared: render.declared, containers, networks });
|
||||
const managedNetworkAttachment = await resolveManagedMeshAttachment(nodeId, stackName);
|
||||
return assembleStackDrift({
|
||||
stack: stackName,
|
||||
declared: render.declared,
|
||||
containers,
|
||||
networks,
|
||||
managedNetworkAttachment,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import net from 'net';
|
||||
import path from 'path';
|
||||
import fs from 'fs/promises';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
import * as YAML from 'yaml';
|
||||
import { ComposeService } from './ComposeService';
|
||||
@@ -20,6 +21,7 @@ import { STREAM_PENDING_DATA_MAX_BYTES } from '../pilot/protocol';
|
||||
import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { isPathWithinBase, isValidStackName, isValidRelativeStackPath } from '../utils/validation';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { PORT as SENCHO_LISTEN_PORT } from '../helpers/constants';
|
||||
import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate';
|
||||
|
||||
@@ -1497,6 +1499,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
// --- Opt-in / opt-out ---
|
||||
|
||||
public async optInStack(nodeId: number, stackName: string, actor: string): Promise<void> {
|
||||
return this.runMeshNodeMutation(nodeId, () => this.optInStackExclusive(nodeId, stackName, actor));
|
||||
}
|
||||
|
||||
private async optInStackExclusive(nodeId: number, stackName: string, actor: string): Promise<void> {
|
||||
this.logDiag('opt-in start', { nodeId, stackName, actor });
|
||||
const t0 = Date.now();
|
||||
if (!isValidStackName(stackName)) {
|
||||
@@ -1541,19 +1547,41 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
}
|
||||
|
||||
db.insertMeshStack(nodeId, stackName, actor);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
try {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} catch (error) {
|
||||
db.deleteMeshStack(nodeId, stackName);
|
||||
try {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} catch (rollbackError) {
|
||||
console.warn('[MeshService] Failed to refresh aliases after opt-in rollback:', sanitizeForLog(getErrorMessage(rollbackError, 'unknown')));
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Push the just-opted-in stack's override loudly. If this fails the
|
||||
// DB state is invalid (alias claimed but remote pilot has no
|
||||
// override file) so roll back rather than leave a half-state that
|
||||
// future opt-in calls would short-circuit on `isMeshStackEnabled`.
|
||||
// Push the just-opted-in stack's override loudly. Explicit target
|
||||
// rejection rolls back the row. A remote transport failure is
|
||||
// ambiguous because the target may already have committed, so retain
|
||||
// authority and let normal regeneration reconcile it.
|
||||
try {
|
||||
await this.pushOverrideToNode(nodeId, stackName);
|
||||
} catch (err) {
|
||||
db.deleteMeshStack(nodeId, stackName);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
const node = db.getNode(nodeId);
|
||||
const explicitlyRejected = node?.type !== 'remote' || err instanceof MeshError;
|
||||
if (explicitlyRejected) {
|
||||
db.deleteMeshStack(nodeId, stackName);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} else {
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'warn', type: 'forwarder.error',
|
||||
nodeId,
|
||||
message: `mesh override push outcome unknown for ${stackName}; retaining opt-in authority for reconciliation`,
|
||||
details: { stackName },
|
||||
});
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
// Regenerate every other meshed stack's override across the fleet
|
||||
@@ -1584,6 +1612,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
}
|
||||
|
||||
public async optOutStack(nodeId: number, stackName: string, actor: string): Promise<void> {
|
||||
return this.runMeshNodeMutation(nodeId, () => this.optOutStackExclusive(nodeId, stackName, actor));
|
||||
}
|
||||
|
||||
private async optOutStackExclusive(nodeId: number, stackName: string, actor: string): Promise<void> {
|
||||
this.logDiag('opt-out start', { nodeId, stackName, actor });
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw new MeshError('denied', `invalid stack name: ${stackName}`);
|
||||
@@ -1591,9 +1623,18 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
const db = DatabaseService.getInstance();
|
||||
if (!db.isMeshStackEnabled(nodeId, stackName)) return;
|
||||
db.deleteMeshStack(nodeId, stackName);
|
||||
await this.removeOverrideFromNode(nodeId, stackName);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
try {
|
||||
await this.removeOverrideFromNode(nodeId, stackName);
|
||||
} catch (error) {
|
||||
db.insertMeshStack(nodeId, stackName, actor);
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} catch (error) {
|
||||
console.warn('[MeshService] Failed to refresh aliases after committed opt-out:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
// The opted-out row is already deleted, so listMeshStacks() will not
|
||||
// include it. Walk the remaining fleet-wide rows so every other
|
||||
// meshed stack regenerates its override without the dropped alias.
|
||||
@@ -1620,6 +1661,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
}
|
||||
|
||||
public async enableForNode(nodeId: number): Promise<void> {
|
||||
return this.runMeshNodeMutation(nodeId, () => this.enableForNodeExclusive(nodeId));
|
||||
}
|
||||
|
||||
private async enableForNodeExclusive(nodeId: number): Promise<void> {
|
||||
this.logDiag('enable-for-node', { nodeId });
|
||||
DatabaseService.getInstance().setNodeMeshEnabled(nodeId, true);
|
||||
this.logActivity({
|
||||
@@ -1642,26 +1687,42 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
nodeId: number,
|
||||
actor: string = 'system:mesh.disable',
|
||||
): Promise<void> {
|
||||
return this.runMeshNodeMutation(nodeId, () => this.disableForNodeExclusive(nodeId, actor));
|
||||
}
|
||||
|
||||
private async disableForNodeExclusive(nodeId: number, actor: string): Promise<void> {
|
||||
this.logDiag('disable-for-node start', { nodeId, actor });
|
||||
const t0 = Date.now();
|
||||
DatabaseService.getInstance().setNodeMeshEnabled(nodeId, false);
|
||||
const stacks = DatabaseService.getInstance().listMeshStacks(nodeId);
|
||||
for (const s of stacks) {
|
||||
DatabaseService.getInstance().deleteMeshStack(nodeId, s.stack_name);
|
||||
}
|
||||
const db = DatabaseService.getInstance();
|
||||
const stacks = db.listMeshStacks(nodeId);
|
||||
// Dispatch DELETE /api/mesh/local-override/:stack for remote nodes
|
||||
// (pilot or proxy) so the override file pushed earlier via
|
||||
// applyLocalOverride is removed; falls back to local deletion for
|
||||
// local nodes. Parallelize per the regenerateOverridesForNode
|
||||
// rationale: each remote call is its own HTTP round-trip, so
|
||||
// awaiting sequentially turns N stacks into N serialised DELETEs.
|
||||
// `allSettled` so a single failure does not abort the others
|
||||
// (removeOverrideFromNode already swallows errors internally).
|
||||
await Promise.allSettled(
|
||||
// `allSettled` so a single failure does not abort the others.
|
||||
const removals = await Promise.allSettled(
|
||||
stacks.map((s) => this.removeOverrideFromNode(nodeId, s.stack_name)),
|
||||
);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
const failed: string[] = [];
|
||||
const removed: typeof stacks = [];
|
||||
removals.forEach((result, index) => {
|
||||
const stack = stacks[index];
|
||||
if (result.status === 'fulfilled') {
|
||||
db.deleteMeshStack(nodeId, stack.stack_name);
|
||||
removed.push(stack);
|
||||
} else {
|
||||
failed.push(stack.stack_name);
|
||||
}
|
||||
});
|
||||
if (failed.length === 0) db.setNodeMeshEnabled(nodeId, false);
|
||||
try {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} catch (error) {
|
||||
console.warn('[MeshService] Failed to refresh aliases after committed node disable changes:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
// Mirror optOutStack: regenerate every remaining node's override
|
||||
// without the dropped aliases, recompose the rest of the fleet so
|
||||
// their containers shed the stale extra_hosts, and redeploy the
|
||||
@@ -1669,9 +1730,15 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
// sencho_mesh network and lose the alias entries they owned.
|
||||
await this.regenerateOverridesAcrossFleet();
|
||||
this.cascadeRecomposeAcrossFleet(undefined, undefined, actor);
|
||||
for (const s of stacks) {
|
||||
for (const s of removed) {
|
||||
this.triggerRedeploy(nodeId, s.stack_name, actor);
|
||||
}
|
||||
if (failed.length > 0) {
|
||||
throw new MeshError(
|
||||
'push_failed',
|
||||
`Could not disable Mesh on node ${nodeId}: override removal failed for ${failed.join(', ')}.`,
|
||||
);
|
||||
}
|
||||
this.logDiag('disable-for-node complete', { nodeId, stacks: stacks.length, ms: Date.now() - t0 });
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'mesh.disable',
|
||||
@@ -1679,6 +1746,24 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
});
|
||||
}
|
||||
|
||||
private readonly meshNodeMutations = new Map<number, Promise<void>>();
|
||||
|
||||
private async runMeshNodeMutation<T>(nodeId: number, operation: () => Promise<T>): Promise<T> {
|
||||
const previous = this.meshNodeMutations.get(nodeId) ?? Promise.resolve();
|
||||
let release!: () => void;
|
||||
const pending = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
this.meshNodeMutations.set(nodeId, pending);
|
||||
await previous;
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
release();
|
||||
if (this.meshNodeMutations.get(nodeId) === pending) this.meshNodeMutations.delete(nodeId);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Override file management ---
|
||||
|
||||
public async ensureStackOverride(nodeId: number, stackName: string): Promise<string | null> {
|
||||
@@ -1690,6 +1775,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
// lives on central per the C-3 design). Use file-presence as the
|
||||
// fallback: if central pushed an override via applyLocalOverride, return
|
||||
// that path so ComposeService picks it up on the next deploy.
|
||||
if (process.env.SENCHO_MODE !== 'pilot') return null;
|
||||
const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`);
|
||||
if (!isPathWithinBase(file, dir)) return null;
|
||||
try {
|
||||
@@ -1754,13 +1840,37 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
portAliases?: MeshGlobalAlias[],
|
||||
): Promise<string | null> {
|
||||
if (!isValidStackName(stackName)) return null;
|
||||
if (!this.senchoIp) {
|
||||
const senchoIp = this.senchoIp;
|
||||
if (!senchoIp) {
|
||||
throw new MeshError(
|
||||
'push_failed',
|
||||
this.networkSetupError || 'mesh data plane unavailable on this node',
|
||||
);
|
||||
}
|
||||
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
localNodeId,
|
||||
stackName,
|
||||
'deploy',
|
||||
'system:mesh.override',
|
||||
() => this.applyLocalOverrideExclusive(localNodeId, stackName, aliases, senchoIp, portAliases),
|
||||
);
|
||||
if (!lock.ran) {
|
||||
throw new MeshError(
|
||||
'push_failed',
|
||||
`Cannot apply Mesh override for "${stackName}": another operation (${lock.existing.action}) is already in progress.`,
|
||||
);
|
||||
}
|
||||
return lock.result;
|
||||
}
|
||||
|
||||
private async applyLocalOverrideExclusive(
|
||||
localNodeId: number,
|
||||
stackName: string,
|
||||
aliases: MeshAlias[],
|
||||
senchoIp: string,
|
||||
portAliases?: MeshGlobalAlias[],
|
||||
): Promise<string | null> {
|
||||
const serviceNames = await this.getDeclaredStackServiceNames(stackName, localNodeId);
|
||||
|
||||
const dir = this.overrideDirFor(localNodeId);
|
||||
@@ -1785,6 +1895,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
message: `mesh override preserved for ${stackName}: declared services unreadable, keeping ${existing.length} existing entries`,
|
||||
details: { stackName, preservedServices: existing },
|
||||
});
|
||||
this.recordLocalOverrideIntent(localNodeId, stackName);
|
||||
return file;
|
||||
}
|
||||
}
|
||||
@@ -1792,13 +1903,24 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
const yaml = generateOverrideYaml({
|
||||
services: serviceNames,
|
||||
aliases,
|
||||
senchoIp: this.senchoIp,
|
||||
senchoIp,
|
||||
});
|
||||
await fs.writeFile(file, yaml, 'utf8');
|
||||
const previousYaml = await this.readOverrideContent(file);
|
||||
await this.writeOverrideAtomically(file, yaml);
|
||||
try {
|
||||
this.recordLocalOverrideIntent(localNodeId, stackName);
|
||||
} catch (error) {
|
||||
await this.restoreOverrideAfterAuthorityFailure(file, previousYaml);
|
||||
throw error;
|
||||
}
|
||||
if (portAliases && portAliases.length > 0) {
|
||||
this.pilotAliasOverlay.set(stackName, portAliases);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
try {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} catch (error) {
|
||||
console.warn('[MeshService] Failed to refresh aliases after committed override apply:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}
|
||||
return file;
|
||||
}
|
||||
@@ -1810,13 +1932,95 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
public async removeLocalOverride(stackName: string): Promise<void> {
|
||||
if (!isValidStackName(stackName)) return;
|
||||
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
localNodeId,
|
||||
stackName,
|
||||
'deploy',
|
||||
'system:mesh.override',
|
||||
() => this.removeLocalOverrideExclusive(localNodeId, stackName),
|
||||
);
|
||||
if (!lock.ran) {
|
||||
throw new MeshError(
|
||||
'push_failed',
|
||||
`Cannot remove Mesh override for "${stackName}": another operation (${lock.existing.action}) is already in progress.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async removeLocalOverrideExclusive(localNodeId: number, stackName: string): Promise<void> {
|
||||
const dir = this.overrideDirFor(localNodeId);
|
||||
const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`);
|
||||
if (!isPathWithinBase(file, dir)) return;
|
||||
try { await fs.unlink(file); } catch { /* ignore not-exist */ }
|
||||
const db = DatabaseService.getInstance();
|
||||
const hadIntent = db.isMeshStackEnabled(localNodeId, stackName);
|
||||
if (hadIntent) db.deleteMeshStack(localNodeId, stackName);
|
||||
try {
|
||||
await fs.unlink(file);
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) {
|
||||
if (hadIntent) db.insertMeshStack(localNodeId, stackName, null);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (this.pilotAliasOverlay.delete(stackName)) {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
try {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} catch (error) {
|
||||
console.warn('[MeshService] Failed to refresh aliases after committed override removal:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async writeOverrideAtomically(file: string, yaml: string): Promise<void> {
|
||||
const dir = path.dirname(file);
|
||||
const tempFile = path.resolve(dir, `.${path.basename(file)}.${randomUUID()}.tmp`);
|
||||
if (!isPathWithinBase(tempFile, dir)) throw new Error('Invalid Mesh override temporary path');
|
||||
|
||||
let handle: Awaited<ReturnType<typeof fs.open>> | null = null;
|
||||
try {
|
||||
handle = await fs.open(tempFile, 'wx');
|
||||
await handle.writeFile(yaml, 'utf8');
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = null;
|
||||
await fs.rename(tempFile, file);
|
||||
} catch (error) {
|
||||
if (handle) {
|
||||
try { await handle.close(); } catch (closeError) {
|
||||
console.warn('[MeshService] Failed to close temporary override:', sanitizeForLog(getErrorMessage(closeError, 'unknown')));
|
||||
}
|
||||
}
|
||||
try {
|
||||
await fs.unlink(tempFile);
|
||||
} catch (cleanupError) {
|
||||
if (!(cleanupError instanceof Error && 'code' in cleanupError && cleanupError.code === 'ENOENT')) {
|
||||
console.warn('[MeshService] Failed to clean up temporary override:', sanitizeForLog(getErrorMessage(cleanupError, 'unknown')));
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async readOverrideContent(file: string): Promise<string | null> {
|
||||
try {
|
||||
return await fs.readFile(file, 'utf8');
|
||||
} catch (error) {
|
||||
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async restoreOverrideAfterAuthorityFailure(file: string, previousYaml: string | null): Promise<void> {
|
||||
try {
|
||||
if (previousYaml === null) {
|
||||
await fs.unlink(file);
|
||||
} else {
|
||||
await this.writeOverrideAtomically(file, previousYaml);
|
||||
}
|
||||
} catch (error) {
|
||||
if (previousYaml === null && error instanceof Error && 'code' in error && error.code === 'ENOENT') return;
|
||||
console.warn('[MeshService] Failed to restore override after authority error:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1825,7 +2029,11 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
const dir = this.overrideDirFor(nodeId);
|
||||
const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`);
|
||||
if (!isPathWithinBase(file, dir)) return;
|
||||
try { await fs.unlink(file); } catch { /* ignore not-exist */ }
|
||||
try {
|
||||
await fs.unlink(file);
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private overrideDirFor(nodeId: number): string {
|
||||
@@ -1833,6 +2041,13 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
return path.join(dataDir, 'mesh', 'overrides', String(nodeId));
|
||||
}
|
||||
|
||||
private recordLocalOverrideIntent(nodeId: number, stackName: string): void {
|
||||
if (process.env.SENCHO_MODE === 'pilot') return;
|
||||
const db = DatabaseService.getInstance();
|
||||
if (db.isMeshStackEnabled(nodeId, stackName)) return;
|
||||
db.insertMeshStack(nodeId, stackName, null);
|
||||
}
|
||||
|
||||
private async regenerateOverridesForNode(nodeId: number, skipStack?: string): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const stacks = db.listMeshStacks(nodeId);
|
||||
@@ -2331,7 +2546,11 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
);
|
||||
}
|
||||
if (!res.ok) {
|
||||
throw new MeshError('push_failed', `HTTP ${res.status} from node ${node.name}`);
|
||||
const message = `HTTP ${res.status} from node ${node.name}`;
|
||||
if (res.status >= 400 && res.status < 500) {
|
||||
throw new MeshError('push_failed', message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2436,15 +2655,20 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
}
|
||||
|
||||
try {
|
||||
await this.proxyFetch(
|
||||
const response = await this.proxyFetch(
|
||||
nodeId,
|
||||
'DELETE',
|
||||
`/api/mesh/local-override/${encodeURIComponent(stackName)}`,
|
||||
undefined,
|
||||
5_000,
|
||||
);
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '');
|
||||
throw new Error(`HTTP ${response.status} from node ${node.name}: ${body.slice(0, 256)}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('[MeshService] removeOverrideFromNode failed:', sanitizeForLog((err as Error).message));
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,8 @@ import type {
|
||||
NetworkDriftFacts, NetworkFactNetwork, NetworkFactService, NetworkRuntimeState, StackNetworkFacts,
|
||||
} from './types';
|
||||
import { classifyMissingExternalNetworks, type MissingExternalNetwork } from './missingExternalNetworks';
|
||||
import { resolveManagedMeshAttachment } from './managedMeshAttachment';
|
||||
import type { ManagedNetworkAttachmentPredicate } from './normalize';
|
||||
|
||||
import { getErrorMessage } from '../../utils/errors';
|
||||
import { redactSensitiveText, sanitizeForLog } from '../../utils/safeLog';
|
||||
@@ -43,6 +45,7 @@ export function assembleStackNetworkFacts(
|
||||
model: EffectiveModel | null,
|
||||
renderError: string | null,
|
||||
snapshot: DependencySnapshot | null,
|
||||
managedNetworkAttachment?: ManagedNetworkAttachmentPredicate,
|
||||
): StackNetworkFacts {
|
||||
const runtime: NetworkRuntimeState = snapshot ? 'available' : 'unavailable';
|
||||
|
||||
@@ -82,7 +85,9 @@ export function assembleStackNetworkFacts(
|
||||
extraHosts: s.extraHosts,
|
||||
}));
|
||||
|
||||
const drift = snapshot ? compareStackNetworks(fromEffectiveModel(model), snapshot, stackName) : EMPTY_DRIFT;
|
||||
const drift = snapshot
|
||||
? compareStackNetworks(fromEffectiveModel(model), snapshot, stackName, managedNetworkAttachment)
|
||||
: EMPTY_DRIFT;
|
||||
const missingExternalNetworks: MissingExternalNetwork[] = snapshot
|
||||
? classifyMissingExternalNetworks(
|
||||
model,
|
||||
@@ -147,5 +152,8 @@ export async function buildStackNetworkFacts(
|
||||
}
|
||||
}
|
||||
|
||||
return assembleStackNetworkFacts(stackName, model, renderError, snapshot);
|
||||
const managedNetworkAttachment = snapshot && model
|
||||
? await resolveManagedMeshAttachment(nodeId, stackName)
|
||||
: undefined;
|
||||
return assembleStackNetworkFacts(stackName, model, renderError, snapshot, managedNetworkAttachment);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { DatabaseService } from '../DatabaseService';
|
||||
import { SENCHO_MESH_NETWORK } from '../MeshComposeOverride';
|
||||
import SelfIdentityService from '../SelfIdentityService';
|
||||
import { getErrorMessage } from '../../utils/errors';
|
||||
import { sanitizeForLog } from '../../utils/safeLog';
|
||||
import { isPathWithinBase, isValidStackName } from '../../utils/validation';
|
||||
import type { ManagedNetworkAttachmentPredicate } from './normalize';
|
||||
|
||||
async function hasPilotMeshOverride(nodeId: number, stackName: string): Promise<boolean> {
|
||||
if (process.env.SENCHO_MODE !== 'pilot' || !isValidStackName(stackName)) return false;
|
||||
|
||||
const dataDir = process.env.DATA_DIR || '/app/data';
|
||||
const overrideDir = path.resolve(dataDir, 'mesh', 'overrides', String(nodeId));
|
||||
const overridePath = path.resolve(overrideDir, `${path.basename(stackName)}.override.yml`);
|
||||
if (!isPathWithinBase(overridePath, overrideDir)) return false;
|
||||
|
||||
try {
|
||||
await fs.access(overridePath);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return false;
|
||||
console.warn(
|
||||
'[NetworkDrift] Could not verify Pilot Mesh override for %s:',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveManagedMeshAttachment(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
): Promise<ManagedNetworkAttachmentPredicate> {
|
||||
let stackManaged = false;
|
||||
try {
|
||||
stackManaged = DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[NetworkDrift] Could not verify Mesh opt-in state for %s:',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
if (!stackManaged) stackManaged = await hasPilotMeshOverride(nodeId, stackName);
|
||||
const selfIdentity = SelfIdentityService.getInstance();
|
||||
|
||||
return (container, networkName) => networkName === SENCHO_MESH_NETWORK && (
|
||||
stackManaged
|
||||
|| selfIdentity.isOwnContainer(container.id)
|
||||
|| selfIdentity.isOwnContainer(container.name)
|
||||
);
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import { FileSystemService } from '../FileSystemService';
|
||||
import { DatabaseService } from '../DatabaseService';
|
||||
import { parseComposeDependencies } from '../../helpers/composeDependencyParse';
|
||||
import { assembleStackDrift } from '../DriftDetectionService';
|
||||
import { resolveManagedMeshAttachment } from './managedMeshAttachment';
|
||||
import { isHostNetwork, isLoopback } from './normalize';
|
||||
import { getErrorMessage } from '../../utils/errors';
|
||||
import { sanitizeForLog } from '../../utils/safeLog';
|
||||
@@ -86,7 +87,14 @@ export async function computeNodeNetworkingSummary(nodeId: number): Promise<Node
|
||||
if (snapshot) {
|
||||
// declared.parseError is already excluded above, so the drift report is authoritative.
|
||||
const containers = snapshot.containers.filter(c => c.stack === stack);
|
||||
const report = assembleStackDrift({ stack, declared, containers, networks: snapshot.networks });
|
||||
const managedNetworkAttachment = await resolveManagedMeshAttachment(nodeId, stack);
|
||||
const report = assembleStackDrift({
|
||||
stack,
|
||||
declared,
|
||||
containers,
|
||||
networks: snapshot.networks,
|
||||
managedNetworkAttachment,
|
||||
});
|
||||
if (report.findings.some(f => f.kind === 'network-undeclared' || f.kind === 'network-missing')) networkDrift.push(stack);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,8 +8,9 @@
|
||||
*/
|
||||
import type { EffectiveModel } from '../preflight/effectiveModel';
|
||||
import type { DeclaredCompose } from '../../helpers/composeDependencyParse';
|
||||
import type { DependencySnapshot } from '../DockerController';
|
||||
import type { DependencyContainer, DependencySnapshot } from '../DockerController';
|
||||
import type { NetworkDriftFacts } from './types';
|
||||
import { SENCHO_MESH_NETWORK } from '../MeshComposeOverride';
|
||||
|
||||
/** Container states that count as "deployed" for drift, matching DriftDetectionService. */
|
||||
const RUNNING_STATES = new Set(['running', 'restarting']);
|
||||
@@ -62,6 +63,11 @@ export interface NormalizedNetworkModel {
|
||||
services: { name: string; networkKeys: string[]; networkMode?: string }[];
|
||||
}
|
||||
|
||||
export type ManagedNetworkAttachmentPredicate = (
|
||||
container: DependencyContainer,
|
||||
networkName: string,
|
||||
) => boolean;
|
||||
|
||||
/** Rendered model: resource names are already resolved by `docker compose config`. */
|
||||
export function fromEffectiveModel(m: EffectiveModel): NormalizedNetworkModel {
|
||||
const networks: NormalizedNetworkModel['networks'] = {};
|
||||
@@ -97,6 +103,7 @@ export function compareStackNetworks(
|
||||
declared: NormalizedNetworkModel,
|
||||
snapshot: DependencySnapshot,
|
||||
stackName: string,
|
||||
isManagedAttachment: ManagedNetworkAttachmentPredicate = () => false,
|
||||
): NetworkDriftFacts {
|
||||
const runtimeOnlyAttachments: NetworkDriftFacts['runtimeOnlyAttachments'] = [];
|
||||
const foreignNetworkAttachments: NetworkDriftFacts['foreignNetworkAttachments'] = [];
|
||||
@@ -118,6 +125,7 @@ export function compareStackNetworks(
|
||||
const net = networkByName.get(attached.name);
|
||||
if (SYSTEM_NETWORK_NAMES.has(attached.name) || net?.isSystem) continue;
|
||||
if (declaredRuntimeNames.has(attached.name)) { usedRuntimeNames.add(attached.name); continue; }
|
||||
if (attached.name === SENCHO_MESH_NETWORK && isManagedAttachment(c, attached.name)) continue;
|
||||
if (net?.stack === stackName || attached.name.startsWith(`${declared.projectName}_`)) {
|
||||
runtimeOnlyAttachments.push({ container: c.name, service: c.service, network: attached.name });
|
||||
} else {
|
||||
|
||||
@@ -153,7 +153,7 @@ When the node is reachable, the tab compares the declared effective model agains
|
||||
| **Declared but unused** | A network is declared in the Compose file but no currently running service is connected to it. Often seen when a service is stopped or removed without `docker compose down`. |
|
||||
| **Missing from runtime** | A network is declared but does not exist in Docker. The stack may not have been deployed, or the network was deleted externally. |
|
||||
|
||||
System-managed networks (`bridge`, `host`, `none`) and Docker's implicit default bridge are excluded from all drift findings.
|
||||
System-managed networks (`bridge`, `host`, `none`) and Docker's implicit default bridge are excluded from all drift findings. Attachments to `sencho_mesh` are also excluded when Sencho verifies that the container is its own instance or that the stack is opted into Sencho Mesh. A manual attachment from an opted-out stack remains visible as drift.
|
||||
|
||||
When the runtime matches the Compose file, the section shows a green **runtime matches compose** card.
|
||||
|
||||
|
||||
@@ -83,7 +83,7 @@ A Compose port range such as `8000-8002:8000-8002` is compared conservatively. B
|
||||
|
||||
### Network findings and the Networking tab
|
||||
|
||||
The **network-undeclared** and **network-missing** findings reuse the same comparison the stack's [Networking](/features/compose-networking) tab uses, so the two surfaces never disagree about a network attachment. The difference is history: this tab persists findings in the drift ledger over time, while the Networking tab always shows the current live state with no history. A stack with an open network finding also counts toward the **Drift** chip in the Fleet Overview's Networking filter group, so a network-attachment mismatch is visible both per-stack here and across the fleet there.
|
||||
The **network-undeclared** and **network-missing** findings reuse the same comparison the stack's [Networking](/features/compose-networking) tab uses, so the two surfaces never disagree about a network attachment. Sencho-verified `sencho_mesh` attachments for its own container and Mesh-opted-in stacks are excluded, while a manual attachment from an opted-out stack remains actionable. The difference is history: this tab persists findings in the drift ledger over time, while the Networking tab always shows the current live state with no history. A stack with an open network finding also counts toward the **Drift** chip in the Fleet Overview's Networking filter group, so a network-attachment mismatch is visible both per-stack here and across the fleet there.
|
||||
|
||||
## When drift is recorded
|
||||
|
||||
@@ -93,6 +93,8 @@ The **network-undeclared** and **network-missing** findings reuse the same compa
|
||||
|
||||
**After every deploy or update**, Sencho automatically records a new baseline hash and runs a full reconciliation. You do not need to click re-check after deploying; the ledger is updated as part of the deploy pipeline.
|
||||
|
||||
An open ledger entry for an attachment that is no longer reported clears during the next re-check or post-deploy reconciliation. Opening the tab refreshes the live report but does not change persisted history.
|
||||
|
||||
<img
|
||||
src="/images/stack-drift/drift-history.png"
|
||||
alt="The Drift tab showing both the Findings section and the Drift history section with an open finding marked just now"
|
||||
|
||||
Reference in New Issue
Block a user