mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-21 15:46:43 +00:00
fix(networking): ignore verified Mesh attachments in drift (#1729)
This commit is contained in:
@@ -5,7 +5,12 @@ import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompos
|
||||
import type { DeclaredCompose, DeclaredService } from '../helpers/composeDependencyParse';
|
||||
import { parseMissingRequiredVars } from '../helpers/envVarParse';
|
||||
import { parseEffectiveModel } from './preflight/effectiveModel';
|
||||
import { compareStackNetworks, fromDeclaredCompose } from './network/normalize';
|
||||
import {
|
||||
compareStackNetworks,
|
||||
fromDeclaredCompose,
|
||||
type ManagedNetworkAttachmentPredicate,
|
||||
} from './network/normalize';
|
||||
import { resolveManagedMeshAttachment } from './network/managedMeshAttachment';
|
||||
import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { isCleanOneShotCompletion } from '../utils/oneShotCompletion';
|
||||
@@ -111,6 +116,8 @@ export interface AssembleStackDriftInput {
|
||||
containers: DependencyContainer[];
|
||||
/** Every network on the node (for resolving foreign vs stack-owned attachments). */
|
||||
networks?: DependencyNetwork[];
|
||||
/** Authoritative runtime attachments that are intentionally absent from authored Compose. */
|
||||
managedNetworkAttachment?: ManagedNetworkAttachmentPredicate;
|
||||
/** Set when the compose file could not be parsed. */
|
||||
parseError?: string;
|
||||
}
|
||||
@@ -137,12 +144,18 @@ function networkDriftFindings(
|
||||
declared: DeclaredCompose,
|
||||
containers: DependencyContainer[],
|
||||
networks: DependencyNetwork[],
|
||||
managedNetworkAttachment?: ManagedNetworkAttachmentPredicate,
|
||||
): StackDriftFinding[] {
|
||||
// Runtime resource names use the Compose project (top-level `name:` when set),
|
||||
// not the stack directory, so a stack with `name:` resolves its networks the
|
||||
// same way Docker does. Containers are still attributed to the stack directory.
|
||||
const normalized = fromDeclaredCompose(declared, declared.projectName ?? stack);
|
||||
const facts = compareStackNetworks(normalized, { containers, networks, volumes: [] }, stack);
|
||||
const facts = compareStackNetworks(
|
||||
normalized,
|
||||
{ containers, networks, volumes: [] },
|
||||
stack,
|
||||
managedNetworkAttachment,
|
||||
);
|
||||
const findings: StackDriftFinding[] = [];
|
||||
|
||||
const serviceByContainer = new Map(containers.map(c => [c.name, c.service ?? c.name]));
|
||||
@@ -293,7 +306,13 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe
|
||||
}
|
||||
}
|
||||
|
||||
findings.push(...networkDriftFindings(stack, declared, containers, networks));
|
||||
findings.push(...networkDriftFindings(
|
||||
stack,
|
||||
declared,
|
||||
containers,
|
||||
networks,
|
||||
input.managedNetworkAttachment,
|
||||
));
|
||||
|
||||
const status: StackDriftStatus = findings.length > 0 ? 'drifted' : 'in-sync';
|
||||
return { stack, status, hasComposeFile: true, hasContainers, findings };
|
||||
@@ -384,5 +403,12 @@ export async function buildStackDriftReport(nodeId: number, stackName: string):
|
||||
};
|
||||
}
|
||||
|
||||
return assembleStackDrift({ stack: stackName, declared: render.declared, containers, networks });
|
||||
const managedNetworkAttachment = await resolveManagedMeshAttachment(nodeId, stackName);
|
||||
return assembleStackDrift({
|
||||
stack: stackName,
|
||||
declared: render.declared,
|
||||
containers,
|
||||
networks,
|
||||
managedNetworkAttachment,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import net from 'net';
|
||||
import path from 'path';
|
||||
import fs from 'fs/promises';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
import * as YAML from 'yaml';
|
||||
import { ComposeService } from './ComposeService';
|
||||
@@ -20,6 +21,7 @@ import { STREAM_PENDING_DATA_MAX_BYTES } from '../pilot/protocol';
|
||||
import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { isPathWithinBase, isValidStackName, isValidRelativeStackPath } from '../utils/validation';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { PORT as SENCHO_LISTEN_PORT } from '../helpers/constants';
|
||||
import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate';
|
||||
|
||||
@@ -1497,6 +1499,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
// --- Opt-in / opt-out ---
|
||||
|
||||
public async optInStack(nodeId: number, stackName: string, actor: string): Promise<void> {
|
||||
return this.runMeshNodeMutation(nodeId, () => this.optInStackExclusive(nodeId, stackName, actor));
|
||||
}
|
||||
|
||||
private async optInStackExclusive(nodeId: number, stackName: string, actor: string): Promise<void> {
|
||||
this.logDiag('opt-in start', { nodeId, stackName, actor });
|
||||
const t0 = Date.now();
|
||||
if (!isValidStackName(stackName)) {
|
||||
@@ -1541,19 +1547,41 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
}
|
||||
|
||||
db.insertMeshStack(nodeId, stackName, actor);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
try {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} catch (error) {
|
||||
db.deleteMeshStack(nodeId, stackName);
|
||||
try {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} catch (rollbackError) {
|
||||
console.warn('[MeshService] Failed to refresh aliases after opt-in rollback:', sanitizeForLog(getErrorMessage(rollbackError, 'unknown')));
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Push the just-opted-in stack's override loudly. If this fails the
|
||||
// DB state is invalid (alias claimed but remote pilot has no
|
||||
// override file) so roll back rather than leave a half-state that
|
||||
// future opt-in calls would short-circuit on `isMeshStackEnabled`.
|
||||
// Push the just-opted-in stack's override loudly. Explicit target
|
||||
// rejection rolls back the row. A remote transport failure is
|
||||
// ambiguous because the target may already have committed, so retain
|
||||
// authority and let normal regeneration reconcile it.
|
||||
try {
|
||||
await this.pushOverrideToNode(nodeId, stackName);
|
||||
} catch (err) {
|
||||
db.deleteMeshStack(nodeId, stackName);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
const node = db.getNode(nodeId);
|
||||
const explicitlyRejected = node?.type !== 'remote' || err instanceof MeshError;
|
||||
if (explicitlyRejected) {
|
||||
db.deleteMeshStack(nodeId, stackName);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} else {
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'warn', type: 'forwarder.error',
|
||||
nodeId,
|
||||
message: `mesh override push outcome unknown for ${stackName}; retaining opt-in authority for reconciliation`,
|
||||
details: { stackName },
|
||||
});
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
// Regenerate every other meshed stack's override across the fleet
|
||||
@@ -1584,6 +1612,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
}
|
||||
|
||||
public async optOutStack(nodeId: number, stackName: string, actor: string): Promise<void> {
|
||||
return this.runMeshNodeMutation(nodeId, () => this.optOutStackExclusive(nodeId, stackName, actor));
|
||||
}
|
||||
|
||||
private async optOutStackExclusive(nodeId: number, stackName: string, actor: string): Promise<void> {
|
||||
this.logDiag('opt-out start', { nodeId, stackName, actor });
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw new MeshError('denied', `invalid stack name: ${stackName}`);
|
||||
@@ -1591,9 +1623,18 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
const db = DatabaseService.getInstance();
|
||||
if (!db.isMeshStackEnabled(nodeId, stackName)) return;
|
||||
db.deleteMeshStack(nodeId, stackName);
|
||||
await this.removeOverrideFromNode(nodeId, stackName);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
try {
|
||||
await this.removeOverrideFromNode(nodeId, stackName);
|
||||
} catch (error) {
|
||||
db.insertMeshStack(nodeId, stackName, actor);
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} catch (error) {
|
||||
console.warn('[MeshService] Failed to refresh aliases after committed opt-out:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
// The opted-out row is already deleted, so listMeshStacks() will not
|
||||
// include it. Walk the remaining fleet-wide rows so every other
|
||||
// meshed stack regenerates its override without the dropped alias.
|
||||
@@ -1620,6 +1661,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
}
|
||||
|
||||
public async enableForNode(nodeId: number): Promise<void> {
|
||||
return this.runMeshNodeMutation(nodeId, () => this.enableForNodeExclusive(nodeId));
|
||||
}
|
||||
|
||||
private async enableForNodeExclusive(nodeId: number): Promise<void> {
|
||||
this.logDiag('enable-for-node', { nodeId });
|
||||
DatabaseService.getInstance().setNodeMeshEnabled(nodeId, true);
|
||||
this.logActivity({
|
||||
@@ -1642,26 +1687,42 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
nodeId: number,
|
||||
actor: string = 'system:mesh.disable',
|
||||
): Promise<void> {
|
||||
return this.runMeshNodeMutation(nodeId, () => this.disableForNodeExclusive(nodeId, actor));
|
||||
}
|
||||
|
||||
private async disableForNodeExclusive(nodeId: number, actor: string): Promise<void> {
|
||||
this.logDiag('disable-for-node start', { nodeId, actor });
|
||||
const t0 = Date.now();
|
||||
DatabaseService.getInstance().setNodeMeshEnabled(nodeId, false);
|
||||
const stacks = DatabaseService.getInstance().listMeshStacks(nodeId);
|
||||
for (const s of stacks) {
|
||||
DatabaseService.getInstance().deleteMeshStack(nodeId, s.stack_name);
|
||||
}
|
||||
const db = DatabaseService.getInstance();
|
||||
const stacks = db.listMeshStacks(nodeId);
|
||||
// Dispatch DELETE /api/mesh/local-override/:stack for remote nodes
|
||||
// (pilot or proxy) so the override file pushed earlier via
|
||||
// applyLocalOverride is removed; falls back to local deletion for
|
||||
// local nodes. Parallelize per the regenerateOverridesForNode
|
||||
// rationale: each remote call is its own HTTP round-trip, so
|
||||
// awaiting sequentially turns N stacks into N serialised DELETEs.
|
||||
// `allSettled` so a single failure does not abort the others
|
||||
// (removeOverrideFromNode already swallows errors internally).
|
||||
await Promise.allSettled(
|
||||
// `allSettled` so a single failure does not abort the others.
|
||||
const removals = await Promise.allSettled(
|
||||
stacks.map((s) => this.removeOverrideFromNode(nodeId, s.stack_name)),
|
||||
);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
const failed: string[] = [];
|
||||
const removed: typeof stacks = [];
|
||||
removals.forEach((result, index) => {
|
||||
const stack = stacks[index];
|
||||
if (result.status === 'fulfilled') {
|
||||
db.deleteMeshStack(nodeId, stack.stack_name);
|
||||
removed.push(stack);
|
||||
} else {
|
||||
failed.push(stack.stack_name);
|
||||
}
|
||||
});
|
||||
if (failed.length === 0) db.setNodeMeshEnabled(nodeId, false);
|
||||
try {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} catch (error) {
|
||||
console.warn('[MeshService] Failed to refresh aliases after committed node disable changes:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
// Mirror optOutStack: regenerate every remaining node's override
|
||||
// without the dropped aliases, recompose the rest of the fleet so
|
||||
// their containers shed the stale extra_hosts, and redeploy the
|
||||
@@ -1669,9 +1730,15 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
// sencho_mesh network and lose the alias entries they owned.
|
||||
await this.regenerateOverridesAcrossFleet();
|
||||
this.cascadeRecomposeAcrossFleet(undefined, undefined, actor);
|
||||
for (const s of stacks) {
|
||||
for (const s of removed) {
|
||||
this.triggerRedeploy(nodeId, s.stack_name, actor);
|
||||
}
|
||||
if (failed.length > 0) {
|
||||
throw new MeshError(
|
||||
'push_failed',
|
||||
`Could not disable Mesh on node ${nodeId}: override removal failed for ${failed.join(', ')}.`,
|
||||
);
|
||||
}
|
||||
this.logDiag('disable-for-node complete', { nodeId, stacks: stacks.length, ms: Date.now() - t0 });
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'mesh.disable',
|
||||
@@ -1679,6 +1746,24 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
});
|
||||
}
|
||||
|
||||
private readonly meshNodeMutations = new Map<number, Promise<void>>();
|
||||
|
||||
private async runMeshNodeMutation<T>(nodeId: number, operation: () => Promise<T>): Promise<T> {
|
||||
const previous = this.meshNodeMutations.get(nodeId) ?? Promise.resolve();
|
||||
let release!: () => void;
|
||||
const pending = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
this.meshNodeMutations.set(nodeId, pending);
|
||||
await previous;
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
release();
|
||||
if (this.meshNodeMutations.get(nodeId) === pending) this.meshNodeMutations.delete(nodeId);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Override file management ---
|
||||
|
||||
public async ensureStackOverride(nodeId: number, stackName: string): Promise<string | null> {
|
||||
@@ -1690,6 +1775,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
// lives on central per the C-3 design). Use file-presence as the
|
||||
// fallback: if central pushed an override via applyLocalOverride, return
|
||||
// that path so ComposeService picks it up on the next deploy.
|
||||
if (process.env.SENCHO_MODE !== 'pilot') return null;
|
||||
const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`);
|
||||
if (!isPathWithinBase(file, dir)) return null;
|
||||
try {
|
||||
@@ -1754,13 +1840,37 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
portAliases?: MeshGlobalAlias[],
|
||||
): Promise<string | null> {
|
||||
if (!isValidStackName(stackName)) return null;
|
||||
if (!this.senchoIp) {
|
||||
const senchoIp = this.senchoIp;
|
||||
if (!senchoIp) {
|
||||
throw new MeshError(
|
||||
'push_failed',
|
||||
this.networkSetupError || 'mesh data plane unavailable on this node',
|
||||
);
|
||||
}
|
||||
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
localNodeId,
|
||||
stackName,
|
||||
'deploy',
|
||||
'system:mesh.override',
|
||||
() => this.applyLocalOverrideExclusive(localNodeId, stackName, aliases, senchoIp, portAliases),
|
||||
);
|
||||
if (!lock.ran) {
|
||||
throw new MeshError(
|
||||
'push_failed',
|
||||
`Cannot apply Mesh override for "${stackName}": another operation (${lock.existing.action}) is already in progress.`,
|
||||
);
|
||||
}
|
||||
return lock.result;
|
||||
}
|
||||
|
||||
private async applyLocalOverrideExclusive(
|
||||
localNodeId: number,
|
||||
stackName: string,
|
||||
aliases: MeshAlias[],
|
||||
senchoIp: string,
|
||||
portAliases?: MeshGlobalAlias[],
|
||||
): Promise<string | null> {
|
||||
const serviceNames = await this.getDeclaredStackServiceNames(stackName, localNodeId);
|
||||
|
||||
const dir = this.overrideDirFor(localNodeId);
|
||||
@@ -1785,6 +1895,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
message: `mesh override preserved for ${stackName}: declared services unreadable, keeping ${existing.length} existing entries`,
|
||||
details: { stackName, preservedServices: existing },
|
||||
});
|
||||
this.recordLocalOverrideIntent(localNodeId, stackName);
|
||||
return file;
|
||||
}
|
||||
}
|
||||
@@ -1792,13 +1903,24 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
const yaml = generateOverrideYaml({
|
||||
services: serviceNames,
|
||||
aliases,
|
||||
senchoIp: this.senchoIp,
|
||||
senchoIp,
|
||||
});
|
||||
await fs.writeFile(file, yaml, 'utf8');
|
||||
const previousYaml = await this.readOverrideContent(file);
|
||||
await this.writeOverrideAtomically(file, yaml);
|
||||
try {
|
||||
this.recordLocalOverrideIntent(localNodeId, stackName);
|
||||
} catch (error) {
|
||||
await this.restoreOverrideAfterAuthorityFailure(file, previousYaml);
|
||||
throw error;
|
||||
}
|
||||
if (portAliases && portAliases.length > 0) {
|
||||
this.pilotAliasOverlay.set(stackName, portAliases);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
try {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} catch (error) {
|
||||
console.warn('[MeshService] Failed to refresh aliases after committed override apply:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}
|
||||
return file;
|
||||
}
|
||||
@@ -1810,13 +1932,95 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
public async removeLocalOverride(stackName: string): Promise<void> {
|
||||
if (!isValidStackName(stackName)) return;
|
||||
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
localNodeId,
|
||||
stackName,
|
||||
'deploy',
|
||||
'system:mesh.override',
|
||||
() => this.removeLocalOverrideExclusive(localNodeId, stackName),
|
||||
);
|
||||
if (!lock.ran) {
|
||||
throw new MeshError(
|
||||
'push_failed',
|
||||
`Cannot remove Mesh override for "${stackName}": another operation (${lock.existing.action}) is already in progress.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async removeLocalOverrideExclusive(localNodeId: number, stackName: string): Promise<void> {
|
||||
const dir = this.overrideDirFor(localNodeId);
|
||||
const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`);
|
||||
if (!isPathWithinBase(file, dir)) return;
|
||||
try { await fs.unlink(file); } catch { /* ignore not-exist */ }
|
||||
const db = DatabaseService.getInstance();
|
||||
const hadIntent = db.isMeshStackEnabled(localNodeId, stackName);
|
||||
if (hadIntent) db.deleteMeshStack(localNodeId, stackName);
|
||||
try {
|
||||
await fs.unlink(file);
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) {
|
||||
if (hadIntent) db.insertMeshStack(localNodeId, stackName, null);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (this.pilotAliasOverlay.delete(stackName)) {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
try {
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
} catch (error) {
|
||||
console.warn('[MeshService] Failed to refresh aliases after committed override removal:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async writeOverrideAtomically(file: string, yaml: string): Promise<void> {
|
||||
const dir = path.dirname(file);
|
||||
const tempFile = path.resolve(dir, `.${path.basename(file)}.${randomUUID()}.tmp`);
|
||||
if (!isPathWithinBase(tempFile, dir)) throw new Error('Invalid Mesh override temporary path');
|
||||
|
||||
let handle: Awaited<ReturnType<typeof fs.open>> | null = null;
|
||||
try {
|
||||
handle = await fs.open(tempFile, 'wx');
|
||||
await handle.writeFile(yaml, 'utf8');
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = null;
|
||||
await fs.rename(tempFile, file);
|
||||
} catch (error) {
|
||||
if (handle) {
|
||||
try { await handle.close(); } catch (closeError) {
|
||||
console.warn('[MeshService] Failed to close temporary override:', sanitizeForLog(getErrorMessage(closeError, 'unknown')));
|
||||
}
|
||||
}
|
||||
try {
|
||||
await fs.unlink(tempFile);
|
||||
} catch (cleanupError) {
|
||||
if (!(cleanupError instanceof Error && 'code' in cleanupError && cleanupError.code === 'ENOENT')) {
|
||||
console.warn('[MeshService] Failed to clean up temporary override:', sanitizeForLog(getErrorMessage(cleanupError, 'unknown')));
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async readOverrideContent(file: string): Promise<string | null> {
|
||||
try {
|
||||
return await fs.readFile(file, 'utf8');
|
||||
} catch (error) {
|
||||
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async restoreOverrideAfterAuthorityFailure(file: string, previousYaml: string | null): Promise<void> {
|
||||
try {
|
||||
if (previousYaml === null) {
|
||||
await fs.unlink(file);
|
||||
} else {
|
||||
await this.writeOverrideAtomically(file, previousYaml);
|
||||
}
|
||||
} catch (error) {
|
||||
if (previousYaml === null && error instanceof Error && 'code' in error && error.code === 'ENOENT') return;
|
||||
console.warn('[MeshService] Failed to restore override after authority error:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1825,7 +2029,11 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
const dir = this.overrideDirFor(nodeId);
|
||||
const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`);
|
||||
if (!isPathWithinBase(file, dir)) return;
|
||||
try { await fs.unlink(file); } catch { /* ignore not-exist */ }
|
||||
try {
|
||||
await fs.unlink(file);
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private overrideDirFor(nodeId: number): string {
|
||||
@@ -1833,6 +2041,13 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
return path.join(dataDir, 'mesh', 'overrides', String(nodeId));
|
||||
}
|
||||
|
||||
private recordLocalOverrideIntent(nodeId: number, stackName: string): void {
|
||||
if (process.env.SENCHO_MODE === 'pilot') return;
|
||||
const db = DatabaseService.getInstance();
|
||||
if (db.isMeshStackEnabled(nodeId, stackName)) return;
|
||||
db.insertMeshStack(nodeId, stackName, null);
|
||||
}
|
||||
|
||||
private async regenerateOverridesForNode(nodeId: number, skipStack?: string): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const stacks = db.listMeshStacks(nodeId);
|
||||
@@ -2331,7 +2546,11 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
);
|
||||
}
|
||||
if (!res.ok) {
|
||||
throw new MeshError('push_failed', `HTTP ${res.status} from node ${node.name}`);
|
||||
const message = `HTTP ${res.status} from node ${node.name}`;
|
||||
if (res.status >= 400 && res.status < 500) {
|
||||
throw new MeshError('push_failed', message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2436,15 +2655,20 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
}
|
||||
|
||||
try {
|
||||
await this.proxyFetch(
|
||||
const response = await this.proxyFetch(
|
||||
nodeId,
|
||||
'DELETE',
|
||||
`/api/mesh/local-override/${encodeURIComponent(stackName)}`,
|
||||
undefined,
|
||||
5_000,
|
||||
);
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '');
|
||||
throw new Error(`HTTP ${response.status} from node ${node.name}: ${body.slice(0, 256)}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('[MeshService] removeOverrideFromNode failed:', sanitizeForLog((err as Error).message));
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,8 @@ import type {
|
||||
NetworkDriftFacts, NetworkFactNetwork, NetworkFactService, NetworkRuntimeState, StackNetworkFacts,
|
||||
} from './types';
|
||||
import { classifyMissingExternalNetworks, type MissingExternalNetwork } from './missingExternalNetworks';
|
||||
import { resolveManagedMeshAttachment } from './managedMeshAttachment';
|
||||
import type { ManagedNetworkAttachmentPredicate } from './normalize';
|
||||
|
||||
import { getErrorMessage } from '../../utils/errors';
|
||||
import { redactSensitiveText, sanitizeForLog } from '../../utils/safeLog';
|
||||
@@ -43,6 +45,7 @@ export function assembleStackNetworkFacts(
|
||||
model: EffectiveModel | null,
|
||||
renderError: string | null,
|
||||
snapshot: DependencySnapshot | null,
|
||||
managedNetworkAttachment?: ManagedNetworkAttachmentPredicate,
|
||||
): StackNetworkFacts {
|
||||
const runtime: NetworkRuntimeState = snapshot ? 'available' : 'unavailable';
|
||||
|
||||
@@ -82,7 +85,9 @@ export function assembleStackNetworkFacts(
|
||||
extraHosts: s.extraHosts,
|
||||
}));
|
||||
|
||||
const drift = snapshot ? compareStackNetworks(fromEffectiveModel(model), snapshot, stackName) : EMPTY_DRIFT;
|
||||
const drift = snapshot
|
||||
? compareStackNetworks(fromEffectiveModel(model), snapshot, stackName, managedNetworkAttachment)
|
||||
: EMPTY_DRIFT;
|
||||
const missingExternalNetworks: MissingExternalNetwork[] = snapshot
|
||||
? classifyMissingExternalNetworks(
|
||||
model,
|
||||
@@ -147,5 +152,8 @@ export async function buildStackNetworkFacts(
|
||||
}
|
||||
}
|
||||
|
||||
return assembleStackNetworkFacts(stackName, model, renderError, snapshot);
|
||||
const managedNetworkAttachment = snapshot && model
|
||||
? await resolveManagedMeshAttachment(nodeId, stackName)
|
||||
: undefined;
|
||||
return assembleStackNetworkFacts(stackName, model, renderError, snapshot, managedNetworkAttachment);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { DatabaseService } from '../DatabaseService';
|
||||
import { SENCHO_MESH_NETWORK } from '../MeshComposeOverride';
|
||||
import SelfIdentityService from '../SelfIdentityService';
|
||||
import { getErrorMessage } from '../../utils/errors';
|
||||
import { sanitizeForLog } from '../../utils/safeLog';
|
||||
import { isPathWithinBase, isValidStackName } from '../../utils/validation';
|
||||
import type { ManagedNetworkAttachmentPredicate } from './normalize';
|
||||
|
||||
async function hasPilotMeshOverride(nodeId: number, stackName: string): Promise<boolean> {
|
||||
if (process.env.SENCHO_MODE !== 'pilot' || !isValidStackName(stackName)) return false;
|
||||
|
||||
const dataDir = process.env.DATA_DIR || '/app/data';
|
||||
const overrideDir = path.resolve(dataDir, 'mesh', 'overrides', String(nodeId));
|
||||
const overridePath = path.resolve(overrideDir, `${path.basename(stackName)}.override.yml`);
|
||||
if (!isPathWithinBase(overridePath, overrideDir)) return false;
|
||||
|
||||
try {
|
||||
await fs.access(overridePath);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return false;
|
||||
console.warn(
|
||||
'[NetworkDrift] Could not verify Pilot Mesh override for %s:',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveManagedMeshAttachment(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
): Promise<ManagedNetworkAttachmentPredicate> {
|
||||
let stackManaged = false;
|
||||
try {
|
||||
stackManaged = DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[NetworkDrift] Could not verify Mesh opt-in state for %s:',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
if (!stackManaged) stackManaged = await hasPilotMeshOverride(nodeId, stackName);
|
||||
const selfIdentity = SelfIdentityService.getInstance();
|
||||
|
||||
return (container, networkName) => networkName === SENCHO_MESH_NETWORK && (
|
||||
stackManaged
|
||||
|| selfIdentity.isOwnContainer(container.id)
|
||||
|| selfIdentity.isOwnContainer(container.name)
|
||||
);
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import { FileSystemService } from '../FileSystemService';
|
||||
import { DatabaseService } from '../DatabaseService';
|
||||
import { parseComposeDependencies } from '../../helpers/composeDependencyParse';
|
||||
import { assembleStackDrift } from '../DriftDetectionService';
|
||||
import { resolveManagedMeshAttachment } from './managedMeshAttachment';
|
||||
import { isHostNetwork, isLoopback } from './normalize';
|
||||
import { getErrorMessage } from '../../utils/errors';
|
||||
import { sanitizeForLog } from '../../utils/safeLog';
|
||||
@@ -86,7 +87,14 @@ export async function computeNodeNetworkingSummary(nodeId: number): Promise<Node
|
||||
if (snapshot) {
|
||||
// declared.parseError is already excluded above, so the drift report is authoritative.
|
||||
const containers = snapshot.containers.filter(c => c.stack === stack);
|
||||
const report = assembleStackDrift({ stack, declared, containers, networks: snapshot.networks });
|
||||
const managedNetworkAttachment = await resolveManagedMeshAttachment(nodeId, stack);
|
||||
const report = assembleStackDrift({
|
||||
stack,
|
||||
declared,
|
||||
containers,
|
||||
networks: snapshot.networks,
|
||||
managedNetworkAttachment,
|
||||
});
|
||||
if (report.findings.some(f => f.kind === 'network-undeclared' || f.kind === 'network-missing')) networkDrift.push(stack);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,8 +8,9 @@
|
||||
*/
|
||||
import type { EffectiveModel } from '../preflight/effectiveModel';
|
||||
import type { DeclaredCompose } from '../../helpers/composeDependencyParse';
|
||||
import type { DependencySnapshot } from '../DockerController';
|
||||
import type { DependencyContainer, DependencySnapshot } from '../DockerController';
|
||||
import type { NetworkDriftFacts } from './types';
|
||||
import { SENCHO_MESH_NETWORK } from '../MeshComposeOverride';
|
||||
|
||||
/** Container states that count as "deployed" for drift, matching DriftDetectionService. */
|
||||
const RUNNING_STATES = new Set(['running', 'restarting']);
|
||||
@@ -62,6 +63,11 @@ export interface NormalizedNetworkModel {
|
||||
services: { name: string; networkKeys: string[]; networkMode?: string }[];
|
||||
}
|
||||
|
||||
export type ManagedNetworkAttachmentPredicate = (
|
||||
container: DependencyContainer,
|
||||
networkName: string,
|
||||
) => boolean;
|
||||
|
||||
/** Rendered model: resource names are already resolved by `docker compose config`. */
|
||||
export function fromEffectiveModel(m: EffectiveModel): NormalizedNetworkModel {
|
||||
const networks: NormalizedNetworkModel['networks'] = {};
|
||||
@@ -97,6 +103,7 @@ export function compareStackNetworks(
|
||||
declared: NormalizedNetworkModel,
|
||||
snapshot: DependencySnapshot,
|
||||
stackName: string,
|
||||
isManagedAttachment: ManagedNetworkAttachmentPredicate = () => false,
|
||||
): NetworkDriftFacts {
|
||||
const runtimeOnlyAttachments: NetworkDriftFacts['runtimeOnlyAttachments'] = [];
|
||||
const foreignNetworkAttachments: NetworkDriftFacts['foreignNetworkAttachments'] = [];
|
||||
@@ -118,6 +125,7 @@ export function compareStackNetworks(
|
||||
const net = networkByName.get(attached.name);
|
||||
if (SYSTEM_NETWORK_NAMES.has(attached.name) || net?.isSystem) continue;
|
||||
if (declaredRuntimeNames.has(attached.name)) { usedRuntimeNames.add(attached.name); continue; }
|
||||
if (attached.name === SENCHO_MESH_NETWORK && isManagedAttachment(c, attached.name)) continue;
|
||||
if (net?.stack === stackName || attached.name.startsWith(`${declared.projectName}_`)) {
|
||||
runtimeOnlyAttachments.push({ container: c.name, service: c.service, network: attached.name });
|
||||
} else {
|
||||
|
||||
Reference in New Issue
Block a user