fix(networking): hold unsafe network deletions and stabilize aggregate reads (#1850)

* fix(networking): hold unsafe network deletions and stabilize aggregate reads

Networking audit hardening:

- Fail closed when deleting unlabeled external networks while any stack
  fails to render or the Docker runtime is unreachable: declarations
  cannot be verified, so the backend 409s with a typed code and the
  inventory table holds the delete affordance instead of letting the
  confirm dialog surprise the operator.
- Bound concurrent compose renders during network delete verification
  with the shared render semaphore and mapWithConcurrency.
- Add a short-TTL memo for the node networking aggregate keyed by node
  and request variant, invalidated eagerly on stack, exposure-intent,
  dossier, and network mutations, with stale-on-error serves flagged as
  degradedCache and surfaced in the overview.
- Resolve node-scoped stack edit permissions against the active node in
  the findings action list.
- Developer-mode debug logs for aggregate serving and delete-guard
  outcomes.

Tests: cache unit suite, hardening integration suite, and component
coverage for the permission threading and delete-affordance holds.

* chore(networking): add missing EOF newline in aggregate cache module
This commit is contained in:
Anso
2026-08-28 12:55:19 +00:00
committed by GitHub
parent 392bc15d91
commit cc4a6571c7
19 changed files with 530 additions and 13 deletions
@@ -0,0 +1,148 @@
/**
* Networking aggregate memo: read endpoints share one computation per node and
* requested variant within the TTL window, a mutation invalidation forces the
* next read to recompute, and a base (no-topology) read never satisfies a
* topology read. In-flight joining itself is covered by CacheService's own
* suite; these tests pin the key derivation this module adds on top.
*/
import { describe, it, expect } from 'vitest';
import {
fetchNodeNetworkingAggregateWithMeta,
invalidateNodeNetworkingAggregate,
networkingAggregateCacheKey,
} from '../services/network/networkingAggregateCache';
import type { NetworkingAggregateOptions } from '../services/network/networkingAggregateCache';
import type { NodeNetworkingAggregate } from '../services/network/networkingTypes';
function fakeAggregate(networkCount: number): NodeNetworkingAggregate {
return {
overview: {
runtimeAvailable: true,
networkCount,
stackCount: 0,
connectedContainerCount: 0,
systemNetworkCount: 0,
senchoManagedNetworkCount: 0,
composeManagedNetworkCount: 0,
unmanagedNetworkCount: 0,
externalDependencyNetworkCount: 0,
exposedStackCount: 0,
unknownExposureStackCount: 0,
missingExternalCount: 0,
networkCollisionCount: 0,
findingCount: 0,
degradedCache: false,
renderFailedStacks: [],
},
networks: [],
findings: [],
stackFacts: [],
runtimeAvailable: true,
recentActivity: [],
};
}
function fakeTopology(): NodeNetworkingAggregate['topology'] {
return { networks: [{ id: 'n1', name: 'net', driver: 'bridge', scope: '', stack: null, isSystem: false, ingress: false, ownership: 'unmanaged', declaredByStacks: [], declaredExternalByStacks: [], isExternalDependency: false, findingIds: [], containers: [] }], includeSystem: false };
}
const NO_OPTIONS: NetworkingAggregateOptions = {};
async function fetchMemo(nodeId: number, options: NetworkingAggregateOptions, compute: () => Promise<NodeNetworkingAggregate>) {
const { value } = await fetchNodeNetworkingAggregateWithMeta(nodeId, options, compute);
return value;
}
describe('networkingAggregateCache', () => {
it('computes once per TTL window per node and variant', async () => {
invalidateNodeNetworkingAggregate(1);
let calls = 0;
const first = await fetchMemo(1, NO_OPTIONS, async () => fakeAggregate(++calls));
const second = await fetchMemo(1, NO_OPTIONS, async () => fakeAggregate(++calls));
expect(first.overview.networkCount).toBe(1);
expect(second.overview.networkCount).toBe(1);
});
it('keeps nodes isolated', async () => {
invalidateNodeNetworkingAggregate(2);
invalidateNodeNetworkingAggregate(3);
const [a, b] = await Promise.all([
fetchMemo(2, NO_OPTIONS, async () => fakeAggregate(2)),
fetchMemo(3, NO_OPTIONS, async () => fakeAggregate(3)),
]);
expect(a.overview.networkCount).toBe(2);
expect(b.overview.networkCount).toBe(3);
});
it('does not serve a base read to a topology request', async () => {
invalidateNodeNetworkingAggregate(5);
const base = await fetchMemo(5, NO_OPTIONS, async () => fakeAggregate(1));
expect(base.topology).toBeUndefined();
let topologyComputes = 0;
const topology = await fetchMemo(5, { includeTopology: true }, async () => {
topologyComputes += 1;
return { ...fakeAggregate(1), topology: fakeTopology() };
});
expect(topologyComputes).toBe(1);
expect(topology.topology?.networks).toHaveLength(1);
// The base entry stays warm and is served for base reads.
let baseComputes = 0;
const warmedBase = await fetchMemo(5, NO_OPTIONS, async () => {
baseComputes += 1;
return fakeAggregate(99);
});
expect(baseComputes).toBe(0);
expect(warmedBase.overview.networkCount).toBe(1);
});
it('treats includeSystem variants as distinct entries', async () => {
invalidateNodeNetworkingAggregate(7);
await fetchMemo(7, { includeTopology: true }, async () => fakeAggregate(1));
let computes = 0;
await fetchMemo(7, { includeTopology: true, includeSystem: true }, async () => {
computes += 1;
return fakeAggregate(1);
});
expect(computes).toBe(1);
});
it('builds a distinct key per variant', () => {
const keys = new Set([
networkingAggregateCacheKey(9, {}),
networkingAggregateCacheKey(9, { includeTopology: true }),
networkingAggregateCacheKey(9, { includeTopology: true, includeSystem: true }),
networkingAggregateCacheKey(10, {}),
]);
expect(keys.size).toBe(4);
});
it('recomputes after an explicit invalidation (mutation path)', async () => {
invalidateNodeNetworkingAggregate(4);
let calls = 0;
await fetchMemo(4, NO_OPTIONS, async () => fakeAggregate(++calls));
invalidateNodeNetworkingAggregate(4);
const after = await fetchMemo(4, NO_OPTIONS, async () => fakeAggregate(++calls));
expect(after.overview.networkCount).toBe(2);
});
it('invalidation drops every variant of the node', async () => {
invalidateNodeNetworkingAggregate(6);
await fetchMemo(6, NO_OPTIONS, async () => fakeAggregate(1));
await fetchMemo(6, { includeTopology: true }, async () => fakeAggregate(1));
invalidateNodeNetworkingAggregate(6);
let calls = 0;
await fetchMemo(6, NO_OPTIONS, async () => { calls += 1; return fakeAggregate(1); });
await fetchMemo(6, { includeTopology: true }, async () => { calls += 1; return fakeAggregate(1); });
expect(calls).toBe(2);
});
});
@@ -0,0 +1,151 @@
/**
* Networking hardening integration tests:
* - the aggregate memo serves repeat reads within its TTL,
* - a base (no-topology) read never satisfies a topology request,
* - exposure-intent writes invalidate the memo so the next overview reflects
* them immediately,
* - the delete guard 409s with stack-declaration-unknown for an unlabeled
* external network while a stack fails to render.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
import fs from 'fs';
import path from 'path';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
import DockerController from '../services/DockerController';
import { ComposeService } from '../services/ComposeService';
import { DatabaseService } from '../services/DatabaseService';
let tmpDir: string;
let app: import('express').Express;
let authHeader: string;
const STACK = 'hardnet';
const NET_ID = 'deadbeefdead';
function stubHealthy() {
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDependencySnapshot: vi.fn().mockResolvedValue({
containers: [],
networks: [
{ id: NET_ID, name: 'shared_ext', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null },
],
volumes: [],
}),
inspectNetwork: vi.fn().mockResolvedValue({ Id: NET_ID, Name: 'shared_ext', Driver: 'bridge', Scope: 'local', Labels: {}, Containers: {} }),
removeNetwork: vi.fn().mockResolvedValue(undefined),
} as unknown as DockerController);
vi.spyOn(ComposeService, 'getInstance').mockReturnValue({
renderConfig: vi.fn().mockResolvedValue({
rendered: JSON.stringify({
name: STACK,
services: { web: { image: 'nginx:latest', ports: ['8080:80'] } },
networks: {},
volumes: {},
}),
stderr: '', code: 0, timedOut: false,
}),
} as unknown as ComposeService);
}
beforeAll(async () => {
tmpDir = await setupTestDb();
({ app } = await import('../index'));
authHeader = `Bearer ${jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '5m' })}`;
});
afterAll(() => cleanupTestDb(tmpDir));
describe('networking hardening', () => {
let stackDir: string;
beforeEach(() => {
stackDir = path.join(process.env.COMPOSE_DIR as string, STACK);
fs.mkdirSync(stackDir, { recursive: true });
fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n web:\n image: nginx:latest\n');
stubHealthy();
});
afterEach(() => {
vi.restoreAllMocks();
fs.rmSync(stackDir, { recursive: true, force: true });
});
it('serves repeat overview reads from one computation within the TTL window', async () => {
const first = await request(app).get('/api/networking/overview').set('Authorization', authHeader);
expect(first.status).toBe(200);
const snapshotSpy = DockerController.getInstance(1).getDependencySnapshot as ReturnType<typeof vi.fn>;
const callsBefore = snapshotSpy.mock.calls.length;
const second = await request(app).get('/api/networking/overview').set('Authorization', authHeader);
expect(second.status).toBe(200);
expect(second.body.schemaVersion).toBe(first.body.schemaVersion);
expect(DockerController.getInstance(1).getDependencySnapshot as ReturnType<typeof vi.fn>).toHaveBeenCalledTimes(callsBefore);
});
it('serves real topology data when overview warmed the cache first', async () => {
await request(app).get('/api/networking/overview').set('Authorization', authHeader);
const res = await request(app)
.get('/api/networking/topology')
.set('Authorization', authHeader);
expect(res.status).toBe(200);
expect((res.body.networks as Array<{ name: string }>)).toContainEqual(
expect.objectContaining({ name: 'shared_ext' }),
);
});
it('reflects an exposure-intent change in the next overview read', async () => {
const initial = await request(app).get('/api/networking/overview').set('Authorization', authHeader);
expect(initial.status).toBe(200);
// The fixture publishes 8080:80 with no intent set, so the pre-write read
// must carry the unclassified finding; asserting both directions makes an
// invalidation regression loud instead of silently vacuous.
const hadUnclassifiedBefore = ((initial.body.findings ?? []) as Array<{ kind: string; stack?: string }>)
.some((f) => f.kind === 'exposure-unclassified' && f.stack === STACK);
expect(hadUnclassifiedBefore).toBe(true);
const put = await request(app)
.put(`/api/stacks/${STACK}/exposure`)
.set('Authorization', authHeader)
.send({ service: '', intent: 'internal' });
expect(put.status).toBe(200);
const res = await request(app).get('/api/networking/overview').set('Authorization', authHeader);
expect(res.status).toBe(200);
const unclassified = (res.body.findings as Array<{ kind: string; stack?: string }>)
.find((f) => f.kind === 'exposure-unclassified' && f.stack === STACK);
expect(unclassified).toBeUndefined();
DatabaseService.getInstance().deleteStackExposureIntents(1, STACK);
});
it('409s an unlabeled external network delete while a stack is unrenderable', async () => {
(ComposeService.getInstance(1).renderConfig as ReturnType<typeof vi.fn>).mockResolvedValue({
rendered: null,
stderr: 'redacted render failure',
code: 1,
timedOut: false,
});
const removeSpy = vi.spyOn(DockerController.getInstance(1), 'removeNetwork');
const res = await request(app)
.post('/api/system/networks/delete')
.set('Authorization', authHeader)
.send({ id: NET_ID });
expect(res.status).toBe(409);
expect(res.body.code).toBe('stack-declaration-unknown');
expect(removeSpy).not.toHaveBeenCalled();
});
it('allows deleting the same network once every stack renders again', async () => {
const removeSpy = vi.spyOn(DockerController.getInstance(1), 'removeNetwork');
const res = await request(app)
.post('/api/system/networks/delete')
.set('Authorization', authHeader)
.send({ id: NET_ID });
expect(res.status).toBe(200);
expect(removeSpy).toHaveBeenCalledOnce();
});
});
@@ -12,6 +12,7 @@ import { ComposeService } from '../services/ComposeService';
import { DatabaseService } from '../services/DatabaseService';
import { evaluateNetworkDeleteGuard } from '../services/network/networkDeleteGuards';
import SelfIdentityService from '../services/SelfIdentityService';
import { invalidateNodeNetworkingAggregate } from '../services/network/networkingAggregateCache';
let tmpDir: string;
let app: import('express').Express;
@@ -85,6 +86,7 @@ describe('networking operator routes', () => {
afterEach(() => {
vi.restoreAllMocks();
invalidateNodeNetworkingAggregate(1);
fs.rmSync(stackDir, { recursive: true, force: true });
});
@@ -196,6 +198,19 @@ describe('evaluateNetworkDeleteGuard', () => {
expect(guard.code).toBe('stack-declaration-unknown');
});
it('fails closed for an unlabeled external network while a stack is unrenderable', () => {
const snapshot = {
containers: [],
networks: [{ id: 'n1', name: 'shared_ext', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null }],
volumes: [],
};
const guard = evaluateNetworkDeleteGuard('n1', snapshot, [
{ stack: STACK, renderable: false, renderError: 'compose file invalid', runtime: 'available', networks: [{ key: 'shared_ext', name: 'shared_ext', external: true, internal: false, createdByStack: false }], services: [], drift: { runtimeOnlyAttachments: [], declaredButUnused: [], missingFromRuntime: [], foreignNetworkAttachments: [] }, missingExternalNetworks: [] },
]);
expect(guard.blocked).toBe(true);
expect(guard.code).toBe('stack-declaration-unknown');
});
it('blocks a system network ahead of every other reason', () => {
const snapshot = {
containers: [], volumes: [],
@@ -13,6 +13,7 @@ import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './he
import DockerController from '../services/DockerController';
import { ComposeService } from '../services/ComposeService';
import { DatabaseService } from '../services/DatabaseService';
import { invalidateNodeNetworkingAggregate } from '../services/network/networkingAggregateCache';
let tmpDir: string;
let app: import('express').Express;
@@ -140,6 +141,7 @@ describe('networking collision correctness', () => {
afterEach(() => {
vi.restoreAllMocks();
invalidateNodeNetworkingAggregate(1);
fs.rmSync(dirA, { recursive: true, force: true });
fs.rmSync(dirB, { recursive: true, force: true });
});
+6
View File
@@ -1,5 +1,6 @@
import { CacheService } from '../services/CacheService';
import { StackFileRootsService } from '../services/StackFileRootsService';
import { invalidateNodeNetworkingAggregate } from '../services/network/networkingAggregateCache';
export const REMOTE_META_NAMESPACE = 'remote-meta';
@@ -17,6 +18,11 @@ export function invalidateNodeCaches(nodeId: number): void {
cache.invalidate(`stats:${nodeId}`);
cache.invalidate(`stack-statuses:${nodeId}`);
cache.invalidate('project-name-map');
// Stack and container mutations (create/delete/edit/deploy/rename/prune),
// auto-created externals during deploy, and network create/delete all reshape
// the networking aggregate; drop the memo (every variant) so the next read
// reflects the change immediately.
invalidateNodeNetworkingAggregate(nodeId);
StackFileRootsService.invalidateNode(nodeId);
}
+17 -3
View File
@@ -27,6 +27,8 @@ import { requirePermission } from '../middleware/permissions';
import { buildStackNetworkFacts } from '../services/network/composeNetworkInspector';
import { evaluateNetworkDeleteGuard } from '../services/network/networkDeleteGuards';
import { loadNetworkingSnapshot } from '../services/network/networkingAggregate';
import { mapWithConcurrency } from '../utils/mapWithConcurrency';
import { withComposeRenderSlot } from '../services/network/composeRenderSemaphore';
// The prune estimate and plan paths are bounded at 12 s. `docker system df`
// cost scales with image-store size (measured ~7.4 s on a 34 GB store), so
@@ -603,16 +605,28 @@ systemMaintenanceRouter.post('/networks/delete', async (req: Request, res: Respo
}
if (rejectIfSelf('network', id, res)) return;
const guardStartedAt = Date.now();
const { stacks, snapshot } = await loadNetworkingSnapshot(req.nodeId);
if (!snapshot) {
return res.status(503).json({ error: 'Docker networking runtime is unavailable' });
}
const stackFacts = await Promise.all(
stacks.map(stack => buildStackNetworkFacts(req.nodeId, stack, snapshot)),
);
const stackFacts = await mapWithConcurrency(stacks, 4, stack => withComposeRenderSlot(
req.nodeId,
() => buildStackNetworkFacts(req.nodeId, stack, snapshot),
));
const baseRow = DockerController.classifySnapshotNetworks(snapshot, stacks)
.find(n => n.id === id);
const guard = evaluateNetworkDeleteGuard(id, snapshot, stackFacts, baseRow);
if (isDebugEnabled()) {
console.debug('[Resources:debug] Network delete guard', {
id: id.substring(0, 12),
ms: Date.now() - guardStartedAt,
stackCount: stacks.length,
unrenderableStacks: stackFacts.filter(f => !f.renderable).length,
blocked: guard.blocked,
code: guard.code ?? null,
});
}
if (guard.blocked) {
return res.status(409).json({ error: guard.error, code: guard.code });
}
+6
View File
@@ -8,6 +8,7 @@ import { applySuppressions } from '../utils/suppression-filter';
import { SENCHO_ROLLBACK_HOLD_SQL_LIKE } from '../utils/senchoRollbackHold';
import type { AuditStatsInput } from './AuditAnomalyService';
import { EXPOSURE_INTENTS, type ExposureIntent } from './network/types';
import { invalidateNodeNetworkingAggregate } from './network/networkingAggregateCache';
import { HIGH_EPSS_THRESHOLD } from './securityPosture';
import type { BackendScheduledAction } from './scheduledActionRegistry';
import { stackPatternMatches } from '../helpers/stackPattern';
@@ -3852,11 +3853,13 @@ export class DatabaseService {
now,
now
);
invalidateNodeNetworkingAggregate(nodeId);
return this.getStackDossier(nodeId, stackName) as StackDossier;
}
public deleteStackDossier(nodeId: number, stackName: string): void {
this.db.prepare('DELETE FROM stack_dossiers WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
invalidateNodeNetworkingAggregate(nodeId);
}
/**
@@ -3947,16 +3950,19 @@ export class DatabaseService {
updated_at = excluded.updated_at,
updated_by = excluded.updated_by`
).run(nodeId, stackName, service, intent, Date.now(), updatedBy);
invalidateNodeNetworkingAggregate(nodeId);
}
/** Clear one intent row, leaving that scope unset; consumers treat a service with no row as inheriting the stack intent. */
public deleteStackExposureIntent(nodeId: number, stackName: string, service: string): void {
this.db.prepare('DELETE FROM stack_exposure_intent WHERE node_id = ? AND stack_name = ? AND service = ?').run(nodeId, stackName, service);
invalidateNodeNetworkingAggregate(nodeId);
}
/** Clear every intent row for a stack (used when the stack is deleted). */
public deleteStackExposureIntents(nodeId: number, stackName: string): void {
this.db.prepare('DELETE FROM stack_exposure_intent WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
invalidateNodeNetworkingAggregate(nodeId);
}
// --- Stack Exposure (Compose reachability descriptor) ---
@@ -59,7 +59,10 @@ export function evaluateNetworkDeleteGuard(
return { blocked: true, code: 'stack-declared', error: 'This network is declared by a Compose stack.' };
}
if (hasUnrenderable && (net.composeProject || net.stack)) {
// Fail closed while any stack is unrenderable: an external network normally
// carries no compose project label, so without this check a broken declaring
// stack would let its declared network slip through as "undeclared".
if (hasUnrenderable) {
return {
blocked: true,
code: 'stack-declaration-unknown',
@@ -19,10 +19,40 @@ import { getErrorMessage } from '../../utils/errors';
import { sanitizeForLog } from '../../utils/safeLog';
import { mapWithConcurrency } from '../../utils/mapWithConcurrency';
import { withComposeRenderSlot } from './composeRenderSemaphore';
import { fetchNodeNetworkingAggregateWithMeta as fetchMemoized, NETWORKING_AGGREGATE_TTL_MS } from './networkingAggregateCache';
import { isDebugEnabled } from '../../utils/debug';
export async function buildNodeNetworkingAggregate(
nodeId: number,
options: { includeTopology?: boolean; includeSystem?: boolean },
): Promise<NodeNetworkingAggregate> {
const startedAt = Date.now();
const { value: aggregate, outcome } = await fetchMemoized(nodeId, options, () => computeNodeNetworkingAggregate(nodeId, options));
if (outcome === 'stale') {
// Stale-on-error fallback: the recompute threw and the memo served the
// last good aggregate. Mark it so the UI can say so instead of
// presenting confidently stale data as fresh.
aggregate.overview.degradedCache = true;
}
if (isDebugEnabled()) {
console.debug('[Networking:debug] Aggregate served', {
nodeId,
outcome,
ms: Date.now() - startedAt,
stackCount: aggregate.stackFacts.length,
networkCount: aggregate.overview.networkCount,
findingCount: aggregate.findings.length,
renderFailedStacks: aggregate.overview.renderFailedStacks.length,
ttlMs: NETWORKING_AGGREGATE_TTL_MS,
variant: options.includeTopology === true ? 'topology' : 'base',
});
}
return aggregate;
}
async function computeNodeNetworkingAggregate(
nodeId: number,
options: { includeTopology?: boolean; includeSystem?: boolean },
): Promise<NodeNetworkingAggregate> {
const fsSvc = FileSystemService.getInstance(nodeId);
const stacks = await fsSvc.getStacks();
@@ -138,6 +168,7 @@ function buildOverview(
f.kind === 'network-name-collision' || f.kind === 'alias-collision' || f.kind === 'service-name-collision',
).length,
findingCount: findings.length,
degradedCache: false,
renderFailedStacks,
};
}
@@ -0,0 +1,39 @@
/**
* Short-TTL memo for the per-node networking aggregate. Read endpoints share one
* computation per window instead of re-rendering every stack per request;
* mutations invalidate eagerly so deletes and creates never serve a stale view
* beyond the request that performed them. Kept as a leaf module so mutation
* helpers and services can invalidate without importing the aggregate pipeline
* back into themselves.
*
* The cache key encodes the requested variant (topology / includeSystem) as
* well as the node: a plain overview read must never satisfy a topology
* request, whose aggregate additionally carries the topology graph.
*/
import { CacheService, type CacheFetchOutcome } from '../CacheService';
import type { NodeNetworkingAggregate } from './networkingTypes';
const NAMESPACE = 'networking-aggregate';
export const NETWORKING_AGGREGATE_TTL_MS = 8_000;
export type NetworkingAggregateOptions = { includeTopology?: boolean; includeSystem?: boolean };
export function networkingAggregateCacheKey(nodeId: number, options: NetworkingAggregateOptions): string {
if (options.includeTopology === true) {
return `${NAMESPACE}:${nodeId}:topology:${options.includeSystem === true}`;
}
return `${NAMESPACE}:${nodeId}:base`;
}
export async function fetchNodeNetworkingAggregateWithMeta(
nodeId: number,
options: NetworkingAggregateOptions,
compute: () => Promise<NodeNetworkingAggregate>,
): Promise<{ value: NodeNetworkingAggregate; outcome: CacheFetchOutcome }> {
return CacheService.getInstance().getOrFetchWithMeta(networkingAggregateCacheKey(nodeId, options), NETWORKING_AGGREGATE_TTL_MS, compute);
}
export function invalidateNodeNetworkingAggregate(nodeId: number): void {
CacheService.getInstance().invalidateNamespace(`${NAMESPACE}:${nodeId}`);
}
@@ -138,6 +138,8 @@ export interface NodeNetworkingOverview {
missingExternalCount: number;
networkCollisionCount: number;
findingCount: number;
/** True when this aggregate was served from the memo after a recompute failure (stale-on-error fallback). */
degradedCache: boolean;
renderFailedStacks: string[];
}
+8
View File
@@ -292,6 +292,14 @@ To re-test connectivity after making changes, open **Settings · Infrastructure
---
## Network deletion is refused while a stack fails to render
**Symptom:** Deleting an external network returns "Cannot verify stack declarations while one or more stacks failed to render." The delete button may also be disabled with a message about render failures.
**Cause:** Sencho checks every stack's effective Compose model before allowing a network delete, so it never removes a network a stack declares as `external:`. When any stack on the node cannot be rendered (usually a YAML error, an unresolved variable, or a broken include), Sencho cannot prove the network is undeclared and refuses the deletion rather than risk removing a network your stacks depend on.
**Fix:** Find which stack cannot render on the Networking page (the Overview tab lists how many stacks failed), then open that stack and check its Anatomy panels: the Networking panel and the deploy preflight check show the Compose error itself. Fix the YAML (an unresolved variable or a broken include produces the same failure) until the model renders again, then retry the deletion. This is a deliberate safety stop, not a malfunction.
## Cannot delete a system network (bridge, host, none)
**Symptom:** The delete button is missing or disabled for certain networks.
+1 -1
View File
@@ -709,7 +709,7 @@ Node-level safety checks and post-deploy observation used during stack deploys a
| **Observe health after updates** | On | After a stack deploy or update succeeds, watch its containers for the observation window and record a passed or failed verdict on the stack timeline. Observational only: nothing is restarted or rolled back automatically. |
| **Observation window** | 90 s | How long to watch containers before declaring the update healthy. Raise it for stacks that take a while to settle. Range 15 to 600 seconds. |
| **Block deploy on missing required env vars** | Off | When on, a deploy or update is refused before it starts if a required `${VAR:?message}` variable is unset or empty, so the stack fails fast with a clear message instead of mid-deploy. |
| **Automatically create missing external networks during deploy** | Off | When on, safe missing external bridge networks are created automatically before deploy continues. When off, interactive deploy prompts first. Advanced drivers and custom options are never auto-created. |
| **Automatically create missing external networks during deploy** | Off | When on, safe missing external bridge networks are created automatically before deploy continues. When off, interactive deploy prompts first. Advanced drivers and custom options are never auto-created. Any deploy that triggers an auto-creation records a `network_auto_created` entry in Recent activity, including deploys started by non-admin operators. |
Click **Save settings** to apply.
@@ -57,14 +57,20 @@ function NetworkTableSkeleton({ rows = 5 }: { rows?: number }) {
/** A network is unsafe to delete without a pre-confirm explanation when it has
* connected containers or is declared by a stack's Compose file; the backend
* 409-guards these, but the UI should explain BEFORE the confirm dialog rather
* than let a generic confirmation surprise the user with a rejection. */
function deleteBlockReason(row: NetworkingNetworkRow): string | null {
* than let a generic confirmation surprise the user with a rejection. When render
* verification is unavailable (a stack failed to render, or the Docker runtime is
* unreachable), declarations cannot be verified at all, so every non-system,
* non-Sencho network is held back the same way the backend does. */
function deleteBlockReason(row: NetworkingNetworkRow, renderVerificationUnavailable: boolean): string | null {
if (row.connectedCount > 0) {
return `Connected to ${row.connectedCount} container${row.connectedCount === 1 ? '' : 's'}; disconnect them first.`;
}
if (row.declaredByStacks.length > 0) {
return `Declared by ${row.declaredByStacks.join(', ')}; remove the declaration first.`;
}
if (renderVerificationUnavailable) {
return 'One or more stacks failed to render; stack declarations cannot be verified right now.';
}
return null;
}
@@ -150,6 +156,7 @@ export function NetworkInventoryTable({
onDelete,
onOpenStack,
onFilterTopology,
renderVerificationUnavailable,
}: {
rows: NetworkingNetworkRow[];
findings: NetworkingFinding[];
@@ -159,6 +166,7 @@ export function NetworkInventoryTable({
onDelete: (id: string, name: string) => void;
onOpenStack: (stack: string) => void;
onFilterTopology: (name: string) => void;
renderVerificationUnavailable: boolean;
}) {
const [filter, setFilter] = useState<NetworkFilter>('all');
const [search, setSearch] = useState('');
@@ -295,7 +303,7 @@ export function NetworkInventoryTable({
<TooltipContent>Show in topology</TooltipContent>
</Tooltip>
{isAdmin && (() => {
const blockReason = deleteBlockReason(row);
const blockReason = deleteBlockReason(row, renderVerificationUnavailable);
const protectedReason = row.isSencho
? 'Protected · running Sencho instance'
: row.isSystem
@@ -23,6 +23,7 @@ export function NetworkingFindingsList({
isAdmin,
onAction,
disabled = false,
nodeId,
}: {
findings: NetworkingFinding[];
loading: boolean;
@@ -30,6 +31,7 @@ export function NetworkingFindingsList({
isAdmin: boolean;
onAction: (action: NetworkingRecommendedAction) => void | Promise<void>;
disabled?: boolean;
nodeId: number | null | undefined;
}) {
if (loading) return <p className="text-sm text-muted-foreground">Loading findings</p>;
if (findings.length === 0) {
@@ -66,7 +68,7 @@ export function NetworkingFindingsList({
{items.map((finding, i) => {
const sourceLabel = findingSourceLabel(finding);
const primary = finding.recommendedActions.find((action) =>
isNetworkingActionVisible(action, isAdmin, (stack) => canEdit('stack:edit', 'stack', stack)),
isNetworkingActionVisible(action, isAdmin, (stack) => canEdit('stack:edit', 'stack', stack, nodeId)),
);
return (
<TableRow
@@ -1,4 +1,4 @@
import { useCallback, useEffect, useState, type ReactNode } from 'react';
import { useCallback, useEffect, useRef, useState, type ReactNode } from 'react';
import {
LayoutDashboard, Network, GitBranch, AlertTriangle, RefreshCw, Plus, Unplug,
} from 'lucide-react';
@@ -14,6 +14,7 @@ import { toast } from '@/components/ui/toast-store';
import { useAuth } from '@/context/AuthContext';
import { useNodes } from '@/context/NodeContext';
import { useIsMobile } from '@/hooks/use-is-mobile';
import { useDeveloperMode } from '@/hooks/useDeveloperMode';
import { Masthead, MobileSubTabs, type Tone } from '@/components/mobile/mobile-ui';
import { springs } from '@/lib/motion';
import { CreateNetworkDialog } from '@/components/resources/CreateNetworkDialog';
@@ -86,6 +87,11 @@ export function NetworkingView({ headerActions }: NetworkingViewProps) {
const { isAdmin, can } = useAuth();
const { activeNode } = useNodes();
const isMobile = useIsMobile();
const developerMode = useDeveloperMode(activeNode?.id);
// Read the flag through a ref inside the load effect so flipping it does not
// retrigger the overview fetch (the flag only gates debug logging).
const developerModeRef = useRef(developerMode);
developerModeRef.current = developerMode;
const nodeId = activeNode?.id;
const [tab, setTab] = useState<NetworkingTab>('overview');
@@ -140,6 +146,7 @@ export function NetworkingView({ headerActions }: NetworkingViewProps) {
setFindings([]);
setRecentActivity([]);
setIsLegacy(false);
const startedAt = Date.now();
const load = async () => {
try {
const response = await apiFetch('/networking/overview', { nodeId, signal: controller.signal });
@@ -150,6 +157,13 @@ export function NetworkingView({ headerActions }: NetworkingViewProps) {
if (!response.ok) throw new Error('Failed to load networking data.');
const body = await response.json() as Partial<NetworkingOverviewEnvelope>;
if (stale) return;
if (developerModeRef.current) {
console.debug('[Networking:debug] overview loaded', {
nodeId,
ms: Date.now() - startedAt,
schemaVersion: body.schemaVersion ?? null,
});
}
const adapted = adaptNetworkingOverview(body);
setIsLegacy(adapted.isLegacy);
setRuntimeAvailable(adapted.runtimeAvailable);
@@ -292,6 +306,11 @@ export function NetworkingView({ headerActions }: NetworkingViewProps) {
<CardContent className="p-3 text-sm text-warning">Docker runtime is unavailable. Compose-model signals remain available.</CardContent>
</Card>
)}
{overview.degradedCache && (
<Card className="border-warning/40 bg-warning/5">
<CardContent className="p-3 text-sm text-warning">Showing cached results from the last successful refresh; a live refresh failed.</CardContent>
</Card>
)}
<div className="grid overflow-hidden rounded-lg border border-card-border bg-card shadow-card-bevel sm:grid-cols-2 xl:grid-cols-4">
{[
{ label: 'Networks', value: overview.networkCount ?? '—', tab: 'networks' as const },
@@ -526,6 +545,7 @@ export function NetworkingView({ headerActions }: NetworkingViewProps) {
setPendingTopologyFilter(networkName);
setTab('topology');
}}
renderVerificationUnavailable={(overview?.renderFailedStacks.length ?? 0) > 0 || !runtimeAvailable}
/>
</TabsContent>
@@ -547,7 +567,7 @@ export function NetworkingView({ headerActions }: NetworkingViewProps) {
</TabsContent>
<TabsContent value="findings" className="mt-4">
<NetworkingFindingsList findings={findings} loading={loading} canEdit={can} isAdmin={isAdmin} onAction={dispatchAction} disabled={isLegacy} />
<NetworkingFindingsList findings={findings} loading={loading} canEdit={can} isAdmin={isAdmin} onAction={dispatchAction} disabled={isLegacy} nodeId={nodeId} />
</TabsContent>
</Tabs>
@@ -29,6 +29,7 @@ describe('NetworkInventoryTable', () => {
onDelete={vi.fn()}
onOpenStack={vi.fn()}
onFilterTopology={vi.fn()}
renderVerificationUnavailable={false}
/>,
);
// Default sort is name ascending, independent of input row order.
@@ -53,6 +54,7 @@ describe('NetworkInventoryTable', () => {
onDelete={vi.fn()}
onOpenStack={vi.fn()}
onFilterTopology={vi.fn()}
renderVerificationUnavailable={false}
/>,
);
const actionButtons = screen.getAllByRole('button').filter((b) =>
@@ -62,4 +64,40 @@ describe('NetworkInventoryTable', () => {
'Open app', 'Inspect a-net', 'Show a-net in topology', 'Delete a-net',
]);
});
it('holds the delete affordance while render verification is unavailable', () => {
const unlabeledExternal = row({ id: '1', name: 'shared_ext', composeProject: null, stack: null });
render(
<NetworkInventoryTable
rows={[unlabeledExternal]}
findings={[]}
loading={false}
isAdmin
onInspect={vi.fn()}
onDelete={vi.fn()}
onOpenStack={vi.fn()}
onFilterTopology={vi.fn()}
renderVerificationUnavailable
/>,
);
expect(screen.getByRole('button', { name: 'Delete shared_ext' })).toBeDisabled();
});
it('keeps delete enabled for the same network when renders succeed', () => {
const unlabeledExternal = row({ id: '1', name: 'shared_ext', composeProject: null, stack: null });
render(
<NetworkInventoryTable
rows={[unlabeledExternal]}
findings={[]}
loading={false}
isAdmin
onInspect={vi.fn()}
onDelete={vi.fn()}
onOpenStack={vi.fn()}
onFilterTopology={vi.fn()}
renderVerificationUnavailable={false}
/>,
);
expect(screen.getByRole('button', { name: 'Delete shared_ext' })).toBeEnabled();
});
});
@@ -2,7 +2,6 @@ import { describe, it, expect, vi } from 'vitest';
import { render, screen } from '@testing-library/react';
import { NetworkingFindingsList } from '../NetworkingFindingsList';
import type { NetworkingFinding } from '@/types/networking';
function finding(overrides: Partial<NetworkingFinding> = {}): NetworkingFinding {
return {
id: overrides.id ?? Math.random().toString(36),
@@ -20,9 +19,18 @@ function finding(overrides: Partial<NetworkingFinding> = {}): NetworkingFinding
const canEdit = () => true;
function exposureFinding(): NetworkingFinding {
return finding({
id: 'exposure',
kind: 'exposure-unclassified',
title: 'Unclassified exposure',
recommendedActions: [{ kind: 'set-exposure-intent', label: 'Set exposure intent', stack: 'proxy' }],
});
}
describe('NetworkingFindingsList', () => {
it('shows a calm empty state when there are no findings', () => {
render(<NetworkingFindingsList findings={[]} loading={false} canEdit={canEdit} isAdmin onAction={vi.fn()} />);
render(<NetworkingFindingsList findings={[]} loading={false} canEdit={canEdit} isAdmin onAction={vi.fn()} nodeId={1} />);
expect(screen.getByText('No networking issues detected.')).toBeInTheDocument();
});
@@ -38,6 +46,7 @@ describe('NetworkingFindingsList', () => {
canEdit={canEdit}
isAdmin
onAction={vi.fn()}
nodeId={1}
/>,
);
expect(screen.getByText(/Needs action/)).toBeInTheDocument();
@@ -45,6 +54,18 @@ describe('NetworkingFindingsList', () => {
expect(screen.getByText(/Informational/)).toBeInTheDocument();
});
it('respects node-scoped stack:edit for the primary action', () => {
const scopedCanEdit = vi.fn((_action: string, _type?: string, _id?: string, nodeId?: number | null) => nodeId === 7);
render(<NetworkingFindingsList findings={[exposureFinding()]} loading={false} canEdit={scopedCanEdit} isAdmin={false} onAction={vi.fn()} nodeId={7} />);
expect(screen.getByRole('button', { name: 'Set exposure intent' })).toBeInTheDocument();
});
it('hides the primary action when the node scope does not match', () => {
const deniedCanEdit = vi.fn((_action: string, _type?: string, _id?: string, nodeId?: number | null) => nodeId === 8);
render(<NetworkingFindingsList findings={[exposureFinding()]} loading={false} canEdit={deniedCanEdit} isAdmin={false} onAction={vi.fn()} nodeId={7} />);
expect(screen.queryByRole('button', { name: 'Set exposure intent' })).not.toBeInTheDocument();
});
it('shows the merged source label for a card found by both engines', () => {
render(
<NetworkingFindingsList
@@ -56,6 +77,7 @@ describe('NetworkingFindingsList', () => {
canEdit={canEdit}
isAdmin
onAction={vi.fn()}
nodeId={1}
/>,
);
expect(screen.getByText('Live · also found by Doctor')).toBeInTheDocument();
+2
View File
@@ -128,6 +128,8 @@ export interface NodeNetworkingOverview {
missingExternalCount: number;
networkCollisionCount: number;
findingCount: number;
/** Served from the backend memo after a recompute failure; data may lag recent changes. */
degradedCache?: boolean;
renderFailedStacks: string[];
}